:jDddlmZddlZddlmZddlmZmZmZm Z m Z m Z ddl m Z erddlZddlZddlmZGddZej(d d d Zd d Zy)) annotationsN)ThreadPoolExecutor) TYPE_CHECKING AbstractSet CollectionLiteralNoReturnSequence) _tiktokenceZdZdd d$dZd%dZd&dZedd d'dZedd d(d Zd d d)d Z d edd  d*dZ edd d+dZ d,dZ d-dZ d.d/dZd0dZd1dZd2dZdd d d3dZd d  d4dZd5dZed6dZej0d7dZd8dZed6dZd9dZd&d Zd:d!Zd;d"Zd)rrs r"__repr__zEncoding.__repr__;sDII=**r$c |jj|S#t$r@|jddj dd}|jj|cYSwxYw)zEncodes a string into tokens, ignoring special tokens. This is equivalent to `encode(text, disallowed_special=())` (but slightly faster). ``` >>> enc.encode_ordinary("hello world") [31373, 995] utf-16 surrogatepassreplace)rencode_ordinaryUnicodeEncodeErrorencodedecodertexts r"r-zEncoding.encode_ordinaryBs] 8>>11$7 7! 8;;x9@@9UD>>11$7 7 8sAA&%A&allallowed_specialdisallowed_specialc|dk(r |j}|dk(r|j|z }|rPt|ts t|}t|j |x}rt |j  |jj||S#t$rA|jddjdd}|jj||cYSwxYw)aHEncodes a string into tokens. Special tokens are artificial tokens used to unlock capabilities from a model, such as fill-in-the-middle. So we want to be careful about accidentally encoding special tokens, since they can be used to trick a model into doing something we don't want it to do. Hence, by default, encode will raise an error if it encounters text that corresponds to a special token. This can be controlled on a per-token level using the `allowed_special` and `disallowed_special` parameters. In particular: - Setting `disallowed_special` to () will prevent this function from raising errors and cause all text corresponding to special tokens to be encoded as natural text. - Setting `allowed_special` to "all" will cause this function to treat all text corresponding to special tokens to be encoded as special tokens. ``` >>> enc.encode("hello world") [31373, 995] >>> enc.encode("<|endoftext|>", allowed_special={"<|endoftext|>"}) [50256] >>> enc.encode("<|endoftext|>", allowed_special="all") [50256] >>> enc.encode("<|endoftext|>") # Raises ValueError >>> enc.encode("<|endoftext|>", disallowed_special=()) [27, 91, 437, 1659, 5239, 91, 29] ``` r3r*r+r,) special_tokens_set isinstance frozenset_special_token_regexsearchraise_disallowed_special_tokengrouprr/r.r0rr2r5r6matchs r"r/zEncoding.encodeRsD e #"55O  &!%!8!8?!J  0)<%./A%B",-?@GGMMuM.u{{}= @>>((? ?! @;;x9@@9UD>>((? ? @s9BACCcl|dk(r |j}|dk(r|j|z }|rPt|ts t|}t|j |x}rt |j ddl}|jj||}|j||jS)zEncodes a string into tokens, returning a numpy array. Avoids the overhead of copying the token buffer into a Python list. r3rN)dtype) r8r9r:r;r<r=r>numpyrencode_to_tiktoken_buffer frombufferuint32)rr2r5r6r@npbuffers r"encode_to_numpyzEncoding.encode_to_numpys e #"55O  &!%!8!8?!J  0)<%./A%B",-?@GGMMuM.u{{}=99$Pr}}V29955r$) num_threadsctj|j}t|5}t |j ||cdddS#1swYyxYw)aDEncodes a list of strings into tokens, in parallel, ignoring special tokens. This is equivalent to `encode_batch(text, disallowed_special=())` (but slightly faster). ``` >>> enc.encode_ordinary_batch(["hello world", "goodbye world"]) [[31373, 995], [11274, 16390, 995]] ``` N) functoolspartialr-rlistmap)rr2rKencoderes r"encode_ordinary_batchzEncoding.encode_ordinary_batchsL##D$8$89  , .gt,- . . .s AA)rKr5r6c>|dk(r |j}|dk(r|j|z }t|ts t|}tj|j ||}t |5}t|j||cdddS#1swYyxYw)aEncodes a list of strings into tokens, in parallel. See `encode` for more details on `allowed_special` and `disallowed_special`. ``` >>> enc.encode_batch(["hello world", "goodbye world"]) [[31373, 995], [11274, 16390, 995]] ``` r3r4N) r8r9r:rMrNr/rrOrP)rr2rKr5r6rQrRs r" encode_batchzEncoding.encode_batchs" e #"55O  &!%!8!8?!J ,i8!*+=!> ## KKM_  , .gt,- . . .s .BBc(|dk(r |j}|dk(r|j|z }|rPt|ts t|}t|j |x}rt |j |jj||S)aEncodes a string into stable tokens and possible completion sequences. Note that the stable tokens will only represent a substring of `text`. See `encode` for more details on `allowed_special` and `disallowed_special`. This API should itself be considered unstable. ``` >>> enc.encode_with_unstable("hello fanta") ([31373], [(277, 4910), (5113, 265), ..., (8842,)]) >>> text = "..." >>> stable_tokens, completions = enc.encode_with_unstable(text) >>> assert text.encode().startswith(enc.decode_bytes(stable_tokens)) >>> assert all(enc.decode_bytes(stable_tokens + seq).startswith(text.encode()) for seq in completions) ``` r3) r8r9r:r;r<r=r>rencode_with_unstabler?s r"rWzEncoding.encode_with_unstables2 e #"55O  &!%!8!8?!J  0)<%./A%B",-?@GGMMuM.u{{}=~~224IIr$czt|tr|jd}|jj |S)aEncodes text corresponding to a single token to its token value. NOTE: this will encode all special tokens. Raises `KeyError` if the token is not in the vocabulary. ``` >>> enc.encode_single_token("hello") 31373 ``` utf-8)r9strr/rencode_single_tokenr text_or_bytess r"r[zEncoding.encode_single_tokens3 mS ))009M~~11-@@r$c8|jj|S)zDecodes a list of tokens into bytes. ``` >>> enc.decode_bytes([31373, 995]) b'hello world' ``` )r decode_bytes)rtokenss r"r_zEncoding.decode_bytes s~~**622r$r,cZ|jj|jd|S)auDecodes a list of tokens into a string. WARNING: the default behaviour of this function is lossy, since decoded bytes are not guaranteed to be valid UTF-8. You can control this behaviour using the `errors` parameter, for instance, setting `errors=strict`. ``` >>> enc.decode([31373, 995]) 'hello world' ``` rYerrors)rr_r0)rr`rcs r"r0zEncoding.decodes)~~**6299'&9QQr$c8|jj|S)zDecodes a token into bytes. NOTE: this will decode all special tokens. Raises `KeyError` if the token is not in the vocabulary. ``` >>> enc.decode_single_token_bytes(31373) b'hello' ``` )rdecode_single_token_bytesrtokens r"rez"Encoding.decode_single_token_bytes!s~~77>>r$cJ|Dcgc]}|j|c}Scc}w)zDecodes a list of tokens into a list of bytes. Useful for visualising tokenisation. >>> enc.decode_tokens_bytes([31373, 995]) [b'hello', b' world'] )re)rr`rgs r"decode_tokens_byteszEncoding.decode_tokens_bytes/s%DJJ%..u5JJJs c |j|}d}g}|D]F}|jtd|d|dcxkxrdkncz |td|Dz }Hdj |j dd}||fS) a.Decodes a list of tokens into a string and a list of offsets. Each offset is the index into text corresponding to the start of each token. If UTF-8 character boundaries do not line up with token boundaries, the offset is the index of the first character that contains bytes from the token. This will currently raise if given tokens that decode to invalid UTF-8; this behaviour may change in the future to be more permissive. >>> enc.decode_with_offsets([31373, 995]) ('hello world', [0, 5]) rc3<K|]}d|cxkrdkrndyw)rkrlrN).0cs r" z/Encoding.decode_with_offsets..KsE!DA4D4DAEs r$rYstrictrb)riappendrsumjoinr0)rr` token_bytestext_lenoffsetsrgr2s r"decode_with_offsetszEncoding.decode_with_offsets8s..v6   FE NN3q(deAh.E.E"FG H EuEE EH F xx $++GH+EW}r$)rcrKctj|j|}t|5}t |j ||cdddS#1swYyxYw)zADecodes a batch (list of lists of tokens) into a list of strings.rbN)rMrNr0rrOrP)rbatchrcrKdecoderrRs r" decode_batchzEncoding.decode_batchQsL##DKK?  , /gu-. / / /s AAct|5}t|j|j|cdddS#1swYyxYw)z?Decodes a batch (list of lists of tokens) into a list of bytes.N)rrOrPr_)rr{rKrRs r"decode_bytes_batchzEncoding.decode_bytes_batchYs< , 9d//78 9 9 9s %;Ac6|jjS)z*Returns the list of all token byte values.)rtoken_byte_valuesr's r"rzEncoding.token_byte_valuesds~~//11r$c |jdS)Nz <|endoftext|>)rr's r" eot_tokenzEncoding.eot_tokenhs##O44r$cHt|jjSN)rrkeysr's r"r8zEncoding.special_tokens_setls4'',,.//r$cBt|tsJ||jvSr)r9intrrfs r"is_special_tokenzEncoding.is_special_tokenps#%%%%2222r$c |jdzS)zEFor backwards compatibility. Prefer to use `enc.max_token_value + 1`.r)rr's r"n_vocabzEncoding.n_vocabts##a''r$czt|tr|jd}|jj |S)zEncodes text corresponding to bytes without a regex split. NOTE: this will not encode any special tokens. ``` >>> enc.encode_single_piece("helloqqqq") [31373, 38227, 38227] ``` rY)r9rZr/rencode_single_piecer\s r"_encode_single_piecezEncoding._encode_single_piece}s3 mS ))009M~~11-@@r$cddl}|j|j}g}|j||D];}|j |j j |jd=|S)z?Encodes a string into tokens, but do regex splitting in Python.rNrY)regexcompilerfindallextendrrr/)rr2r _unused_patretpieces r"_encode_only_native_bpez Encoding._encode_only_native_bpesc mmDMM2 ]];5 RE JJt~~99%,,w:OP Q R r$c8|jj|Sr)r _encode_bytesr1s r"rzEncoding._encode_bytess~~++D11r$cddl}||jjj|jur |jS|j|j |j |jdS)Nr)rrr r!)tiktoken.registryregistry ENCODINGSgetrrrr)rtiktokens r" __getstate__zEncoding.__getstate__s^  8$$..22499= =99 II}}#44"22   r$cddl}t|tr+|jj |j |_y|j di|y)Nrrn)rr9rZr get_encoding__dict__r#)rvaluers r" __setstate__zEncoding.__setstate__sA eS !$--::5AJJDM  r$) rrZrrZr zdict[bytes, int]r!zdict[str, int]rz int | None)returnrZ)r2rZr list[int])r2rZr5!Literal['all'] | AbstractSet[str]r6 Literal['all'] | Collection[str]rr)r2rZr5rr6rrznpt.NDArray[np.uint32])r2 list[str]rKrrlist[list[int]]) r2rrKrr5rr6rrr)r2rZr5rr6rrz!tuple[list[int], list[list[int]]])r] str | bytesrr)r` Sequence[int]rbytes)r,)r`rrcrZrrZ)rgrrr)r`rr list[bytes])r`rrztuple[str, list[int]])r{Sequence[Sequence[int]]rcrZrKrrr)r{rrKrrr)rr)rr)rzset[str])rgrrbool)r]rrr)r2rrr)robject)rrrNone)!__name__ __module__ __qualname__r#r(r-rr/rIrSrUrWr[r_r0reriryr}rrpropertyrrMcached_propertyr8rrrrrrrrnr$r"r r s$(,(U(U (U * (U ' (U%(UT+8(>AU?D 6@6@; 6@ = 6@  6@x>AU?D 66; 6 = 6 64LM .$=@U?D .. . ; . = . .D>AU?D #J#J; #J = #J + #JJA(3 R ?K4@I]^/,/9</WZ/ /EF9,9>A9 9255003(( A 2  r$r rk)maxsizec ddldjfd|D}jd|dS#t$rddlY?wxYw)Nr|c3BK|]}j|ywr)escape)rorgres r"rqz'_special_token_regex..s:%YRYYu%:s())r ImportErrorrrur)r`innerrs @r"r;r;sN HH:6: :E 2::%l ## s7 AAc ,td|d|d|d)Nz;Encountered text corresponding to disallowed special token zo. If you want this text to be encoded as a special token, pass it to `allowed_special`, e.g. `allowed_special={z, ...}`. If you want this text to be encoded as normal text, disable the check for this token by passing `disallowed_special=(enc.special_tokens_set - {zQ})`. To disable this check for all special tokens, pass `disallowed_special=()`. ) ValueError)rgs r"r=r=s;  EeYOAAF JFFKYOX  X r$)r`zfrozenset[str]rzre.Pattern[str])rgrZrr ) __future__rrMconcurrent.futuresrtypingrrrrr r rr rrCrG numpy.typingnptr lru_cacher;r=rnr$r"rsV"1VV \\~ S!$"$r$