Ë
    ýÇ:jí  ã                   ój   — d dl mZmZ dZdZdZdZdZdZd„ Z	d	„ Z
d
„ Zd„ Zd„ Zd„ Zd„ Ze	e
eeeeedœZy)é    )Údata_gym_to_mergeable_bpe_ranksÚload_tiktoken_bpeú<|endoftext|>z<|fim_prefix|>z<|fim_middle|>z<|fim_suffix|>z<|endofprompt|>zO'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\sc                  óD   — t        dddd¬«      } ddt        | t        did	œS )
NzIhttps://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpezLhttps://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.jsonÚ@1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5Ú@196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783)Úvocab_bpe_fileÚencoder_json_fileÚvocab_bpe_hashÚencoder_json_hashÚgpt2éQÄ  éPÄ  ©ÚnameÚexplicit_n_vocabÚpat_strÚmergeable_ranksÚspecial_tokens)r   Úr50k_pat_strÚ	ENDOFTEXT©r   s    úo/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/tiktoken_ext/openai_public.pyr   r      s8   € Ü5ØbØhØYØ\ô	€Oð Ø!ÜØ*Ü$ eÐ,ñð ó    c                  ó@   — t        dd¬«      } ddt        | t        didœS )NzGhttps://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktokenÚ@306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930©Úexpected_hashÚ	r50k_baser   r   r   ©r   r   r   r   s    r   r   r   !   ó2   € Ü'ØQØXô€Oð
 Ø!ÜØ*Ü$ eÐ,ñð r   c                  ó@   — t        dd¬«      } ddt        | t        didœS )NúGhttps://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktokenÚ@94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069r   Ú	p50k_baseéiÄ  r   r   r    r   s    r   r%   r%   /   r!   r   c                  óf   — t        dd¬«      } t        dt        dt        dt        di}dt
        | |d	œS )
Nr#   r$   r   r   r&   ijÄ  ikÄ  Ú	p50k_edit©r   r   r   r   )r   r   Ú
FIM_PREFIXÚ
FIM_MIDDLEÚ
FIM_SUFFIXr   ©r   r   s     r   r(   r(   =   sA   € Ü'ØQØXô€Oô   ¬
°E¼:ÀuÌjÐZ_Ð`€NàÜØ*Ø(ñ	ð r   c            
      ój   — t        dd¬«      } t        dt        dt        dt        dt
        di}d	d
| |dœS )NzIhttps://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktokenÚ@223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7r   i¡‡ i¢‡ i£‡ i¤‡ i´‡ Úcl100k_basezu'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\sr)   )r   r   r*   r+   r,   ÚENDOFPROMPTr-   s     r   r0   r0   K   sO   € Ü'ØSØXô€Oô
 	�6Ü�FÜ�FÜ�FÜ�Vð€Nð ð PØ*Ø(ñ	ð r   c                  ól   — t        dd¬«      } t        dt        di}dj                  g d¢«      }d|| |d	œS )
NzHhttps://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktokenÚ@446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2dr   é? iR ú|)zi[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?zi[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?z
\p{N}{1,3}z ?[^\s\p{L}\p{N}]+[\r\n/]*z
\s*[\r\n]+z	\s+(?!\S)z\s+Ú
o200k_baser)   )r   r   r1   Újoin)r   r   r   s      r   r6   r6   _   sO   € Ü'ØRØXô€Oô   ¬°fÐ=€Nð �h‰hò	
ó
€Gð ØØ*Ø(ñ	ð r   c                  ó¸   — t        «       } d}| d   }| d   }i | d   ¥ddddd	d
ddddddddddœ¥t        dd«      D �ci c]	  }d|› d�|“Œ c}z  }||||dœS c c}w )NÚo200k_harmonyr   r   r   i> r4   i@ iA iB iC iD iE iF iG iH iI iJ iK iL )z<|startoftext|>r   z<|reserved_200000|>z<|reserved_200001|>z
<|return|>z<|constrain|>z<|reserved_200004|>z<|channel|>z	<|start|>z<|end|>z<|message|>z<|reserved_200009|>z<|reserved_200010|>z<|reserved_200011|>z<|call|>iM i€ z<|reserved_z|>r)   )r6   Úrange)Úbase_encr   r   r   Úir   s         r   r9   r9   {   s³   € Ü‹|€HØ€DØ�yÑ!€GØÐ0Ñ1€OðØ
Ð#Ñ
$ðà!ØØ%Ø%ØØØ%ØØØØØ%Ø%Ø%Øò!ô" */¨v°vÓ)>Ö? Aˆ;�q�c˜Ð	˜aÑ	Ò?ñ#@€Nð& ØØ*Ø(ñ	ð ùò 	@s   ½A)r   r   r%   r(   r0   r6   r9   N)Útiktoken.loadr   r   r   r*   r+   r,   r1   r   r   r   r%   r(   r0   r6   r9   ÚENCODING_CONSTRUCTORS© r   r   ú<module>r@      so   ðß Là€	Ø€
Ø€
Ø€
Ø€ð [ð ò
ò òòòò(ò8ð@ ØØØØØØ"ñÑ r   