Ë
    ýÇ:jÿ  ã                  óz   — d dl mZ d dlZd dlZd dlZd	d„Zd
d„Zddd„Z	 	 	 d	 	 	 	 	 	 	 	 	 	 	 dd„Zdd„Z	ddd„Z
y)é    )ÚannotationsNc                ó\  — d| vr(t        | dd¬«      5 }|j                  «       cd d d «       S | j                  d«      r1dd l}|j	                  | «      }|j                  «        |j                  S 	 dd l}|j                  | «      S # 1 sw Y   ŒaxY w# t        $ r}t        d«      |‚d }~ww xY w)Nz://Úrbr   ©Ú	buffering)zhttp://zhttps://úOblobfile is not installed. Please install it by running `pip install blobfile`.)
ÚopenÚreadÚ
startswithÚrequestsÚgetÚraise_for_statusÚcontentÚblobfileÚImportErrorÚ
read_bytes)ÚblobpathÚfr   Úrespr   Úes         úb/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/tiktoken/load.pyÚ	read_filer      s²   € Ø�HÑÜ�(˜D¨AÔ.ð 	°!Ø—6‘6“8÷	ñ 	ð ×ÑÐ2Ô3ãà�|‰|˜HÓ%ˆØ×ÑÔØ�|‰|ÐðÛð
 ×Ñ˜xÓ(Ð(÷#	ð 	ûô ò ÜØ]ó
àð	ûðús#   “BÁ0B ÂBÂ	B+ÂB&Â&B+c                óR   — t        j                  | «      j                  «       }||k(  S ©N)ÚhashlibÚsha256Ú	hexdigest)ÚdataÚexpected_hashÚactual_hashs      r   Ú
check_hashr!      s$   € Ü—.‘. Ó&×0Ñ0Ó2€KØ˜-Ñ'Ð'ó    c                ób  — d}dt         j                  v rt         j                  d   }nZdt         j                  v rt         j                  d   }n4dd l}t         j                  j	                  |j                  «       d«      }d}|dk(  rt        | «      S t        j                  | j                  «       «      j                  «       }t         j                  j	                  ||«      }t         j                  j                  |«      rMt        |dd¬	«      5 }|j                  «       }d d d «       |�t        |«      rS 	 t        j                  |«       t        | «      }	|rt        |	|«      st#        d
| › d|› d�«      ‚dd l}
	 t        j&                  |d¬«       |dz   t)        |
j+                  «       «      z   dz   }t        |d«      5 }|j-                  |	«       d d d «       t        j.                  ||«       |	S # 1 sw Y   ŒÖxY w# t         $ r Y Œ¾w xY w# 1 sw Y   Œ<xY w# t         $ r |r‚ Y |	S w xY w)NTÚTIKTOKEN_CACHE_DIRÚDATA_GYM_CACHE_DIRr   zdata-gym-cacheFÚ r   r   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)Úexist_okú.z.tmpÚwb)ÚosÚenvironÚtempfileÚpathÚjoinÚ
gettempdirr   r   Úsha1Úencoder   Úexistsr	   r
   r!   ÚremoveÚOSErrorÚ
ValueErrorÚuuidÚmakedirsÚstrÚuuid4ÚwriteÚrename)r   r   Úuser_specified_cacheÚ	cache_dirr,   Ú	cache_keyÚ
cache_pathr   r   Úcontentsr6   Útmp_filenames               r   Úread_file_cachedrB   #   sù  € ØÐØœrŸz™zÑ)Ü—J‘JÐ3Ñ4‰	Ø	¤§¡Ñ	+Ü—J‘JÐ3Ñ4‰	ãä—G‘G—L‘L ×!4Ñ!4Ó!6Ð8HÓIˆ	Ø$Ðà�B‚ä˜Ó"Ð"ä—‘˜XŸ_™_Ó.Ó/×9Ñ9Ó;€Iä—‘—‘˜i¨Ó3€JÜ	‡w�w‡~�~�jÔ!Ü�*˜d¨aÔ0ð 	°AØ—6‘6“8ˆD÷	àÐ ¤J¨t°]Ô$CØˆKð	Ü�I‰I�jÔ!ô ˜Ó"€HÙœZ¨°-Ô@ÜØ5°h°Z¸{È=È/ð ZHð Ió
ð 	
ó
 ð	Ü
�‰�I¨Õ-Ø! CÑ'¬#¨d¯j©j«lÓ*;Ñ;¸fÑDˆÜ�, Ó%ð 	¨Ø�G‰G�HÔ÷	ä
�	‰	�, 
Ô+ð €O÷?	ð 	ûô ò 	Ùð	ú÷	ð 	ûô ò áØð  ð €OðúsI   ÄG7Ä;H Æ AH ÇHÇH Ç7H È	HÈHÈHÈH ÈH.È-H.c                ó  ‡— t        d«      D �cg c]-  }t        |«      j                  «       sŒt        |«      dk7  sŒ,|‘Œ/ }}|D �ci c]  }t        |«      |“Œ c}Šd}t        d«      D ].  }||vsŒ|j                  |«       |‰t        d|z   «      <   |dz  }Œ0 t	        |«      dk(  sJ ‚t        | |«      j                  «       }|j                  d«      dd D �	cg c]  }	t        |	j                  «       «      ‘Œ }
}	d
ˆfd„}t        |«      D ��ci c]  \  }}t        |g«      |“Œ }}}~t	        |«      }|
D ]  \  }}|| ||«       ||«      z   <   |dz  }Œ  dd l}|j                  t        ||«      «      }|j                  «       D ��ci c]  \  }} ||«      |“Œ }}}|j                  dd «       |j                  d	d «       |r|D ]  }t	        |«      dk(  sŒ||   ||<   Œ ||k(  sJ ‚|S c c}w c c}w c c}	w c c}}w c c}}w )Né   ú r   é   ú
éÿÿÿÿc                ó,   •— t        ˆfd„| D «       «      S )Nc              3  ó(   •K  — | ]	  }‰|   –— Œ y ­wr   © )Ú.0ÚbÚdata_gym_byte_to_bytes     €r   ú	<genexpr>zKdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>q   s   øè ø€ Ò=°!Ð*¨1Õ-Ñ=ùs   ƒ)Úbytes)ÚvaluerN   s    €r   Údecode_data_gymz8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymp   s   ø€ ÜÓ=°uÔ=Ó=Ð=r"   s   <|endoftext|>s   <|startoftext|>)rQ   r8   ÚreturnrP   )ÚrangeÚchrÚisprintableÚappendÚlenrB   ÚdecodeÚsplitÚtupleÚ	enumeraterP   ÚjsonÚloadsÚitemsÚpop)Úvocab_bpe_fileÚencoder_json_fileÚvocab_bpe_hashÚencoder_json_hashÚclobber_one_byte_tokensrM   Úrank_to_intbyteÚnÚvocab_bpe_contentsÚ	merge_strÚ
bpe_mergesrR   ÚiÚ	bpe_ranksÚfirstÚsecondr]   Úencoder_jsonÚkÚvÚencoder_json_loadedrN   s                        @r   Údata_gym_to_mergeable_bpe_ranksrs   Y   s4  ø€ ô #(¨£+ÖX˜Q´°Q³×1CÑ1CÕ1EÌ#ÈaË&ÐTWË-’qÐX€OÐXà0?Ö@¨1œS ›V Q™YÒ@ÐØ	€AÜ�4‹[ò ˆØ�OÒ#Ø×"Ñ" 1Ô%Ø34Ð!¤# d¨Q¡h£-Ñ0Ø�‰F‰Að	ô
 ˆÓ 4Ò'Ð'Ð'ô *¨.¸.ÓI×PÑPÓRÐØ<N×<TÑ<TÐUYÓ<ZÐ[\Ð]_Ð<`Öa¨y”%˜	Ÿ™Ó)Õ*Ða€JÐaõ>ô
 ,5°_Ó+E×F¡4 1 a”˜�s“˜Q‘ÐF€IÑFØô 	ˆI‹€AØ#ò ‰ˆˆvØFGˆ	‘/ %Ó(©?¸6Ó+BÑBÑCØ	ˆQ‰‰ðó ð
 —:‘:Ô.Ð/@ÐBSÓTÓU€LØ=I×=OÑ=OÓ=Q×R±T°Q¸™?¨1Ó-¨qÑ0ÐRÐÑRà×ÑÐ,¨dÔ3Ø×ÑÐ.°Ô5áØ$ò 	6ˆAÜ�1‹v˜‹{Ø2°1Ñ5�	˜!’ð	6ð Ð+Ò+Ð+Ð+àÐùò_ Yùâ@ùò bùó Gùó Ss'   �G/®G/½G/ÁG4Ã G9ÄG>ÆHc                óh  — 	 dd l }|j                  |d«      5 }t        | j	                  «       d„ ¬«      D ]I  \  }}|j                  t        j                  |«      dz   t        |«      j                  «       z   dz   «       ŒK 	 d d d «       y # t        $ r}t        d«      |‚d }~ww xY w# 1 sw Y   y xY w)Nr   r   r)   c                ó   — | d   S )NrF   rK   )Úxs    r   ú<lambda>z#dump_tiktoken_bpe.<locals>.<lambda>›   s
   € À1ÀQÁ4€ r"   )Úkeyó    ó   
)
r   r   ÚBlobFileÚsortedr_   r:   Úbase64Ú	b64encoder8   r1   )rl   Útiktoken_bpe_filer   r   r   ÚtokenÚranks          r   Údump_tiktoken_bper‚   “   s¹   € ðÛð
 
×	Ñ	Ð,¨dÓ	3ð Q°qÜ! )§/¡/Ó"3¹ÔHò 	Q‰KˆE�4Ø�G‰G”F×$Ñ$ UÓ+¨dÑ2´S¸³Y×5EÑ5EÓ5GÑGÈ%ÑOÕPñ	Q÷Qð Qøô	 ò ÜØ]ó
àð	ûðú÷Qð Qús#   ‚B ˜A)B(Â	B%ÂB Â B%Â(B1c                ó  — t        | |«      }i }|j                  «       D ]:  }|sŒ	 |j                  «       \  }}t        |«      |t	        j
                  |«      <   Œ< |S # t        $ r}t        d|›d| › �«      |‚d }~ww xY w)NzError parsing line z in )rB   Ú
splitlinesrZ   Úintr}   Ú	b64decodeÚ	Exceptionr5   )r   r   r@   ÚretÚliner€   r�   r   s           r   Úload_tiktoken_bperŠ   Ÿ   s›   € äÐ 1°=ÓA€HØ
€CØ×#Ñ#Ó%ò [ˆÙØð	[ØŸ*™*›,‰KˆE�4Ü+.¨t«9ˆC”× Ñ  Ó'Ò(ð[ð €Jøô ò 	[ÜÐ2°4°(¸$Ð?PÐ>QÐRÓSÐYZÐZûð	[ús   ¦4AÁ	A>Á'A9Á9A>)r   r8   rS   rP   )r   rP   r   r8   rS   Úboolr   )r   r8   r   ú
str | NonerS   rP   )NNF)ra   r8   rb   r8   rc   rŒ   rd   rŒ   re   r‹   rS   údict[bytes, int])rl   r�   r   r8   rS   ÚNone)r   r8   r   rŒ   rS   r�   )Ú
__future__r   r}   r   r*   r   r!   rB   rs   r‚   rŠ   rK   r"   r   ú<module>r�      su   ðÝ "ã Û Û 	ó)ó,(ô
3ðr "&Ø$(Ø$)ð7Øð7àð7ð ð7ð "ð	7ð
 "ð7ð ó7ót	Qõr"   