Ë
    ýÇ:j%   ã                  ó†   — d Z ddlmZ ddlZddlZddlZ G d„ d«      Z	 d
	 	 	 	 	 	 	 dd„Z	 d
	 	 	 	 	 	 	 	 	 dd„Zdd„Z	d	„ Z
y)zJThis is an educational implementation of the byte pair encoding algorithm.é    )ÚannotationsNc                  óX   — e Zd Zd	d„Zd
dd„Zdd„Zdd„Zdd„Zedd„«       Z	ed„ «       Z
y)ÚSimpleBytePairEncodingc               ó²   — || _         || _        |j                  «       D ��ci c]  \  }}||“Œ
 c}}| _        t	        j
                  |«      | _        yc c}}w )zCreates an Encoding object.N)Úpat_strÚmergeable_ranksÚitemsÚ_decoderÚregexÚcompileÚ_pat)Úselfr   r   Útoken_bytesÚtokens        új/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/tiktoken/_educational.pyÚ__init__zSimpleBytePairEncoding.__init__   sN   € ð ˆŒà.ˆÔàFU×F[ÑF[ÓF]×^Ñ0B°¸U˜ Ñ+Ó^ˆŒÜ—M‘M 'Ó*ˆ�	ùó _s   ¢Ac                óÂ   — | j                   j                  |«      }g }|D ]<  }|j                  d«      }t        | j                  ||¬«      }|j                  |«       Œ> |S )z`Encodes a string into tokens.

        >>> enc.encode("hello world")
        [388, 372]
        úutf-8)Ú	visualise)r   ÚfindallÚencodeÚ
bpe_encoder   Úextend)r   Útextr   ÚwordsÚtokensÚwordÚ
word_bytesÚword_tokenss           r   r   zSimpleBytePairEncoding.encode   sb   € ð —	‘	×!Ñ! $Ó'ˆØˆØò 	'ˆDàŸ™ WÓ-ˆJÜ$ T×%9Ñ%9¸:ÐQZÔ[ˆKØ�M‰M˜+Õ&ð		'ð
 ˆó    c                ó8   ‡ — dj                  ˆ fd„|D «       «      S )znDecodes a list of tokens into bytes.

        >>> enc.decode_bytes([388, 372])
        b'hello world'
        r    c              3  ó<   •K  — | ]  }‰j                   |   –— Œ y ­w©N©r
   )Ú.0r   r   s     €r   ú	<genexpr>z6SimpleBytePairEncoding.decode_bytes.<locals>.<genexpr>-   s   øè ø€ ÒA°˜Ÿ™ eÕ,ÑAùs   ƒ)Újoin©r   r   s   ` r   Údecode_bytesz#SimpleBytePairEncoding.decode_bytes'   s   ø€ ð �x‰xÓA¸&ÔAÓAÐAr    c                óF   — | j                  |«      j                  dd¬«      S )u   Decodes a list of tokens into a string.

        Decoded bytes are not guaranteed to be valid UTF-8. In that case, we replace
        the invalid bytes with the replacement character "ï¿½".

        >>> enc.decode([388, 372])
        'hello world'
        r   Úreplace©Úerrors)r)   Údecoder(   s     r   r.   zSimpleBytePairEncoding.decode/   s%   € ð × Ñ  Ó(×/Ñ/°À	Ð/ÓJÐJr    c                óF   — |D �cg c]  }| j                   |   ‘Œ c}S c c}w )zÁDecodes a list of tokens into a list of bytes.

        Useful for visualising how a string is tokenised.

        >>> enc.decode_tokens_bytes([388, 372])
        [b'hello', b' world']
        r$   )r   r   r   s      r   Údecode_tokens_bytesz*SimpleBytePairEncoding.decode_tokens_bytes:   s!   € ð 39Ö9¨�—‘˜eÓ$Ò9Ð9ùÒ9s   …c                ó8   — t        | ||¬«      }t        ||¬«      S )z#Train a BPE tokeniser on some data!)ÚdataÚ
vocab_sizer   ©r   r   )Ú	bpe_trainr   )Útraining_datar3   r   r   s       r   ÚtrainzSimpleBytePairEncoding.trainD   s    € ô $¨À:ÐW^Ô_ˆÜ%¨gÀÔWÐWr    c                óŽ   — t        | t        «      rt        j                  | «      } t	        | j
                  | j                  ¬«      S )Nr4   )Ú
isinstanceÚstrÚtiktokenÚget_encodingr   Ú_pat_strÚ_mergeable_ranks)Úencodings    r   Úfrom_tiktokenz$SimpleBytePairEncoding.from_tiktokenJ   s:   € ä�h¤Ô$Ü×,Ñ,¨XÓ6ˆHÜ%Ø×%Ñ%°x×7PÑ7Pô
ð 	
r    N)r   r:   r   údict[bytes, int]ÚreturnÚNone©Úcolour)r   r:   r   ú
str | NonerB   ú	list[int])r   rG   rB   Úbytes)r   rG   rB   r:   )r   rG   rB   úlist[bytes])r6   r:   r3   Úintr   r:   )Ú__name__Ú
__module__Ú__qualname__r   r   r)   r.   r0   Ústaticmethodr7   r@   © r    r   r   r      sG   „ ó+ôó Bó	Kó:ð òXó ðXð
 ñ
ó ñ
r    r   c                óº  — |D �cg c]  }t        |g«      ‘Œ }}	 |r |dv rt        |«       n|dk(  rt        |«       d }d }t        t	        |d d |dd  «      «      D ].  \  }}| j                  |d   |d   z   «      }	|	€Œ#|�|	|k  sŒ+|}|	}Œ0 |€n#|€J ‚|d | ||   ||dz      z   gz   ||dz   d  z   }Œ˜|r
t        «        |D �
cg c]  }
| |
   ‘Œ	 }}
|S c c}w c c}
w )N©rE   ÚcolorÚsimpleéÿÿÿÿé   r   é   )rH   Úvisualise_tokensÚprintÚ	enumerateÚzipÚget)r   Úinputr   ÚbÚpartsÚmin_idxÚmin_rankÚiÚpairÚrankÚpartr   s               r   r   r   S   s:  € ð "'Ö'˜AŒU�A�3�ZÐ'€EÐ'Ø
áØÐ/Ñ/Ü  Õ'Ø˜hÒ&Ü�e”ð ˆØˆÜ ¤ U¨3¨B Z°°q°r°Ó!;Ó<ò 	 ‰GˆAˆtØ"×&Ñ& t¨A¡w°°a±Ñ'8Ó9ˆDØÑ XÐ%5¸À»Ø�Ø‘ð		 ð ÐØØÐ"Ð"Ð"ð �h�w� 5¨¡>°E¸'ÀA¹+Ñ4FÑ#FÐ"GÑGÈ%ÐPWÐZ[ÑP[ÐP]ÐJ^Ñ^ˆð/ ñ2 ÜŒà05Ö6¨ˆo˜dÓ#Ð6€FÐ6Ø€Mùò= (ùò: 7s   …CÃCc           
     ó¸  ‡— |dk  rt        d«      ‚i }t        d«      D ]  }||t        |g«      <   Œ t        j                  || «      D ��cg c]+  }|j                  d«      D �cg c]  }t        |g«      ‘Œ c}‘Œ- }}}t        |«      |k  �r»t        j                  «       Š|D ]&  }	t        |	d d |	dd  «      D ]  }
‰|
xx   dz  cc<   Œ Œ( t        ‰ˆfd„¬«      }|d   |d   z   }t        |«      }|||<   g }|D ]Ÿ  }g }d}|t        |«      dz
  k  rR||   ||dz      f|k(  r|j                  |«       |d	z  }n|j                  ||   «       |dz  }|t        |«      dz
  k  rŒR|t        |«      dz
  k(  r|j                  ||   «       |j                  |«       Œ¡ |}|r—t        d
|d   › d|d   › �«       t        d|› dt        |«      › d�«       |dv r1t        d«       t        |d d D ��cg c]  }|D ]  }|‘Œ Œ c}}«       n%|dk(  r t        d«       |d d D ]  }t        |«       Œ t        d«       t        |«      |k  r�Œ»|S c c}w c c}}w c c}}w )Né   z;vocab_size must be at least 256, so we can encode all bytesr   rT   rU   c                ó   •— ‰|    S r#   rO   )ÚxÚstatss    €r   ú<lambda>zbpe_train.<locals>.<lambda>“   s   ø€ °E¸!±H€ r    )Úkeyr   rV   z The current most common pair is z + zSo we made z our zth tokenrQ   z9Now the first fifty words in our training data look like:é2   rS   z:Now the first twenty words in our training data look like:é   ú
)Ú
ValueErrorÚrangerH   r   r   r   ÚlenÚcollectionsÚCounterrZ   ÚmaxÚappendrX   rW   )r2   r3   r   r   Úranksra   r   r]   r   Úpiecerb   Úmost_common_pairr   r   Ú	new_wordsÚnew_wordri   s                   @r   r5   r5   w   s²  ø€ ð �DÒÜÐVÓWÐWØ€EÜ�4‹[ò ˆØˆŒe�Q�C‹jÒðô @E¿}¹}ÈWÐVZÓ?[÷ Ø7;˜TŸ[™[¨Ó1Ö2˜Œ�ˆs�Ô2ð €Eñ  ô
 ˆe‹*�zÓ
!ä×#Ñ#Ó%ˆØò 	!ˆEÜ˜E # 2˜J¨¨a¨b¨	Ó2ò !�Ø�d“˜qÑ ”ñ!ð	!ô ˜uÓ*<Ô=ÐØ& qÑ)Ð,<¸QÑ,?Ñ?ˆÜ�E“
ˆà"ˆˆkÑð ˆ	Øò 	'ˆDØˆHØˆAØ”c˜$“i !‘mÒ#Ø˜‘G˜T ! a¡%™[Ð)Ð-=Ò=à—O‘O KÔ0Ø˜‘F‘Aà—O‘O D¨¡GÔ,Ø˜‘F�Að ”c˜$“i !‘mÓ#ð ”C˜“I ‘MÒ!Ø—‘  Q¡Ô(Ø×Ñ˜XÕ&ð	'ð ˆñ ÜÐ4Ð5EÀaÑ5HÐ4IÈÐM]Ð^_ÑM`ÐLaÐbÔcÜ�K ˜}¨E´#°e³*°¸XÐFÔGØÐ/Ñ/ÜÐQÔRÜ °E¸#¸2°J×!Q¨DÈDÒ!QÀ5¢%Ð!Q %Ó!QÕRØ˜hÒ&ÜÐRÔSØ! # 2˜Jò  �DÜ˜$•Kð ä�$ŒKôW ˆe‹*�zÔ
!ðZ €Lùòc 	3ùó ùóV "Rs   ÁIÁ$IÁ7IÇ0I
ÉIc                óL  — dD �cg c]  }d|› d�‘Œ
 }}| D �cg c]  }|j                  dd¬«      ‘Œ }}d}d }|D ]S  }||t        |«      z     }||k(  r||dz   t        |«      z     }||k7  sJ ‚|}|t        |«      z  }t        ||z   d	¬
«       ŒU t        d«       y c c}w c c}w )N)é§   é³   é¹   éM   éP   éD   é†   z[48;5;Úmr   r+   r,   r   rU   Ú )Úendz[0m)r.   rq   rX   )	Útoken_valuesra   Ú
backgroundrh   Úunicode_token_valuesÚrunning_lengthÚ
last_colorr   rR   s	            r   rW   rW   ¼   sÔ   € Ø/OÖP¨!�L   1Ò%ÐP€JÐPð JVÖVÀA˜AŸH™H W°Y˜HÕ?ÐVÐÐVà€NØ€JØ%ò %ˆØ˜>¬C°
«OÑ;Ñ<ˆØ�JÒØ °Ñ 2´c¸*³oÑEÑFˆEØ˜JÒ&Ð&Ð&Øˆ
Øœ#˜e›*Ñ$ˆÜˆe�e‰m Ö$ð%ô 
ˆ+Õùò! Qùò Ws
   …B˜B!c                 ób  — d} t        t        «      5 }|j                  «       }d d d «       t        j	                  d| ¬«      }t        d«       |j                  d«      }|j                  |«      dk(  sJ ‚|j                  |«      dk(  sJ ‚|j                  |«      ddgk(  sJ ‚|S # 1 sw Y   ŒƒxY w)	NzN's|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+iX  )r3   r   zJThis is the sequence of merges performed in order to encode 'hello world':zhello worlds   hello worlds   hellos    world)
ÚopenÚ__file__Úreadr   r7   rX   r   r.   r)   r0   )Úgpt2_patternÚfr2   Úencr   s        r   Útrain_simple_encodingr’   Ð   s¸   € à]ð ô 
Œh‹ð ˜1Ø�v‰v‹xˆ÷ô !×
&Ñ
& t¸À\Ð
&Ó
R€Cä	Ð
VÔWØ�Z‰Z˜Ó&€FØ�:‰:�fÓ Ò.Ð.Ð.Ø×Ñ˜FÓ# ~Ò5Ð5Ð5Ø×"Ñ" 6Ó*¨x¸Ð.CÒCÐCÐCà€J÷ð ús   ’B%Â%B.rD   )r   rA   r\   rH   r   rF   rB   rG   )
r2   r:   r3   rJ   r   r:   r   rF   rB   rA   )r†   rI   rB   rC   )Ú__doc__Ú
__future__r   rr   r   r;   r   r   r5   rW   r’   rO   r    r   ú<module>r•      s“   ðÙ På "ã ã ã ÷D
ñ D
ðP NVð!Ø%ð!Ø.3ð!Ø@Jð!àó!ðJ GOðBØ
ðBØðBØ),ðBØ9CðBàóBóJó(r    