Ë
    îÍ:j‚  ã                   ód   — d dl mZmZmZmZmZ d dlmZ d dlm	Z	 d dl
mZ  G d„ d«      Zdefd„Zy	)
é    )ÚRegexÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚBPE)ÚLlamaTokenizerFast)Úbytes_to_unicodec                   ó>   — e Zd ZdZ	 	 	 	 d	d„Zdefd„Zd„ Zdefd„Z	y)
ÚMistralConverterz'
    A general tiktoken converter.
    Nc                 ó<   — || _         || _        || _        || _        y )N)ÚvocabÚpatternÚadd_prefix_spaceÚadditional_special_tokens)Úselfr   r   r   r   Úkwargss         úv/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/transformers/integrations/mistral.pyÚ__init__zMistralConverter.__init__   s"   € ð ˆŒ
ØˆŒØ 0ˆÔØ)BˆÕ&ó    r   c                 ó&  ‡‡— |Št        «       Šˆfd„}g }i }t        ‰j                  «       «      D ]¤  \  }\  }}|| j                  vr‰|| ||«      <   t	        |«      dk(  rŒ1g }t        dt	        |«      «      D ]2  }|d | ||d  }
}	|	‰v sŒ|
‰v sŒ|	|
z   ‰v sŒ|j                  |	|
|f«       Œ4 t        |ˆfd„d¬«      }|j                  |«       Œ |||<   Œ¦ t        |d„ d¬«      }|D �cg c]  } ||d   «       ||d   «      f‘Œ }}||fS c c}w )Nc           	      ó‚   •— dj                  | j                  d«      D �cg c]  }‰t        |«         ‘Œ c}«      S c c}w )NÚ zlatin-1)ÚjoinÚdecodeÚord)ÚbÚcharÚbyte_encoders     €r   Útoken_bytes_to_stringzOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string   s2   ø€ Ø—7‘7ÀÇÁÈÓ@SÖT¸˜L¬¨T«Ó3ÒTÓUÐUùÒTs    <é   c                 ó$   •— ‰| d      ‰| d      fS )Nr   r!   © )ÚxÚ	bpe_rankss    €r   ú<lambda>zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>-   s   ø€ °Y¸qÀ¹t±_ÀiÐPQÐRSÑPTÁoÐ4V€ r   F)ÚkeyÚreversec                 ó   — | d   S )Né   r#   )Úvals    r   r&   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>1   s
   € °°A±€ r   r   )	r
   Ú	enumerateÚitemsr   ÚlenÚrangeÚappendÚsortedÚextend)r   r   r    ÚmergesÚidxÚtokenÚrankÚlocalÚindexÚpiece_lÚpiece_rr+   r%   r   s               @@r   Úextract_vocab_merges_from_modelz0MistralConverter.extract_vocab_merges_from_model   sK  ù€ Øˆ	Ü'Ó)ˆô	Vð ˆØˆÜ"+¨I¯O©OÓ,=Ó">ò 	#ÑˆC‘�%˜Ø˜D×:Ñ:Ñ:Ø69�Ñ+¨EÓ2Ñ3Ü�u“: ’?ØØ�Ü" 1¤c¨%£jÓ1ò ?�EØ',¨V¨e }°e¸E¸F°m˜W�GØ )Ò+°¸9Ò0DÈ'ÐT[ÑJ[Ð`iÒIiØŸ™ g¨w¸Ð%=Õ>ð?ô ˜uÓ*VÐ`eÔf�Ø—‘˜eÕ$à"��e’ð	#ô ˜Ñ$6ÀÔFˆØ\bÖcÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÒPÐcˆÐcØ�fˆ}Ðùò ds   Ã+Dc                 ó¾   — | j                  | j                  «      \  }}t        t        ||d¬«      «      }t	        |j
                  d«      rd|j
                  _        |S )NF)Úfuse_unkÚignore_mergesT)r;   r   r   r   ÚhasattrÚmodelr>   )r   Úvocab_scoresr3   Ú	tokenizers       r   rB   zMistralConverter.tokenizer5   sN   € Ø#×CÑCÀDÇJÁJÓOÑˆ�fÜœc ,°ÀÔGÓHˆ	Ü�9—?‘? OÔ4Ø,0ˆI�O‰OÔ)ØÐr   Úreturnc                 óŒ  — | j                  «       }t        j                  t        j                  t	        | j
                  «      dd¬«      t        j                  | j                  d¬«      g«      |_        t        j                  «       |_
        |j                  | j                  «       t        j                  d¬«      |_        |S )NÚisolatedF)ÚbehaviorÚinvert)r   Ú	use_regex)Útrim_offsets)rB   r   ÚSequenceÚSplitr   r   Ú	ByteLevelr   Úpre_tokenizerr   ÚdecoderÚadd_special_tokensr   r   Úpost_processor)r   rB   s     r   Ú	convertedzMistralConverter.converted<   s™   € Ø—N‘NÓ$ˆ	Ü"0×"9Ñ"9ä×$Ñ$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ñ(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ñ.Ó0ˆ	ÔØ×$Ñ$ T×%CÑ%CÔDä#-×#7Ñ#7ÀUÔ#Kˆ	Ô àÐr   )Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ústrr;   rB   r   rQ   r#   r   r   r   r      s;   „ ñð ð KØØ"&óCð°Só ò6ð˜9ô r   r   Útokenizer_filec                 óæ  — ddl m} |j                  | «      }|j                  j                  j
                  }|j                  j                  j                  D �cg c]  }t        |d«      r|j                  n|‘Œ }}|D �ci c]  }||j                  |«      “Œ }}|j                  |«       |}t        t        ||¬«      j                  «       ¬«      }|j                  d|i«       |S c c}w c c}w )z1Convert a "tekken" tokenizer to a fast Tokenizer.r   )ÚMistralTokenizerÚvalue)r   r   )Útokenizer_objectr   )Ú(mistral_common.tokens.tokenizers.mistralrY   Ú	from_fileÚinstruct_tokenizerrB   Ú_tekken_token2id_nospecialÚ_all_special_tokensr?   rZ   r8   Úupdater	   r   rQ   rO   )rW   rY   Úmistral_tokenizerr   r5   Úall_specialÚspecials_tokensrB   s           r   Úconvert_tekken_tokenizerre   L   sø   € õ Jð )×2Ñ2°>ÓBÐð ×0Ñ0×:Ñ:×UÑU€Eð '×9Ñ9×CÑC×WÑWöàô ˜u gÔ.ˆ�Š°EÑ9ð€Kð ð EPÖP¸5�u˜k×/Ñ/°Ó6Ñ6ÐP€OÐPØ×Ñ˜5Ô!Ø€Eô #Ü)°ÐQ\Ô]×gÑgÓiô€Ið
 × Ñ Ð"=¸{Ð!KÔLàÐùò!ùò Qs   Á!C)ÂC.N)Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr   Útransformersr	   Ú#transformers.convert_slow_tokenizerr
   r   rV   re   r#   r   r   ú<module>rj      s-   ðß MÕ MÝ !å +Ý @÷Añ AðH¨Sô r   