Ë
    îÍ:jZ ã                   ó(  — d Z ddlZddlmZ ddlmZ ddlmZ ddlm	Z	m
Z
mZmZmZmZmZ ddlmZmZmZ ddlmZ d	d
lmZmZmZmZ d	dlmZ  ej8                  e«      Zd£d„Zde de!fd„Z"d„ Z# G d„ d«      Z$ G d„ de$«      Z%de!de fd„Z& G d„ d«      Z' G d„ de'«      Z( G d„ de'«      Z) G d„ de'«      Z* G d„ d e'«      Z+ G d!„ d"e'«      Z, G d#„ d$e'«      Z- G d%„ d&e'«      Z. G d'„ d(e'«      Z/ G d)„ d*e'«      Z0 G d+„ d,e'«      Z1 G d-„ d.e'«      Z2 G d/„ d0e'«      Z3 G d1„ d2e3«      Z4 G d3„ d4e3«      Z5 G d5„ d6e3«      Z6 G d7„ d8e3«      Z7 G d9„ d:e3«      Z8 G d;„ d<e3«      Z9 G d=„ d>e3«      Z: G d?„ d@e3«      Z; G dA„ dBe3«      Z< G dC„ dDe3«      Z= G dE„ dFe3«      Z> G dG„ dHe3«      Z? G dI„ dJe3«      Z@ G dK„ dLe3«      ZA G dM„ dNe3«      ZB G dO„ dPe3«      ZC G dQ„ dRe'«      ZD G dS„ dTe3«      ZE G dU„ dVe'«      ZF G dW„ dXe'«      ZG G dY„ dZe'«      ZH G d[„ d\e3«      ZI G d]„ d^e3«      ZJ G d_„ d`e3«      ZK G da„ dbe'«      ZL G dc„ dde3«      ZM G de„ dfe3«      ZN G dg„ dhe3«      ZOdi„ ZP G dj„ dk«      ZQ G dl„ dm«      ZRi dne4“doe0“dpe5“dqe(“dreE“dseH“dte6“dueF“dve-“dwe(“dxe2“dye7“dze(“d{e(“d|e(“d}e(“d~e(“i de4“d€e*“d�e-“d‚e.“dƒe(“d„e(“d…e0“d†e<“d‡e0“dˆe0“d‰e(“dŠeL“d‹e8“dŒe9“d�e+“dŽe(“d�e0“¥i d�e:“d‘e,“d’eA“d“e/“d”e(“d•e>“d–e?“d—e(“d˜e0“d™e1“dše;“d›e(“dœeB“d�eC“džeD“dŸe<“d e=“¥e)eIeKeKeJeKd¡œ¥ZSd¤defd¢„ZTy)¥zù
Utilities to convert slow tokenizers in their fast tokenizers counterparts.

All the conversions are grouped here to gather SentencePiece dependencies outside of the fast tokenizers files and
allow to make our dependency on SentencePiece optional.
é    N)Ú	lru_cache)ÚOptional)Úversion)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramÚ	WordPiece)Útqdmé   )Úis_protobuf_availableÚis_sentencepiece_availableÚloggingÚrequires_backends)ÚPROTOBUF_IMPORT_ERRORc                 ó  — t        «       rddlm} |S t        «       rSdd l}t        j                  |j                  j                  «      t        j                  d«      k  rddl	m} |S ddl	m
} |S t        t        j                  | «      «      ‚)Nr   )Úsentencepiece_model_pb2z4.0.0)Úsentencepiece_model_pb2_new)r   Úsentencepiecer   r   Úgoogle.protobufr   ÚparseÚprotobufÚ__version__Útransformers.utilsr   ÚImportErrorr   Úformat)Úerror_messager   Úgoogles      úx/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/transformers/convert_slow_tokenizer.pyÚimport_protobufr%   %   sl   € Ü!Ô#Ý9à&Ð&ÜÔÛä�=‰=˜Ÿ™×4Ñ4Ó5¼¿¹ÀgÓ8NÒNÝBð 'Ð&õ bØ&Ð&äÔ/×6Ñ6°}ÓEÓFÐFó    Úadd_prefix_spaceÚreturnc                 ó4   — | rd}t        |dd«      sd}|S d}|S )NÚalwaysÚlegacyTÚfirstÚnever)Úgetattr)r'   Úoriginal_tokenizerÚprepend_schemes      r$   Ú_get_prepend_schemer1   6   s1   € ÙØ!ˆÜÐ)¨8°TÔ:Ø$ˆNð Ðð !ˆØÐr&   c                 ó‚  ‡ — |d u}|rt        |«      n‰ }g }|j                  «       D ]j  \  }}g }t        dt        |«      «      D ]*  }|d | ||d  }	}|‰ v sŒ|	‰ v sŒ|j	                  ||	|f«       Œ, t        |ˆ fd„¬«      }|j                  |«       Œl t        |d„ |¬«      }|D �
cg c]  }
|
d   |
d   f‘Œ }}
|S c c}
w )Nr   c                 ó$   •— ‰| d      ‰| d      fS ©Nr   r   © )ÚxÚvocabs    €r$   ú<lambda>z!generate_merges.<locals>.<lambda>K   s   ø€ ¨U°1°Q±4©[¸%ÀÀ!Á¹+Ð,F€ r&   ©Úkeyc                 óB   — | d   t        | d   «      t        | d   «      fS )Né   r   r   )Úlen©Úvals    r$   r8   z!generate_merges.<locals>.<lambda>N   s!   € ¨S°©V´S¸¸Q¹³[Ä#ÀcÈ!ÁfÃ+Ð,N€ r&   ©r:   Úreverser   )ÚdictÚitemsÚranger=   ÚappendÚsortedÚextend)r7   Úvocab_scoresrA   ÚmergesÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rr?   s   `          r$   Úgenerate_mergesrP   @   sê   ø€ Ø $Ð&€GÙ)0”4˜Ô%°e€Là€FØ*×0Ñ0Ó2ò Ñˆˆ{ØˆÜ˜1œc %›jÓ)ò 	>ˆEØ$ V e˜}¨e°E°F¨m�WˆGØ˜%Ò G¨uÒ$4Ø—‘˜g w°Ð<Õ=ð	>ô �uÓ"FÔGˆØ�‰�eÕðô �FÑ NÐX_Ô`€FØ*0Ö1 3ˆs�1‰v�s˜1‘vÒÐ1€FÐ1Ø€Mùò 2s   Â'B<c                   óD   — e Zd ZdZdefd„Zddeeeef   e	e   f   fd„Z
y)ÚSentencePieceExtractorzl
    Extractor implementation for SentencePiece trained models. https://github.com/google/sentencepiece
    Úmodelc                 óv   — t        | d«       ddlm}  |«       | _        | j                  j	                  |«       y )Nr   r   )ÚSentencePieceProcessor)r   r   rU   ÚspÚLoad)ÚselfrS   rU   s      r$   Ú__init__zSentencePieceExtractor.__init__X   s)   € Ü˜$ Ô0Ý8á(Ó*ˆŒØ�‰�‰�UÕr&   Nr(   c                 ó°   — | j                   }t        |j                  «       «      D �ci c]  }|j                  |«      |“Œ }}t	        ||«      }||fS c c}w )úÂ
        By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
        order the merges with respect to the piece scores instead.
        )rV   rD   ÚGetPieceSizeÚid_to_piecerP   ©rX   rH   rV   rM   r7   rI   s         r$   ÚextractzSentencePieceExtractor.extract_   sV   € ð
 �W‰WˆÜ;@ÀÇÁÓARÓ;SÖT°%�—‘ Ó&¨Ñ-ÐTˆÐTä  ¨Ó5ˆà�fˆ}Ðùò	 Us   ¨A©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrrY   ÚtuplerB   ÚintÚlistr_   r5   r&   r$   rR   rR   S   s5   „ ñð˜có ñ
¨E°$°s¸C°x±.À$ÀuÁ+Ð2MÑ,Nô 
r&   rR   c                   ó4   — e Zd Zddeeeef   ee   f   fd„Zy)ÚGemmaSentencePieceExtractorNr(   c                 óà   — | j                   }t        |j                  «       «      D �ci c]  }|j                  |«      |“Œ }}d|vr|j	                  d«      |d<   t        ||«      }||fS c c}w )r[   ú	ú<0x09>)rV   rD   r\   r]   ÚgetrP   r^   s         r$   r_   z#GemmaSentencePieceExtractor.extractm   sr   € ð
 �W‰WˆÜ;@ÀÇÁÓARÓ;SÖT°%�—‘ Ó&¨Ñ-ÐTˆÐTð �uÑØŸ)™) HÓ-ˆE�$‰KÜ  ¨Ó5ˆØ�fˆ}Ðùò Us   ¨A+r`   )	ra   rb   rc   rf   rB   re   rg   rh   r_   r5   r&   r$   rj   rj   l   s$   „ ñ¨E°$°s¸C°x±.À$ÀuÁ+Ð2MÑ,Nô r&   rj   Úpiecec                 ó^   — t        | «      dk  xs | d   dk7  xs | d   j                  «        S )Nr<   éÿÿÿÿú,éþÿÿÿ)r=   Úisdigit)ro   s    r$   Úcheck_number_commaru   }   s3   € Üˆu‹:˜‰>ÒH˜U 2™Y¨#Ñ-ÒH°U¸2±Y×5FÑ5FÓ5HÐ1HÐHr&   c                   ó   — e Zd Zd„ Zdefd„Zy)Ú	Converterc                 ó   — || _         y r`   )r/   )rX   r/   s     r$   rY   zConverter.__init__‚   s
   € Ø"4ˆÕr&   r(   c                 ó   — t        «       ‚r`   )ÚNotImplementedError©rX   s    r$   Ú	convertedzConverter.converted…   s   € Ü!Ó#Ð#r&   N)ra   rb   rc   rY   r   r|   r5   r&   r$   rw   rw   �   s   „ ò5ð$˜9ô $r&   rw   c                   ó   — e Zd Zdefd„Zy)ÚBertConverterr(   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )N©Ú	unk_tokenFÚbasic_tokenizerT©Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaseú:0 $A:0 ú:0ú:0 $B:1 ú:1©ÚsingleÚpairÚspecial_tokensú##©Úprefix©r/   r7   r   r   re   r�   Úhasattrr‚   Útokenize_chinese_charsr†   Údo_lower_caser
   ÚBertNormalizerÚ
normalizerr   ÚBertPreTokenizerÚpre_tokenizerÚ	cls_tokenÚ	sep_tokenÚcls_token_idÚsep_token_idr   ÚTemplateProcessingÚpost_processorr	   Údecoder©
rX   r7   Ú	tokenizerr•   r†   r–   ÚclsÚsepr�   rž   s
             r$   r|   zBertConverter.convertedŠ   ó…  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐr&   N©ra   rb   rc   r   r|   r5   r&   r$   r~   r~   ‰   ó   „ ð#˜9ô #r&   r~   c                   ó   — e Zd Zdefd„Zy)ÚSplinterConverterr(   c           
      ó‚  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }t	        | j                   j&                  «      }d}	| j                   j(                  }
| j                   j*                  }| j                   j,                  }| j                   j/                  d«      }| j                   j0                  dk(  r|› d|› d	|	› d	|› d
|› d�
}n|› d|› d
|› d	|	› d	|› d�
}t3        j4                  |› d|› d�|||
f||f||f|	|fg¬«      |_        t9        j                  d¬«      |_        |S )Nr€   Fr‚   Trƒ   ú.Úrightrˆ   ú rŠ   r‹   r‰   rŒ   r�   r‘   )r/   r7   r   r   re   r�   r”   r‚   r•   r†   r–   r
   r—   r˜   r   r™   rš   r›   rœ   Úquestion_tokenr�   rž   Úquestion_token_idÚconvert_tokens_to_idsÚpadding_sider   rŸ   r    r	   r¡   )rX   r7   r£   r•   r†   r–   r¤   r¥   ÚquestionÚdotr�   rž   r°   Údot_token_idrŽ   s                  r$   r|   zSplinterConverter.converted±   s"  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆÜ�t×.Ñ.×=Ñ=Ó>ˆØˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆØ ×3Ñ3×EÑEÐØ×.Ñ.×DÑDÀSÓIˆà×"Ñ"×/Ñ/°7Ò:Ø�U˜( 8 *¨A¨c¨U°!°C°5¸ÀÀÀRÐH‰Dà�U˜( 3 % x°¨z¸¸3¸%¸qÀÀÀRÐHˆDä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Øà�lÐ#Ø�lÐ#ØÐ,Ð-Ø�lÐ#ð	ô	$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐr&   Nr§   r5   r&   r$   rª   rª   °   s   „ ð.˜9ô .r&   rª   c                   ó   — e Zd Zdefd„Zy)ÚFunnelConverterr(   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )Nr€   Fr‚   Trƒ   z:2 $A:0 r‰   rŠ   r‹   rŒ   r�   r‘   r“   r¢   s
             r$   r|   zFunnelConverter.convertedã   r¦   r&   Nr§   r5   r&   r$   r·   r·   â   r¨   r&   r·   c                   ó   — e Zd Zdefd„Zy)ÚMPNetConverterr(   c                 ór  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	|› d
�||f||	fg¬«      |_        t1        j                  d¬«      |_        |S )Nr€   Fr‚   Trƒ   rˆ   r‰   z:0 rŠ   r‹   rŒ   r�   r‘   r“   r¢   s
             r$   r|   zMPNetConverter.converted
  s‹  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   S¨¨¨X°c°U¸"Ð=à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐr&   Nr§   r5   r&   r$   rº   rº   	  r¨   r&   rº   c                   ó   — e Zd Zdefd„Zy)ÚOpenAIGPTConverterr(   c           
      ó  — | j                   j                  }t        | j                   j                  j	                  «       «      }| j                   j
                  }t        t        ||d t        |«      dd¬«      «      }|j                  t        |«      «      �|j                  t        |«      g«       t        j                  d¬«      |_        t        j                  «       |_        t#        j$                  d¬«      |_        |S )Nú</w>F)r7   rI   Údropoutr�   Úend_of_word_suffixÚfuse_unkT)r‡   ©Úsuffix)r/   Úencoderrh   Ú	bpe_ranksÚkeysr�   r   r   re   Útoken_to_idÚadd_special_tokensr
   r—   r˜   r   r™   rš   r	   Ú
BPEDecoderr¡   ©rX   r7   rI   r�   r£   s        r$   r|   zOpenAIGPTConverter.converted1  sÔ   € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØÜ˜i›.Ø#)Øôó	
ˆ	ð × Ñ ¤ Y£Ó0Ð<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä*×9Ñ9ÀDÔIˆ	ÔÜ"0×"AÑ"AÓ"Cˆ	ÔÜ$×/Ñ/°vÔ>ˆ	ÔàÐr&   Nr§   r5   r&   r$   r½   r½   0  s   „ ð˜9ô r&   r½   c            	       óJ   — e Zd Z	 ddeeeef      deeeeef         de	fd„Z
y)ÚGPT2ConverterNr7   rI   r(   c           
      óN  — |s| j                   j                  }|st        | j                   j                  «      }t	        t        ||d ddd¬«      «      }t        | j                   dd«      }t        j                  |¬«      |_	        t        j                  «       |_        t        | j                   dd«      rT| j                   j                  }| j                   j                  }t        j                  |› d�|› d�||fg¬	«      |_        |S t        j                  d¬
«      |_        |S )NÚ F©r7   rI   rÀ   Úcontinuing_subword_prefixrÁ   rÂ   r'   ©r'   Úadd_bos_tokenz:0 $A:0z:0 $A:0 $B:1rŒ   ©Útrim_offsets)r/   rÅ   rh   rÆ   r   r   r.   r   Ú	ByteLevelrš   r	   r¡   Ú	bos_tokenÚbos_token_idr   rŸ   r    )rX   r7   rI   r£   r'   ÚbosrØ   s          r$   r|   zGPT2Converter.convertedL  s  € ñ Ø×+Ñ+×3Ñ3ˆEÙÜ˜$×1Ñ1×;Ñ;Ó<ˆFäÜØØØØ*,Ø#%Øôó	
ˆ	ô # 4×#:Ñ#:Ð<NÐPUÓVÐÜ"0×":Ñ":ÐL\Ô"]ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ�4×*Ñ*¨O¸UÔCØ×)Ñ)×3Ñ3ˆCØ×2Ñ2×?Ñ?ˆLÜ'1×'DÑ'DØ˜˜g�Ø�u˜LÐ)à˜,Ð'ð ô(ˆIÔ$ð Ðô (2×';Ñ';ÈÔ'OˆIÔ$ØÐr&   ©NN©ra   rb   rc   r   rB   re   rg   rh   rf   r   r|   r5   r&   r$   rÍ   rÍ   K  sD   „ à`dñ$Ø˜d 3¨ 8™nÑ-ð$Ø>FÀtÈEÐRUÐWZÐRZÉOÑG\Ñ>]ð$à	ô$r&   rÍ   c                   ó   — e Zd Zdefd„Zy)ÚHerbertConverterr(   c           	      óŒ  — d}d}| j                   j                  }t        | j                   j                  j	                  «       «      }||d   d   v r|dd  }t        t        ||d | j                   j                  |¬«      «      }t        j                  dd¬«      |_
        t        j                  «       |_        t        j                  |¬«      |_        t#        j$                  | j                   j&                  | j                   j(                  f| j                   j*                  | j                   j,                  f¬	«      |_        |S )
Nz	#version:r¿   r   r   )rÀ   r�   rÁ   F)r‡   r†   rÃ   )r¥   r¤   )r/   rÅ   rh   rÆ   rÇ   r   r   r�   r
   r—   r˜   r   r™   rš   r	   rÊ   r¡   r   ÚBertProcessingrœ   rž   r›   r�   r    )rX   Útokenizer_info_strÚtoken_suffixr7   rI   r£   s         r$   r|   zHerbertConverter.convertedt  s  € Ø(ÐØˆà×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ ¨¡¨1¡Ñ-Ø˜A˜B�ZˆFäÜØØØØ×1Ñ1×;Ñ;Ø#/ôó
ˆ	ô  +×9Ñ9ÀEÐY^Ô_ˆ	ÔÜ"0×"AÑ"AÓ"Cˆ	ÔÜ$×/Ñ/°|ÔDˆ	ÔÜ#-×#<Ñ#<Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYô$
ˆ	Ô ð
 Ðr&   Nr§   r5   r&   r$   rÝ   rÝ   s  ó   „ ð˜9ô r&   rÝ   c            	       óJ   — e Zd Z	 ddeeeef      deeeeef         de	fd„Z
y)ÚQwen2ConverterNr7   rI   r(   c                 ó0  — |s| j                   j                  }|s-t        | j                   j                  j	                  «       «      }t        t        ||d d dddd¬«      «      }t        j                  «       |_	        t        j                  t        j                  t        d«      dd¬«      t        j                  t        | j                   dd«      d¬«      g«      |_        t#        j                  «       |_        t'        j                  d¬	«      |_        |S )
NrÏ   F)r7   rI   rÀ   r�   rÑ   rÁ   rÂ   Úbyte_fallbackzn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+Úisolated©ÚbehaviorÚinvertr'   ©r'   Ú	use_regexrÔ   )r/   rÅ   rh   rÆ   rÇ   r   r   r
   ÚNFCr˜   r   ÚSequenceÚSplitr   rÖ   r.   rš   r	   r¡   r   r    )rX   r7   rI   r£   s       r$   r|   zQwen2Converter.converted“  s  € ñ Ø×+Ñ+×3Ñ3ˆEÙÜ˜$×1Ñ1×;Ñ;×@Ñ@ÓBÓCˆFäÜØØØØØ*,Ø#%ØØ#ô	ó
ˆ	ô  +Ÿ™Ó0ˆ	Ôä"0×"9Ñ"9ä×$Ñ$Üð Nóð (Ø ôô ×(Ñ(Ü%,¨T×-DÑ-DÐFXÐZ_Ó%`Ø#ôðó#
ˆ	Ôô  %×.Ñ.Ó0ˆ	ÔÜ#-×#7Ñ#7ÀUÔ#Kˆ	Ô àÐr&   rÚ   rÛ   r5   r&   r$   rä   rä   ’  sD   „ à`dñ*Ø˜d 3¨ 8™nÑ-ð*Ø>FÀtÈEÐRUÐWZÐRZÉOÑG\Ñ>]ð*à	ô*r&   rä   c                   ó   — e Zd Zdefd„Zy)ÚRobertaConverterr(   c           
      óÔ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  |j                  |j                   f|j"                  |j$                  f|j                  d¬«      |_        |S )NrÏ   FrÐ   rÒ   T©r¥   r¤   r'   rÕ   )r/   rÅ   rh   rÆ   rÇ   r   r   r   rÖ   r'   rš   r	   r¡   r   ÚRobertaProcessingrœ   rž   r›   r�   r    ©rX   Úotr7   rI   r£   s        r$   r|   zRobertaConverter.convertedÁ  sÄ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#?Ñ#?Ø—‘˜rŸ™Ð/Ø—‘˜rŸ™Ð/Ø×0Ñ0Øô	$
ˆ	Ô ð Ðr&   Nr§   r5   r&   r$   rñ   rñ   À  ó   „ ð˜9ô r&   rñ   c                   ó   — e Zd Zdefd„Zy)ÚRoFormerConverterr(   c           	      óV  — ddl m} | j                  j                  }t	        t        |t        | j                  j                  «      ¬«      «      }d}d}t        | j                  d«      r@| j                  j                  j                  }| j                  j                  j                  }t        j                  dd||¬«      |_        t        j                   j#                   ||«      «      |_        t        | j                  j&                  «      }t        | j                  j(                  «      }| j                  j*                  }| j                  j,                  }	t/        j0                  |› d|› d	�|› d|› d
|› d�||f||	fg¬«      |_        t5        j
                  d¬«      |_        |S )Nr   )ÚJiebaPreTokenizerr€   Fr‚   Trƒ   rˆ   r‰   rŠ   r‹   rŒ   r�   r‘   )Ú"models.roformer.tokenization_utilsrû   r/   r7   r   r   re   r�   r”   r‚   r†   r–   r
   r—   r˜   r   ÚPreTokenizerÚcustomrš   r›   rœ   r�   rž   r   rŸ   r    r	   r¡   )
rX   rû   r7   r£   r†   r–   r¤   r¥   r�   rž   s
             r$   r|   zRoFormerConverter.convertedÞ  sy  € ÝIà×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	àˆØˆÜ�4×*Ñ*Ð,=Ô>Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!&Ø'Ø#ô	 
ˆ	Ôô #1×"=Ñ"=×"DÑ"DÑEVÐW\ÓE]Ó"^ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐr&   Nr§   r5   r&   r$   rù   rù   Ý  r¨   r&   rù   c                   ó   — e Zd Zdefd„Zy)ÚDebertaConverterr(   c           
      óÖ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  ddd| j                   j                  d«      fd| j                   j                  d«      fg¬	«      |_        |S )
NrÏ   FrÐ   rÒ   ú[CLS]:0 $A:0 [SEP]:0ú![CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1ú[CLS]ú[SEP]rŒ   )r/   rÅ   rh   rÆ   rÇ   r   r   r   rÖ   r'   rš   r	   r¡   r   rŸ   r±   r    rõ   s        r$   r|   zDebertaConverter.converted  sÕ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#@Ñ#@Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô$
ˆ	Ô ð Ðr&   Nr§   r5   r&   r$   r   r     râ   r&   r   c                   ó`   ‡ — e Zd ZdZeZi Zˆ fd„Zd„ Zd„ Z	d„ Z
d„ Zd„ Zd„ Zd	„ Zd
efd„Zˆ xZS )ÚSpmConverterFc                 ó¦  •— t        | d«       t        ‰| �  |Ž  t        «       }|j	                  «       }t        | j                  j                  d«      5 }|j                  |j                  «       «       d d d «       || _
        | j                  j                  j                  r#| j                  st        j                  d«       y y y # 1 sw Y   ŒTxY w)Nr   Úrba  The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.)r   ÚsuperrY   r%   Ú
ModelProtoÚopenr/   Ú
vocab_fileÚParseFromStringÚreadÚprotoÚtrainer_specræ   Úhandle_byte_fallbackÚwarningsÚwarn)rX   ÚargsÚ	model_pb2ÚmÚfÚ	__class__s        €r$   rY   zSpmConverter.__init__(  s°   ø€ Ü˜$ 
Ô+ä‰Ñ˜$Ñô $Ó%ˆ	à× Ñ Ó"ˆÜ�$×)Ñ)×4Ñ4°dÓ;ð 	(¸qØ×Ñ˜aŸf™f›hÔ'÷	(àˆŒ
à�:‰:×"Ñ"×0Ò0¸×9RÒ9RÜ�M‰Mðeõð :SÐ0÷		(ð 	(ús   Á CÃCc                 ól   — |j                   D �cg c]  }|j                  |j                  f‘Œ c}S c c}w r`   ©Úpiecesro   Úscore©rX   r  ro   s      r$   r7   zSpmConverter.vocab=  s'   € Ø8=¿¹ÖE¨u�—‘˜eŸk™kÒ*ÒEÐEùÒEs   �1c                 ó.   — |j                   j                  S r`   )r  Úunk_id©rX   r  s     r$   r   zSpmConverter.unk_id@  s   € Ø×!Ñ!×(Ñ(Ð(r&   c                 ó~  — |j                   j                  }| j                  |«      }|dk(  r1t        t	        || j                  |«      | j                  ¬«      «      }n£|dk(  r“| j                  | j                  j                  «      j                  |«      \  }}t        |«      D ���	ci c]  \  }\  }}	||“Œ }
}}}	t        t        |
||j                   j                  d| j                  d ¬«      «      }nt        d«      ‚t        |j                  «      D ��cg c]I  \  }}|j                   dv r6||j"                  |j                   dk(  xs |j"                  | j$                  v f‘ŒK }}}|j'                  t)        |d	„ ¬
«      D ���cg c]  \  }}}t+        |d|¬«      ‘Œ c}}}«       |S c c}	}}w c c}}w c c}}}w )Nr   ©r   ræ   r<   T©r�   rÂ   ræ   rÀ   z]You're trying to run a `Unigram` model but you're file was trained with a different algorithm©é   é   r&  c                 ó   — | d   S ©Nr   r5   ©r6   s    r$   r8   z(SpmConverter.tokenizer.<locals>.<lambda>o  ó   € ÐQRÐSTÑQU€ r&   r9   F©Ú
normalizedÚspecial)r  Ú
model_typer7   r   r   r   r  ÚSpmExtractorr/   r  r_   Ú	enumerater   Ú	unk_pieceÚ	Exceptionr  Útypero   r�   Ú
add_tokensrF   r   )rX   r  r/  rH   r£   Ú_rI   ÚiÚwordr  Ú	bpe_vocabÚidÚpÚspm_added_tokensÚtokenr.  s                   r$   r£   zSpmConverter.tokenizerC  s´  € Ø×'Ñ'×2Ñ2ˆ
Ø—z‘z %Ó(ˆà˜Š?Ü!ÜØ ØŸ;™; uÓ-Ø"&×";Ñ";ôó‰Ið ˜1Š_Ø×)Ñ)¨$×*AÑ*A×*LÑ*LÓM×UÑUÐVbÓc‰IˆAˆvÜ9BÀ<Ó9P×QÐQÑ%5 Q©¨¨u˜˜q™ÐQˆIÒQÜ!ÜØØØ#×0Ñ0×:Ñ:Ø!Ø"&×";Ñ";Ø ôó	‰Iô Øoóð ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÖDôô	
ð ÐùôC Rùó*
ùôs   Â)F+ÄAF2ÆF8c                 ó   — |j                   j                  }t        j                  dd¬«      t        j                  t        d«      d«      g}|st        j                  |«      S t        j                  t        j                  |«      g|z   «      S )NFT)Úleftr­   ú {2,}õ   â–�)Únormalizer_specÚprecompiled_charsmapr
   ÚStripÚReplacer   rî   ÚPrecompiled©rX   r  rC  Ú_normalizerss       r$   r˜   zSpmConverter.normalizeru  s{   € Ø$×4Ñ4×IÑIÐä×Ñ 5°Ô5Ü×Ñ¤ g£°Ó6ð
ˆñ $Ü×'Ñ'¨Ó5Ð5ä×'Ñ'¬×)@Ñ)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgr&   c                 ó\   — t        || j                  «      }t        j                  ||¬«      S ©N©Úreplacementr0   )r1   r/   r   Ú	Metaspace©rX   rL  r'   r0   s       r$   rš   zSpmConverter.pre_tokenizer€  s)   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×'Ñ'°KÐP^Ô_Ð_r&   c                  ó   — y r`   r5   r{   s    r$   r    zSpmConverter.post_processor„  s   € Ør&   c                 ó\   — t        || j                  «      }t        j                  ||¬«      S rJ  )r1   r/   r	   rM  rN  s       r$   r¡   zSpmConverter.decoder‡  s(   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×!Ñ!¨kÈ.ÔYÐYr&   r(   c                 óz  — | j                  | j                  «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j
                  }| j                  ||«      }|�||_        | j                  ||«      |_        | j                  «       }|r||_        |S )NrA  Tr'   )	r£   r  r˜   r”   r/   r'   rš   r¡   r    )rX   r£   r˜   rL  r'   rš   r    s          r$   r|   zSpmConverter.converted‹  s¸   € Ø—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$àÐr&   )ra   rb   rc   r  rR   r0  r�   rY   r7   r   r£   r˜   rš   r    r¡   r   r|   Ú__classcell__)r  s   @r$   r  r  #  sL   ø„ Ø ÐØ)€LØ€Nôò*Fò)ò0òd	hò`òòZð˜9÷ r&   r  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚAlbertConverterc                 óÌ   — |j                   D �cg c]J  }t        |j                  «      r|j                  |j                  fn|j                  |j                  dz
  f‘ŒL c}S c c}w ©Néd   ©r  ru   ro   r  r  s      r$   r7   zAlbertConverter.vocab¥  ó^   € ð Ÿ™ö
àô +=¸U¿[¹[Ô*IˆU�[‰[˜%Ÿ+™+Ñ&ÐPU×P[ÑP[Ð]b×]hÑ]hÐknÑ]nÐOoÑoò
ð 	
ùò 
ó   �AA!c                 ó‚  — t        j                  dd«      t        j                  dd«      g}| j                  j                  sF|j	                  t        j
                  «       «       |j	                  t        j                  «       «       | j                  j                  r#|j	                  t        j                  «       «       |j                  j                  }|r$|j	                  t        j                  |«      «       |j	                  t        j                  t        d«      d«      «       t        j                  |«      S ©Nz``ú"z''r@  r®   ©r
   rE  r/   Úkeep_accentsrE   ÚNFKDÚStripAccentsr–   Ú	LowercaserB  rC  rF  r   rî   ©rX   r  Úlist_normalizersrC  s       r$   r˜   zAlbertConverter.normalizer«  óï   € ä×Ñ  cÓ*Ü×Ñ  cÓ*ð
Ðð ×&Ñ&×3Ò3Ø×#Ñ#¤K×$4Ñ$4Ó$6Ô7Ø×#Ñ#¤K×$<Ñ$<Ó$>Ô?Ø×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<à$×4Ñ4×IÑIÐáØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRà×Ñ¤× 3Ñ 3´E¸'³NÀCÓ HÔIÜ×#Ñ#Ð$4Ó5Ð5r&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S ©Nr  r  r  r  rŒ   ©r   rŸ   r/   r±   r{   s    r$   r    zAlbertConverter.post_processor¾  óR   € Ü×,Ñ,Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô
ð 	
r&   N©ra   rb   rc   r7   r˜   r    r5   r&   r$   rT  rT  ¤  ó   „ ò
ò6ó&
r&   rT  c                   ó   — e Zd Zd„ Zd„ Zy)ÚBarthezConverterc                 ó
   — d}|S ©Nr&  r5   ©rX   r  r   s      r$   r   zBarthezConverter.unk_idÊ  ó   € ØˆØˆr&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S ©Nz<s> $A </s>z<s> $A </s> </s> $B </s>ú<s>ú</s>rŒ   rh  r{   s    r$   r    zBarthezConverter.post_processorÎ  óR   € Ü×,Ñ,Ø Ø+à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r&   N)ra   rb   rc   r   r    r5   r&   r$   rm  rm  É  s   „ òó
r&   rm  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚCamembertConverterc                 ó�   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|dgz  }|S c c}w )N))z
<s>NOTUSEDç        ©ú<pad>rz  )z</s>NOTUSEDrz  ©z<unk>rz  )z<unk>NOTUSEDiœÿÿÿr   ©z<mask>rz  r  ©rX   r  r7   ro   s       r$   r7   zCamembertConverter.vocabÚ  sP   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØ�/Ð"Ñ"ˆØˆùò Ló   —Ac                  ó   — yro  r5   r!  s     r$   r   zCamembertConverter.unk_idç  s   € àr&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rs  rh  r{   s    r$   r    z!CamembertConverter.post_processorë  rv  r&   N©ra   rb   rc   r7   r   r    r5   r&   r$   rx  rx  Ù  s   „ òòó
r&   rx  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚDebertaV2Converterc                 ó  — g }| j                   j                  r%|j                  t        j                  d¬«      «       t        || j                   «      }|j                  t        j                  ||¬«      «       t        j                  |«      S )Nrç   )ré   rK  )r/   Úsplit_by_punctrE   r   ÚPunctuationr1   rM  rî   )rX   rL  r'   Úlist_pretokenizersr0   s        r$   rš   z DebertaV2Converter.pre_tokenizer÷  sq   € ØÐØ×"Ñ"×1Ò1Ø×%Ñ%¤n×&@Ñ&@È*Ô&UÔVÜ,Ð-=¸t×?VÑ?VÓWˆØ×!Ñ!¤.×":Ñ":À{ÐcqÔ"rÔsÜ×&Ñ&Ð'9Ó:Ð:r&   c                 ó¼  — g }| j                   j                  r#|j                  t        j                  «       «       |j                  t        j
                  «       «       |j                  j                  }|r$|j                  t        j                  |«      «       |j                  t        j                  t        d«      d«      «       t        j                  |«      S )Nr@  r®   )r/   r–   rE   r
   rb  rD  rB  rC  rF  rE  r   rî   rc  s       r$   r˜   zDebertaV2Converter.normalizerÿ  s¥   € ØÐØ×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<Ø×Ñ¤× 1Ñ 1Ó 3Ô4à$×4Ñ4×IÑIÐÙØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRØ×Ñ¤× 3Ñ 3´E¸'³NÀCÓ HÔIä×#Ñ#Ð$4Ó5Ð5r&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rg  rh  r{   s    r$   r    z!DebertaV2Converter.post_processor  ri  r&   N)ra   rb   rc   rš   r˜   r    r5   r&   r$   r…  r…  ö  s   „ ò;ò6ó
r&   r…  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚMBartConverterc                 óž   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|dgz  }|S c c}w )N©©rt  rz  r{  ©ru  rz  r}  r&  )©Úar_ARrz  ©Úcs_CZrz  ©Úde_DErz  ©Úen_XXrz  ©Úes_XXrz  ©Úet_EErz  ©Úfi_FIrz  ©Úfr_XXrz  ©Úgu_INrz  ©Úhi_INrz  ©Úit_ITrz  ©Úja_XXrz  ©Úkk_KZrz  ©Úko_KRrz  ©Últ_LTrz  ©Úlv_LVrz  ©Úmy_MMrz  ©Úne_NPrz  ©Únl_XXrz  ©Úro_ROrz  ©Úru_RUrz  ©Úsi_LKrz  ©Útr_TRrz  ©Úvi_VNrz  ©Úzh_CNrz  r~  r  r  s       r$   r7   zMBartConverter.vocab  sa   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò 
ñ 	
ˆð6 	�/Ð"Ñ"ˆØˆùò; Ló   —A
c                  ó   — yro  r5   r!  s     r$   r   zMBartConverter.unk_id>  ó   € Ør&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz$A </s> en_XXz$A $B </s> en_XXr™  ru  rŒ   rh  r{   s    r$   r    zMBartConverter.post_processorA  óR   € Ü×,Ñ,Ø"Ø#à˜$×1Ñ1×GÑGÈÓPÐQØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r&   Nrƒ  r5   r&   r$   r�  r�    s   „ ò$òLó
r&   r�  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚMBart50Converterc                 óž   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|dgz  }|S c c}w )Nr�  r&  )4r’  r”  r–  r˜  rš  rœ  rž  r   r¢  r¤  r¦  r¨  rª  r¬  r®  r°  r²  r´  r¶  r¸  rº  r¼  r¾  rÀ  rÂ  )Úaf_ZArz  )Úaz_AZrz  )Úbn_INrz  )Úfa_IRrz  )Úhe_ILrz  )Úhr_HRrz  )Úid_IDrz  )Úka_GErz  )Úkm_KHrz  )Úmk_MKrz  )Úml_INrz  )Úmn_MNrz  )Úmr_INrz  )Úpl_PLrz  )Úps_AFrz  )Úpt_XXrz  )Úsv_SErz  )Úsw_KErz  )Úta_INrz  )Úte_INrz  )Úth_THrz  )Útl_XXrz  )Úuk_UArz  )Úur_PKrz  )Úxh_ZArz  )Úgl_ESrz  )Úsl_SIrz  r~  r  r  s       r$   r7   zMBart50Converter.vocabM  sa   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò  Rñ  	RˆØ�/Ð"Ñ"ˆØˆùò LrÄ  c                  ó   — yro  r5   r!  s     r$   r   zMBart50Converter.unk_idY  rÆ  r&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nzen_XX $A </s>zen_XX $A $B </s>r™  ru  rŒ   rh  r{   s    r$   r    zMBart50Converter.post_processor\  rÈ  r&   Nrƒ  r5   r&   r$   rÊ  rÊ  L  s   „ ò
òó
r&   rÊ  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚNllbConverterc                 ó„   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )Nr�  r&  r  r  s       r$   r7   zNllbConverter.vocabh  óC   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò Ló   —=c                  ó   — yro  r5   r!  s     r$   r   zNllbConverter.unk_idr  rÆ  r&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nzeng_Latn $A </s>zeng_Latn $A $B </s>Úeng_Latnru  rŒ   rh  r{   s    r$   r    zNllbConverter.post_processoru  sR   € Ü×,Ñ,Ø%Ø&à˜T×4Ñ4×JÑJÈ:ÓVÐWØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r&   Nrƒ  r5   r&   r$   rê  rê  g  s   „ òòó
r&   rê  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚSeamlessM4TConverterc                 ó„   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )N)r{  r}  r�  r‘  r&  r  r  s       r$   r7   zSeamlessM4TConverter.vocab�  rì  rí  c                 ó.   — | j                   j                  S r`   )r/   Úunk_token_idr!  s     r$   r   zSeamlessM4TConverter.unk_id‹  s   € Ø×&Ñ&×3Ñ3Ð3r&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz__eng__ $A </s>z__eng__ $A $B </s>Ú__eng__ru  rŒ   rh  r{   s    r$   r    z#SeamlessM4TConverter.post_processorŽ  sR   € Ü×,Ñ,Ø$Ø%à˜D×3Ñ3×IÑIÈ)ÓTÐUØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r&   Nrƒ  r5   r&   r$   rò  rò  €  s   „ òò4ó
r&   rò  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚXLMRobertaConverterc                 ó�   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|dgz  }|S c c}w )Nr�  r&  r~  r  r  s       r$   r7   zXLMRobertaConverter.vocabš  sP   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØ�/Ð"Ñ"ˆØˆùò Lr€  c                 ó
   — d}|S ro  r5   rp  s      r$   r   zXLMRobertaConverter.unk_id¥  rq  r&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rs  rh  r{   s    r$   r    z"XLMRobertaConverter.post_processor©  rv  r&   Nrƒ  r5   r&   r$   rù  rù  ™  ó   „ ò	òó
r&   rù  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚXLNetConverterc                 óÌ   — |j                   D �cg c]J  }t        |j                  «      r|j                  |j                  fn|j                  |j                  dz
  f‘ŒL c}S c c}w rV  rX  r  s      r$   r7   zXLNetConverter.vocabµ  rY  rZ  c                 ó‚  — t        j                  dd«      t        j                  dd«      g}| j                  j                  sF|j	                  t        j
                  «       «       |j	                  t        j                  «       «       | j                  j                  r#|j	                  t        j                  «       «       |j                  j                  }|r$|j	                  t        j                  |«      «       |j	                  t        j                  t        d«      d«      «       t        j                  |«      S r\  r^  rc  s       r$   r˜   zXLNetConverter.normalizer»  re  r&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz$A:0 <sep>:0 <cls>:2z!$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2z<sep>z<cls>rŒ   rh  r{   s    r$   r    zXLNetConverter.post_processorÎ  ri  r&   Nrj  r5   r&   r$   rÿ  rÿ  ´  rk  r&   rÿ  c                   ó   — e Zd Zy)ÚReformerConverterN©ra   rb   rc   r5   r&   r$   r  r  Ù  ó   „ Ør&   r  c                   ó   — e Zd Zd„ Zd„ Zy)ÚRemBertConverterc                 ób  — t        j                  dd«      t        j                  dd«      t        j                  t        d«      d«      g}| j                  j                  sF|j                  t        j                  «       «       |j                  t        j                  «       «       | j                  j                  r#|j                  t        j                  «       «       |j                  j                  }|r$|j                  t        j                  |«      «       t        j                  |«      S r\  )r
   rE  r   r/   r_  rE   r`  ra  r–   rb  rB  rC  rF  rî   rc  s       r$   r˜   zRemBertConverter.normalizerß  sä   € ä×Ñ  cÓ*Ü×Ñ  cÓ*Ü×Ñ¤ g£°Ó4ð
Ðð
 ×&Ñ&×3Ò3Ø×#Ñ#¤K×$4Ñ$4Ó$6Ô7Ø×#Ñ#¤K×$<Ñ$<Ó$>Ô?Ø×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<à$×4Ñ4×IÑIÐáØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRä×#Ñ#Ð$4Ó5Ð5r&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rg  rh  r{   s    r$   r    zRemBertConverter.post_processorò  ri  r&   N)ra   rb   rc   r˜   r    r5   r&   r$   r  r  Ý  s   „ ò6ó&
r&   r  c                   ó   — e Zd Zy)ÚBertGenerationConverterNr  r5   r&   r$   r  r  ý  r  r&   r  c                   ó$   — e Zd Zd„ Zd„ Zd„ Zd„ Zy)ÚPegasusConverterc                 óv  — | j                   j                  df| j                   j                  dfg}| j                   j                  �|| j                   j                  dfgz  }| j                   j                  �I| j                   j
                  | j                   j                  k  r|| j                   j                  dfgz  }|t        d| j                   j                  «      D �cg c]
  }d|› d�df‘Œ c}z  }||j                  dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w c c}w )Nrz  r<   z<unk_ú>g      YÀ)r/   Ú	pad_tokenÚ	eos_tokenÚmask_token_sentÚ
mask_tokenÚmask_token_idÚoffsetrD   r  ro   r  )rX   r  r7   r7  ro   s        r$   r7   zPegasusConverter.vocab  s%  € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 ×"Ñ"×2Ñ2Ð>Ø�t×.Ñ.×>Ñ>ÀÐDÐEÑEˆEð ×#Ñ#×.Ñ.Ð:Ø×'Ñ'×5Ñ5¸×8OÑ8O×8VÑ8VÒVà�t×.Ñ.×9Ñ9¸3Ð?Ð@Ñ@ˆEà´%¸¸4×;RÑ;R×;YÑ;YÓ2ZÖ[¨Q�U˜1˜#˜Q�< Ò(Ò[Ñ[ˆØ¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò \ùÚKs   Ã%D1ÄD6c                 ó\   — |j                   j                  | j                  j                  z   S r`   )r  r   r/   r  r!  s     r$   r   zPegasusConverter.unk_id  s%   € Ø×!Ñ!×(Ñ(¨4×+BÑ+B×+IÑ+IÑIÐIr&   c                 óª   — t        || j                  «      }t        j                  t        j                  «       t        j
                  ||¬«      g«      S rJ  )r1   r/   r   rî   ÚWhitespaceSplitrM  rN  s       r$   rš   zPegasusConverter.pre_tokenizer  sJ   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×&Ñ&ä×.Ñ.Ó0Ü×(Ñ(°[ÐQ_Ô`ðó
ð 	
r&   c                 óš   — | j                   j                  }|| j                   j                  fg}t        j                  d|gdd|g|¬«      S )Nú$Aú$BrŒ   )r/   r  Úeos_token_idr   rŸ   )rX   Úeosr�   s      r$   r    zPegasusConverter.post_processor!  sR   € Ø×%Ñ%×/Ñ/ˆà�$×)Ñ)×6Ñ6Ð7ð
ˆô ×,Ñ,°T¸3°KÀtÈTÐSVÐFWÐhvÔwÐwr&   N)ra   rb   rc   r7   r   rš   r    r5   r&   r$   r  r    s   „ òò&Jò
óxr&   r  c                   ó   — e Zd Zd„ Zd„ Zy)ÚT5Converterc                 óò   — | j                   j                  }|j                  D �cg c]  }|j                  |j                  f‘Œ }}|t        |dz
  dd«      D �cg c]
  }d|› d�df‘Œ c}z  }|S c c}w c c}w )Nr   rq   z
<extra_id_r  rz  )r/   Ú
_extra_idsr  ro   r  rD   )rX   r  Únum_extra_idsro   r7   r7  s         r$   r7   zT5Converter.vocab*  sw   € Ø×/Ñ/×:Ñ:ˆØ9>¿¹ÖF°�%—+‘+˜uŸ{™{Ò+ÐFˆÐFØ´E¸-È!Ñ:KÈRÐQSÓ4TÖU¨q�Z ˜s !Ð$ cÒ*ÒUÑUˆØˆùò GùÚUs   ¥A/ÁA4c                 ór   — t        j                  ddgg d¢d| j                  j                  d«      fg¬«      S ©Nr  ru  )r  ru  r  ru  rŒ   rh  r{   s    r$   r    zT5Converter.post_processor0  ó=   € Ü×,Ñ,Ø˜&�>Ú-à˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r&   N)ra   rb   rc   r7   r    r5   r&   r$   r   r   )  s   „ òó
r&   r   c                   ó   — e Zd Zd„ Zy)ÚUdopConverterc                 ór   — t        j                  ddgg d¢d| j                  j                  d«      fg¬«      S r%  rh  r{   s    r$   r    zUdopConverter.post_processor;  r&  r&   N©ra   rb   rc   r    r5   r&   r$   r(  r(  :  s   „ ó
r&   r(  c                   ó   — e Zd Zdefd„Zy)ÚWhisperConverterr(   c           
      óÈ  — | j                   j                  }t        | j                   j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  | j                   j                  ¬«      |_
        t        j                  «       |_        | j                   j                  }| j                   j                  |«      }| j                   j                  }| j                   j                   }dj#                  |D �cg c]  }|› d�‘Œ	 c}«      }	t%        j&                  |	› d|› d�|	› d|› d	�||fgt)        ||«      ¢¬
«      |_        |S c c}w )NrÏ   FrÐ   rÒ   r®   r‰   z $A:0 z $A:0 $B:1 r‹   rŒ   )r/   rÅ   rh   rÆ   rÇ   r   r   r   rÖ   r'   rš   r	   r¡   Úprefix_tokensÚconvert_ids_to_tokensr  r  Újoinr   rŸ   Úzipr    )
rX   r7   rI   r£   Úprefix_token_idsÚprefixesr  r  r=  Úprefix_templates
             r$   r|   zWhisperConverter.convertedF  sR  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈD×LcÑLc×LtÑLtÔ"uˆ	ÔÜ$×.Ñ.Ó0ˆ	Ôà×2Ñ2×@Ñ@ÐØ×*Ñ*×@Ñ@ÐAQÓRˆØ×%Ñ%×/Ñ/ˆØ×.Ñ.×;Ñ;ˆØŸ(™(¸hÖ#G°U u g¨R¢LÒ#GÓHˆÜ#-×#@Ñ#@Ø%Ð& f¨S¨E°Ð4Ø#Ð$ K°¨u°BÐ7à�lÐ#ðä�XÐ/Ó0ðô$
ˆ	Ô ð Ðùò $Hs   ÄENr§   r5   r&   r$   r,  r,  E  s   „ ð ˜9ô  r&   r,  c                   ó   — e Zd Zd„ Zy)ÚBigBirdConverterc           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rg  rh  r{   s    r$   r    zBigBirdConverter.post_processorj  ri  r&   Nr*  r5   r&   r$   r6  r6  i  s   „ ó
r&   r6  c                   ó   — e Zd Zdefd„Zy)ÚCLIPConverterr(   c                 óp  — | j                   j                  }t        | j                   j                  j	                  «       «      }| j                   j
                  }t        t        ||d dddt        |«      ¬«      «      }t        j                  t        j                  «       t        j                  t        d«      d«      t        j                  «       g«      |_        t!        j                  t!        j"                  t        d«      dd	¬
«      t!        j$                  d¬«      g«      |_        t)        j$                  «       |_        t-        j.                  | j                   j0                  | j                   j2                  f| j                   j4                  | j                   j6                  fdd¬«      |_        |S )NrÏ   r¿   F©r7   rI   rÀ   rÑ   rÁ   rÂ   r�   z\s+r®   z9's|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedTrè   rÒ   ró   )r/   rÅ   rh   rÆ   rÇ   r�   r   r   re   r
   rî   rí   rE  r   rb  r˜   r   rï   rÖ   rš   r	   r¡   r   rô   r  r  r×   rØ   r    rË   s        r$   r|   zCLIPConverter.convertedv  sk  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØØ*,Ø#)ØÜ˜i›.ôó

ˆ	ô  +×3Ñ3Ü�_‰_Ó¤× 3Ñ 3´E¸&³MÀ3Ó GÌ×I^ÑI^ÓI`Ðaó 
ˆ	Ôô #1×"9Ñ"9ä×$Ñ$ÜÐZÓ[Ø&Øôô
 ×(Ñ(¸%Ô@ðó	#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôô $.×#?Ñ#?Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ"Øô	$
ˆ	Ô ð Ðr&   Nr§   r5   r&   r$   r9  r9  u  s   „ ð'˜9ô 'r&   r9  c                   ó   — e Zd Zdefd„Zy)ÚLayoutLMv2Converterr(   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )Nr€   FTr‚   rƒ   rˆ   r‰   rŠ   r‹   rŒ   r�   r‘   r“   r¢   s
             r$   r|   zLayoutLMv2Converter.converted¡  s…  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐr&   Nr§   r5   r&   r$   r>  r>     r¨   r&   r>  c                   ó   — e Zd Zdefd„Zy)ÚBlenderbotConverterr(   c           
      ó®  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  d|j                  › d�|j                  |j                   fg¬«      |_        |S )NrÏ   FrÐ   rÒ   z$A:0 r‰   )r�   r�   )r/   rÅ   rh   rÆ   rÇ   r   r   r   rÖ   r'   rš   r	   r¡   r   rŸ   r  r  r    rõ   s        r$   r|   zBlenderbotConverter.convertedÈ  s½   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#@Ñ#@Ø˜2Ÿ<™<˜.¨Ð+à—‘˜rŸ™Ð/ðô$
ˆ	Ô ð Ðr&   Nr§   r5   r&   r$   rA  rA  Ç  r÷   r&   rA  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚXGLMConverterc                 ó’   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|S c c}w )Nr�  r&  ))z<madeupword0>rz  )z<madeupword1>rz  )z<madeupword2>rz  )z<madeupword3>rz  )z<madeupword4>rz  )z<madeupword5>rz  )z<madeupword6>rz  r  r  s       r$   r7   zXGLMConverter.vocabå  sT   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò  zñ  	zˆØˆùò Ls   —Ac                 ó
   — d}|S ro  r5   rp  s      r$   r   zXGLMConverter.unk_idð  rq  r&   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz</s> $Az</s> $A </s> </s> $Brt  ru  rŒ   rh  r{   s    r$   r    zXGLMConverter.post_processorô  sR   € Ü×,Ñ,ØØ'à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r&   Nrƒ  r5   r&   r$   rD  rD  ä  rý  r&   rD  c                   ó<   — e Zd ZdZeZddhZ	 d„ Zd„ Zd„ Z	d„ Z
d„ Zy	)
ÚGemmaConverterTz<start_of_turn>z<end_of_turn>c                 ó.   — t        j                  dd«      S ©Nr®   rA  )r
   rE  r!  s     r$   r˜   zGemmaConverter.normalizer  s   € Ü×"Ñ" 3¨Ó.Ð.r&   c                 ót  — | j                   j                  df| j                   j                  df| j                   j                  dfg}||j                  dd  D �cg c]  }|j
                  |j                  f‘Œ c}z  }t        d„ |D «       «      s#t        d„ t        |«      D «       d «      }|�d||<   |S c c}w )Nrz  r&  c              3   ó,   K  — | ]  }|d    dk(  –— Œ y­w)r   rl   Nr5   )Ú.0r6   s     r$   ú	<genexpr>z'GemmaConverter.vocab.<locals>.<genexpr>  s   è ø€ Ò/ A�1�Q‘4˜4•<Ñ/ùs   ‚c              3   ó8   K  — | ]  \  }}|d    dk(  sŒ|–— Œ y­w)r   rm   Nr5   )rN  r7  r6   s      r$   rO  z'GemmaConverter.vocab.<locals>.<genexpr>  s   è ø€ Ò"V©¨¨AÀQÀqÁTÈXÓEU¤1Ñ"Vùs   ‚“)rl   rz  )
r/   r  r  r×   r  ro   r  ÚanyÚnextr1  )rX   r  r7   ro   Úoverride_indexs        r$   r7   zGemmaConverter.vocab  s·   € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆô Ñ/¨Ô/Ô/Ü!Ñ"V´¸5Ó1AÔ"VÐX\Ó]ˆNØÐ)Ø(3��nÑ%àˆùò Ls   ÁB5c                 ó.   — t        j                  dd«      S )Nr®   Úmerged_with_previous)r   rï   ©rX   rL  r'   s      r$   rš   zGemmaConverter.pre_tokenizer"  s   € Ü×#Ñ# CÐ)?Ó@Ð@r&   c                 ó
   — d}|S ro  r5   rp  s      r$   r   zGemmaConverter.unk_id%  rq  r&   c                 ó¢   — t        j                  t        j                  dd«      t        j                  «       t        j                  «       g«      S )NrA  r®   )r	   rî   rE  ÚByteFallbackÚFuserV  s      r$   r¡   zGemmaConverter.decoder)  s?   € Ü× Ñ ä× Ñ  ¨Ó,Ü×%Ñ%Ó'Ü—‘“ðó
ð 	
r&   N)ra   rb   rc   r  rj   r0  r�   r˜   r7   rš   r   r¡   r5   r&   r$   rI  rI  ÿ  s6   „ ØÐØ.€Là'¨Ð9€Nðò/òò Aòó
r&   rI  c                   ó4   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	y)	ÚLlamaConverterTc                 ó(  — | j                   j                  d«      df| j                   j                  d«      df| j                   j                  d«      dfg}||j                  dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )Nr   rz  r   r<   r&  )r/   r/  r  ro   r  r  s       r$   r7   zLlamaConverter.vocab6  sŽ   € à×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCð
ˆð
 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò Ls   Á)Bc                 ó
   — d}|S r)  r5   rp  s      r$   r   zLlamaConverter.unk_id?  rq  r&   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S ©NrA  r®   r   )Úcontentr?  ©r	   rE  rY  rZ  rD  rî   ©rX   rL  r'   Úsequences       r$   r¡   zLlamaConverter.decoderC  ó\   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆñ
 ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r&   c                 óø   — t        | j                  dd«      rcg }t        | j                  dd«      r|t        j                  d¬«      gz  }|t        j                  dd¬«      gz  }t        j
                  |«      S y )Nr+   Tr'   rA  )Úprependr®   )Úpatternra  )r.   r/   r
   ÚPrependrE  rî   )rX   r  rd  s      r$   r˜   zLlamaConverter.normalizerM  sr   € Ü�4×*Ñ*¨H°dÔ;ØˆHÜ�t×.Ñ.Ð0BÀDÔIØœ[×0Ñ0¸Ô?Ð@Ñ@�Øœ×,Ñ,°SÀ%ÔHÐIÑIˆHÜ×'Ñ'¨Ó1Ð1Ør&   c                 óŽ   — t        | j                  dd«      s.t        || j                  «      }t        j                  ||d¬«      S y )Nr+   TF©rL  r0   Úsplit)r.   r/   r1   r   rM  rN  s       r$   rš   zLlamaConverter.pre_tokenizerV  sA   € Ü�t×.Ñ.°¸$Ô?Ü0Ð1AÀ4×CZÑCZÓ[ˆNÜ!×+Ñ+¸ÐTbÐjoÔpÐpØr&   c                  ó   — y r`   r5   r{   s    r$   r    zLlamaConverter.post_processor\  s   € àr&   N)
ra   rb   rc   r  r7   r   r¡   r˜   rš   r    r5   r&   r$   r\  r\  3  s&   „ ØÐòòò+òòór&   r\  c                   ó   — e Zd Zdefd„Zy)ÚMarkupLMConverterr(   c                 óŠ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd| j                   j                  ¬«      «      }t        j                  |j                  ¬«      |_        t        j                  «       |_        t        | j                   j                  «      }t        | j                   j                   «      }| j                   j"                  }| j                   j$                  }t'        j(                  |› d|› �|› d|› d|› �||f||fg¬«      |_        |S )NrÏ   Fr;  rÒ   z $A z $B rŒ   )r/   rÅ   rh   rÆ   rÇ   r   r   r�   r   rÖ   r'   rš   r	   r¡   re   r›   rœ   r�   rž   r   rŸ   r    )	rX   rö   r7   rI   r£   r¤   r¥   r�   rž   s	            r$   r|   zMarkupLMConverter.convertedb  s,  € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%ØØ×1Ñ1×;Ñ;ôó

ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜$˜s˜eÐ$Ø�5˜˜S˜E  c UÐ+à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ð Ðr&   Nr§   r5   r&   r$   ro  ro  a  s   „ ð"˜9ô "r&   ro  c                   ó*   — e Zd ZdZdd„Zd„ Zd„ Zd„ Zy)ÚMoshiConverterTNc                 ó  — t        | d«       t        j                  | |«       t        «       }|j	                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _        y # 1 sw Y   || _        y xY w©Nr   r	  ©	r   rw   rY   r%   r  r  r  r  r  )rX   r  Úmodel_max_lengthÚkwargsr  r  r  s          r$   rY   zMoshiConverter.__init__Š  sr   € Ü˜$ 
Ô+ä×Ñ˜4 Ô,ô $Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
úó   Á	 A9Á9B	c                 óâ   — |j                   j                  }t        j                  dd«      g}|st        j                  |«      S t        j                  t        j
                  |«      g|z   «      S rK  )rB  rC  r
   rE  rî   rF  rG  s       r$   r˜   zMoshiConverter.normalizer—  sg   € Ø$×4Ñ4×IÑIÐä×Ñ  UÓ+ð
ˆñ $Ü×'Ñ'¨Ó5Ð5ä×'Ñ'¬×)@Ñ)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgr&   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S r`  rb  rc  s       r$   r¡   zMoshiConverter.decoder¡  re  r&   c                 ó6   — d}t        j                  ||d¬«      S )Nr,   Frk  )r   rM  rN  s       r$   rš   zMoshiConverter.pre_tokenizer«  s   € Ø ˆÜ×'Ñ'°KÐP^ÐfkÔlÐlr&   r`   )ra   rb   rc   r  rY   r˜   r¡   rš   r5   r&   r$   rr  rr  ‡  s   „ ØÐóòhò+ómr&   rr  c                   óB   — e Zd ZdZdd„Zd„ Zd„ Zd„ Zd„ Zd„ Z	d	„ Z
d
„ Zy)ÚHeliumConverterTNc                 ó  — t        | d«       t        j                  | |«       t        «       }|j	                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _        y # 1 sw Y   || _        y xY wrt  ru  )rX   r  rw  r  r  r  s         r$   rY   zHeliumConverter.__init__³  sp   € Ü˜$ 
Ô+ä×Ñ˜4 Ô,ä#Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
úrx  c                 óV  — | j                  |«      }t        t        || j                  |«      | j                  ¬«      «      }t        |j                  «      D ��cg c]I  \  }}|j                  dv r6||j                  |j                  dk(  xs |j                  | j                  v f‘ŒK }}}|j                  t        |d„ ¬«      D ���cg c]  \  }}}t        |d|d¬«      ‘Œ c}}}«       |j                  t        d	dd¬
«      g«       |j                  dd¬«       |S c c}}w c c}}}w )Nr#  r%  r&  c                 ó   — | d   S r)  r5   r*  s    r$   r8   z+HeliumConverter.tokenizer.<locals>.<lambda>Ô  r+  r&   r9   FT)r-  r.  Úsingle_wordú
r,  r|  )r  Úpad_id)r7   r   r   r   r  r1  r  r4  ro   r�   r5  rF   r   Úenable_padding)	rX   r  rH   r£   r:  r;  r<  r=  r.  s	            r$   r£   zHeliumConverter.tokenizer¿  s   € Ø—z‘z %Ó(ˆÜÜØØ—{‘{ 5Ó)Ø"×7Ñ7ôó
ˆ	ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÐQUÖVôô	
ð 	×Ñœj¨¸%ÈÔOÐPÔQØ× Ñ ¨7¸1Ð Ô=ØÐùó
ùôs   ÁADÃD$c                 ó¨   — g }|j                   D ]@  }|j                  dk(  r|d|j                  fgz  }Œ%||j                  |j                  fgz  }ŒB |S )Nz<0x0A>r‚  r  r  s       r$   r7   zHeliumConverter.vocabÛ  s]   € ØˆØ—\‘\ò 	6ˆEØ�{‰{˜hÒ&Ø˜4 §¡Ð-Ð.Ñ.‘à˜5Ÿ;™;¨¯©Ð4Ð5Ñ5‘ð		6ð
 ˆr&   c                 ó
   — d}|S r)  r5   rp  s      r$   r   zHeliumConverter.unk_idä  rq  r&   c                 óÜ   — t        j                  dd«      t        j                  «       t        j                  «       g}|t        j                  dd¬«      gz  }t        j
                  |«      S r`  rb  rc  s       r$   r¡   zHeliumConverter.decoderè  sY   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆð
 	”X—^‘^¨C°aÔ8Ð9Ñ9ˆÜ× Ñ  Ó*Ð*r&   c                 ó~   — t        j                  t        j                  d«      t        j                  dd«      g«      S rK  )r
   rî   ri  rE  r!  s     r$   r˜   zHeliumConverter.normalizerñ  s2   € Ü×#Ñ#¤[×%8Ñ%8¸Ó%=¼{×?RÑ?RÐSWÐY^Ó?_Ð$`ÓaÐar&   c                 óV   — t        j                  t        j                  dd«      g«      S )Nr‚  Ú
contiguous)r   rî   rï   rV  s      r$   rš   zHeliumConverter.pre_tokenizerô  s#   € Ü×&Ñ&¬×(<Ñ(<¸TÀ<Ó(PÐ'QÓRÐRr&   c                 ó<   — t        j                  ddgg d¢dg¬«      S )Nrt  r  )rt  r  rt  r  )rt  r   rŒ   )r   rŸ   r{   s    r$   r    zHeliumConverter.post_processor÷  s/   € Ü×,Ñ,àØðòð ðô
ð 	
r&   r`   )ra   rb   rc   r  rY   r£   r7   r   r¡   r˜   rš   r    r5   r&   r$   r}  r}  °  s2   „ ØÐó
òò8òò+òbòSó
r&   r}  c                   ó   — e Zd ZdZdd„Zd„ Zy)ÚParakeetConverterTNc                 ó&  — || _         t        | d«       t        j                  | |«       t	        «       }|j                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _	        y # 1 sw Y   || _	        y xY wrt  )
r  r   rw   rY   r%   r  r  r  r  r  )rX   r  r  r  r  r  s         r$   rY   zParakeetConverter.__init__  sw   € Ø$ˆŒä˜$ 
Ô+ä×Ñ˜4 Ô,ä#Ó%ˆ	Ø× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
ús   Á B Â Bc                 ó°  — | j                  |«      }| j                  | j                  «      j                  |«      \  }}t	        |«      D ���ci c]  \  }\  }}||“Œ }}}}t        t        |||j                  j                  d| j                  d ¬«      «      }	t	        |j                  «      D �
�cg c]I  \  }
}|j                  dv r6|
|j                  |j                  dk(  xs |j                  | j                  v f‘ŒK }}
}|	j                  t        |d„ ¬«      D �
��cg c]  \  }
}}t!        |d|¬«      ‘Œ c}}}
«       |	S c c}}}w c c}}
w c c}}}
w )	NTr$  r%  r&  c                 ó   — | d   S r)  r5   r*  s    r$   r8   z-ParakeetConverter.tokenizer.<locals>.<lambda>2  r+  r&   r9   Fr,  )r7   r0  r  r_   r1  r   r   r  r2  r  r  r4  ro   r�   r5  rF   r   )rX   r  rH   r6  rI   r7  r8  r  r9  r£   r:  r;  r<  r=  r.  s                  r$   r£   zParakeetConverter.tokenizer  sM  € Ø—z‘z %Ó(ˆà×%Ñ% d§o¡oÓ6×>Ñ>¸|ÓL‰	ˆˆ6Ü5>¸|Ó5L×MÐMÑ!1 ¡M T¨5�T˜1‘WÐMˆ	ÒMÜÜØØØ×,Ñ,×6Ñ6ØØ"×7Ñ7Øôó	
ˆ	ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÖDôô	
ð Ðùô3 Nùó
ùôs   ÁEÂ2AEÄ Er`   )ra   rb   rc   r  rY   r£   r5   r&   r$   r�  r�  	  s   „ ØÐóór&   r�  c            	      óò  — t        t        t        d«      t        d«      dz   «      «      t        t        t        d«      t        d«      dz   «      «      z   t        t        t        d«      t        d«      dz   «      «      z   } | dd }d	}t        d
«      D ]1  }|| vsŒ| j                  |«       |j                  d
|z   «       |dz  }Œ3 |D �cg c]  }t	        |«      ‘Œ }}t        t        | |«      «      S c c}w )a8  
    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
    characters the bpe code barfs on.

    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
    tables between utf-8 bytes and unicode strings.
    ú!ú~r   õ   Â¡õ   Â¬õ   Â®õ   Ã¿Nr   é   )rh   rD   ÚordrE   ÚchrrB   r1  )ÚbsÚcsÚnÚbs       r$   Úbytes_to_unicoderŸ  :  sæ   € ô 	ŒU”3�s“8œS ›X¨™\Ó*Ó+¬d´5¼¸T»ÄCÈÃIÐPQÁMÓ3RÓ.SÑSÔVZÔ[`ÔadÐeiÓajÔloÐptÓluÐxyÑlyÓ[zÓV{Ñ{ð ð 
‰Aˆ€BØ	€AÜ�4‹[ò ˆØ�BŠ;Ø�I‰I�aŒLØ�I‰I�d˜Q‘hÔØ�‰F‰Að	ð
 Ö	�QŒ#ˆa�&Ð	€BÐ	Ü”�B˜“ÓÐùò 
s   ÃC4c                   ó>   — e Zd ZdZ	 	 	 	 d	d„Zdefd„Zd„ Zdefd„Z	y)
ÚTikTokenConverterz'
    A general tiktoken converter.
    Nc                 óˆ   — || _         || _        || _        t        |t        «      r|j                  «       | _        y || _        y r`   ©r  rh  r'   Ú
isinstancerB   rÇ   Úadditional_special_tokens©rX   r  rh  r'   r¥  rw  s         r$   rY   zTikTokenConverter.__init__W  óJ   € ð %ˆŒØˆŒØ 0ˆÔô Ð3´TÔ:ð &×*Ñ*Ó,ð 	Õ&ð +ð 	Õ&r&   Útiktoken_urlc                 ó0  ‡‡— 	 ddl m}  ||«      Št	        «       Šˆfd„}g }i }‰j                  «       D ]�  \  }}|| ||«      <   t        |«      dk(  rŒ g }t        dt        |«      «      D ]2  }	|d |	 ||	d  }}
|
‰v sŒ|‰v sŒ|
|z   ‰v sŒ|j                  |
||f«       Œ4 t        |ˆfd„d¬«      }|j                  |«       Œ� t        |d	„ d¬«      }|D �cg c]  } ||d   «       ||d   «      f‘Œ }}||fS # t        $ r t        d«      ‚w xY wc c}w )
Nr   )Úload_tiktoken_bpezY`tiktoken` is required to read a `tiktoken` file. Install it with `pip install tiktoken`.c           	      ó‚   •— dj                  | j                  d«      D �cg c]  }‰t        |«         ‘Œ c}«      S c c}w ©NrÏ   zlatin-1©r0  Údecoder™  ©rž  ÚcharÚbyte_encoders     €r$   Útoken_bytes_to_stringzPTikTokenConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_strings  s2   ø€ Ø—7‘7ÀÇÁÈÓ@SÖT¸˜L¬¨T«Ó3ÒTÓUÐUùÒTó    <r   c                 ó$   •— ‰| d      ‰| d      fS r4   r5   ©r6   rÆ   s    €r$   r8   zCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>�  s   ø€ °¸1¸Q¹4±À)ÈAÈaÉDÁ/Ð0R€ r&   Fr@   c                 ó   — | d   S ©Nr<   r5   r>   s    r$   r8   zCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>ƒ  ó
   € °°A±€ r&   )Útiktoken.loadrª  r3  Ú
ValueErrorrŸ  rC   r=   rD   rE   rF   rG   )rX   r¨  rª  r²  rI   r7   r=  ÚrankrL   rM   rN   rO   r?   rÆ   r±  s                @@r$   Úextract_vocab_merges_from_modelz1TikTokenConverter.extract_vocab_merges_from_modelh  sY  ù€ ð	Ý7ñ & lÓ3ˆ	Ü'Ó)ˆô	Vð ˆØˆØ$Ÿ?™?Ó,ò 
	!‰KˆE�4Ø26ˆEÑ'¨Ó.Ñ/Ü�5‹z˜QŠØØˆEÜ˜q¤# e£*Ó-ò ;�Ø#(¨¨% =°%¸¸°-˜�Ø˜iÒ'¨G°yÒ,@ÀgÐPWÑFWÐ\eÒEeØ—L‘L '¨7°DÐ!9Õ:ð;ô ˜5Ó&RÐ\aÔbˆEØ�M‰M˜%Õ ð
	!ô ˜Ñ$6ÀÔFˆØ\bÖcÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÒPÐcˆÐcØ�fˆ}Ðøô5 ò 	ÜØkóð ð	üò2 ds   „C; ÃDÃ;Dc                 ó¾   — | j                  | j                  «      \  }}t        t        ||d¬«      «      }t	        |j
                  d«      rd|j
                  _        |S ©NF)rÂ   Úignore_mergesT©r¼  r  r   r   r”   rS   r¿  ©rX   rH   rI   r£   s       r$   r£   zTikTokenConverter.tokenizer‡  óN   € Ø#×CÑCÀDÇOÁOÓTÑˆ�fÜœc ,°ÀÔGÓHˆ	Ü�9—?‘? OÔ4Ø,0ˆI�O‰OÔ)ØÐr&   r(   c           
      óÊ  — | j                  «       }t        j                  t        j                  t	        | j
                  «      dd¬«      t        j                  | j                  d¬«      g«      |_        t        j                  «       |_
        |j                  | j                  D �cg c]  }t        |dd¬«      ‘Œ c}«       t        j                  d¬«      |_        |S c c}w )Nrç   Frè   rë   Tr,  rÔ   )r£   r   rî   rï   r   rh  rÖ   r'   rš   r	   r¡   rÉ   r¥  r   r   r    )rX   r£   r=  s      r$   r|   zTikTokenConverter.convertedŽ  s¸   € Ø—N‘NÓ$ˆ	Ü"0×"9Ñ"9ä×$Ñ$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ñ(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôà×$Ñ$ØLP×LjÑLjÖkÀ5ŒZ˜¨%¸Ö>Òkô	
ô $.×#7Ñ#7ÀUÔ#Kˆ	Ô àÐùò ls   Â'C ©Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)
ra   rb   rc   rd   rY   re   r¼  r£   r   r|   r5   r&   r$   r¡  r¡  R  s:   „ ñð ð KØØ"&ó
ð"¸Có ò>ð˜9ô r&   r¡  c                   ó:   — e Zd Z	 	 	 	 dd„Zdefd„Zd„ Zdefd„Zy)	ÚMistralConverterNc                 óˆ   — || _         || _        || _        t        |t        «      r|j                  «       | _        y || _        y r`   r£  r¦  s         r$   rY   zMistralConverter.__init__¢  r§  r&   r¨  c                 óÊ  ‡‡— dd l }dd l}t        | j                  dd¬«      5 }|j	                  |«      }d d d «       d   d   | _        |d   D �cg c]  }t        |d   |d	   ¬
«      ‘Œ c}| _        |d   Št        «       Št        ˆfd„«       }g }i }	t        | j                  «      D ]  \  }
}|
|	|j                  <   Œ ‰D �cg c]  }|j                  |d   «      ‘Œ c}Št        ‰«      }t        t        ‰d¬«      «      D ]�  \  }}||	 ||«      <   t        |«      dk(  rŒ g }t!        dt        |«      «      D ]2  }|d | ||d  }}||v sŒ||v sŒ||z   |v sŒ|j#                  |||f«       Œ4 t%        |ˆfd„d¬«      }|j'                  |«       Œ� t%        |d„ d¬«      }|D �cg c]  } ||d   «       ||d   «      f‘Œ }}|	|fS # 1 sw Y   �Œ–xY wc c}w c c}w c c}w )Nr   Úrzutf-8)ÚencodingÚconfigrh  r�   Ú	token_strÚ
is_control)r.  r7   c           	      ó‚   •— dj                  | j                  d«      D �cg c]  }‰t        |«         ‘Œ c}«      S c c}w r¬  r­  r¯  s     €r$   r²  zOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_stringÀ  s2   ø€ à—7‘7ÀÇÁÈÓ@SÖT¸˜L¬¨T«Ó3ÒTÓUÐUùÒTr³  Útoken_bytesz(Converting tekken.json to tokenizer.json)Údescr   c                 óT   •— ‰j                  | d   «      ‰j                  | d   «      fS r4   )rM   rµ  s    €r$   r8   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>Ó  s)   ø€ °·±ÀÀ1ÁÓ1FÈ	ÏÉÐXYÐZ[ÑX\ÓH]Ð0^€ r&   Fr@   c                 ó   — | d   S r·  r5   r>   s    r$   r8   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>Õ  r¸  r&   )Úbase64Újsonr  r  Úloadrh  r   r¥  rŸ  r   r1  ra  Ú	b64decodeÚsetr   r=   rD   rE   rF   rG   )rX   r¨  rÓ  rÔ  r  ÚuntypedÚkr²  rI   r7   Úidxr=  Úrank_setr»  rL   rM   rN   rO   r?   rÆ   r±  s                      @@r$   r¼  z0MistralConverter.extract_vocab_merges_from_model³  s  ù€ ÛÛä�$—/‘/ 3°Ô9ð 	#¸QØ—i‘i “lˆG÷	#à˜xÑ(¨Ñ3ˆŒàIPÐQaÑIbö*
ØDEŒJ�q˜‘~¨q°©Ö?ò*
ˆÔ&ð ˜GÑ$ˆ	Ü'Ó)ˆä	ó	Vó 
ð	Vð ˆØˆÜ# D×$BÑ$BÓCò 	'‰JˆC�Ø#&ˆE�%—-‘-Ò ð	'àAJÖK¸A�V×%Ñ% a¨Ñ&6Õ7ÒKˆ	Ü�y“>ˆÜ$¤T¨)Ð:dÔ%eÓfò 
	!‰KˆD�%Ø26ˆEÑ'¨Ó.Ñ/Ü�5‹z˜QŠØØˆEÜ˜q¤# e£*Ó-ò ;�Ø#(¨¨% =°%¸¸°-˜�Ø˜hÒ&¨7°hÒ+>ÀGÈgÑDUÐZbÒCbØ—L‘L '¨7°DÐ!9Õ:ð;ô ˜5Ó&^ÐhmÔnˆEØ�M‰M˜%Õ ð
	!ô ˜Ñ$6ÀÔFˆØ\bÖcÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÒPÐcˆÐcØ�fˆ}Ð÷A	#ñ 	#üò*
ùò Lùò ds   £G	ÁGÃGÆ&G Ç	Gc                 ó¾   — | j                  | j                  «      \  }}t        t        ||d¬«      «      }t	        |j
                  d«      rd|j
                  _        |S r¾  rÀ  rÁ  s       r$   r£   zMistralConverter.tokenizerÙ  rÂ  r&   r(   c                 óŒ  — | j                  «       }t        j                  t        j                  t	        | j
                  «      dd¬«      t        j                  | j                  d¬«      g«      |_        t        j                  «       |_
        |j                  | j                  «       t        j                  d¬«      |_        |S )Nrç   Frè   rë   rÔ   )r£   r   rî   rï   r   rh  rÖ   r'   rš   r	   r¡   r5  r¥  r   r    )rX   r£   s     r$   r|   zMistralConverter.convertedà  s™   € Ø—N‘NÓ$ˆ	Ü"0×"9Ñ"9ä×$Ñ$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ñ(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôà×Ñ˜T×;Ñ;Ô<Ü#-×#7Ñ#7ÀUÔ#Kˆ	Ô àÐr&   rÄ  )	ra   rb   rc   rY   re   r¼  r£   r   r|   r5   r&   r$   rÆ  rÆ  ¡  s6   „ ð ð KØØ"&ó
ð"$¸Có $òLð˜9ô r&   rÆ  ÚAlbertTokenizerÚBartTokenizerÚBarthezTokenizerÚBertTokenizerÚBigBirdTokenizerÚBlenderbotTokenizerÚCamembertTokenizerÚCLIPTokenizerÚCodeGenTokenizerÚConvBertTokenizerÚDebertaTokenizerÚDebertaV2TokenizerÚDistilBertTokenizerÚDPRReaderTokenizerÚDPRQuestionEncoderTokenizerÚDPRContextEncoderTokenizerÚElectraTokenizerÚFNetTokenizerÚFunnelTokenizerÚGPT2TokenizerÚHerbertTokenizerÚLayoutLMTokenizerÚLayoutLMv2TokenizerÚLayoutLMv3TokenizerÚLayoutXLMTokenizerÚLongformerTokenizerÚLEDTokenizerÚLxmertTokenizerÚMarkupLMTokenizerÚMBartTokenizerÚMBart50TokenizerÚMPNetTokenizerÚMobileBertTokenizerÚMvpTokenizerÚNllbTokenizerÚOpenAIGPTTokenizerÚPegasusTokenizerÚQwen2TokenizerÚRealmTokenizerÚReformerTokenizerÚRemBertTokenizerÚRetriBertTokenizerÚRobertaTokenizerÚRoFormerTokenizerÚSeamlessM4TTokenizerÚSqueezeBertTokenizerÚT5TokenizerÚUdopTokenizerÚWhisperTokenizerÚXLMRobertaTokenizerÚXLNetTokenizer)ÚSplinterTokenizerÚXGLMTokenizerÚLlamaTokenizerÚCodeLlamaTokenizerÚGemmaTokenizerÚPhi3Tokenizerc                 ó*  — | j                   j                  }|t        v r!|st        |   } || «      j                  «       S | j                  j                  d«      r?| | _        t        j                  d«       t        | j                  «      j                  «       S 	 t        j                  d«       t        | j                  | j                  ¬«      j                  «       S # t        $ r* t        dt        t        j                  «       «      › �«      ‚w xY w)aˆ  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].
       from_tiktoken (bool, optional): Whether to use the `tiktoken` library to convert the tokenizer instead of sentencepiece.
            Defaults to False.

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    ztekken.jsonz#Converting from Mistral tekken.jsonzConverting from Tiktoken)r  r¥  zÌConverting from SentencePiece and Tiktoken failed, if a converter for SentencePiece is available, provide a model path with a SentencePiece tokenizer.model file.Currently available slow->fast converters: )r  ra   ÚSLOW_TO_FAST_CONVERTERSr|   r  Úendswithr/   ÚloggerÚinforÆ  r¡  r¥  r3  rº  rh   rÇ   )Útransformer_tokenizerÚfrom_tiktokenÚtokenizer_class_nameÚconverter_classs       r$   Úconvert_slow_tokenizerr   -  sþ   € ð  1×:Ñ:×CÑCÐØÔ6Ñ6¹}Ü1Ð2FÑGˆÙÐ4Ó5×?Ñ?ÓAÐAØ	×	)Ñ	)×	2Ñ	2°=Ô	AØ3HÐÔ0Ü�‰Ð9Ô:ÜÐ 5× @Ñ @ÓA×KÑKÓMÐMð	Ü�K‰KÐ2Ô3Ü$Ø0×;Ñ;Ø*?×*YÑ*Yô÷ ‰i‹kðøô ò 	Üð>ä>BÔCZ×C_ÑC_ÓCaÓ>bÐ=cðeóð ð	ús   ÂAC Ã3D)rÏ   )F)Urd   r  Ú	functoolsr   Útypingr   Ú	packagingr   Ú
tokenizersr   r   r   r	   r
   r   r   Útokenizers.modelsr   r   r   r   Úutilsr   r   r   r   Úutils.import_utilsr   Ú
get_loggerra   r  r%   Úboolre   r1   rP   rR   rj   ru   rw   r~   rª   r·   rº   r½   rÍ   rÝ   rä   rñ   rù   r   r  rT  rm  rx  r…  r�  rÊ  rê  rò  rù  rÿ  r  r  r  r  r   r(  r,  r6  r9  r>  rA  rD  rI  r\  ro  rr  r}  r�  rŸ  r¡  rÆ  r  r   r5   r&   r$   ú<module>r*     sŠ  ðñó Ý Ý å ß f× fÑ fß 5Ñ 5Ý ç `Ó `Ý 5ð 
ˆ×	Ñ	˜HÓ	%€óGð"¨$ð Àsó ò÷&ñ ô2Ð"8ô ð"I˜cð I dó I÷$ñ $ô$�Iô $ôN/˜	ô /ôd$�iô $ôN$�Yô $ôN˜ô ô6%�Iô %ôP�yô ô>+�Yô +ô\�yô ô:$˜	ô $ôN�yô ô>~�9ô ~ôB"
�lô "
ôJ
�|ô 
ô 
˜ô 
ô:
˜ô 
ôB2
�\ô 2
ôj
�|ô 
ô6
�Lô 
ô2
˜<ô 
ô2
˜,ô 
ô6"
�\ô "
ôJ	˜ô 	ô
�|ô 
ô@	˜lô 	ô%x�|ô %xôP
�,ô 
ô"
�Lô 
ô!�yô !ôH	
�|ô 	
ô(�Iô (ôV$˜)ô $ôN˜)ô ô:
�Lô 
ô61
�\ô 1
ôh+�\ô +ô\#˜	ô #ôL&m�\ô &môRV
�lô V
ôr-˜ô -òb÷0Lñ L÷^Lñ Lð^:Ø�ð:àÐ%ð:ð Ð(ð:ð �]ð	:ð
 Ð(ð:ð Ð.ð:ð Ð,ð:ð �]ð:ð ˜ð:ð ˜ð:ð Ð(ð:ð Ð,ð:ð ˜=ð:ð ˜-ð:ð " =ð:ð  ! -ð!:ð" ˜ñ#:ð$ �_ð%:ð& �ð':ð( �]ð):ð* Ð(ð+:ð, ˜ð-:ð. ˜=ð/:ð0 Ð+ð1:ð2 Ð-ð3:ð4 Ð+ð5:ð6 Ð$ð7:ð8 �}ð9:ð: Ð*ð;:ð< �nð=:ð> Ð(ð?:ð@ �nðA:ðB ˜=ðC:ðD Ð$òE:ðF �]ðG:ðH Ð,ðI:ðJ Ð(ðK:ðL �nðM:ðN �mðO:ðP Ð*ðQ:ðR Ð(ðS:ðT ˜-ðU:ðV Ð(ðW:ðX Ð*ðY:ðZ Ð0ð[:ð\ ˜Mð]:ð^ �;ð_:ð` �]ða:ðb Ð(ðc:ðd Ð.ðe:ðf �nñg:ðh +Ø"Ø$Ø(Ø$Ø#òs:Ð ñz$È)ô $r&   