Ë
    îÍ:jŽv  ã                   óD  — d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ  ej2                  e«      Zi ddddœ“ddddddddddddœ
“ddddddddddddœ
“ddddddddddddœ
“dddddddddddddd œ“d!ddddddddddd"d#œ“d$dddddddddddœ
“d%dddddddddddddd&œ“d'dddddddddddœ
“d(d)d*d+d,d-œ“d.dddddddddddœ
“d/d0dd1dd2d3œ“d4d5d6d7d8d9d:dd2d;d<dd=œ“d>dddddddd?dd@œ	“dAd0dBdCdDd1d2dEœ“dFdddddddGdHœ“dIdddd2ddJdKdLddMœ	“dddddddddNddœ
dddddddddddOddPœdddddddddddOddPœd5d6d7d8d9d:dd2d;d<dd=œdddddddddddœ
dQœ¥ZdRdSdTdUdVd)d*d+d,dWdXœ
dYdd)d*d+d,dZœd[œZd\„ Z G d]„ d^«      Z G d_„ d`e«      Z  G da„ dbe«      Z! G dc„ dde«      Z" G de„ dfe«      Z# G dg„ dhe«      Z$ G di„ dje«      Z%i de “de!“dke!“d$e!“d%e!“d.e"“d/e#“d'e#“d>e#“dAe#“dFe#“d4e$“dIe#“dle#“dme%“dne%“doe$“e e dpœ¥Z&dqe'dre(ee)f   fds„Z*y)tz
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
é    )ÚarrayN)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramé   )Ú
AddedToken)ÚGemmaConverterÚGPT2ConverterÚLlamaConverterÚQwen2ConverterÚT5Converter)Úlogging)ÚtqdmÚgeneralÚ
model_typeÚ_model_name_or_path)ÚarchitectureÚnameÚllamaÚmax_position_embeddingsÚnum_hidden_layersÚintermediate_sizeÚhidden_sizeÚhead_dimÚ
rope_thetaÚnum_attention_headsÚnum_key_value_headsÚrms_norm_epsÚ
vocab_size)
Úcontext_lengthÚblock_countÚfeed_forward_lengthÚembedding_lengthúrope.dimension_countúrope.freq_baseúattention.head_countúattention.head_count_kvú attention.layer_norm_rms_epsilonr#   ÚmistralÚqwen2Úqwen2moeÚnum_expertsÚnum_experts_per_tok)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   Úexpert_countÚexpert_used_countÚlfm2Úconv_L_cache)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   zshortconv.l_cacheÚqwen3Ú	qwen3_moe)r$   r%   r&   r'   r(   r)   úattention.key_lengthr*   r+   r,   r#   r2   r3   ÚfalconÚ	tokenizerÚbos_token_idÚeos_token_idÚunk_token_idÚpad_token_id)úggml.bos_token_idúggml.eos_token_idúggml.unknown_token_idúggml.padding_token_idÚphi3ÚbloomÚn_layerÚn_headÚlayer_norm_epsilon)r%   r'   r*   r#   úattention.layer_norm_epsilonÚt5Ún_positionsÚ
num_layersÚd_ffÚd_modelÚd_kvÚ	num_headsÚrelative_attention_num_bucketsÚdecoder_start_token_id)r$   r%   r&   r'   r8   r*   r+   rH   z attention.relative_buckets_countrQ   r#   ÚstablelmÚlayer_norm_eps)	r$   r%   r&   r'   r(   r*   r+   rH   r#   Úgpt2Ún_ctxÚn_embdr&   )r%   r$   r'   r&   r*   rH   Ú
starcoder2Únorm_epsilon)r%   r$   r'   r&   r*   r+   rH   ÚmambaÚconv_kernelÚ
state_sizeÚtime_step_rank)	r#   r$   r'   r,   r%   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizeÚnorm_epsÚsliding_window)r$   r%   r&   r'   r(   r)   r8   r*   r+   r,   zattention.sliding_windowr#   )ÚnemotronÚgemma2Úgemma3Úumt5ÚdeciÚtokenizer_typeÚtokensÚscoresÚ
token_typeÚmergesÚadd_prefix_space)
ú
ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr?   r@   rA   rB   zggml.add_space_prefixÚchat_template)rk   rj   r?   r@   rA   rB   )r:   Útokenizer_configc                 ó”  — t        |t        «      s|g}t        |«      dk(  r|d   }d }n|d   dk7  rt        d«      ‚|\  }}|dv rt	        | d   «      } | S |dv rt        | d   «      } | S |dk(  rt        | d   «      } | S |dk(  r3t        d	t        | «      «      j                  «       j                  «       } | S |dk(  rt        | |«      } | S )
Né   r   é	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rn   r   é   é   é   é
   é   )é   é   é   é   ÚB)Ú
isinstanceÚlistÚlenÚ
ValueErrorÚintÚfloatÚboolr   ÚtobytesÚdecodeÚ_gguf_parse_value)Ú_valueÚ	data_typeÚarray_data_types      ús/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/transformers/integrations/ggml.pyrƒ   rƒ   =  sô   € Ü�i¤Ô&Ø�Kˆ	Ü
ˆ9ƒ~˜ÒØ˜a‘Lˆ	Ø‰à�Q‰<˜1ÒÜÐoÓpÐpØ%.Ñ"ˆ	�?àÐ.Ñ.Ü�V˜A‘Y“ˆð €Mð 
�gÑ	Ü�v˜a‘yÓ!ˆð €Mð 
�aŠÜ�f˜Q‘i“ˆð
 €Mð	 
�aŠÜ�sœD ›LÓ)×1Ñ1Ó3×:Ñ:Ó<ˆð €Mð 
�aŠÜ" 6¨?Ó;ˆØ€Mó    c                   ó   — e Zd Zd„ Zy)ÚGGUFTokenizerSkeletonc                 óp  ‡— |j                  «       D ]  \  }}t        | ||«       Œ t        | d«      �s)t        | d«      rt        | d«      st        d«      ‚| j                  }| j
                  }t        |«      D ��ci c]  \  }}|||   “Œ c}}Št        j                  d«       g }t        ‰j                  «       «      D ]k  \  }	}
g }t        dt        |	«      «      D ]*  }|	d | |	|d  }}||v sŒ||v sŒ|j                  |||
f«       Œ, t        |ˆfd„d¬	«      }|j                  |«       Œm t        |d
„ d¬	«      }|D �cg c]  }|d   |d   f‘Œ }}|| _        nt| j                  D �	cg c]  }	t!        |	j#                  d«      «      ‘Œ c}	| _        t        | d«      s1t        t        | j                  «      «      D �cg c]  }d ‘Œ c}| _        t        | d«      sg | _        t        | d«      sd | _        t        | d«      r| j&                  €| j(                  | _        y y y c c}}w c c}w c c}	w c c}w )Nrh   re   rf   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.z:Merges were not in checkpoint, building merges on the fly.rn   c                 ó$   •— ‰| d      ‰| d      fS )Nr   rn   © )ÚxÚvocabs    €r‡   ú<lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>k  s   ø€ °U¸1¸Q¹4±[À%ÈÈ!ÉÁ+Ð4N€ rˆ   T)ÚkeyÚreversec                 ó   — | d   S )Nr   r�   )Úvals    r‡   r�   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>m  s
   € °C¸±F€ rˆ   r   ú Úadded_tokensr=   Úunknown_token_id)ÚitemsÚsetattrÚhasattrr}   re   rf   Ú	enumerateÚloggerÚwarningr   Úranger|   ÚappendÚsortedÚextendrh   ÚtupleÚsplitr–   r=   r—   )ÚselfÚdict_ÚkÚvre   rf   ÚiÚtrh   ÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rr”   Ú_r�   s                    @r‡   Ú__init__zGGUFTokenizerSkeleton.__init__V  s   ø€ Ø—K‘K“Mò 	 ‰DˆAˆqÜ�D˜!˜QÕð	 ô �t˜XÕ&Ü˜4 Ô*´'¸$ÀÔ2IÜ Øróð ð —[‘[ˆFØ—[‘[ˆFÜ.7¸Ó.?×@¡d a¨�Q˜˜q™	‘\Ó@ˆEä�N‰NÐWÔXØˆFÜ&*¨5¯;©;«=Ó&9ò %Ñ"��{Ø�Ü" 1¤c¨%£jÓ1ò F�EØ',¨V¨e }°e¸E¸F°m˜W�GØ &Ò(¨W¸Ò->ØŸ™ g¨w¸Ð%DÕEðFô ˜uÓ*NÐX\Ô]�Ø—‘˜eÕ$ð%ô ˜FÑ(:ÀDÔIˆFØ28Ö9¨3�s˜1‘v˜s 1™vÒ&Ð9ˆFÐ9Ø ˆD�Kà@DÇÁÖL°uœ5 §¡¨SÓ!1Õ2ÒLˆDŒKÜ˜4 Ô*Ü-2´3°t·{±{Ó3CÓ-DÖE¨štÒE�”ä�t˜^Ô,Ø "ˆDÔä�t˜^Ô,Ø $ˆDÔô �4Ð+Ô,°×1BÑ1BÐ1JØ $× 5Ñ 5ˆDÕð 2KÐ,ùó7 Aùò :ùò MùâEs   Á=H#ÅH)Å+!H.Ç 	H3N)Ú__name__Ú
__module__Ú__qualname__r±   r�   rˆ   r‡   rŠ   rŠ   U  s   „ ó'6rˆ   rŠ   c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zy)ÚGGUFLlamaConverterc                 ó’   — t        |«      | _        | j                  | _        i | _        t	        | j                  dd«      dk7  | _        y )Nrd   r   )rŠ   ÚprotoÚoriginal_tokenizerÚadditional_kwargsÚgetattrÚis_llama_3_tokenizer©r¤   Útokenizer_dicts     r‡   r±   zGGUFLlamaConverter.__init__�  s>   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÔÜ$+¨D¯J©JÐ8HÈ'Ó$RÐV]Ñ$]ˆÕ!rˆ   c                 óT   — t        t        |j                  |j                  «      «      S ©N©r{   Úzipre   rf   ©r¤   r¸   s     r‡   r�   zGGUFLlamaConverter.vocab‡  ó   € Ü”C˜Ÿ™ e§l¡lÓ3Ó4Ð4rˆ   c                 ó   — |j                   S rÀ   ©rh   rÃ   s     r‡   rh   zGGUFLlamaConverter.mergesŠ  ó   € Ø�|‰|Ðrˆ   c           
      óÄ  — | j                  | j                  «      }| j                  | j                  «      }t        |«      D ���ci c]  \  }\  }}||“Œ }}}}|j                  �|j
                  |j                     nd }t        |dd «      �|j
                  |j                     nd }	t        |dd «      �|j
                  |j                     nd }
t        t        |||dd¬«      «      }g }t        | j                  d«      s^|�|j                  t        |dd¬«      «       |	�|j                  t        |	dd¬«      «       |
�›|j                  t        |
dd¬«      «       n}t        j                  t        j                  | j                  j                   «      dk(  «      d	   }|D ]6  }|j                  t        | j                  j
                  |   dd¬«      «       Œ8 t#        |«      d	k7  r|j%                  |«       t#        | j                  j&                  «      d	k7  r?|j)                  | j                  j&                  D �cg c]  }t        |dd¬«      ‘Œ c}«       || j*                  d
<   |	| j*                  d<   |
| j*                  d<   | j,                  r>d | j*                  d<   d| j*                  d<   d| j*                  d<   d| j.                  _        |S c c}}}w c c}w )Nr;   r<   T)Ú	unk_tokenÚfuse_unkÚbyte_fallbackrg   F©Ú
normalizedÚspecialrp   r   rÉ   Ú	eos_tokenÚ	bos_tokenri   Úclean_up_tokenization_spacesÚlegacy)r�   r¸   rh   r›   r=   re   r»   r;   r   r	   rš   rŸ   r   ÚnpÚwherer   rg   r|   Úadd_special_tokensr–   Ú
add_tokensrº   r¼   r¹   rÒ   )r¤   r¸   Úvocab_scoresrh   r¨   ÚwordÚ_scoreÚ	bpe_vocabrÉ   rÐ   rÏ   r:   Úspecial_tokensÚspecial_tokens_idxÚidxÚadded_tokens                   r‡   r:   zGGUFLlamaConverter.tokenizer�  s¨  € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÓ6M×NÐNÑ!2 ¡N T¨6�T˜1‘WÐNˆ	ÒNà8=×8JÑ8JÐ8V�E—L‘L ×!3Ñ!3Ò4Ð\`ˆ	Ü8?ÀÀ~ÐW[Ó8\Ð8h�E—L‘L ×!3Ñ!3Ò4Ðnrˆ	Ü8?ÀÀ~ÐW[Ó8\Ð8h�E—L‘L ×!3Ñ!3Ò4Ðnrˆ	äÜØØØ#ØØ"ôó
ˆ	ð ˆä�t—z‘z <Ô0ØÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\àÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\àÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Õ\ô "$§¡¬"¯(©(°4·:±:×3HÑ3HÓ*IÈQÑ*NÓ!OÐPQÑ!RÐà)ò j�Ø×%Ñ%¤j°·±×1BÑ1BÀ3Ñ1GÐTYÐcgÔ&hÕiðjô ˆ~Ó !Ò#Ø×(Ñ(¨Ô8äˆt�z‰z×&Ñ&Ó'¨1Ò,Ø× Ñ Ø]a×]gÑ]g×]tÑ]tÖuÈk”˜K°EÀ5ÖIÒuôð /8ˆ×Ñ˜{Ñ+Ø.7ˆ×Ñ˜{Ñ+Ø.7ˆ×Ñ˜{Ñ+à×$Ò$Ø9=ˆD×"Ñ"Ð#5Ñ6ØEIˆD×"Ñ"Ð#AÑBà/4ˆD×"Ñ" 8Ñ,Ø-2ˆD×#Ñ#Ô*àÐùôk OùòN vs   ÁKÉKc                 ó0  — t        j                  «       t        j                  «       t        j                  dd«      g}| j                  r|t        j
                  ddd¬«      gz  }|r|t        j                  dd¬«      gz  }t        j                  |«      S )Nõ   â–�r•   FT©ri   Útrim_offsetsÚ	use_regexrn   ©ÚcontentÚleft)r   ÚByteFallbackÚFuseÚReplacer¼   Ú	ByteLevelÚStripÚSequence©r¤   Úreplacementri   Úsequences       r‡   ÚdecoderzGGUFLlamaConverter.decoderÇ  sƒ   € ä×!Ñ!Ó#Ü�M‰M‹OÜ×Ñ˜U CÓ(ð
ˆð ×$Ò$Øœ×+Ñ+¸UÐQVÐbfÔgÐhÑhˆHáØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*rˆ   c                 ó   — | j                  | j                  «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j
                  }| j                  ||«      }|�||_        | j                  ||«      |_        | j                  «       }|r||_        | j                  r7t        j                  ddd¬«      |_        t        j                  g «      |_        |S )Nrà   Tri   Frá   )r:   r¸   Ú
normalizerrš   r¹   ri   Úpre_tokenizerrð   Úpost_processorr¼   r   rê   r   rì   )r¤   r:   rò   rî   ri   ró   rô   s          r‡   Ú	convertedzGGUFLlamaConverter.convertedÕ  sò   € à—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$ð ×$Ò$Ü&4×&>Ñ&>Ø!&°UÀdô'ˆIÔ#ô
 $/×#7Ñ#7¸Ó#;ˆIÔ àÐrˆ   N)	r²   r³   r´   r±   r�   rh   r:   rð   rõ   r�   rˆ   r‡   r¶   r¶   €  s"   „ ò^ò5òò8òt+ó!rˆ   r¶   c                   ó*   ‡ — e Zd Zd„ Zdefˆ fd„Zˆ xZS )ÚGGUFQwen2Converterc                 ó2   — t        |«      | _        i | _        y rÀ   ©rŠ   r¹   rº   r½   s     r‡   r±   zGGUFQwen2Converter.__init__ú  ó   € Ü"7¸Ó"GˆÔØ!#ˆÕrˆ   Úreturnc           	      ó4  •— t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 }}}| j                  j                  }t        ‰| �  ||«      }|j                  t        ddd¬«      t        ddd¬«      t        ddd¬«      g«       |S c c}}w )Nú<|endoftext|>FTrÌ   z<|im_start|>z
<|im_end|>)r›   r¹   re   rh   Úsuperrõ   rÕ   r   ©r¤   r¨   rØ   r�   rh   r:   Ú	__class__s         €r‡   rõ   zGGUFQwen2Converter.convertedþ  s–   ø€ Ü(1°$×2IÑ2I×2PÑ2PÓ(Q×R™W˜Q ��q‘ÐRˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% e¨VÓ4ˆ	à×$Ñ$ä˜?°uÀdÔKÜ˜>°eÀTÔJÜ˜<°EÀ4ÔHðô	
ð Ðùó Ss   ¤B©r²   r³   r´   r±   r   rõ   Ú__classcell__©r   s   @r‡   r÷   r÷   ù  s   ø„ ò$ð˜9÷ ñ rˆ   r÷   c                   ó6   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zdefd„Z	y)	ÚGGUFPhi3Converterc                 óT   — t        |«      | _        | j                  | _        i | _        y rÀ   ©rŠ   r¸   r¹   rº   r½   s     r‡   r±   zGGUFPhi3Converter.__init__  s"   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÕrˆ   c                 óT   — t        t        |j                  |j                  «      «      S rÀ   rÁ   rÃ   s     r‡   r�   zGGUFPhi3Converter.vocab  rÄ   rˆ   c                 ó   — |j                   S rÀ   rÆ   rÃ   s     r‡   rh   zGGUFPhi3Converter.merges  rÇ   rˆ   c                 ó  — | j                  | j                  «      }| j                  | j                  «      }t        |«      D ���ci c]  \  }\  }}||“Œ }}}}t	        t        ||«      «      }|j                  t        ddddd¬«      t        ddd¬«      t        dddd¬«      t        d	ddd¬«      t        d
ddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      g«       |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |S c c}}}w )Nú</s>TF)ÚrstripÚlstriprÍ   rÎ   rý   rÌ   z<|assistant|>)r  rÍ   rÎ   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>rÉ   rÏ   rÐ   Ú	pad_token)r�   r¸   rh   r›   r   r	   rÕ   r   r=   re   rº   r<   r;   r>   )	r¤   r¸   r×   rh   r¨   rØ   rÙ   rÚ   r:   s	            r‡   r:   zGGUFPhi3Converter.tokenizer  s   € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÓ6M×NÐNÑ!2 ¡N T¨6�T˜1‘WÐNˆ	ÒNäœc )¨VÓ4Ó5ˆ	à×$Ñ$ä˜6¨$°uÈÐX\Ô]Ü˜?°uÀdÔKÜ˜?°4ÀEÐSWÔXÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[Ü˜<°ÀÐPTÔUÜ˜9¨T¸eÈTÔRÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[Ü˜:¨d¸uÈdÔSðô	
ð$ 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð ÐùôE Os   ÁG:c                 óà   — t        j                  «       t        j                  «       t        j                  |d«      g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S )Nr•   rn   rä   )r   rç   rè   ré   rë   rì   rí   s       r‡   rð   zGGUFPhi3Converter.decoder@  s\   € ä×!Ñ!Ó#Ü�M‰M‹OÜ×Ñ˜[¨#Ó.ð
ˆñ ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*rˆ   rû   c                 óÊ   — | j                  | j                  «      }d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      |_        |S )Nrà   Tri   )r:   r¸   rš   r¹   ri   rð   )r¤   r:   rî   ri   s       r‡   rõ   zGGUFPhi3Converter.convertedK  s\   € Ø—N‘N 4§:¡:Ó.ˆ	àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà ŸL™L¨Ð6FÓGˆ	ÔàÐrˆ   N)
r²   r³   r´   r±   r�   rh   r:   rð   r   rõ   r�   rˆ   r‡   r  r    s(   „ ò$ò
5òò%òN	+ð
˜9ô 
rˆ   r  c                   ó*   ‡ — e Zd Zd„ Zdefˆ fd„Zˆ xZS )ÚGGUFGPTConverterc                 ó2   — t        |«      | _        i | _        y rÀ   rù   r½   s     r‡   r±   zGGUFGPTConverter.__init__Y  rú   rˆ   rû   c                 óÄ   •— t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 }}}| j                  j                  }t        ‰| �  ||«      }|S c c}}w rÀ   )r›   r¹   re   rh   rþ   rõ   rÿ   s         €r‡   rõ   zGGUFGPTConverter.converted]  s]   ø€ Ü(1°$×2IÑ2I×2PÑ2PÓ(Q×R™W˜Q ��q‘ÐRˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% e¨VÓ4ˆ	ØÐùó Ss   ¤Ar  r  s   @r‡   r  r  X  s   ø„ ò$ð˜9÷ ñ rˆ   r  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zdefd„Zy)ÚGGUFT5Converterc                 óÜ   — dg|d<   t        |«      | _        t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 c}}| _        | j                  | _        i | _        y c c}}w ©Nz
dummy textrh   )rŠ   r¸   r›   re   Útoken2idr¹   rº   )r¤   r¾   r§   r¦   s       r‡   r±   zGGUFT5Converter.__init__e  s]   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ü*3°D·J±J×4EÑ4EÓ*F×G¡$ ! Q˜˜A™ÓGˆŒØ"&§*¡*ˆÔØ!#ˆÕùó Hs   ¹A(c                 óT   — t        t        |j                  |j                  «      «      S rÀ   rÁ   rÃ   s     r‡   r�   zGGUFT5Converter.vocabn  rÄ   rˆ   c                 óø   — t        | j                  dd«      rcg }t        | j                  dd«      r|t        j                  d¬«      gz  }|t        j                  dd¬«      gz  }t        j
                  |«      S y )NrÒ   Tri   rà   )Úprependr•   )Úpatternrå   )r»   r¹   r   ÚPrependré   rì   )r¤   r¸   rï   s      r‡   rò   zGGUFT5Converter.normalizerq  sr   € Ü�4×*Ñ*¨H°dÔ;ØˆHÜ�t×.Ñ.Ð0BÀDÔIØœ[×0Ñ0¸Ô?Ð@Ñ@�Øœ×,Ñ,°SÀ%ÔHÐIÑIˆHÜ×'Ñ'¨Ó1Ð1Ørˆ   c                 óZ   — t        j                  ddgg d¢d| j                  d   fg¬«      S )Nú$Ar  )r   r  z$Br  )ÚsingleÚpairrÛ   )r   ÚTemplateProcessingr  )r¤   s    r‡   rô   zGGUFT5Converter.post_processorz  s5   € Ü×,Ñ,Ø˜&�>Ú-à˜Ÿ™ vÑ.Ð/ðô
ð 	
rˆ   rû   c                 óÐ  — | j                  | j                  «      }t        t        || j                  j                  d¬«      «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      }|�||_	        | j                  ||«      |_
        | j                  «       }|r||_        |S )NF©Úunk_idrË   rà   Tri   )r�   r¸   r   r
   r=   rò   rš   r¹   ri   ró   rð   rô   )r¤   r×   r:   rò   rî   ri   ró   rô   s           r‡   rõ   zGGUFT5Converter.convertedƒ  sÝ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø#ôó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$àÐrˆ   N)	r²   r³   r´   r±   r�   rò   rô   r   rõ   r�   rˆ   r‡   r  r  d  s"   „ ò$ò5òò
ð˜9ô rˆ   r  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zdefd„Zy)ÚGGUFGemmaConverterc                 ó`   — dg|d<   t        |«      | _        | j                  | _        i | _        y r  r  r½   s     r‡   r±   zGGUFGemmaConverter.__init__¤  s.   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÕrˆ   c                 óL  — t        t        |j                  |j                  «      «      }g }|D ]s  \  }}|dk(  r|j	                  d|f«       Œd|v r>t        |j                  «       «      dk(  r"dt        |«      z  }|j	                  ||f«       Œa|j	                  ||f«       Œu |S )Nz<0x09>ú	r•   r   rà   )r{   rÂ   re   rf   rŸ   r|   Ústrip)r¤   r¸   Úoriginal_vocabÚupdated_vocabÚtokenÚscoreÚunderscoress          r‡   r�   zGGUFGemmaConverter.vocab¬  s¡   € Üœc %§,¡,°·±Ó=Ó>ˆØˆà*ò 	5‰LˆE�5Ø˜Ò Ø×$Ñ$ d¨E ]Õ3Ø˜‘¤# e§k¡k£mÓ"4¸Ò"9Ø#¤c¨%£jÑ0�Ø×$Ñ$ k°5Ð%9Õ:à×$Ñ$ e¨U ^Õ4ð	5ð Ðrˆ   c                 ó.   — t        j                  dd«      S )Nr•   rà   )r   ré   rÃ   s     r‡   rò   zGGUFGemmaConverter.normalizer»  s   € Ü×"Ñ" 3¨Ó.Ð.rˆ   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S )Nrà   r•   rn   rä   )r   ré   rç   rè   rë   rì   rí   s       r‡   rð   zGGUFGemmaConverter.decoder¾  s\   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆñ ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*rˆ   rû   c                 ó²  — | j                  | j                  «      }t        t        || j                  j                  | j
                  ¬«      «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      |_
        | j                  ||«      }|�||_        |S )Nr%  rà   Tri   )r�   r¸   r   r
   r=   Úhandle_byte_fallbackrò   rš   r¹   ri   rð   ró   )r¤   r×   r:   rò   rî   ri   ró   s          r‡   rõ   zGGUFGemmaConverter.convertedÉ  sÊ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø"×7Ñ7ôó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà ŸL™L¨Ð6FÓGˆ	ÔØ×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#àÐrˆ   N)	r²   r³   r´   r±   r�   rò   rð   r   rõ   r�   rˆ   r‡   r(  r(  £  s"   „ ò$òò/ò	+ð˜9ô rˆ   r(  Ú	qwen2_moer_   r`   Úgemma3_textrb   )rc   Údecilmr   rû   c                 ó^   — | }t        |   |«      }|j                  «       }||j                  fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )ÚGGUF_TO_FAST_CONVERTERSrõ   rº   )r   r¾   Útokenizer_class_nameÚ	converterÚfast_tokenizers        r‡   Úconvert_gguf_tokenizerr>  û  s9   € ð (ÐÜ'Ð(<Ñ=¸nÓM€IØ×(Ñ(Ó*€NØ˜9×6Ñ6Ð6Ð6rˆ   )+Ú__doc__r   ÚnumpyrÓ   Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr	   r
   Ú r   Úconvert_slow_tokenizerr   r   r   r   r   Úutilsr   Úutils.loggingr   Ú
get_loggerr²   rœ   ÚGGUF_CONFIG_MAPPINGÚGGUF_TOKENIZER_MAPPINGrƒ   rŠ   r¶   r÷   r  r  r  r(  r:  Ústrr¢   Údictr>  r�   rˆ   r‡   ú<module>rL     s²  ðñ õ
 ã ß SÕ Sß *å ß oÕ oÝ Ý  ð 
ˆ×	Ñ	˜HÓ	%€ðØØ$Ø%ñðð
 Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ñðð$ Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ñð%ð> Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñð?ðV Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø%Ø2ñðWðr Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø+ñðsðL Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðMðd Ø3Ø*Ø2Ø)Ø $Ø&Ø *Ø 5Ø#8Ø,:Ø"Ø%Ø2ñðeðB Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðCðZ Ø+Ø+Ø!/Ø!/ñ	ð[ðf Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðgð~ Ø Ø)Ø (Ø"Ø(<ñððL 	Ø'Ø#Ø%Ø%Ø &Ø +Ø#8Ø(<Ø,LØ":Ø"ñðMðf Ø3Ø*Ø2Ø)Ø $Ø 5Ø#8Ø(8Ø"ñ
ðgð| Ø Ø!Ø$Ø4Ø (Ø(<ñð}ðL Ø*Ø3Ø)Ø2Ø 5Ø#8Ø(6ñðMð^ Ø"Ø3Ø)Ø,@Ø*Ø(Ø&Ø.Ø-ñ
ð_ðv 4Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,6Ø"ñð 4Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø$4Ø"ñð" 4Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø$4Ø"ñð" (Ø#Ø%Ø%Ø &Ø +Ø#8Ø(<Ø,LØ":Ø"ñð 4Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñògÐ ðF 'ØØØ'ØØ+Ø+Ø!/Ø!/Ø!3ñð )Ø"Ø+Ø+Ø!/Ø!/ññÐ ò0÷0(6ñ (6ôVv˜ô vôr˜ô ô(H˜ô HôV	�}ô 	ô<�kô <ô~>˜ô >ðBØÐðàÐðð Ð#ðð Ðð	ð
 Ð#ðð Ððð Ððð Ððð Ð ðð Ððð Ð"ðð 	ˆ/ðð Ððð Ð ðð Ð ðð  Ð%ð!ð" ˆOð#ð$ Ø ò'Ð ð.7¨ð 7ÀÀyÐRVÀÑAWô 7rˆ   