Ë
    èÍ:jÏ*  ã                   ó„  — d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dl	m
Z
 d dlmZmZ ej                  j                  ej                  j                   ej                  j"                  dœZh d£Z G d„ d	e
«      Zd
„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zdd„Zd„ Zd„ Zdd„Z dd„Z!d„ Z"dd„Z#dd„Z$d„ Z%e&dk(  r e%«        yy) é    N)ÚList)Úutils)Ú	Converter)Úcommon_specÚtransformer_spec)ÚgeluÚreluÚswish>   Ú ÚdÚnÚndc                   ó,   — e Zd ZdZdedee   fd„Zd„ Zy)ÚMarianConverterz$Converts models trained with Marian.Ú
model_pathÚvocab_pathsc                 ó    — || _         || _        y)z¹Initializes the Marian converter.

        Arguments:
          model_path: Path to the Marian model (.npz file).
          vocab_paths: Paths to the vocabularies (.yml files).
        N)Ú_model_pathÚ_vocab_paths)Úselfr   r   s      úr/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/ctranslate2/converters/marian.pyÚ__init__zMarianConverter.__init__   s   € ð &ˆÔØ'ˆÕó    c           
      ó  — t        j                  | j                  «      }t        |«      }t	        t        t        | j                  «      «      }|d   }d|d   v }|d   }t        j                  «       } ||d   dk(  d«        ||d   d	k(  d
«        ||d    d«        ||t        v d|›ddj                  t        j                  «       «      ›d�«        ||t        v d|›ddj                  t        «      ›d�«       |r1 ||d   dk(  xr |d   dk(  xr |j                  dd«      dk(  d«       n0 ||d   dk(  xr |d   dk(  xr |j                  dd«      dk(  d«       |j                  «        |d   }|dk(  rdnt!        |«      dz
  }d|v }	t"        j$                  j'                  |d   |d   f|d    |t        |   |d|	¬!«      }
t)        |
|«       |
j+                  |d"   «       |
j-                  |d   «       d#|
j.                  _        |
S )$Nztransformer-ffn-activationr   ztransformer-preprocessztransformer-postprocess-embÚtypeÚtransformerz#Option --type must be 'transformer'ztransformer-decoder-autoregzself-attentionz=Option --transformer-decoder-autoreg must be 'self-attention'ztransformer-no-projectionz3Option --transformer-no-projection is not supportedz$Option --transformer-ffn-activation z. is not supported (supported activations are: z, ú)z%Option --transformer-postprocess-emb z) is not supported (supported values are: ztransformer-postprocessÚdaztransformer-postprocess-topr   zµUnsupported pre-norm Transformer architecture, expected the following combination of options: --transformer-preprocess n --transformer-postprocess da --transformer-postprocess-top nÚdanz¹Unsupported post-norm Transformer architecture, excepted the following combination of options: --transformer-preprocess '' --transformer-postprocess dan --transformer-postprocess-top ''z"transformer-guided-alignment-layerÚlastéÿÿÿÿé   z	enc-depthz	dec-depthztransformer-heads)Úpre_normÚ
activationÚalignment_layerÚalignment_headsÚlayernorm_embeddingr   T)ÚnpÚloadr   Ú_get_model_configÚlistÚmapÚ
load_vocabr   r   ÚConfigurationCheckerÚ_SUPPORTED_ACTIVATIONSÚjoinÚkeysÚ_SUPPORTED_POSTPROCESS_EMBÚgetÚvalidateÚintr   ÚTransformerSpecÚfrom_configÚset_transformer_specÚregister_source_vocabularyÚregister_target_vocabularyÚconfigÚadd_source_eos)r   Úmodelr;   Úvocabsr$   r#   Úpostprocess_embÚcheckr%   r'   Ú
model_specs              r   Ú_loadzMarianConverter._load#   sp  € Ü—‘˜×(Ñ(Ó)ˆÜ" 5Ó)ˆÜ”cœ* d×&7Ñ&7Ó8Ó9ˆàÐ8Ñ9ˆ
Ø˜&Ð!9Ñ:Ð:ˆØ Ð!>Ñ?ˆä×*Ñ*Ó,ˆÙˆf�V‰n Ñ-Ð/TÔUÙØÐ0Ñ1Ð5EÑEØKô	
ñ 	ØÐ2Ñ3Ð3ØAô	
ñ 	ØÔ0Ñ0ò ˜4Ÿ9™9Ô%;×%@Ñ%@Ó%BÕCðEô	
ñ 	ØÔ9Ñ9â §	¡	Ô*DÕ EðGô	
ñ ÙØÐ/Ñ0°CÑ7ò IØÐ4Ñ5¸Ñ=òIà—J‘JÐ<¸bÓAÀSÑHð2õ		ñ ØÐ/Ñ0°BÑ6ò HØÐ4Ñ5¸Ñ>òHà—J‘JÐ<¸bÓAÀRÑGð3ô		ð 	�‰Ôà Ð!EÑFˆØ /°6Ò 9™"¼sÀ?Ó?SÐVWÑ?WˆØ! _Ð4Ðä%×5Ñ5×AÑAØ�KÑ  &¨Ñ"5Ð6ØÐ&Ñ'ØÜ-¨jÑ9Ø+ØØ 3ð Bó 
ˆ
ô 	˜Z¨Ô/Ø×-Ñ-¨f°Q©iÔ8Ø×-Ñ-¨f°R©jÔ9Ø+/ˆ
×ÑÔ(ØÐr   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústrr   r   rB   © r   r   r   r      s"   „ Ù.ð( 3ð (°T¸#±Yó (óIr   r   c                 ó`   — | d   }|d d j                  «       }t        j                  |«      }|S )Nzspecial:model.ymlr!   )ÚtobytesÚyamlÚ	safe_load)r=   r;   s     r   r*   r*   o   s4   € ØÐ&Ñ'€FØ�C�Rˆ[× Ñ Ó"€FÜ�^‰^˜FÓ#€FØ€Mr   c           	      óV  — t        | d¬«      5 }g }d }d }t        |«      D �]&  \  }}|j                  d«      }|sŒ|j                  d«      r|dd  }n|�|dd  }n|j	                  dd«      \  }}|�¢|j                  d«      rX|j                  d«      rGt        j                  d	d
|«      }|dd }|j                  d«      rSt        t        |dd  d¬«      «      }n9|j                  d«      r(|j                  d«      r|dd }|j                  dd«      }|€Œö	 t        |j                  «       «      }|j                  ||f«       d }d }�Œ) 	 d d d «       t        d„ ¬«      D ��cg c]  \  }}|‘Œ	 c}}S # t        $ r}t        d|dz   |fz  «      |‚d }~ww xY w# 1 sw Y   ŒOxY wc c}}w )Nzutf-8)Úencodingz
z? é   ú:r"   ú"z\\([^x])z\1r!   z\xé   )Úbaseú'z''z"Unexpected format at line %d: '%s'c                 ó   — | d   S )Nr   rH   )Úitems    r   ú<lambda>zload_vocab.<locals>.<lambda>¡   s
   € À$ÀqÁ'€ r   )Úkey)ÚopenÚ	enumerateÚrstripÚ
startswithÚrsplitÚendswithÚreÚsubÚchrr5   ÚreplaceÚstripÚ
ValueErrorÚappendÚsorted)	ÚpathÚvocabÚtokensÚtokenÚidxÚiÚlineÚeÚ_s	            r   r-   r-   v   s¾  € ä	ˆd˜WÔ	%ð '¨ØˆØˆØˆÜ  Ó'ó #	‰GˆAˆtØ—;‘;˜vÓ&ˆDÙØà�‰˜tÔ$Ø˜Q˜R˜‘ØÐ"Ø˜1˜2�h‘à!Ÿ[™[¨¨aÓ0‘
��sàÐ Ø×#Ñ# CÔ(¨U¯^©^¸CÔ-@äŸF™F ;°°uÓ=�EØ! ! B˜K�EØ×'Ñ'¨Ô.ä #¤C¨¨a¨b¨	¸Ô$;Ó <™Ø×%Ñ% cÔ*¨u¯~©~¸cÔ/BØ! ! B˜K�EØ!ŸM™M¨$°Ó4�Eà‰ðÜ˜cŸi™i›kÓ*�Cð —‘˜s E˜lÔ+à�Ø’ñG#	÷	'ôR #)¨Ñ5IÔ"J×K‘h�a˜ŠEÓKÐKøô "ò Ü$Ø<ÀÀAÁÀt¸}ÑLóàðûðú÷?'ð 'üóR Ls6   ŽDFÄE4Ä0FÅ$F%Å4	FÅ=FÆFÆFÆF"c                 ó`   — t        | j                  |d«       t        | j                  |d«       y )NÚencoderÚdecoder)Úset_transformer_encoderrq   Úset_transformer_decoderrr   )ÚspecÚweightss     r   r8   r8   ¤   s"   € Ü˜DŸL™L¨'°9Ô=Ü˜DŸL™L¨'°9Õ=r   c           	      ó‚   — t        | ||«       t        | j                  «      D ]  \  }}t        ||d||dz   fz  «       Œ y )Nú%s_l%dr"   )Úset_common_layersrZ   ÚlayerÚset_transformer_encoder_layer©ru   rv   Úscoperl   Ú
layer_specs        r   rs   rs   ©   sF   € Ü�d˜G UÔ+Ü" 4§:¡:Ó.ò V‰ˆˆ:Ü% j°'¸8ÀuÈaÐRSÉeÀnÑ;TÕUñVr   c           	      óð   — d| _         t        | ||«       t        | j                  «      D ]  \  }}t	        ||d||dz   fz  «       Œ t        | j                  |d|z  | j                  j                  ¬«       y )NTrx   r"   z%s_ff_logit_out)Úreuse_weight)	Ústart_from_zero_embeddingry   rZ   rz   Úset_transformer_decoder_layerÚ
set_linearÚ
projectionÚ
embeddingsÚweightr|   s        r   rt   rt   ¯   sw   € Ø%)€DÔ"Ü�d˜G UÔ+Ü" 4§:¡:Ó.ò V‰ˆˆ:Ü% j°'¸8ÀuÈaÐRSÉeÀnÑ;TÕUðVô Ø�‰ØØ˜EÑ!Ø—_‘_×+Ñ+ö	r   c                 ód  — | j                   }t        |t        «      s|g}t        |d   ||«       t	        | j
                  ||d   j                  j                  d   ¬«       t        | d«      rt        | j                  |d|z  d¬«       t        | d«      rt        | j                  |d	|z  «       y y )
Nr   r"   )Údimr'   z%s_embT©r#   Ú
layer_normz%s_top)r…   Ú
isinstancer+   Úset_embeddingsÚset_position_encodingsÚposition_encodingsr†   ÚshapeÚhasattrÚset_layer_normr'   rŠ   )ru   rv   r}   Úembeddings_specss       r   ry   ry   ½   s°   € Ø—‘ÐÜÐ&¬Ô-Ø,Ð-ÐäÐ# AÑ&¨°Ô7ÜØ×Ñ Ð.>¸qÑ.A×.HÑ.H×.NÑ.NÈqÑ.Qõô ˆtÐ*Ô+ÜØ×$Ñ$ØØ�uÑØõ		
ô ˆt�\Ô"Ü�t—‘¨°¸EÑ1AÕBð #r   c                 óp   — t        | j                  |d|z  «       t        | j                  |d|z  d¬«       y )Nú%s_ffnú%s_selfT©Úself_attention)Úset_ffnÚffnÚset_multi_head_attentionr—   ©ru   rv   r}   s      r   r{   r{   Ñ   s3   € ÜˆD�H‰H�g˜x¨%Ñ/Ô0ÜØ×Ñ˜W i°%Ñ&7Èör   c                 ó¤   — t        | j                  |d|z  «       t        | j                  |d|z  d¬«       t        | j                  |d|z  «       y )Nr”   r•   Tr–   z
%s_context)r˜   r™   rš   r—   Ú	attentionr›   s      r   r‚   r‚   Ø   sJ   € ÜˆD�H‰H�g˜x¨%Ñ/Ô0ÜØ×Ñ˜W i°%Ñ&7Èõô ˜TŸ^™^¨W°lÀUÑ6JÕKr   c                 óN  — t        d«      D �cg c]  }t        j                  «       ‘Œ }}t        |d   ||d«       t        |d   ||d«       t        |d   ||d«       |r$t	        j
                  | j                  d   |«       nh|d   j                  | j                  d   _        |d   j                  | j                  d   _        t	        j
                  | j                  d   |dd  «       t        | j                  d   ||d	«       t        | j                  |d
|z  «       y c c}w )Né   r   Úqr"   ÚkrO   Úvr!   Úoz%s_Wo)Úranger   Ú
LinearSpecrƒ   r   Úfuse_linearÚlinearr†   ÚbiasÚset_layer_norm_autorŠ   )ru   rv   r}   r—   ro   Úsplit_layerss         r   rš   rš   à   sö   € Ü6;¸A³hÖ?°”K×*Ñ*Õ,Ð?€LÐ?Üˆ|˜A‰ ¨°Ô4Üˆ|˜A‰ ¨°Ô4Üˆ|˜A‰ ¨°Ô4áÜ×Ñ˜$Ÿ+™+ a™.¨,Õ7à ,¨Q¡× 6Ñ 6ˆ�‰�A‰ÔØ*¨1™o×2Ñ2ˆ�‰�A‰ÔÜ×Ñ˜$Ÿ+™+ a™.¨,°q°rÐ*:Ô;äˆt�{‰{˜2‰ ¨°Ô4Ü˜Ÿ™¨°'¸E±/ÕBùò @s   ŽD"c                 ó˜   — t        | j                  |d|z  «       t        | j                  ||d«       t        | j                  ||d«       y )Nr”   Ú1Ú2)r©   rŠ   rƒ   Úlinear_0Úlinear_1r›   s      r   r˜   r˜   ñ   s;   € Ü˜Ÿ™¨°(¸UÑ2BÔCÜˆt�}‰}˜g u¨cÔ2Üˆt�}‰}˜g u¨cÕ2r   c                 ó\   — 	 t        | ||d¬«       y # t        $ r t        | ||«       Y y w xY w)NTr‰   )r‘   ÚKeyErrorr›   s      r   r©   r©   ÷   s0   € ð-Ü�t˜W e°dÖ;øÜò -Ü�t˜W eÖ,ð-ús   ‚ ’+ª+c                 ó„   — |rdnd}||›d|›�   j                  «       | _        ||›d|›�   j                  «       | _        y )NÚ_prer   Ú	_ln_scaleÚ_ln_bias)ÚsqueezeÚgammaÚbeta)ru   rv   r}   r#   Úsuffixs        r   r‘   r‘   þ   s?   € Ù‰V R€FØªE±6Ð:Ñ;×CÑCÓE€D„JØª%±Ð8Ñ9×AÑAÓC€D…Ir   c                 óî   — |j                  |›d|›�«      }|€|j                  |›d|›�|«      }n|j                  «       }|| _        |j                  |›d|›�«      }|�|j                  «       | _        y y )NÚ_WÚ_WtÚ_b)r3   Ú	transposer†   r¶   r¨   )ru   rv   r}   r¹   r€   r†   r¨   s          r   rƒ   rƒ     so   € Ø�[‰[¢U©FÐ3Ó4€Fà€~Ø—‘ª%±Ð8¸,ÓG‰à×!Ñ!Ó#ˆà€D„Kà�;‰;¢5©&Ð1Ó2€DØÐØ—L‘L“Nˆ�	ð r   c                 ó|   — |j                  d|z  «      | _        | j                  €|j                  d«      | _        y y )Nz%s_WembÚWemb)r3   r†   r›   s      r   rŒ   rŒ     s6   € Ø—+‘+˜i¨%Ñ/Ó0€D„KØ‡{�{ÐØ—k‘k &Ó)ˆ�ð r   c                 óD   — |j                  dt        |«      «      | _        y )NÚWpos)r3   Ú#_make_sinusoidal_position_encodingsÚ	encodings)ru   rv   rˆ   s      r   r�   r�     s   € Ø—[‘[ Ô)LÈSÓ)QÓR€D…Nr   c                 óÄ  — t        j                  |«      }t        j                  ddt        j                  | «      dz  z  | z  «      }t        j                  |d«      t        j                  |d«      z  }t        j                  |«      }t        j
                  |d d …dd d…f   «      |d d …d | dz  …f<   t        j                  |d d …dd d…f   «      |d d …| dz  d …f<   |S )Ni'  rO   r"   r   )r(   ÚarangeÚpowerÚexpand_dimsÚ
zeros_likeÚsinÚcos)rˆ   Únum_positionsÚ	positionsÚ
timescalesÚposition_encÚtables         r   rÃ   rÃ     sÄ   € Ü—	‘	˜-Ó(€IÜ—‘˜% ¤b§i¡i°£n¸Ñ&9Ñ!:¸SÑ!@ÓA€JÜ—>‘> )¨QÓ/´"·.±.ÀÈQÓ2OÑO€LÜ�M‰M˜,Ó'€EÜŸ6™6 ,ªq°!°$°Q°$¨wÑ"7Ó8€EŠ!ˆZˆs�a‰xˆZˆ-ÑÜŸ6™6 ,ªq°!°$°Q°$¨wÑ"7Ó8€EŠ!ˆS�A‰X‰Zˆ-ÑØ€Lr   c                  óJ  — t        j                  t         j                  ¬«      } | j                  ddd¬«       | j                  dddd¬	«       t	        j
                  | «       | j                  «       }t        |j                  |j                  «      }|j                  |«       y )
N)Úformatter_classz--model_pathTzPath to the model .npz file.)ÚrequiredÚhelpz--vocab_pathsú+z'List of paths to the YAML vocabularies.)rÓ   ÚnargsrÔ   )ÚargparseÚArgumentParserÚArgumentDefaultsHelpFormatterÚadd_argumentr   Údeclare_argumentsÚ
parse_argsr   r   r   Úconvert_from_args)ÚparserÚargsÚ	converters      r   Úmainrá   '  s›   € Ü×$Ñ$Ü ×>Ñ>ô€Fð ×ÑØ Ð,Jð ô ð ×ÑØØØØ6ð	 ô ô ×Ñ Ô'Ø×ÑÓ€DÜ §¡°×1AÑ1AÓB€IØ×Ñ Õ%r   Ú__main__)F)r   N)N)i   )'r×   r_   Útypingr   Únumpyr(   rK   Úctranslate2.convertersr   Ú ctranslate2.converters.converterr   Úctranslate2.specsr   r   Ú
ActivationÚGELUSigmoidÚRELUÚSWISHr/   r2   r   r*   r-   r8   rs   rt   ry   r{   r‚   rš   r˜   r©   r‘   rƒ   rŒ   r�   rÃ   rá   rC   rH   r   r   ú<module>rì      sÜ   ðÛ Û 	å ã Û å (Ý 6ß ;ð ×"Ñ"×.Ñ.Ø×"Ñ"×'Ñ'Ø×#Ñ#×)Ñ)ñÐ ò 2Ð ôV�iô Vòrò+Lò\>ò
VòòCò(òLóCò"3ò-óDó#ò*óSóò&ð& ˆzÒÙ…Fð r   