Ë
    îÍ:j+  ã                   óú  — d dl mZ d dlZddlmZmZmZ ddlmZ  ej                  e
«      Z edd¬«      Z ed	d¬«      Z e«       Z e«       Zd
ej                   dedej                   fd„Zdeej                      dej                   defd„Z	 	 	 ddej*                  j,                  dej                   dej                   dej                   deej                      dedee   dee   deej                   df   fd„Zy)é    )ÚOptionalNé   )Úis_torch_npu_availableÚis_torch_xpu_availableÚlogging)Úis_torch_greater_or_equalz2.5T)Ú
accept_devz2.8Úhidden_statesÚn_repÚreturnc                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    é   N)ÚshapeÚexpandÚreshape)r
   r   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         ú}/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/transformers/integrations/sdpa_attention.pyÚ	repeat_kvr      so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTó    Úattention_maskÚkeyc                 óÜ   — t         r-t        xr% t        |t        j                  j
                  «       S t        ryt        xr+ | d u xr% t        |t        j                  j
                  «       S )NF)Ú_is_torch_xpu_availableÚ#_is_torch_greater_or_equal_than_2_8Ú
isinstanceÚtorchÚfxÚProxyÚ_is_torch_npu_availableÚ#_is_torch_greater_or_equal_than_2_5)r   r   s     r   Úuse_gqa_in_sdpar$      sX   € õ Ü2ÒZ¼:ÀcÌ5Ï8É8Ï>É>Ó;ZÐ7ZÐZÝØÜ.Òq°>ÀTÐ3IÒqÔR\Ð]`Ôbg×bjÑbj×bpÑbpÓRqÐNqÐqr   ÚmoduleÚqueryÚvalueÚdropoutÚscalingÚ	is_causalc                 ó´  — |j                  dd«      s|j                  d«      �t        j                  d«       i }	t        | d«      r=t	        ||«      s-t        || j                  «      }t        || j                  «      }nddi}	|�-|j                  dk(  r|d d …d d …d d …d |j                  d	   …f   }|€'|j                  d
   dkD  xr |d u xr t        | dd«      }t        j                  j                  «       r*t        |t        j                  «      r|j                  «       }t         r[|�Y|j"                  t        j$                  k7  r<t        j&                  |j%                  «       «      j)                  |j*                  «      }t        j,                  j.                  j0                  |||f||||dœ|	¤Ž}
|
j3                  dd
«      j5                  «       }
|
d fS )NÚoutput_attentionsFÚ	head_maskz’`sdpa` attention does not support `output_attentions=True` or `head_mask`. Please set your attention to `eager` if you want any of these features.Únum_key_value_groupsÚ
enable_gqaTé   éþÿÿÿr   r   r*   )Ú	attn_maskÚ	dropout_pÚscaler*   )ÚgetÚloggerÚwarning_onceÚhasattrr$   r   r.   Úndimr   Úgetattrr   ÚjitÚ
is_tracingr   ÚTensorÚitemr"   ÚdtypeÚboolÚlogical_notÚtoÚdeviceÚnnÚ
functionalÚscaled_dot_product_attentionÚ	transposeÚ
contiguous)r%   r&   r   r'   r   r(   r)   r*   ÚkwargsÚsdpa_kwargsÚattn_outputs              r   Úsdpa_attention_forwardrL   0   s¾  € ð ‡z�zÐ% uÔ-°·±¸KÓ1HÐ1TÜ×ÑðWô	
ð €KÜˆvÐ-Ô.Ü˜~¨sÔ3Ü˜C ×!<Ñ!<Ó=ˆCÜ˜e V×%@Ñ%@ÓA‰Eà'¨Ð.ˆKàÐ! n×&9Ñ&9¸QÒ&>Ø'ªª1ªa°°3·9±9¸R±=°Ð(@ÑAˆð
 Ðð —K‘K ‘N QÑ&Òh¨>¸TÐ+AÒhÄgÈfÐVaÐcgÓFhˆ	ô ‡y�y×ÑÔ¤*¨Y¼¿¹Ô"EØ—N‘NÓ$ˆ	õ
 ØÐ%¨.×*>Ñ*>Ä%Ç*Á*Ò*Lä"×.Ñ.¨~×/BÑ/BÓ/DÓE×HÑHÈÏÉÓVˆNä—(‘(×%Ñ%×BÑBØØØð	ð !ØØØñ	ð ñ	€Kð ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜ÐÐr   )g        NN)Útypingr   r   Úutilsr   r   r   Úutils.import_utilsr   Ú
get_loggerÚ__name__r6   r#   r   r   r"   r=   Úintr   r@   r$   rD   ÚModuleÚfloatÚtuplerL   © r   r   ú<module>rW      sD  ðÝ ã ç KÑ KÝ :ð 
ˆ×	Ñ	˜HÓ	%€ñ '@ÀÐRVÔ&WÐ #Ù&?ÀÐRVÔ&WÐ #Ù0Ó2Ð Ù0Ó2Ð ð	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uðr H¨U¯\©\Ñ$:ð rÀÇÁð rÐRVó rð0 Ø#Ø $ñ<Ø�H‰H�O‰Oð<à�<‰<ð<ð 
�‰ð<ð �<‰<ð	<ð
 ˜UŸ\™\Ñ*ð<ð ð<ð �e‰_ð<ð ˜‰~ð<ð ˆ5�<‰<˜ÐÑô<r   