Ë
    îÍ:j‘  ã                   ó$  — d dl mZ d dlZdej                  dedej                  fd„Z	 	 ddej                  j                  dej                  d	ej                  d
ej                  deej                     dedee   de	ej                  df   fd„Z
y)é    )ÚOptionalNÚhidden_statesÚn_repÚreturnc                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    é   N)ÚshapeÚexpandÚreshape)r   r   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         úy/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/transformers/integrations/sdpa_paged.pyÚ	repeat_kvr      so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTó    ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚdropoutÚscalingc           	      ón  — |j                  dd «      }|�d |j                  ||| j                  fi |¤Ž\  }}|j                  dd«      j	                  d«      }|j                  dd«      j	                  d«      }t        | d«      r,t        || j                  «      }t        || j                  «      }|}	|j                  «       }|j                  «       }|j                  «       }t        j                  j                  j                  ||||	||d¬«      }
|
j                  dd«      j                  «       }
|
d fS )NÚcacher   r   Únum_key_value_groupsF)Ú	attn_maskÚ	dropout_pÚscaleÚ	is_causalé   )ÚpopÚupdateÚ	layer_idxÚ	transposeÚ	unsqueezeÚhasattrr   r   Ú
contiguousÚtorchÚnnÚ
functionalÚscaled_dot_product_attention)r   r   r   r   r   r   r   Úkwargsr   Úcausal_maskÚattn_outputs              r   Úsdpa_attention_paged_forwardr0      s2  € ð �J‰J�w Ó%€EØÐà!�U—\‘\ # u¨f×.>Ñ.>ÑIÀ&ÑI‰
ˆˆUØ�m‰m˜A˜qÓ!×+Ñ+¨AÓ.ˆØ—‘  1Ó%×/Ñ/°Ó2ˆô ˆvÐ-Ô.Ü˜˜V×8Ñ8Ó9ˆÜ˜% ×!<Ñ!<Ó=ˆð !€Kð ×ÑÓ€EØ
�.‰.Ó
€CØ×ÑÓ€EÜ—(‘(×%Ñ%×BÑBØØØØØØàð Có 	€Kð ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜ÐÐr   )g        N)Útypingr   r)   ÚTensorÚintr   r*   ÚModuleÚfloatÚtupler0   © r   r   ú<module>r8      s¹   ðÝ ã ð	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð$ Ø#ñ*Ø�H‰H�O‰Oð*à�<‰<ð*ð 
�‰ð*ð �<‰<ð	*ð
 ˜UŸ\™\Ñ*ð*ð ð*ð �e‰_ð*ð ˆ5�<‰<˜ÐÑô*r   