Ë
    îÍ:j  ã                   óê   — d dl mZ d dlZd dlmZ dej                  dedej                  fd„Zdej                  d	ej                  d
ej                  dej                  deej                     defd„Z	y)é    )ÚOptionalN)ÚnnÚhidden_statesÚn_repÚreturnc                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    é   N)ÚshapeÚexpandÚreshape)r   r   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         úz/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/transformers/integrations/eager_paged.pyÚ	repeat_kvr      so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTó    ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingc                 óì  — |j                  dd «      }|�d |j                  ||| j                  fi |¤Ž\  }}|j                  dd«      j	                  d«      }|j                  dd«      j	                  d«      }t        | d«      r,t        || j                  «      }t        || j                  «      }t        |t        «      rt        | dd«      }|dk(  s|€dnd}	||	   }
n|}
t        j                  ||j                  dd	«      «      |z  }|
�||
z   }t        | d
«      rÖ| j                  j                  dddd«      j                  |j                   d   d|j                   d   d«      }t        j"                  ||gd¬«      }||j%                  dd¬«      j&                  z
  }t(        j*                  j-                  |dt        j.                  ¬«      j1                  |j2                  «      }|dd d…f   }nIt(        j*                  j-                  |dt        j.                  ¬«      j1                  |j2                  «      }t        j                  ||«      }|j                  dd«      j5                  «       }||fS )NÚcacher   r	   Únum_key_value_groupsÚsliding_windowÚfull_attentionÚsliding_attentioné   é   Úsinkséÿÿÿÿéþÿÿÿ)ÚdimT)r%   Úkeepdim)r%   Údtype.)ÚpopÚupdateÚ	layer_idxÚ	transposeÚ	unsqueezeÚhasattrr   r   Ú
isinstanceÚdictÚgetattrÚtorchÚmatmulr"   r   r   r
   ÚcatÚmaxÚvaluesr   Ú
functionalÚsoftmaxÚfloat32Útor'   Ú
contiguous)r   r   r   r   r   r   Úkwargsr   r   Ú
layer_typeÚcausal_maskÚattn_weightsr"   Úattn_outputs                 r   Úeager_paged_attention_forwardr@      s;  € ð �J‰J�w Ó%€EØÐà!�U—\‘\ # u¨f×.>Ñ.>ÑIÀ&ÑI‰
ˆˆUØ�m‰m˜A˜qÓ!×+Ñ+¨AÓ.ˆØ—‘  1Ó%×/Ñ/°Ó2ˆô ˆvÐ-Ô.Ü˜˜V×8Ñ8Ó9ˆÜ˜% ×!<Ñ!<Ó=ˆô �.¤$Ô'Ü  Ð)9¸1Ó=ˆØ)7¸1Ò)<ÀÐ@VÑ%Ð\oˆ
Ø$ ZÑ0‰à$ˆä—<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LØÐØ# kÑ1ˆô ˆv�wÔà—‘×$Ñ$ Q¨¨A¨qÓ1×8Ñ8¸¿¹ÀQ¹ÈÈUÏ[É[ÐY[É_Ð^`ÓaˆÜ—y‘y ,°Ð!6¸BÔ?ˆà# l×&6Ñ&6¸2ÀtÐ&6Ó&L×&SÑ&SÑSˆä—}‘}×,Ñ,¨\¸rÌÏÉÐ,ÓW×ZÑZÐ[`×[fÑ[fÓgˆØ# C¨¨"¨ HÑ-‰ä—}‘}×,Ñ,¨\¸rÌÏÉÐ,ÓW×ZÑZÐ[`×[fÑ[fÓgˆä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r   )
Útypingr   r1   r   ÚTensorÚintr   ÚModuleÚfloatr@   © r   r   ú<module>rG      sŠ   ðÝ ã Ý ð	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð2%Ø�I‰Ið2%à�<‰<ð2%ð 
�‰ð2%ð �<‰<ð	2%ð
 ˜UŸ\™\Ñ*ð2%ð ô2%r   