Ë
    îÍ:j5  ã                   óN  — d dl mZ d dlZddlmZmZ ddlmZ  ej                  e	«      Z
 e«       Z	 	 	 	 ddej                  j                  dej                  dej                  d	ej                  d
eej                     dedee   dee   dee   deej                  df   fd„Zy)é    )ÚOptionalNé   )Ú_flash_attention_forwardÚ!flash_attn_supports_top_left_mask)ÚloggingÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚdropoutÚscalingÚsliding_windowÚsoftcapÚreturnc	                 óx  — |	j                  dd«      s|	j                  d«      �t        j                  d«       |j                  d   }
t	        d„ |j                  D «       «      rt        d«      ‚|j                  dd«      }|j                  dd«      }|j                  dd«      }d }|j                  t        j                  k(  rŠt        j                  «       rt        j                  «       }nat        | j                  d	«      r| j                  j                  }n4t        d
„ | j!                  «       D «       «      j"                  j                  }|	j%                  dd «      }|€| j&                  }t)        ||||f|
|||||t*        || j                  j,                  t        | d«      r| j.                  nd dœ
|	¤Ž}|d fS )NÚoutput_attentionsFÚ	head_maskz•`flash_attention_2` does not support `output_attentions=True` or `head_mask`. Please set your attention to `eager` if you want any of these features.r   c              3   ó&   K  — | ]	  }|d k(  –— Œ y­w)r   N© )Ú.0Údims     ú~/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/transformers/integrations/flash_attention.pyú	<genexpr>z*flash_attention_forward.<locals>.<genexpr>#   s   è ø€ Ò
+˜ˆ3�!�8Ñ
+ùs   ‚z•Tensor query has shape  with a zero dimension.
FlashAttention does not support inputs with dim=0.
Please check your input shapes or use SDPA instead.é   Ú_pre_quantization_dtypec              3   ój   K  — | ]+  }t        |t        j                  j                  «      sŒ(|–— Œ- y ­w)N)Ú
isinstanceÚtorchÚnnÚLinear)r   Úlayers     r   r   z*flash_attention_forward.<locals>.<genexpr>;   s'   è ø€ Òj¨%ÄzÐRWÔY^×YaÑYa×YhÑYhÕGi¤Ñjùs   ‚)3¬3Ú	is_causalÚ	layer_idx)
Úquery_lengthr#   r   Úsoftmax_scaler   r   Úuse_top_left_maskÚtarget_dtypeÚattn_implementationr$   )ÚgetÚloggerÚwarning_onceÚshapeÚanyÚ
ValueErrorÚ	transposeÚdtyper   Úfloat32Úis_autocast_enabledÚget_autocast_gpu_dtypeÚhasattrÚconfigr   ÚnextÚmodulesÚweightÚpopr#   r   Ú_use_top_left_maskÚ_attn_implementationr$   )r   r	   r
   r   r   r   r   r   r   ÚkwargsÚseq_lenr(   r#   Úattn_outputs                 r   Úflash_attention_forwardr@      s�  € ð ‡z�zÐ% uÔ-°·±¸KÓ1HÐ1TÜ×ÑðWô	
ð �k‰k˜!‰n€Gä
Ñ
+˜uŸ{™{Ô
+Ô+ÜðBó
ð 	
ð �O‰O˜A˜qÓ!€EØ
�-‰-˜˜1Ó
€CØ�O‰O˜A˜qÓ!€Eð €LØ‡{�{”e—m‘mÒ#Ü×$Ñ$Ô&Ü ×7Ñ7Ó9‰Lä�V—]‘]Ð$=Ô>Ø!Ÿ=™=×@Ñ@‰LäÑj°6·>±>Ó3CÔjÓj×qÑq×wÑwˆLð —
‘
˜;¨Ó-€IØÐØ×$Ñ$ˆ	ä*ØØØØð	ð
 ØØØØ%ØÜ,Ø!Ø"ŸM™M×>Ñ>Ü&-¨f°kÔ&B�&×"Ò"Èñð ñ€Kð$ ˜ÐÐó    )g        NNN)Útypingr   r   Úmodeling_flash_attention_utilsr   r   Úutilsr   Ú
get_loggerÚ__name__r+   r;   r    ÚModuleÚTensorÚfloatÚintÚtupler@   r   rA   r   ú<module>rL      sã   ðÝ ã ç hÝ ð 
ˆ×	Ñ	˜HÓ	%€á6Ó8Ð ð Ø#Ø$(Ø#ñFØ�H‰H�O‰OðFà�<‰<ðFð 
�‰ðFð �<‰<ð	Fð
 ˜UŸ\™\Ñ*ðFð ðFð �e‰_ðFð ˜S‘MðFð �e‰_ðFð ˆ5�<‰<˜ÐÑôFrA   