Ë
    ÝÍ:jE  ã                   óZ   — d dl mZ d dlmZ d dlmZ d dlmZ  ee«      Z	 G d„ de«      Z
y)é    )Ú	getLogger)ÚFusion)Úhelper)Ú	OnnxModelc                   ó”   ‡ — e Zd Zdefˆ fd„Zdedefd„Zdedz  fd„Zdedededz  fd	„Z	dedededz  fd
„Z
dedededz  fd„Zˆ xZS )ÚFusionFastGeluÚmodelc                 ó(   •— t         ‰| �  |dd«       y )NÚFastGeluÚTanh)ÚsuperÚ__init__)Úselfr	   Ú	__class__s     €ú}/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/onnxruntime/transformers/fusion_fastgelu.pyr   zFusionFastGelu.__init__   s   ø€ Ü‰Ñ˜ 
¨FÕ3ó    Úinput_name_to_nodesÚoutput_name_to_nodec                 ó¤   — | j                  |||«      ry | j                  |||«      ry | j                  |||«      ry | j                  |||«      ry y )N)Úfuse_1Úfuse_2Úfuse_3Úfuse_4)r   Ú	tanh_noder   r   s       r   ÚfusezFusionFastGelu.fuse   s\   € Ø�;‰;�yÐ"5Ð7JÔKØà�;‰;�yÐ"5Ð7JÔKØà�;‰;�yÐ"5Ð7JÔKØà�;‰;�yÐ"5Ð7JÔKØð Lr   ÚreturnNc                 ó*  — |j                   d   |vry||j                   d      }t        |«      dk7  s|d   j                  dk7  ry|d   }| j                  j	                  |d«      sy|j                   d   |vry||j                   d      }t        |«      dk7  s|d   j                  dk7  ry|d   }| j                  j                  |dd|«      }|€y| j                  j                  |d«      }|dk  ry|j                  |dk(  rdnd   }	| j                  j                  ||dk(  rdnd|«      }
| j                  j                  |dd|«      }|€y| j                  j                  |dd	¬
«      }|dk  ry| j                  j                  |d|dk(  rdnd|«      }|€y| j                  j                  |dd||
r|
gng ¬«      }|€y| j                  j                  |dd	¬
«      }|dk  ry| j                  j                  |d|dk(  rdnd|«      }|€y| j                  j	                  |d«      sy|j                  d   |	k7  ry||||||||g}| j                  j                  ||j                   d   g||«      sy| j                  j                  |«       t        j                  d|	g|j                   | j                  j                  d«      ¬«      }d|_        | j                   j#                  |«       | j$                  | j&                  |j(                  <   y)aj  
        Fuse Gelu with tanh into one node:
              +---------------------------+
              |                           |
              |                           v
            [root] --> Pow --> Mul -----> Add  --> Mul --> Tanh --> Add --> Mul
              |       (Y=3)   (B=0.0447...)       (B=0.7978...)    (B=1)     ^
              |                                                              |
              +------> Mul(B=0.5)--------------------------------------------+
        Note that constant input for Add and Mul could be first or second input: like either A=0.5 or B=0.5 is fine.
        r   Né   ÚAddç      ð?ÚMulç      à?ç+ö—Ý“‡é?ç-Cëâ6?©Údelta©Úexcludeç¥,Cëâ¦?ÚPowç      @r   ©ÚinputsÚoutputsÚnameúcom.microsoftT)ÚoutputÚlenÚop_typer	   Úhas_constant_inputÚmatch_parentÚfind_constant_inputÚinputÚ
get_parentÚis_safe_to_fuse_nodesÚnodes_to_removeÚextendr   Ú	make_nodeÚcreate_node_nameÚdomainÚnodes_to_addÚappendÚthis_graph_nameÚnode_name_to_graph_namer/   )r   r   r   r   ÚchildrenÚadd_after_tanhÚmul_after_tanhÚmul_halfÚiÚ
root_inputÚ	root_nodeÚmul_before_tanhÚadd_before_tanhÚmul_after_powÚpowÚsubgraph_nodesÚ
fused_nodes                    r   r   zFusionFastGelu.fuse_1   sB  € ð ×Ñ˜AÑÐ&9Ñ9ØØ& y×'7Ñ'7¸Ñ':Ñ;ˆÜˆx‹=˜AÒ ¨!¡×!4Ñ!4¸Ò!=ØØ! !™ˆà�z‰z×,Ñ,¨^¸SÔAØà× Ñ  Ñ#Ð+>Ñ>ØØ& ~×'<Ñ'<¸QÑ'?Ñ@ˆÜˆx‹=˜AÒ ¨!¡×!4Ñ!4¸Ò!=ØØ! !™ˆà—:‘:×*Ñ*¨>¸5À$ÐH[Ó\ˆØÐØà�J‰J×*Ñ*¨8°SÓ9ˆØˆqŠ5Øà—^‘^¨¨aª¡A°QÑ7ˆ
ð —J‘J×)Ñ)¨(¸¸aº±AÀQÐH[Ó\ˆ	àŸ*™*×1Ñ1°)¸UÀAÐGZÓ[ˆØÐ"Øà�J‰J×*Ñ*¨?¸FÈ&Ð*ÓQˆØˆqŠ5ØàŸ*™*×1Ñ1°/À5ÈqÐTUÊvÉ!Ð[\Ð^qÓrˆØÐ"ØàŸ
™
×/Ñ/ØØØØÙ#,�Y‘K°"ð 0ó 
ˆð Ð Øà�J‰J×*Ñ*¨=¸&ÈÐ*ÓOˆØˆqŠ5Øà�j‰j×%Ñ% m°UÀÀaÂ¹AÈQÐPcÓdˆØˆ;Øà�z‰z×,Ñ,¨S°#Ô6Øà�9‰9�Q‰<˜:Ò%Øð ØØØØØØØð	
ˆð �z‰z×/Ñ/ØØ×"Ñ" 1Ñ%Ð&ØØô	
ð à×Ñ×#Ñ# NÔ3Ü×%Ñ%ØØ�<Ø"×)Ñ)Ø—‘×,Ñ,¨ZÓ8ô	
ˆ
ð ,ˆ
ÔØ×Ñ× Ñ  Ô,Ø8<×8LÑ8Lˆ×$Ñ$ Z§_¡_Ñ5Ør   c                 óä  — |j                   d   |vry||j                   d      }t        |«      dk7  s|d   j                  dk7  ry|d   }| j                  j	                  |d«      sy|j                   d   |vry||j                   d      }t        |«      dk7  s|d   j                  dk7  ry|d   }| j                  j                  |d«      }|dk  ry|j                   d   |vry||j                   d      }t        |«      dk7  s|d   j                  dk7  ry|d   }| j                  j                  ||j                  d   |j                   d   k(  rdnd|«      }	| j                  j                  |dd|«      }
|
€y| j                  j                  |
dd	¬
«      }|dk  ry| j                  j                  |
d|dk(  rdnd|«      }|€y| j                  j                  |dd||	r|	gng ¬«      }|€y| j                  j                  |dd	¬
«      }|dk  ry| j                  j                  |d|dk(  rdnd|«      }|€y| j                  j	                  |d«      sy|j                  |j                  d   |j                   d   k(  rdnd   }|j                  d   |k7  ry|||||
|||g}| j                  j                  ||j                   d   g||«      sy| j                  j                  |«       t        j                  d|g|j                   | j                  j                  d«      ¬«      }d|_        | j                   j#                  |«       | j$                  | j&                  |j(                  <   y)aÐ  
        This pattern is from Tensorflow model.
        Fuse Gelu with tanh into one node:
              +---------------------------+
              |                           |
              |                           v
            [root] --> Pow --> Mul -----> Add  --> Mul --> Tanh --> Add --> Mul(B=0.5)-->Mul-->
              |       (Y=3)   (B=0.0447...)       (B=0.7978...)    (B=1)                  ^
              |                                                                           |
              +---------------------------------------------------------------------------+
        Note that constant input for Add and Mul could be first or second input: like either A=0.5 or B=0.5 is fine.
        r   Nr   r   r    r!   r"   r#   r$   r%   r'   r)   r*   r+   r   r,   r0   T)r1   r2   r3   r	   r4   r6   r8   r7   r5   r9   r:   r;   r   r<   r=   r>   r?   r@   rA   rB   r/   )r   r   r   r   rC   rD   rF   rG   Úmul_after_mul_halfrI   rJ   rK   rL   rM   rH   rN   rO   s                    r   r   zFusionFastGelu.fuse_2‹   s¨  € ð ×Ñ˜AÑÐ&9Ñ9ØØ& y×'7Ñ'7¸Ñ':Ñ;ˆÜˆx‹=˜AÒ ¨!¡×!4Ñ!4¸Ò!=ØØ! !™ˆà�z‰z×,Ñ,¨^¸SÔAØà× Ñ  Ñ#Ð+>Ñ>ØØ& ~×'<Ñ'<¸QÑ'?Ñ@ˆÜˆx‹=˜AÒ ¨!¡×!4Ñ!4¸Ò!=ØØ˜A‘;ˆà�J‰J×*Ñ*¨8°SÓ9ˆØˆqŠ5Øà�?‰?˜1ÑÐ%8Ñ8ØØ& x§¡°qÑ'9Ñ:ˆÜˆx‹=˜AÒ ¨!¡×!4Ñ!4¸Ò!=ØØ% a™[Ðð —J‘J×)Ñ)ØØ#×)Ñ)¨!Ñ,°·±ÀÑ0BÒB‰AÈØó
ˆ	ð Ÿ*™*×1Ñ1°)¸UÀAÐGZÓ[ˆØÐ"Øà�J‰J×*Ñ*¨?¸FÈ&Ð*ÓQˆØˆqŠ5ØàŸ*™*×1Ñ1°/À5ÈqÐTUÊvÉ!Ð[\Ð^qÓrˆØÐ"ØàŸ
™
×/Ñ/ØØØØÙ#,�Y‘K°"ð 0ó 
ˆð Ð Øà�J‰J×*Ñ*¨=¸&ÈÐ*ÓOˆØˆqŠ5Øà�j‰j×%Ñ% m°UÀÀaÂ¹AÈQÐPcÓdˆØˆ;Øà�z‰z×,Ñ,¨S°#Ô6Øà'×-Ñ-Ð3E×3KÑ3KÈAÑ3NÐRZ×RaÑRaÐbcÑRdÒ3d©aÐjkÑlˆ
à�9‰9�Q‰<˜:Ò%Øð ØØØØØØØð	
ˆð �z‰z×/Ñ/ØØ×&Ñ& qÑ)Ð*ØØô	
ð à×Ñ×#Ñ# NÔ3Ü×%Ñ%ØØ�<Ø&×-Ñ-Ø—‘×,Ñ,¨ZÓ8ô	
ˆ
ð ,ˆ
ÔØ×Ñ× Ñ  Ô,Ø8<×8LÑ8Lˆ×$Ñ$ Z§_¡_Ñ5Ør   c           	      ó¬  — |j                   d   |vry||j                   d      }t        |«      dk7  s|d   j                  dk7  ry|d   }| j                  j	                  |d«      sy|j                   d   |vry||j                   d      }t        |«      dk7  s|d   j                  dk7  ry|d   }| j                  j                  |dd|«      }|€y| j                  j                  |d«      }|dk  ry|j                  |dk(  rdnd   }	| j                  j                  |dd|«      }
|
€y| j                  j                  |
dd|«      }|€y| j                  j                  |d«      }|dk  ry| j                  j                  |
dd|«      }|€y| j                  j                  |dd	¬
«      }|dk  ry|j                  |dk(  rdnd   |	k7  ry| j                  j                  |d|dk(  rdnd|«      }|€y|j                  d   |	k(  rd}n|j                  d   |	k(  rd}ny| j                  j                  |d||«      }|€y| j                  j                  |dd	¬
«      }|dk  ry|j                  |dk(  rdnd   |	k7  ry||||
|||||g	}| j                  j                  ||j                   d   g||«      sy| j                  j                  |«       t        j                  d|	g|j                   | j                  j                  d«      ¬«      }d|_        | j                  j!                  |«       | j"                  | j$                  |j&                  <   y)až  
        OpenAI's gelu implementation, also used in Megatron:
           Gelu(x) = x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1.0 + 0.044715 * x * x)))

        Fuse subgraph into a FastGelu node:
            +------------ Mul (B=0.79788456) -------------------+
            |                                                   |
            +-------------------------------+                   |
            |                               |                   |
            |                               v                   v
          [root] --> Mul (B=0.044715) --> Mul --> Add(B=1) --> Mul --> Tanh --> Add(B=1) --> Mul-->
            |                                                                                 ^
            |                                                                                 |
            +-----------> Mul (B=0.5) --------------------------------------------------------+
        r   Nr   r   r    r!   r"   r#   r$   r%   r)   r   r,   r0   T)r1   r2   r3   r	   r4   r5   r6   r7   r9   r:   r;   r   r<   r=   r>   r?   r@   rA   rB   r/   )r   r   r   r   rC   rD   Úmul_lastrF   rG   rH   rJ   Úadd_1ÚjÚmul_7978ÚkÚmul_before_add_1ÚanotherÚmul_0447ÚmrN   rO   s                        r   r   zFusionFastGelu.fuse_3ÿ   s‡  € ð  ×Ñ˜AÑÐ&9Ñ9Øà& y×'7Ñ'7¸Ñ':Ñ;ˆÜˆx‹=˜AÒ ¨!¡×!4Ñ!4¸Ò!=ØØ! !™ˆà�z‰z×,Ñ,¨^¸SÔAØà× Ñ  Ñ#Ð+>Ñ>ØØ& ~×'<Ñ'<¸QÑ'?Ñ@ˆÜˆx‹=˜AÒ ¨!¡×!4Ñ!4¸Ò!=ØØ˜A‘;ˆà—:‘:×*Ñ*¨8°U¸DÐBUÓVˆØÐØà�J‰J×*Ñ*¨8°SÓ9ˆØˆqŠ5Øà—^‘^¨¨aª¡A°QÑ7ˆ
àŸ*™*×1Ñ1°)¸UÀAÐGZÓ[ˆØÐ"Øà—
‘
×'Ñ'¨¸ÀÐFYÓZˆØˆ=ØØ�J‰J×*Ñ*¨5°#Ó6ˆØˆqŠ5Øà—:‘:×*Ñ*¨?¸EÀ4ÐI\Ó]ˆØÐØØ�J‰J×*Ñ*¨8°VÀ6Ð*ÓJˆØˆqŠ5ØØ�>‰>˜q Ašv™!¨1Ñ-°Ò;ØàŸ:™:×2Ñ2°5¸%ÀaÈ1ÂfÁÐRSÐUhÓiÐØÐ#Øà×!Ñ! !Ñ$¨
Ò2Ø‰GØ×#Ñ# AÑ&¨*Ò4Ø‰Gàà—:‘:×*Ñ*Ð+;¸UÀGÐM`ÓaˆØÐØØ�J‰J×*Ñ*¨8°VÀ6Ð*ÓJˆØˆqŠ5Øà�>‰>˜q Ašv™!¨1Ñ-°Ò;Øð ØØØØØØØØð

ˆð �z‰z×/Ñ/ØØ�_‰_˜QÑÐ ØØô	
ð à×Ñ×#Ñ# NÔ3Ü×%Ñ%ØØ�<Ø—O‘OØ—‘×,Ñ,¨ZÓ8ô	
ˆ
ð ,ˆ
ÔØ×Ñ× Ñ  Ô,Ø8<×8LÑ8Lˆ×$Ñ$ Z§_¡_Ñ5Ør   c           	      óL  — |j                   d   |vry||j                   d      }t        |«      dk7  s|d   j                  dk7  ry|d   }| j                  j	                  |d«      sy|j                   d   |vry||j                   d      }t        |«      dk7  s|d   j                  dk7  ry|d   }|j                   d   |vry||j                   d      }t        |«      dk7  s|d   j                  dk7  ry|d   }| j                  j	                  |d«      sy|j
                  |j
                  d   |j                   d   k(  rdnd   }| j                  j                  |dd|«      }	|	€y| j                  j                  |	dd	¬
«      }
|
dk  ry| j                  j                  |	d|
dk(  rdnd|«      }|€y|j
                  d   |k(  rd}n|j
                  d   |k(  rd}ny| j                  j                  |d||«      }|€y| j                  j                  |dd	¬
«      }|dk  ry| j                  j                  |d|dk(  rdnd|«      }|€y|j
                  d   |k(  rd}n|j
                  d   |k(  rd}ny| j                  j                  |d||«      }|€y|j
                  d   |k7  s|j
                  d   |k7  ry|||||	||||g	}| j                  j                  ||j                   d   g||«      sy| j                  j                  |«       t        j                  d|g|j                   | j                  j                  d«      ¬«      }d|_        | j                  j!                  |«       | j"                  | j$                  |j&                  <   | j)                  d«       y)aR  
        PyTorch's gelu implementation with tanh approximation:
           Gelu(x) = 0.5 * x * (1 + torch.tanh(0.7978845834732056 * (x + 0.044714998453855515 * x * x * x)))

        Fuse Gelu with tanh into one node:
              +-----------------+------------------+
              |                 |                  |
              |                 v                  v
            [root] ==> Mul --> Mul --> Mul -----> Add  --> Mul --> Tanh --> Add -----> Mul --> Mul -->
              |                       (A=0.0447)          (A=0.7978)        (A=1)       ^     (A=0.5)
              |                                                                         |
              +-------------------------------------------------------------------------+
        Note that constant input for Add and Mul could be first or second input.
        r   Nr   r   r    r!   r"   r#   g{®Gáz„?r%   r)   r   r,   r0   T)r1   r2   r3   r	   r4   r7   r5   r6   r9   r:   r;   r   r<   r=   r>   r?   r@   rA   rB   r/   Úincrease_counter)r   r   r   r   rC   rD   rE   rF   rH   rJ   rW   rK   rY   rL   r[   Ú	mul_cubedÚmul_squaredrN   rO   s                      r   r   zFusionFastGelu.fuse_4s  sà  € ð ×Ñ˜AÑÐ&9Ñ9Øà& y×'7Ñ'7¸Ñ':Ñ;ˆÜˆx‹=˜AÒ ¨!¡×!4Ñ!4¸Ò!=ØØ! !™ˆà�z‰z×,Ñ,¨^¸SÔAØà× Ñ  Ñ#Ð+>Ñ>ØØ& ~×'<Ñ'<¸QÑ'?Ñ@ˆÜˆx‹=˜AÒ ¨!¡×!4Ñ!4¸Ò!=ØØ! !™ˆà× Ñ  Ñ#Ð+>Ñ>ØØ& ~×'<Ñ'<¸QÑ'?Ñ@ˆÜˆx‹=˜AÒ ¨!¡×!4Ñ!4¸Ò!=ØØ˜A‘;ˆà�z‰z×,Ñ,¨X°sÔ;Øà#×)Ñ)¨~×/CÑ/CÀAÑ/FÈ.×J_ÑJ_Ð`aÑJbÒ/b©!ÐhiÑjˆ
àŸ*™*×1Ñ1°)¸UÀAÐGZÓ[ˆØÐ"Øà�J‰J×*Ñ*¨?¸FÈ$Ð*ÓOˆØˆqŠ5ØàŸ*™*×1Ñ1°/À5ÈqÐTUÊvÉ!Ð[\Ð^qÓrˆØÐ"Øà× Ñ  Ñ# zÒ1Ø‰GØ×"Ñ" 1Ñ%¨Ò3Ø‰GààŸ
™
×/Ñ/°ÀÈÐQdÓeˆØÐ Øà�J‰J×*Ñ*¨=¸&ÈÐ*ÓMˆØˆqŠ5Øà—J‘J×+Ñ+¨M¸5ÀqÈAÂvÁ!ÐSTÐViÓjˆ	ØÐØà�?‰?˜1Ñ Ò+Ø‰GØ�_‰_˜QÑ :Ò-Ø‰Gàà—j‘j×-Ñ-¨i¸ÀÐI\Ó]ˆØÐØà×Ñ˜QÑ :Ò-°×1BÑ1BÀ1Ñ1EÈÒ1SØð ØØØØØØØØð

ˆð �z‰z×/Ñ/ØØ�_‰_˜QÑÐ ØØô	
ð à×Ñ×#Ñ# NÔ3Ü×%Ñ%ØØ�<Ø—O‘OØ—‘×,Ñ,¨ZÓ8ô	
ˆ
ð ,ˆ
ÔØ×Ñ× Ñ  Ô,Ø8<×8LÑ8Lˆ×$Ñ$ Z§_¡_Ñ5Ø×Ñ˜jÔ)Ør   )Ú__name__Ú
__module__Ú__qualname__r   r   Údictr   Úboolr   r   r   r   Ú__classcell__)r   s   @r   r   r      sª   ø„ ð4˜iõ 4ð°4ð Èdó ðjÈTÐTXÉ[ó jðXr°Tð rÐPTð rÐY]Ð`dÑYdó rðhr°Tð rÐPTð rÐY]Ð`dÑYdó rðhy°Tð yÐPTð yÐY]Ð`dÑYd÷ yr   r   N)Úloggingr   Úfusion_baser   Úonnxr   Ú
onnx_modelr   r`   Úloggerr   © r   r   ú<module>rl      s+   ðõ
 å Ý Ý  á	�8Ó	€ô^�Võ ^r   