:jddlmZddlmZddlmZddlmZmZm Z ddl m Z ee Z GddeZGdd eZy ) ) getLogger)Fusion) FusionUtils) NodeProto TensorProtohelper) OnnxModelceZdZdZd dedeffd ZdeddeeefzfdZ d ed e ee efd e de fd Z d ZdZdZdZdZdedeedezffdZ d!ded edededdezdedzf dZdZdZ d"dZdZdZdZxZS)#FusionEmbedLayerNoMaskz Fuse embedding layer into one node (EmbedLayerNormalization). It supports the following model types: BERT, DistilBert, ALBert. model descriptionct||dddg|t||_d|_d|_d|_d|_y)NEmbedLayerNormalizationLayerNormalizationSkipLayerNormalizationF)super__init__rutils shape_infershape_infer_done attention embed_node)selfr r __class__s /home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/onnxruntime/transformers/fusion_embedlayer.pyrzFusionEmbedLayerNoMask.__init__sP   % !#; <   !'  %addreturnNc|jj|dgdg}|y|jj|dgdg}|y|d|dfS)NGatherr)r match_parent_path)rr gather_0_path gather_1_paths rmatch_two_gatherz'FusionEmbedLayerNoMask.match_two_gather%sa 44S8*qcJ   44S8*qcJ  Qq!111r layernorminput_name_to_nodesis_distil_bertc4|jj|d|d|_|jy|jd|vry||jd}t |Dcgc]}|j c}}|gdk(ri|D]d}|j dk(s|jj |gdgd }|7|d jd|jdk(sZ|d |_yt|d k(r|dj d k(r|djd|vr||djd} t| d k(r}| dj dk(rk| djd|vrW|| djd} | D]}|j dk(s||_yt | Dcgc]}|j c}}|r,|gdk7r$|gdk7r|gdk7rtjdyy|gdk7r|gdk7rtjdyycc}wcc}w)aCheck that LayerNormalization has a child of Attention node or subgraph like Attention. Args: layernorm (NodeProto): LayerNormalization node input_name_to_nodes (Dict[str, List[NodeProto]]): map from input name to nodes is_distil_bert (bool): whether it is DistilBert or not Returns: bool: whether there is Attention node or subgraph like Attention AttentionF) recursiveTr)MatMulr,r,rr)Addr,MultiHeadAttentionr,)NNrrr!r,r-)r,r,r,Shaper)r-r,r,r,r1r1)r-r,r,r,r1z> %   A &9 9&y'7'7':;H E5 EF U U $<<#;; JJ88I*E (U2Y__Q-?9CSCSTUCV-V/4Qx,# $ x=A (1+"5"5"AhqkFXFXYZF[_rFr/ 0B0B10EFMM"a'!!$,,5!!$++A.2EE+M!,<,C,CA,FG!$D||{2)-#$"(E(J5(J!K "cc"&]]"&TT [\ " !%  [\q!F:)Ks H#Hc&|jj|ddgddg}|$|jj|gdgd}|y|d|d}}|jd|k7ry|jj|gd gd fgd gd fg|\}}}|y|d} |jj | ddr|jj | d dsy|d} |jj | ddsy|d} | jd|k7ryy)az Match position embedding path from input_ids to Gather for DistilBert. Pattern is like the following: (input_ids) | Shape | | Gather (indices=1) | | | Cast (optional) | | | Range (start=0, end=*, delta=1) | | | Unsqueeze | / Expand | Gather Expandr1r!)rDWhereReshaper1)r!r!r0rFrr/) UnsqueezeRangeCastr r1)rrr!rr)rGrHr r1)rrr!rr0T)r r"r6match_parent_pathsrcheck_node_input_value) rposition_embedding_gather input_idsoutput_name_to_noder?expandshape_path2 range_node gather_node shape_nodes r#match_position_embedding_distilbertz:FusionEmbedLayerNoMask.match_position_embedding_distilberts9* ,,-FSZH[^_ab]cd =JJ00)7E }a%) ;;q>Y &jj33 BOT:LI   5! =1X JJ - -j!Q ?DJJDeDefprsuvDwBi  11+q!D2Y   A ) +rcy)aYMatch position embedding path from input_ids to Gather for Roberta. Roberta Embedding Layer Pattern (* is optional since it might be removed by ORT, ? is the padding word id): (input_ids) --> Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Mul -- Cast(to=7) -- Add(B=1) -- Cast(to=7)* --> Gather | ^ V | +------------------------------+ Roberta new pattern from transformers v4.9: (input_ids) --> Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Add(B=0) -- Mul -- Cast(to=7) -- Add(B=1) --> Gather | ^ V | +-------------------------------------------+ start_node = position_embedding_gather start_index = 1 # match optional Cast node. parent = self.model.get_parent(start_node, start_index, output_name_to_node) if parent is None: return if parent.op_type == "Cast": if OnnxModel.get_node_attribute(parent, "to") != 7: return start_node = parent start_index = 0 i, path, return_indices = self.model.match_parent_paths( start_node, [ (['Add', 'Cast', 'Mul', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0]), (['Add', 'Cast', 'Mul', 'Add', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0, 0])], output_name_to_node) if path is not None: # constant input of Add shall be 1. i, value = self.model.get_constant_input(path[0]) if value != 1: return False _, self.padding_word_id = self.model.get_constant_input(path[-1]) return input_ids == path[-1].input[0] FrrMrNrOs r match_position_embedding_robertaz7FusionEmbedLayerNoMask.match_position_embedding_robertasZrcN|jj|ddgddg|}|y|\}}|jj|jd}|t |j dk(r|j ddk(rr|j j|ddgrT|j j|ddgr6t |jdk(s|j j|ddgsy|jj}|d krtj|d dgs y|j j|ddgsy|jj|d|} | y| jd k(r<|j j| ddsy|jj| d|} n| } | | jd k7ry|j j| ddsy|jj| d|} | | jd k7ry|| jdk(S)a  Match position embedding path from input_ids to Gather for BERT. BERT Embedding Layer Pattern: (input_ids) / / Shape / | / Gather (indices=1) / | / Add (optional, B=0) / | Gather (segment_ids) Unsqueeze (axes=0) \ | | \ Gather Slice (data[1,512], starts=0, ends=*, axes=1, steps=1) \ / | Add Gather \ / Add | LayerNormalization SlicerGr!r0Fr axesr-r r1) r r"get_constant_valuer6r8rQrrLget_opset_versionrcheck_node_attribute get_parentr5) rrMrNrOpathslice unsqueeze slice_weight opset_versionr>gatherrQs rmatch_position_embedding_bertz4FusionEmbedLayerNoMask.match_position_embedding_berts,zz++ % k " F    <yzz44U[[^D  $L&&'1,""1%* 11%QC@ 11%QC@U[[!Q&$***K*KESTWXVY*Z 446 2 33IvsK::44YA3Gzz$$Y3FG < <<5 ::44T1a@ZZ**44GHFF >V^^x7 11&!Q? %%fa1DE =EMMW4EKKN**rcT|j|||ry|j|||ryy)NTF)rlrWrZs rmatch_position_embeddingz/FusionEmbedLayerNoMask.match_position_embedding9s5  - -.GTg h  3 34MyZm nrc |jd}|r|jdnd}|jd}|js(|jjd|_d|_|j|jj |}|jj |}|r|sJt |dk(rt |dk(r |d|dk(stjd|d|y|rQ|jj||s5tjd |d |jj |y|jj|jd } | t | jdk7rtjd y|jj|jd } | 7t | jdk7s| jd| jdk7rtjd y|rw|jj|jd } | 7t | jdk7s| jd| jdk7rtjdy| jd | jd krUtjd|jd d| jd d|jd d| jd |r| jd  jd krUtjd|jd d| jd d|jd d| jd | jd | jd krUtjd|jd d| jd d|jd d| jd y)zXSanity check of embedding weights, and match hidden_size of weights and shape of inputs.r!NT)updater0z^Cannot fuse EmbedLayerNormalization: input_ids and position_ids not matched in 2nd dimension: z vs FzYCannot fuse EmbedLayerNormalization: input_ids and segment_ids does not have same shape: z != rzICannot fuse EmbedLayerNormalization: word embedding table is not expectedzMCannot fuse EmbedLayerNormalization: position embedding table is not expectedzLCannot fuse EmbedLayerNormalization: segment embedding table is not expectedzword_embedding_table (z) size z <= position_embedding_table (z <= segment_embedding_table (zposition_embedding_table () r6rr infer_runtime_shaperget_edge_shaper8r9info compare_shaperbrQwarning) rword_embedding_gathersegment_embedding_gatherrMrN segment_ids position_idsinput_ids_shapeposition_ids_shapeword_embedding_tableposition_embedding_tablesegment_embedding_tables rcheck_embeddingz&FusionEmbedLayerNoMask.check_embeddingGs)//2 ;S.44Q7Y] 066q9 $$#zz==T=JD $(D !    '"..==iHO!%!1!1!@!@!N "'9 99O$)*+q0#A&*)D)DQ)G(HPdPjPjklPmOnoMNgNmNmnoNpMqqxyQyWyWXYyZx[\  #))!,0G0M0Ma0PP,-B-H-H-K,LGThTnTnopTqSrsPQiQoQopqQrPssz{R{X{XYZ{[z\](--a04K4Q4QRS4TT01J1P1PQR1S0TT[\t\z\z{|\}[~\]u]{]{|}]~\FG^GdGdefGgFhir input_namec6d}|jj|}|Y|jjjt j k7r"|jj|\}}||fS|}||fS|jj|\}}||fS)aCast a graph input or node input to int32. Args: input_name (str): name of graph input or node input Returns: A tuple of casted input name and the cast node. int32_output (str): If input is int32, it is the input name, Otherwise it is output name of Cast node. input_cast_node (Union[None, NodeProto]): Cast node. It could be None if input is int32. N) r find_graph_inputtype tensor_type elem_typerINT32rcast_input_to_int32)rrinput_cast_node graph_input int32_outputs r cast_to_int32z$FusionEmbedLayerNoMask.cast_to_int32sjj11*=  "++559J9JJ04 0N0Nz0Z- o _,, * _,,-1JJ,J,J:,V )L/_,,rrNrvrMrwryc g} |j|\}} |jjd} |jdk(r|jd} |jd} n|jd} |jd} d}|R|j|jd\}} |||jd|jd|jd| | g}n#|d|jd|jdd| | g}|6|j d|j|\}} |j || d z| d zg}|r||n| d z}|j |t jd||| }d |_|jD].}|jdk(s|jj|g0t|jdk(r0|jjt jddg| j || D]%}|j|j|j<'|j j| ||_|S)agCreate an EmbedLayerNormalization node. Note that segment embedding is optional. Args: input_ids (str): input_ids for word embeddings layernorm (NodeProto): LayerNormalization or SkipLayerNormalization node. word_embedding_gather (NodeProto): the Gather node for word embedding position_embedding_gather (NodeProto): the Gather node for position embedding segment_embedding_gather (Union[None, NodeProto]): the Gather node for segment embedding, or None. Returns: NodeProto: the EmbedLayerNormalization node created. rrr!r0r^Nr_output_dummy_mask_index_embedding_sum)outputsnamez com.microsoftepsilong-q=)rr create_node_namer5r6appendr make_nodedomain attributerextendr8make_attributethis_graph_namenode_name_to_graph_name nodes_to_addr)rrNr&rvrMrwryembedding_sum_outputembedding_sum_namerrR node_namegammabetaembed_node_inputsrxembed_node_outputsrrattr>s rcreate_fused_nodez(FusionEmbedLayerNoMask.create_fused_nodes. )))4 1JJ//0IJ    4 4OOA&E??1%DOOA&E??1%D  # /!//0H0N0Nq0QRNK%++A.)//2(..q1! %++A.)//2!   #  $ $R ("00>OL!  $ $\ 2')3YAT5TU );)G%YYiMiD  % %d +%% % &  , && 3Cxx9$$$++SE2 3 z## $ )  ' ')>)>y')R(S T J'  KD6:6J6JD ( ( 3 K   .$rc~|jj|jd|jdd|_y)NrT)r replace_input_of_all_nodesr3 prune_graph)rr&rs r finish_fusionz$FusionEmbedLayerNoMask.finish_fusion s5 --i.>.>q.A:CTCTUVCWXrc|jdk(xr5t|jdkDxrt|jddkDS)Nrr^r)r5r8r3)rr>s r"is_skip_layer_norm_with_sum_outputz9FusionEmbedLayerNoMask.is_skip_layer_norm_with_sum_outputsD 88nc$++>NQR>RnWZ[_[f[fgh[iWjmnWnnrc |j|}|y|\}}|jd} |jd} |j||dsy|j|d|sy|jdk(rL|j |} d} |} | r|j dnd}|duxr|jj|du}n|} |jdk(rdnd} t|j | kDr|j | nd}|duxr|jj|du}|xr||vxrt||dkD}|duxr|jdk7xs|xs|} |j| ||||| | |r|nd}| r:d | j | <|s)|jj||j d |j||y ) NFr!r(rr^r-r)rr_no_use__to_be_removed_r0T) r%r6rBrr5rr3r find_graph_outputr8rrr)rr&add_before_layernormr'rOoptional_segment_gather two_gatherrvrMrNryneed_embedding_sum_outputsum_output_indexnode_with_sum_output sum_outputis_sum_graph_outputis_sum_used_by_multiple_nodesrs r fuse_gpt2z FusionEmbedLayerNoMask.fuse_gpt2s/(**+?@  ;E88)//2 066q9 ,,Y8K\a,b##$94AZ[    8 8(,(O(OPY(Z % #, 0I))!,tJ#-T#9"u @\@\]g@hpt@t #7 $8$@$@E$Iqq +2236FF%++,<=  $.T#9"u @\@\]g@hpt@t o .A AosK^_iKjGknoGo **44)?)$,,5m9LmPm & ++   ! % # !:-@zd,   %">?SV^U_bcad"e " *$;A$>!,,-F Sfg001I9Vij+D'@ $(, %##$9;SUno++   ! % $   9j1rc|jj|dgdg}|jdk(r |y|d}d}n|jj|dgdg}|jj|dgdg}|||y|d}|d}n5|/|-|jj|dgdg}|y|d}|d}n|}d}|j|||||ry|j ||||ry|j ||||ryy)Nr-rrr r!)r r"r5rrr) rr>r'rOfirst_add_pathrrr#r$s rfusezFusionEmbedLayerNoMask.fusesQ55dUGaSI <</ /%#1!#4 &* # JJ88zA3OM JJ88zA3OM$)B!)'5a'8$*7*:'*}/D!%!=!=dUGaS!Q!)'5a'8$*7*:''+$*.' >> &(;=PRi     &:>$ 46IK^ _  `r)zno mask)NFN)N)__name__ __module__ __qualname____doc__r strrrtupler%dictlistboolrBrWr[rlrnrrrrrrrrr __classcell__rs@rr r sB i c  2I 2$y)?S9T2T 2RR"#tI"67R R  Rh<|-^F+P HT--c4);K6K0L-<$("``` ) ` $- ` #'"2 `Dj`D orvOb'R0d"rr c6eZdZddeffd ZdZfdZxZS)FusionEmbedLayerNormalizationr c4t||d||_y)Nz with mask)rruse_mask_index)rr rrs rrz&FusionEmbedLayerNormalization.__init__s  ,,rc|j}t|jdk(r<|jj|tj d|j nxt|jdkDr?|jds0||jd<tj d|j n!tj d|j y|D]z}tj d|j |jdk(r|jd|jd<O|jd k(s_|jd|jd <|y) Nzappend mask to %szreplace mask in %szskip mask in %szupdate mask_index in %sr*r!r^r.r_) rr8r6rr9r:rr5r3)r mask_int32attention_nodesrattention_nodes r replace_maskz*FusionEmbedLayerNormalization.replace_masks__ z A %    # #J / LL,joo > !! "Q &z/?/?/B",J  Q  LL-z ? LL*JOO < - ?N LL2N4G4G H%%4*4*;*;A*>$$Q'''+??*4*;*;A*>$$Q'  ?rcHd|_d|_d|_t|||||jy|j s't jd|jdy|j3|j't jd|jdy|jr|jjd}n|jjd}||}|jj|rB|Dcgc]}|jdvs|}}|j|||jdy||vr(t jd||jdy||}|jd vr|Dcgc]}|jdvs|}}jd k(rA|jd }t|t|k(r|jj!||j|||jdyycc}wcc}w) NzG--use_mask_index is not set: EmbedLayerNormalization will not have maskz EmbedLayerNormalization(no mask)zLEmbedLayerNormalization will not have mask since attention node is not foundr^r_)r*r.z"EmbedLayerNormalization(with mask)zHEmbedLayerNormalization will not have mask since %s is not a node output) ReduceSumrIrr)rr7rrrrr9r:increase_counterr6r rr5rr8nodes_to_remover)rr>r'rOrchildren_nodesrrs rrz"FusionEmbedLayerNormalization.fuses#  T.0CD ?? " "" LLb c  ! !"D E  >> !d&:&:&B LLg h  ! !"D E  >>--a0J--33A6J,Z8 :: & &z 20>v$,,RuButvOv   j/ :  ! !"F G  0 0 LLceo p  ! !"D E ":. <<0 00>v$,,RuButvOv||{*!ZZ] ~&#o*>>((//5   j/ :  ! !"F G 1wws HH HH)F)rrrr rrrrrs@rrrs -i-?*-H-HrrN)loggingr fusion_baser fusion_utilsronnxrrr onnx_modelr rr9r rrYrrrsC $// 8 P VP fGH$:GHr