:ji`ddlZddlZddlZddlZddlZddlZddlmZmZm Z m Z m Z m Z m Z ddlmZmZddlmZej&eZdZddZddZdZgd Zd gd d gd d gd ZGd dZ ddZddZy)N)AttributeProto GraphProto ModelProto NodeProto TensorProtohelper numpy_helper) infer_shapesinfer_shapes_path)versionc |Dcgc]8}tt|jdddjdd:c}Scc}w)z| Convert numpy float16 to python int. :param np_list: numpy float16 list :return int_list: python int list HN)intbinviewzfill)np_list_s u/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/onnxruntime/transformers/float16.py_npfloat16_to_intrs?=D DqCAFF3K $**2. 2 DD Ds=Ac Rd}|tj|dkDjddkDr|tj|dkDj}|tj|dkDj }||k\rt j d|d|||krt j d|d||tj|dkjddkDr|tj|dkj}|tj|dkj }|| krt j d|d| || k\rt j d|d| tj|d||||}tj|| |d| |}tj|||td||}tj|td|| | |}tj|S)a? Convert float32 numpy array to float16 without changing sign or finiteness. Positive values less than min_positive_val are mapped to min_positive_val. Positive finite values greater than max_finite_val are mapped to max_finite_val. Similar for negative values. NaN, 0, inf, and -inf are unchanged. c:tj||k||kSN)np logical_and)abcs rbetweenz&convert_np_to_float16..between/s~~a!eQU++rzthe float32 number z will be truncated to infz-inf) rwhereshapemaxminloggerdebugfloatfloat16)np_arraymin_positive_valmax_finite_valr! positive_max positive_min negative_max negative_mins rconvert_np_to_float16r3's ,A&'--a014A 67;;= A 67;;= > ) LL.|n@PRZ[Hxx!1 18Q?BRART\]Hxx%,GYabHxxf x.IN?\deH ::h r"ct|tstdt||jtj k(rtj |_|jrSttj|j||}t|}||jddg|jdd|jrCtj|jd}t|||}|j|_ |S)aConvert tensor float to float16. Args: tensor (TensorProto): the tensor to convert. min_positive_val (float, optional): minimal positive value. Defaults to 1e-7. max_finite_val (float, optional): maximal finite value. Defaults to 1e4. Raises: ValueError: input type is not TensorProto. Returns: TensorProto: the converted tensor. 3Expected input type is an ONNX TensorProto but got Nfloat32dtype) isinstancer ValueErrortype data_typeFLOATFLOAT16 float_datar3rarrayr int32_dataraw_data frombuffertobytes)tensorr-r. float16_dataint_list float32_list float16_lists rconvert_tensor_float_to_float16rJIs fk *NtTZ|n]^^ ;,,,&..   0&:K:K1LN^`noL(6H#+F  a #%F  a ??== JL0?OQ_`L*224FO Mr"ctj|j}tj|j |j |Sr)r to_arrayr%rmake_tensor_value_infonamer<)rEr%s rmake_value_info_from_tensorrOns7  ! !& ) / /E  ( (f6F6F NNr")ArrayFeatureExtractor BinarizerCastMapCategoryMapperDictVectorizerFeatureVectorizerImputer LabelEncoderLinearClassifierLinearRegressor Normalizer OneHotEncoderRandomUniformLike SVMClassifier SVMRegressorScalerTreeEnsembleClassifierTreeEnsembleRegressor TreeEnsembleZipMapNonMaxSuppressionTopKRoiAlignRangeCumSumMinMaxUpsampler)Resize GroupNorm SkipGroupNormc(eZdZdZdefdZdefdZy)InitializerTrackerz'Class for keeping track of initializer. initializerc.||_g|_g|_yr)rr fp32_nodes fp16_nodes)selfrrs r__init__zInitializerTracker.__init__s&r"nodecv|r|jj|y|jj|yr)rtappendru)rvrxis_node_blockeds radd_nodezInitializerTracker.add_nodes)  OO " "4 ( OO " "4 (r"N)__name__ __module__ __qualname____doc__rrwrr|r"rrqrqs1K )Y)r"rqc d|dk\sJd|ttjtjjksJd|in|} t |t r|} tjtjtjdk\rn|sltjtjj| 5} | j } t#| | tj$| }d}dddntj$| }t |t&st)dt+|d}|sAtjtjtjd k\r t,} |t.}|g}t1|}t1|}t d |j2Dd }t5t6}|d krdg|d<t8j;d|d|d|d|d|d|d|g}g}g}g}|||}|j=|i}t1}t1}|j>j@Dcgc]@}|j*jBjDtFjHk(s5|j B}}|j>jJDcgc]@}|j*jBjDtFjHk(s5|j B}}t |tLr)|Dcgc] }||vs| }}|Dcgc] }||vs| }}n|sg}g}tO|j>j@D]<\}}|j |vsdt |z}|||j <|jQ|j dt |z}|j>jRjQ}|jU|||_tFjV|j*jB_"tYjZd|j g|gtFjV|g}|j>j\j_||j=||jQ|?tO|j>jJD].\}}|j |vsdt |z} | ||j <|jQ|j dt |z}|j>jRjQ}|jU|| |_tFjV|j*jB_"tYjZd| g|j gd|g}|j>j\j_||j=||jQ|1i}!|rg}"|D]}#t |#t&r|"j=|#j>t |#t`r6|#jbD]H}|jdtFjHk(s!|j |!vsJtg||!|j <J|#j\D]}|j |vrtitk|j@D]3}|j@||vs||j@||j@|<5titk|jJD]3}|jJ||vs||jJ||jJ|<5|jl|vxs|j |v}$tO|j@D]a\}} | |!vs |$xs>||jo|jlgvxr|| jo|jlgv}%|!| jq||%c|$r|j=|g|jldk(rU|jrD]F}&|&j dk(s|&jttFjHk(s1tFjV|&_:n|jldvrd}'|jrD]G}&|&j d k(sd}'|&jttFjHk(s3tFjV|&_:I|jld!vr@|'s>|jrj_tYjvd tFjVg|jl|vs|jl| vr$|jrD]}&|"j=|&|j=|t |#txr|"j=|#jz|#j|D]}|"j=||#j~jUt|#j~|||#jD]}t|||}t |#t`s#tj|#j@|#jJ|#jRD];}|j*jBjDtFjHk(rH|j |vr:tFjV|j*jB_"|j=||j*jd"s|j*jjDjBjDtFjHk(s|j |vstFjV|j*jjDjB_"|j=|>|"}|r|!jD]}(|s |(jst|(jb|||(_1|j=t|(jb|(js_|rbt8jd#|(j|D]+})tO|)j@D]\}} |||)jlvs|| jo|)jlgvr7|D]}*| |*j k(s|j>jRjQ}|jU|*| d$z}||_tFjH|j*jB_"| d%z}tYjZd| g|gd|g}|j>j\j_|||)j@|<.| rtFjntFjH}+|D]})titk|)j@D]}|)j@|} |D]}*| |*j k(s|j>jRjQ}|jU|*| d$z}||_|+|j*jB_"| d%z}tYjZd| g|g|+|g}|j>j\j_|||)j@|<titk|)jJD]}|)jJ|},|D]}*|,|*j k(s|j>jRjQ}|jU|*|,d&z}-|-|_|+|j*jB_"|,d'z}tYjZd|-g|,gd |g}|j>j\j_||-|)jJ|<|S#1swYuxYw#wxYwcc}wcc}wcc}wcc}w)(aConvert tensor float type in the input ONNX model to tensor float16. Args: model (ModelProto or str): The ONNX model or path of the model to convert. min_positive_val (float, optional): minimal positive value. Defaults to 5.96e-08. max_finite_val (float, optional): maximal finite value of float16. Defaults to 65504. keep_io_types (Union[bool, List[str]], optional): It could be boolean or a list of float32 input/output names. If True, model inputs/outputs should be left as float32. Defaults to False. disable_shape_infer (bool, optional): Skips running onnx shape/type inference. Useful if shape inference has been done. Defaults to False. op_block_list (List[str], optional): List of op types to leave as float32. Defaults to None, which will use `float16.DEFAULT_OP_BLOCK_LIST`. node_block_list (List[str], optional): List of node names to leave as float32. Defaults to None. force_fp16_initializers(bool): force converting all float initializers to float16. Default to false, which will convert only the one needed to avoid precision loss. force_fp16_inputs(Dict[str, List[int]]): Force the conversion of the inputs of some operators to float16, even if this script's preference it to keep them in float32. Raises: ValueError: input type is not ModelProto. Returns: ModelProto: converted model. "\o>zginvalid min_positive_val. smallest positive float16 value: subnormal 5.96e-08, and normalized 6.104e-05z4invalid max_finite_val. largest float16 value: 65504Nz1.8.0)dirTz$Expected an ONNX ModelProto but got z1.2.0c3RK|]}|jdvs|j!yw))zai.onnxN)domainr ).0os r z+convert_float_to_float16..s[Aqxx??Zaii[s'' )default rlrmz"fp16 parameters: min_positive_val=z max_finite_val=z keep_io_types=z disable_shape_infer=z op_block_list=z node_block_list=z force_fp16_initializers=graph_input_cast_graph_input_castCast)torNgraph_output_cast_graph_output_castr)EyeLike Multinomial RandomNormalRandomNormalLike RandomUniformr\ SequenceEmpty BernoulliFr8)rrr sequence_typezXinitializer is used by both fp32 and fp16 nodes. Consider add these nodes to block list: _cast_to_fp32_cast_to_fp32_node _cast_to_fp16_cast_to_fp16_node)Lr*rfinfor+r&r9strr parseonnx __version__tempfileNamedTemporaryFileospathdirnamerNr loadrr:r;r DEFAULT_OP_BLOCK_LISTset opset_importdictALWAYS_FLOAT_INPUTSr(r)rzgraphinput tensor_type elem_typerr=outputlist enumerateadd value_infoCopyFromr>r make_noderxextendrrrr<rqrangelenop_typegetr| attributeimake_attributerggraphstrJtensors itertoolschainHasFieldrvaluesrurOrtinfoBFLOAT16).modelr-r. keep_io_typesdisable_shape_infer op_block_listnode_block_listforce_fp16_initializersforce_fp16_inputs#use_bfloat16_as_blocked_nodes_dtypeforce_fp16_inputs_dict model_pathtmpfileshape_infer_model_pathfunc_infer_shape onnx_opsetalways_float_inputsqueuevalue_info_list node_listmixed_float_type_node_list name_mappinggraph_io_to_skipio_castsn fp32_inputs fp32_outputsr output_name node_namenew_value_infonew_node input_namefp32_initializers next_levelqr{use_fp32_weightattr has_dtypevaluerxr accuracy_typeroutput_cast_names. rconvert_float_to_float16rs H x 'q ' U288BJJ#7#;#;< <t>tt <#4#>N~N^^mn{m|}RSfRggvwDvEEVWfVgg@AX@Y Z EOI "$# ' LLLuuH#(;;#4#4ja8J8J8T8TXcXiXi8i166jKj$)KK$6$6lq!&&:L:L:V:VZeZkZk:kAFFlLl-&"-DQm1CqD D#/Fa1 3EF F   %++++,$1 66[ -A6K#.L   (*SV3I"[[33779N  # #A &"-N 8C8K8KN   + + 5((!&&K=[M`M`gpqrH KK   # #H -  " "> 2 LL #$"%++,,-$1 66\ !-A6J#-L   (+c!f4I"[[33779N  # #A &",N 8C8K8KN   + + 5((*xAT]^_H KK   # #H -  " "> 2 LL #$"8:  ^ :A!Z(!!!''*!Z(JA{{k&7&77 vv->>>>4Fq4I)!&&1J :AAvv) "3qww<0B771:5)5aggaj)AAGGAJB#3qxx=1D88A;,6*6qxx{*CAHHQKD'(ii=&@&]AFFoD]O)2177);W :%)::.=/ !%8%<% > J J T / 6 6q 9 :k^ :@E H#))+ "e&6&6 ?@Q@QSces tE   " "#>u?P?P#Q R(? notoooAB +&tzz2 MAz+DLL99QBXB\B\]a]i]ikmBn=n-  0%*[[%;%;%?%?%AN"++J7",">K*5N'@K@Q@QN''33= *-A AI & 0 0* }YZaj klHKK$$++H5$/DJJqM  (-PK((U`UfUfM's4::' AAJ-  0%*[[%;%;%?%?%AN"++J7",">K*5N'@MN''33= *-A AI & 0 0* }Yfmv wxHKK$$++H5$/DJJqM  &s4;;'( A[[^F-  Z__,%*[[%;%;%?%?%AN"++J7'-'?$*:N'@MN''33= &)= =I & 0 0:J9KfXZ\cl mnHKK$$++H5%5DKKN  /'P Le + +" PklDFsH0~ ~"6~~6~#8~# ~(&~(0 ~-:~- ~~ct|tstdt||jtj k7r tdd}|j rtj|j }|jr!tj|jd}| tdt|||}tjtj|tj|z S)zSMeasure the maximum absolute difference after converting a float tensor to float16.r5z#Expected tensor data type is float.Nr6r7zexternal data not loaded!)r9rr:r;r<r=r?rr@rBrC RuntimeErrorr3amaxabsr6)rEr-r. float32_datarFs rfloat_to_float16_max_diffrs fk *NtTZ|n]^^ ;,,,>??L xx 1 12  }}V__IF 677(7GXL 77266,L)AAB CCr")r@) rrFFNNFNF)rloggingrrnumpyrrrrrrrrr onnx.shape_inferencer r packagingr getLoggerr}r(rr3rJrOrrrqrrrr"rrs  eee@   8 $E D"JO D#$Aq6QPQFS ) )"!(-AH Dr"