Ë
    ÝÍ:jD  ã                   óÀ  — d Z ddlZddlZddlZddlZddlZddlmZ ddlZddlZddl	m
Z
mZmZmZmZmZmZmZmZmZmZ ddlmZ ddlmZmZ ddlmZmZmZmZ ddlm Z  dd	l!m"Z"  ejF                  d
«      Z$ ejJ                  d¬«      Z%dejL                  vr e'e%«      ejL                  d<   ddl(Z(ddl)m*Z*m+Z+m,Z, d„ Z-d„ Z.de/de/fd„Z0d„ Z1d„ Z2d„ Z3e4dk(  r e3«        yy)a>  Benchmarking the inference of pretrained transformer models.
PyTorch/TorchScript benchmark is based on https://github.com/huggingface/transformers/blob/master/examples/benchmarks.py.
One difference is that random input_ids is generated in this benchmark.

For onnxruntime, this script will convert a pretrained model to ONNX, and optimize it when -o parameter is used.

Example commands:
    Export all models to ONNX, optimize and validate them:
        python benchmark.py -b 0 -o -v -i 1 2 3
    Run OnnxRuntime on GPU for all models:
        python benchmark.py -g
    Run OnnxRuntime on GPU for all models with fp32 optimization:
        python benchmark.py -g -o
    Run OnnxRuntime on GPU with fp16 optimization:
        python benchmark.py -g -o -p "fp16"
    Run TorchScript on GPU for all models:
        python benchmark.py -e torchscript -g
    Run TorchScript on GPU for all models with fp16:
        python benchmark.py -e torchscript -g -p "fp16"
    Run ONNXRuntime and TorchScript on CPU for all models with quantization:
        python benchmark.py -e torchscript onnxruntime -p "int8" -o
    Run OnnxRuntime with bfloat16 fastmath mode kernels on aarch64 platforms with bfloat16 support:
        python benchmark.py --enable_arm64_bfloat16_fastmath_mlas_gemm

It is recommended to use run_benchmark.sh to launch benchmark.
é    N)Údatetime)ÚConfigModifierÚOptimizerInfoÚ	PrecisionÚcreate_onnxruntime_sessionÚget_latency_resultÚinference_ortÚinference_ort_with_io_bindingÚoutput_detailsÚoutput_fusion_statisticsÚoutput_summaryÚsetup_logger)ÚFusionOptions)ÚMODEL_CLASSESÚMODELS)Úcreate_onnxruntime_inputÚexport_onnx_model_from_ptÚexport_onnx_model_from_tfÚload_pretrained_model)Úversion)ÚQuantizeHelperÚ F)ÚlogicalÚOMP_NUM_THREADS)Ú
AutoConfigÚAutoTokenizerÚLxmertConfigc                 óÐ  — dd l }g }| rMd|j                  «       vr;d|j                  «       vr)d|j                  «       vrt        j                  d«       |S d}|dk(  r;t        j
                  }d}d|j                  «       vrt        j                  d	«       |S |t        j
                  k(  rt        j                  d
|› d�«       |D �]Š  }t        |   d   }|
D �]u  }|t        |«      kD  r Œ&|d | }t        |   d   |_	        t        j                  |«      }d|v r[t        j                  «       5  t        |t        |   d   t        |   d   t        |   d   |||||| |||||||«      \  }} }!}"d d d «       d|v r>t        |t        |   d   t        |   d   t        |   d   |||||| |||||||«      \  }} }!}" sŒãt!        | |d|||¬«      }#|#€Œø|#j#                  «       D �$cg c]  }$|$j$                  ‘Œ }%}$g }&| rdnd}'t'        j(                  ||¬«      }(t+        j,                  t/        |«      t/        |«      t/        !|(j0                  «      g«      })t+        j,                  t/        |«      |(j0                  g«      }*|D �]Ê  }+|+dk  rŒ
|D �]»  },"�|,|"kD  rŒd|v rt*        j2                  nt*        j4                  }-t7        |!|+|,||(|-«      }.d|j8                  ||'||| ||||+|,|j;                  «       t=        t?        j@                  «       «      dœ}/|(j                  dv r4t        jC                  d|› d|+d|(jD                  |(jD                  g› �«       nt        jC                  d|› d|+|,g› �«       |rtG        |#|.|/|	|+|«      }0nŸ|#jI                  |%|.«      }1|)g}2tK        t        |1«      «      D ]9  }3|3dk(  r!t        |   d   dk(  r|2jM                  |*«       Œ)|2jM                  |)«       Œ; d|v rt*        jN                  nt*        jP                  }4tS        |#|.|/|	|%|1|&|2|+|'|4|«      }0t        jC                  |0«       |jM                  |0«       �Œ¾ �ŒÍ �Œx �Œ� |S # 1 sw Y   �ŒêxY wc c}$w )Nr   ÚCUDAExecutionProviderÚMIGraphXExecutionProviderÚDmlExecutionProviderzŽPlease install onnxruntime-gpu or onnxruntime-directml package instead of onnxruntime, and use a machine with GPU for testing gpu performance.Útensorrté   ÚTensorrtExecutionProviderzhPlease install onnxruntime-gpu-tensorrt package, and use a machine with GPU for testing gpu performance.zOptimizerInfo is set to zA, graph optimizations specified in FusionOptions are not applied.é   Úpté   é   ÚtfT)Úenable_all_optimizationÚnum_threadsÚverboseÚ(enable_mlas_gemm_fastmath_arm64_bfloat16ÚcudaÚcpu©Ú	cache_dirÚonnxruntime©Úenginer   Ú	providersÚdeviceÚ	optimizerÚ	precisionÚ
io_bindingÚ
model_nameÚinputsÚthreadsÚ
batch_sizeÚsequence_lengthÚcustom_layer_numr   ©ÚvitÚswinzRun onnxruntime on ú with input shape Úgpt)*r2   Úget_available_providersÚloggerÚerrorr   ÚNOOPTÚwarningr   ÚlenÚ
model_typer   ÚparseÚtorchÚno_gradr   r   r   Úget_outputsÚnamer   Úfrom_pretrainedÚnumpyÚprodÚmaxÚhidden_sizeÚint64Úint32r   Ú__version__Úget_layer_numÚstrr   ÚnowÚinfoÚ
image_sizer	   ÚrunÚrangeÚappendÚlonglongÚintcr
   )5Úuse_gpuÚproviderÚmodel_namesÚmodel_classÚconfig_modifierr8   r+   Úbatch_sizesÚsequence_lengthsÚrepeat_timesÚinput_countsÚoptimizer_infoÚvalidate_onnxr1   Úonnx_dirr,   Ú	overwriteÚdisable_ort_io_bindingÚuse_raw_attention_maskÚmodel_fusion_statisticsÚmodel_sourceÚ(enable_arm64_bfloat16_fastmath_mlas_gemmÚargsr2   ÚresultsÚwarm_up_repeatr:   Úall_input_namesÚ
num_inputsÚinput_namesÚfusion_optionsÚonnx_model_fileÚis_valid_onnx_modelÚ
vocab_sizeÚmax_sequence_lengthÚort_sessionÚnode_argÚort_output_namesÚoutput_buffersr6   ÚconfigÚmax_last_state_sizeÚmax_pooler_sizer=   r>   Úinput_value_typeÚ
ort_inputsÚresult_templateÚresultÚort_outputsÚoutput_buffer_max_sizesÚiÚ	data_types5                                                        úw/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/onnxruntime/transformers/benchmark.pyÚrun_onnxruntimer�   X   sH  € ó2 à€GáØ$¨K×,OÑ,OÓ,QÑQØ(°×0SÑ0SÓ0UÑUØ#¨;×+NÑ+NÓ+PÑPä�‰ð ]ô	
ð ˆà€NØ�:ÒÜ&×,Ñ,ˆØˆØ&¨k×.QÑ.QÓ.SÑSÜ�L‰LØzôð ˆNàœ×,Ñ,Ò,Ü�‰Ø& ~Ð&6Ð6wÐxô	
ð "ó e+ˆ
Ü  Ñ,¨QÑ/ˆØ&ó c	+ˆJØœC Ó0Ò0Ùà)¨+¨:Ð6ˆKÜ$ ZÑ0°Ñ3ˆDŒOÜ*×0Ñ0°Ó6ˆNà�|Ñ#Ü—]‘]“_ñ ô 2Ø"Ü˜zÑ*¨1Ñ-Ü˜zÑ*¨1Ñ-Ü˜zÑ*¨1Ñ-Ø#Ø'Ø!Ø Ø#ØØ!Ø&Ø%Ø.Ø!Ø/Ø&ó#ñØ'Ø+Ø"Ø+÷ð2 �|Ñ#ô .ØÜ˜:Ñ& qÑ)Ü˜:Ñ& qÑ)Ü˜:Ñ& qÑ)ØØ#ØØØØØØ"Ø!Ø*ØØ+Ø"ó#ñØ#Ø'ØØ'ñ* 'Øä4ØØØØ(,Ø'ØØ9aôˆKð Ð"Øà>I×>UÑ>UÓ>WÖX°( §£ÐXÐÐXØˆNÙ&‘V¨EˆFÜ×/Ñ/°
ÀiÔPˆFÜ"'§*¡*ä˜Ó$ÜÐ(Ó)Ü˜
 F×$6Ñ$6Ó7ðó#Ðô $Ÿj™j¬#¨kÓ*:¸F×<NÑ<NÐ)OÓPˆOØ)ó L+�
Ø ’?ØØ'7ó I+�OØ*Ð6¸?ÐM`Ò;`Ø à6:¸lÑ6J¤u§{¢{ÔPU×P[ÑP[Ð$Ü!9Ø"Ø"Ø'Ø#ØØ(ó"�Jð #0Ø#.×#:Ñ#:Ø%-Ø"(Ø%3Ø%.Ø*@Ð&@Ø&0Ø",Ø#.Ø&0Ø+:Ø,;×,IÑ,IÓ,KÜ$'¬¯©«Ó$7ñ'�Oð" ×(Ñ(¨OÑ;ÜŸ™Ø1°*°Ð=OÐQ[Ð]^Ð`f×`qÑ`qÐsy÷  tEñ  tEð  QFð  PGð  Hõô Ÿ™Ð&9¸*¸ÐEWÐYcÐetÐXuÐWvÐ$wÔxá-Ü!.Ø'Ø&Ø+Ø(Ø&Ø*ó"™ð '2§o¡oÐ6FÈ
Ó&S˜Ø3FÐ2GÐ/Ü!&¤s¨;Ó'7Ó!8ò T˜AØ  Ašv¬&°Ñ*<¸QÑ*?À5Ò*Hà 7× >Ñ >¸Õ Oà 7× >Ñ >Ð?RÕ SðTð 7;¸lÑ6J¤E§N¢NÔPU×PZÑPZ˜	Ü!>Ø'Ø&Ø+Ø(Ø,Ø'Ø*Ø3Ø&Ø"Ø%Ø*ó"˜ô —K‘K Ô'Ø—N‘N 6Ö*òSI+òL+òoc	+ðe+ðN €N÷yñ üòD  Ys   Ä/?QÇ"Q#ÑQ c                 óª  ‡‡— g }| r5t         j                  j                  «       st        j	                  d«       |S t        j
                  d«       |D �]2  }t        j                  ||	|¬«      }|j                  |«       t        ||||¬«      }|j                  dv r|d   g}n#t        j                  ||¬«      }|j                  }t        j                  d|› �«       t        j                  d	|j                  «       › �«       |t        j                   k(  r|j#                  «        t        j$                  | rd
nd«      }|j'                  |«       |t        j(                  k(  rt+        j,                  |«      }|D �]  }|dk  rŒ
|D �]÷  }|j                  dv r•t        j/                  d|› d|d|j0                  |j0                  g› �«       t        j2                  |d|j0                  |j0                  f|t        j                   k(  rt         j4                  nt         j6                  |¬«      Šn\�||kD  rŒ¯t        j/                  d|› d||g› �«       t        j8                  d|j:                  dz
  ||ft         j<                  |¬«      Š	 |	r t         j>                  jA                  |‰«      n|
rt        jB                  |«      n|Š ‰‰«       tE        jF                  ˆˆfd„|d¬«      }|	rdn|
rdndt         jH                  d| rdndd|d|d||||jK                  «       tM        tO        jP                  «       «      dœ}|jS                  tU        ||«      «       t        j/                  |«       |jW                  |«       �Œú �Œ	 �Œ5 |S # tX        $ r>}t        j[                  |«       t         j                  j]                  «        Y d }~�ŒEd }~ww xY w)NzYPlease install PyTorch with Cuda, and use a machine with GPU for testing gpu performance.F)Útorchscriptr1   )r„   r1   Úcustom_model_classr@   r   r0   zModel zNumber of parameters zcuda:0r/   zRun PyTorch on rC   r%   )ÚsizeÚdtyper6   r'   )ÚlowÚhighr”   r•   r6   c                  ó   •—  ‰ ‰«      S ©N© ©Ú	inferenceÚ	input_idss   €€r�   ú<lambda>zrun_pytorch.<locals>.<lambda>Œ  ó   ø€ ±Y¸yÓ5I€ ó    ©ÚrepeatÚnumberr’   Útorch2rM   ÚNAr.   r   r3   )/rM   r.   Úis_availablerF   rG   Úset_grad_enabledr   rQ   Úmodifyr   rK   r   Úmodel_max_lengthÚdebugÚnum_parametersr   ÚFLOAT16Úhalfr6   ÚtoÚINT8r   Úquantize_torch_modelr\   r]   ÚrandnÚfloat16Úfloat32Úrandintr~   ÚlongÚjitÚtraceÚcompileÚtimeitr¢   rX   rY   rZ   r   r[   Úupdater   r`   ÚRuntimeErrorÚ	exceptionÚempty_cache)rc   re   rf   rg   r8   r+   rh   ri   rj   r’   r¤   r1   r,   rv   r:   r„   ÚmodelÚ	tokenizerÚmax_input_sizer6   r=   r>   ÚruntimesrŠ   Úerœ   r�   s                            @@r�   Úrun_pytorchrÃ   8  sp  ù€ ð €GÙ”u—z‘z×.Ñ.Ô0Ü�‰ÐpÔqØˆä	×Ñ˜5Ô!à!ó U-ˆ
Ü×+Ñ+¨JÀKÐ[dÔeˆØ×Ñ˜vÔ&Ü%ØØØØ*ô	
ˆð ×Ñ Ñ/à 0°Ñ 3Ð4Ñä%×5Ñ5°jÈIÔVˆIà&×7Ñ7ˆNä�‰�v˜e˜WÐ%Ô&Ü�‰Ð,¨U×-AÑ-AÓ-CÐ,DÐEÔFàœ	×)Ñ)Ò)Ø�J‰JŒLä—‘©'™h°uÓ=ˆØ�‰�Ôàœ	Ÿ™Ò&Ü"×7Ñ7¸Ó>ˆEà%ó 7	-ˆJØ˜QŠØà#3ó 3-�Ø×$Ñ$¨Ñ7Ü—K‘KØ)¨*¨Ð5GÈÐUVÐX^×XiÑXiÐkq×k|Ñk|ÐH}ÐG~Ðôô !&§¡Ø(¨!¨V×->Ñ->À×@QÑ@QÐRØ/8¼I×<MÑ<MÒ/MœeŸmšmÔSX×S`ÑS`Ø%ô!‘Ið &Ð1°oÈÒ6VØ ä—K‘K /°*°Ð=OÐQ[Ð]lÐPmÐOnÐ oÔpÜ %§¡ØØ#×.Ñ.°Ñ2Ø(¨/Ð:Ü#Ÿj™jØ%ô!�Ið-á=HœŸ	™	Ÿ™¨¨yÔ9ÑflÌeÏmÉmÐ\aÔNbÐrwð ñ ˜iÔ(ä%Ÿ}™}Ô-IÐR^ÐghÔi�Hñ 4?¡-ÑPVÁHÐ\cÜ#(×#4Ñ#4Ø%)Ù,3¡&¸Ø%'Ø%.Ø&(Ø&0Ø"#Ø#.Ø&0Ø+:Ø,;×,IÑ,IÓ,KÜ$'¬¯©«Ó$7ñ�Fð  —M‘MÔ"4°X¸zÓ"JÔKÜ—K‘K Ô'Ø—N‘N 6Ö*òa3-ò	7	-ð=U-ðn €Nøô	 $ò -Ü×$Ñ$ QÔ'Ü—J‘J×*Ñ*×,Ò,ûð-ús   ÊC4NÎ	O	Î3O	ÏO	Údo_eager_modeÚuse_xlac                 ó2   ‡ ‡‡‡— ddl mŠ dd lŠˆ ˆˆˆfd„}|S )Nr   )Úwrapsc                 ó–   •‡ —  ‰‰ «      ˆ fd„«       } ‰‰ «      ‰j                  ‰¬«      ˆ fd„«       «       }‰du r‰du sJ d«       ‚|S |S )Nc                  ó   •—  ‰| i |¤ŽS r™   rš   ©ru   ÚkwargsÚfuncs     €r�   Úrun_in_eager_modezFrun_with_tf_optimizations.<locals>.run_func.<locals>.run_in_eager_mode®  s   ø€ á˜Ð( Ñ(Ð(r    )Újit_compilec                  ó   •—  ‰| i |¤ŽS r™   rš   rÊ   s     €r�   Úrun_in_graph_modezFrun_with_tf_optimizations.<locals>.run_func.<locals>.run_in_graph_mode²  s   ø€ ñ ˜Ð( Ñ(Ð(r    TFzcCannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.)Úfunction)rÌ   rÍ   rÐ   rÄ   r)   rÅ   rÇ   s   `  €€€€r�   Úrun_funcz+run_with_tf_optimizations.<locals>.run_func­  st   ù€ Ù	ˆt‹ó	)ó 
ð	)ñ 
ˆt‹Ø	�‰ ˆÓ	)ó	)ó 
*ó 
ð	)ð ˜DÑ Ø˜eÑ#ð ØuóÐ#ð %Ð$à$Ð$r    )Ú	functoolsrÇ   Ú
tensorflow)rÄ   rÅ   rÒ   r)   rÇ   s   `` @@r�   Úrun_with_tf_optimizationsrÕ   ¨  s   û€ Ýã÷%ð$ €Or    c                 ó´  ‡‡‡‡ ‡!— g }dd l Š!‰!j                  j                  j                  |«       | s‰!j                  j	                  g d«       | r1‰!j
                  j                  «       st        j                  d«       |S | r€‰!j                  j                  d«      }	 ‰!j                  j	                  |d   d«       ‰!j                  j                  j                  |d   d«       ‰!j                  j                  d¬«       |t         j"                  k(  s|t         j$                  k(  rt'        d«      ‚|D �]  }t)        j*                  ||	¬«      Š|j-                  ‰«       t/        |‰|	|d¬	«      Š t1        j*                  ||	¬«      }|j2                  }t5        d
d
¬«      ˆ fd„«       }t5        d
d
¬«      ˆ fd„«       }t5        d
d
¬«      ˆˆ ˆ!fd„«       }‰j6                  r|Šnt9        ‰t:        «      r|Šn|Š|D �]Q  }|dk  rŒ
|D �]B  }|�||kD  rŒt        j=                  d|› d||g› �«       t?        j@                  «       }tC        ||z  «      D �cg c]!  }|jE                  d‰jF                  dz
  «      ‘Œ# }}‰!jI                  |||f‰!jJ                  ¬«      Š	  ‰‰«       tM        jN                  ˆˆfd„|d¬«      }d‰!jP                  d| rdndd|d|d||||jS                  «       tU        tW        jX                  «       «      dœ}|j[                  t]        ||«      «       t        j=                  |«       |j_                  |«       �ŒE �ŒT �Œ |S # t        $ r }t        j                  |«       Y d }~�Œwd }~ww xY wc c}w # t        $ rF}t        j                  |«       ddl0m1} |je                  «       }|jg                  «        Y d }~�ŒÉd }~ww xY w)Nr   ÚGPUzVPlease install Tensorflow-gpu, and use a machine with GPU for testing gpu performance.Tz/gpu:0)r6   z+Mixed precision is currently not supported.r0   )r„   r1   r“   Úis_tf_modelF)rÄ   rÅ   c                 ó   •—  ‰| d¬«      S )NF)Útrainingrš   ©r�   r¾   s    €r�   Úencoder_forwardz'run_tensorflow.<locals>.encoder_forwardü  s   ø€ á˜¨UÔ3Ð3r    c                 ó   •—  ‰| | d¬«      S )NF)Údecoder_input_idsrÚ   rš   rÛ   s    €r�   Úencoder_decoder_forwardz/run_tensorflow.<locals>.encoder_decoder_forward   s   ø€ á˜°iÈ%ÔPÐPr    c                 ó¼   •— ‰j                   j                  dd‰j                  g«      }‰j                   j                  dd‰j                  g«      } ‰| ||d¬«      S )Nr'   F)Úvisual_featsÚ
visual_posrÚ   )ÚrandomÚnormalÚvisual_feat_dimÚvisual_pos_dim)r�   ÚfeatsÚposr„   r¾   r)   s      €€€r�   Úlxmert_forwardz&run_tensorflow.<locals>.lxmert_forward  s^   ø€ à—I‘I×$Ñ$ a¨¨F×,BÑ,BÐ%CÓDˆEØ—)‘)×"Ñ" A q¨&×*?Ñ*?Ð#@ÓAˆCÙØØ"ØØô	ð r    zRun Tensorflow on rC   r'   )Úshaper•   c                  ó   •—  ‰ ‰«      S r™   rš   r›   s   €€r�   rž   z run_tensorflow.<locals>.<lambda>'  rŸ   r    r¡   rÔ   r¥   r.   r/   r   r3   )r.   )4rÔ   r„   Ú	threadingÚ set_intra_op_parallelism_threadsÚset_visible_devicesÚtestÚis_built_with_cudarF   rG   Úlist_physical_devicesÚexperimentalÚset_memory_growthÚ
distributeÚOneDeviceStrategyr»   r¼   r   r¬   r¯   ÚNotImplementedErrorr   rQ   r¨   r   r   r©   rÕ   Úis_encoder_decoderÚ
isinstancer   r\   rã   ÚRandomr_   r´   r~   ÚconstantrW   r¹   r¢   rX   rY   rZ   r   r[   rº   r   r`   Únumbar.   Úget_current_deviceÚreset)"rc   re   rf   rg   r8   r+   rh   ri   rj   r1   r,   rv   Úphysical_devicesrÂ   r:   r¿   rÀ   rÜ   rß   ré   r=   r>   Úrngr�   ÚvaluesrÁ   rŠ   r.   r6   r„   rœ   r�   r¾   r)   s"                                @@@@@r�   Úrun_tensorflowr  Â  sŸ  ü€ ð €Gãà‡I�I×Ñ×8Ñ8¸ÔEáØ
�	‰	×%Ñ% b¨%Ô0á�r—w‘w×1Ñ1Ô3Ü�‰ÐmÔnØˆáØŸ9™9×:Ñ:¸5ÓAÐð	 Ø�I‰I×)Ñ)Ð*:¸1Ñ*=¸uÔEØ�I‰I×"Ñ"×4Ñ4Ð5EÀaÑ5HÈ$ÔOØ�M‰M×+Ñ+°8Ð+Ô<ð ”I×%Ñ%Ò%¨´i·n±nÒ)DÜ!Ð"OÓPÐPà!ó Y#ˆ
Ü×+Ñ+¨JÀ)ÔLˆØ×Ñ˜vÔ&ä%ØØØØ*Øô
ˆô "×1Ñ1°*È	ÔRˆ	à"×3Ñ3ˆô 
#°ÀÔ	Fó	4ó 
Gð	4ô 
#°ÀÔ	Fó	Qó 
Gð	Qô 
#°ÀÔ	Fõ	ó 
Gð	ð ×$Ò$Ø/‰IÜ˜¤Ô-Ø&‰Ià'ˆIà%ó +	#ˆJØ˜QŠØà#3ó '#�Ø!Ð-°/ÀNÒ2RØä—‘Ð0°°Ð<NÐPZÐ\kÐOlÐNmÐnÔoä—m‘m“o�ÜINÈzÐ\kÑOkÓIlÖmÀA˜#Ÿ+™+ a¨×):Ñ):¸QÑ)>Õ?Ðm�ÐmØŸK™K¨°zÀ?Ð6SÐ[]×[cÑ[c˜KÓd�	ð#Ù˜iÔ(ä%Ÿ}™}Ô-IÐR^ÐghÔi�Hð #/Ø#%§>¡>Ø%)Ù,3¡&¸Ø%'Ø%.Ø&(Ø&0Ø"#Ø#.Ø&0Ø+:Ø,;×,IÑ,IÓ,KÜ$'¬¯©«Ó$7ñ�Fð  —M‘MÔ"4°X¸zÓ"JÔKÜ—K‘K Ô'Ø—N‘N 6Ö*òC'#ò	+	#ð]Y#ðv €NøôC ò 	 Ü×Ñ˜Q×Òûð	 üò~ nøô6 $ò #Ü×$Ñ$ QÔ'Ý*à!×4Ñ4Ó6�FØ—L‘L—N’Nûð#ús8   Â A$M É&N
ÊB-NÍ	N Í M;Í;N Î	O	Î;O	ÏO	c                  ó8  — t        j                  «       } | j                  ddddt        g d¢t	        t        j                  «       «      ddj                  t        j                  «       «      z   ¬«       | j                  d	dd
t        dddgd¬«       | j                  ddt        d t	        t        «      ddj                  t        «      z   ¬«       | j                  ddddt        dgg d¢d¬«       | j                  dddt        t        j                  j                  dd«      d¬«       | j                  ddt        t        j                  j                  dd«      d¬«       | j                  dd dd!d"¬#«       | j                  d$dt        d d%¬«       | j                  d&d't        t        j                  t	        t        «      d(¬)«       | j                  d*dd!d+¬#«       | j                  d,dd!d-¬#«       | j                  d.d/t        t        j                  t	        t        «      d0¬)«       | j                  d1d2dd!d3¬#«       | j                  d4d5dd d6¬7«       | j                  d8d9dd d:¬7«       | j                  d;d<dd d=¬7«       | j                  d>d?ddd
gt        g d@¢dA¬B«       | j                  dCdDddEt        dF¬G«       | j                  dHdIdt        d
g¬J«       | j                  dKdLdt        g dM¢¬J«       | j                  dNdd!dO¬#«       | j!                  d¬P«       | j                  dQdRddt        dSgdT¬U«       | j                  dVdt        d dW¬«       | j                  dXdd!dY¬#«       | j!                  d¬Z«       t#        j$                  | «       | j'                  «       }|S )[Nz-mz--modelsFú+)zbert-base-casedzroberta-baseÚgpt2z Pre-trained models in the list: z, )ÚrequiredÚnargsÚtypeÚdefaultÚchoicesÚhelpz--model_sourcer'   r&   r)   zExport onnx from pt or tfz--model_classz!Model type selected in the list: )r  r  r  r	  r
  z-ez	--enginesr2   )r2   rM   r¤   r’   rÔ   zEngines to benchmarkz-cz--cache_dirú.Úcache_modelsz%Directory to cache pre-trained models)r  r  r  r
  z
--onnx_dirÚonnx_modelszDirectory to store onnx modelsz-gz	--use_gpuÚ
store_truezRun on gpu device)r  Úactionr
  z
--providerzExecution provider to usez-pz--precisionzfPrecision of model to run. fp32 for full precision, fp16 for half precision, and int8 for quantization)r  r  r	  r
  z	--verbosezPrint more informationz--overwritezOverwrite existing modelsz-oz--optimizer_infozjOptimizer info: Use optimizer.py to optimize onnx model as default. Can also choose from by_ort and no_optz-vz--validate_onnxzValidate ONNX modelz-fz--fusion_csvz:CSV file for saving summary results of graph optimization.)r  r  r
  z-dz--detail_csvz#CSV file for saving detail results.z-rz--result_csvz$CSV file for saving summary results.z-iz--input_counts)r'   r(   r%   zXNumber of ONNX model inputs. Please use 1 for fair comparison with Torch or TorchScript.)r  r  r  r  r	  r
  z-tz--test_timeséd   z8Number of repeat times to get average inference latency.)r  r  r  r
  z-bz--batch_sizes)r  r  r  z-sz--sequence_lengths)é   é   é   é    é@   é€   é   z--disable_ort_io_bindingz=Disable running ONNX Runtime with binded inputs and outputs. )rp   z-nz--num_threadsr   zThreads to use)r  r  r  r  r
  z--force_num_layersz%Manually set the model's layer numberz*--enable_arm64_bfloat16_fastmath_mlas_gemmzHEnable bfloat16 mlas gemm kernels on aarch64. Supported only for CPU EP )rt   )ÚargparseÚArgumentParserÚadd_argumentrZ   Úlistr   ÚkeysÚjoinr   ÚosÚpathr   ÚFLOAT32r   ÚBYSCRIPTÚintÚset_defaultsr   Úadd_argumentsÚ
parse_args)Úparserru   s     r�   Úparse_argumentsr'  F  s0  € Ü×$Ñ$Ó&€Fà
×ÑØØØØÜÚ;Ü”V—[‘[“]Ó#Ø/°$·)±)¼F¿K¹K»MÓ2JÑJð ô 	ð ×ÑØØØÜØØ�t�Ø(ð ô ð ×ÑØØÜØÜ”]Ó#Ø0°4·9±9¼]Ó3KÑKð ô ð ×ÑØØØØÜØ�ÚOØ#ð ô 	ð ×ÑØØØÜÜ—‘—‘˜S .Ó1Ø4ð ô ð ×ÑØØÜÜ—‘—‘˜S -Ó0Ø-ð ô ð ×Ñ˜˜k°EÀ,ÐUhÐÔià
×ÑØØÜØØ(ð ô ð ×ÑØØÜÜ×!Ñ!Ü”Y“Øuð ô ð ×Ñ˜¨e¸LÐOgÐÔhà
×ÑØØØØ(ð	 ô ð ×ÑØØÜÜ×&Ñ&Ü”]Ó#Øyð ô ð ×ÑØØØØØ"ð ô ð ×ÑØØØØØIð ô ð ×ÑØØØØØ2ð ô ð ×ÑØØØØØ3ð ô ð ×ÑØØØØØ�ÜÚØgð ô 	ð ×ÑØØØØÜØGð ô ð ×Ñ˜˜o°S¼sÈQÈCÐÔPà
×ÑØØØÜÚ,ð ô ð ×ÑØ"ØØØLð	 ô ð ×Ñ¨uÐÔ5à
×ÑØØØØÜØ�Øð ô ð ×ÑØØÜØØ4ð ô ð ×ÑØ4ØØØWð	 ô ð ×ÑÀÐÔGä×Ñ Ô'à×ÑÓ€DØ€Kr    c                  ó,  — t        «       } t        | j                  «       | j                  t        j
                  k(  r"| j                  st        j                  d«       y | j                  t        j                  k(  r0| j                  r$| j                  dvrt        j                  d«       y t        | j                  «      dk(  r#t        | j                  d      d   dv rdg| _        t        | j                   D �ch c]  }|dk  rt"        n|’Œ c}«      | _        t        j%                  d	| › �«       t&        j(                  j+                  | j,                  «      s 	 t'        j.                  | j,                  «       d| j2                  v }d| j2                  v }d| j2                  v }d| j2                  v }d| j2                  v }|r`t5        j6                  t8        j:                  «      t5        j6                  d«      k  r't        j                  dt8        j:                  › �«       y t=        | j>                  «      }g }| j                   D �]7  }	t9        j@                  |	«       t        jC                  t8        jD                  jG                  «       «       |s|s|�r‡| jH                  dgk7  rt        jK                  d«       |rt|tM        | j                  | j                  | jN                  || j                  |	| jP                  | j                  | jR                  dd| j,                  | j                  «      z  }|rt|tM        | j                  | j                  | jN                  || j                  |	| jP                  | j                  | jR                  dd| j,                  | j                  «      z  }|rt|tM        | j                  | j                  | jN                  || j                  |	| jP                  | j                  | jR                  dd| j,                  | j                  «      z  }|rr|tU        | j                  | j                  | jN                  || j                  |	| jP                  | j                  | jR                  | j,                  | j                  «      z  }i }
|s�ŒR	 | jV                   }|tY        | j                  | j                  | j                  | jN                  || j                  |	| jP                  | j                  | jR                  | jH                  | jZ                  | j\                  | j,                  | j^                  | j                  | j`                  | jb                  ||
| jd                  | jf                  | «      z  }�Œ: tm        jn                  «       jq                  d«      }
r | jr                  xs d|› d�}tu        |
|«       t        |«      dk(  r&| jP                  dgk7  rt        jK                  d«       y | jv                  xs d|› d�}ty        ||«       | jz                  xs d|› d�}t}        ||| «       y c c}w # t0        $ r$ t        j                  d
| j,                  «       Y �Œòw xY w# th        $ r t        jk                  d«       Y �ŒLw xY w)Nzfp16 is for GPU only)Úmigraphxzint8 is for CPU onlyr'   r   r%   )rA   Úswimr   zArguments: z#Creation of the directory %s failedrM   r¤   r’   r2   rÔ   z2.0.0z2PyTorch version must be >=2.0.0 and you are using zB--input_counts is not implemented for torch or torchscript engine.TFÚ	Exceptionz%Y%m%d-%H%M%SÚbenchmark_fusion_z.csvzNo any result available.Úbenchmark_detail_Úbenchmark_summary_)?r'  r   r,   r8   r   r¬   rc   rF   rG   r¯   rd   rJ   Úmodelsr   ri   Úsortedr+   Ú	cpu_countr\   r  r  Úexistsr1   ÚmkdirÚOSErrorÚenginesr   rL   rM   rX   r   Úforce_num_layersÚset_num_threadsrª   Ú
__config__Úparallel_infork   rI   rÃ   rf   rh   Ú
test_timesr  Úuse_mask_indexr�   rl   rm   rn   ro   rp   rs   rt   r+  r¼   r   r[   ÚstrftimeÚ
fusion_csvr   Ú
detail_csvr   Ú
result_csvr   )ru   ÚxÚenable_torchÚenable_torch2Úenable_torchscriptÚenable_onnxruntimeÚenable_tensorflowrg   rv   r+   rr   rq   Ú
time_stampÚcsv_filenames                 r�   ÚmainrH    sp  € ÜÓ€Dä�—‘Ôà‡~�~œ×*Ñ*Ò*°4·<²<Ü�‰Ð+Ô,Øà‡~�~œŸ™Ò'¨D¯LªL¸T¿]¹]ÐR^Ñ=^Ü�‰Ð+Ô,Øä
ˆ4�;‰;Ó˜1Ò¤¨¯©°A©Ñ!7¸Ñ!:¸oÑ!MØ!# ˆÔäÀ4×CSÑCSÖT¸a¨A°ªF�y¸Ñ9ÒTÓU€DÔä
‡K�K�+˜d˜VÐ$Ô%ä�7‰7�>‰>˜$Ÿ.™.Ô)ð	PÜ�H‰H�T—^‘^Ô$ð ˜dŸl™lÐ*€LØ §¡Ð,€MØ&¨$¯,©,Ð6ÐØ&¨$¯,©,Ð6ÐØ$¨¯©Ð4ÐáœŸ™¤u×'8Ñ'8Ó9¼G¿M¹MÈ'Ó<RÒRÜ�‰ÐIÌ%×J[ÑJ[ÐI\Ð]Ô^Øä$ T×%:Ñ%:Ó;€Oà€Gà×'Ñ'ó g.ˆÜ×Ñ˜kÔ*Ü�‰”U×%Ñ%×3Ñ3Ó5Ô6Ù™=Ò,>Ø× Ñ  Q CÒ'Ü—‘ÐcÔdá!Øœ;Ø—L‘LØ—K‘KØ×$Ñ$Ø#Ø—N‘NØØ×$Ñ$Ø×)Ñ)Ø—O‘OØØØ—N‘NØ—L‘Lóñ �ñ  Øœ;Ø—L‘LØ—K‘KØ×$Ñ$Ø#Ø—N‘NØØ×$Ñ$Ø×)Ñ)Ø—O‘OØØØ—N‘NØ—L‘Lóñ �ñ  Øœ;Ø—L‘LØ—K‘KØ×$Ñ$Ø#Ø—N‘NØØ×$Ñ$Ø×)Ñ)Ø—O‘OØØØ—N‘NØ—L‘Lóñ �ñ  Ø”~Ø—‘Ø—‘Ø× Ñ ØØ—‘ØØ× Ñ Ø×%Ñ%Ø—‘Ø—‘Ø—‘óñ ˆGð #%ÐÛð.Ø-1×-@Ñ-@Ð)@Ð&Øœ?Ø—L‘LØ—M‘MØ—K‘KØ×$Ñ$Ø#Ø—N‘NØØ×$Ñ$Ø×)Ñ)Ø—O‘OØ×%Ñ%Ø×'Ñ'Ø×&Ñ&Ø—N‘NØ—M‘MØ—L‘LØ—N‘NØ×/Ñ/Ø*Ø+Ø×%Ñ%Ø×AÑAØó/ñ ’ð[g.ôR —‘“×(Ñ(¨Ó9€JÙØ—‘ÒNÐ,=¸j¸\ÈÐ*NˆÜ Ð!8¸,ÔGä
ˆ7ƒ|�qÒØ×Ñ ˜sÒ"Ü�N‰NÐ5Ô6Øà—?‘?ÒJÐ(9¸*¸ÀTÐ&J€LÜ�7˜LÔ)à—?‘?ÒKÐ(:¸:¸,ÀdÐ&K€LÜ�7˜L¨$Õ/ùò_ Uøô ò 	PÜ�L‰LÐ>ÀÇÁ×Oð	Pûôn ò .Ü× Ñ  ×-ð.ús+   Ã:X<ÅY ÒC%Y1Ù)Y.Ù-Y.Ù1ZÚZÚ__main__)5Ú__doc__r  Úloggingr  rã   r¹   r   rR   ÚpsutilÚbenchmark_helperr   r   r   r   r   r	   r
   r   r   r   r   r{   r   Úhuggingface_modelsr   r   Úonnx_exporterr   r   r   r   Ú	packagingr   Úquantize_helperr   Ú	getLoggerrF   r1  ÚenvironrZ   rM   Útransformersr   r   r   r�   rÃ   ÚboolrÕ   r  r'  rH  Ú__name__rš   r    r�   ú<module>rW     sê   ðñ ó6 Û Û 	Û Û Ý ã Û ÷÷ ÷ ñ õ )ß 4÷ó õ Ý *à	ˆ×	Ñ	˜2Ó	€àˆF×Ñ UÔ+€	ð ˜BŸJ™JÑ&Ù$'¨	£N€B‡J�JÐ Ñ!ã ß @Ñ @ò]ò@mð`¨Tð ¸Dó ò4AòHEòP_0ðD ˆzÒÙ…Fð r    