From abcd1576c900889917939abf87bd7ad8b08ea0a9 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Tue, 26 May 2020 16:42:12 -0700 Subject: [PATCH] Add Linux bash and Windows batch scripts for running transformers benchmarks (#3997) --- .../tools/transformers/dev_benchmark.cmd | 150 +++++++++++++++++ .../tools/transformers/run_benchmark.sh | 153 ++++++++++++++++++ 2 files changed, 303 insertions(+) create mode 100644 onnxruntime/python/tools/transformers/dev_benchmark.cmd create mode 100644 onnxruntime/python/tools/transformers/run_benchmark.sh diff --git a/onnxruntime/python/tools/transformers/dev_benchmark.cmd b/onnxruntime/python/tools/transformers/dev_benchmark.cmd new file mode 100644 index 0000000000..7ef3f7dd00 --- /dev/null +++ b/onnxruntime/python/tools/transformers/dev_benchmark.cmd @@ -0,0 +1,150 @@ +@echo off + +REM Run benchmark in Windows for developing purpose. For official benchmark, please use run_benchmark.sh. +REM Settings are different from run_benchmark.sh: no cli, batch and sequence, input counts, average over 100, no fp16, less models etc. + +REM Please install PyTorch (see https://pytorch.org/) before running this benchmark. Like the following: +REM GPU: conda install pytorch torchvision cudatoolkit=10.1 -c pytorch +REM CPU: conda install pytorch torchvision cpuonly -c pytorch + +REM When run_cli=true, this script is self-contained and you need not copy other files to run benchmarks +REM it will use onnxruntime-tools package. +REM If run_cli=false, it depends on other python script (*.py) files in this directory. +set run_cli=false + +REM only need once +set run_install=false + +REM Engines to test +set run_ort=true +set run_torch=false +set run_torchscript=false + +REM Devices to test. +REM Attention: You cannot run both CPU and GPU at the same time: gpu need onnxruntime-gpu, and CPU need onnxruntime. +set run_gpu_fp32=true +set run_gpu_fp16=false +set run_cpu=false + +set average_over=100 + +REM Enable optimizer (use script instead of OnnxRuntime for graph optimization) +set use_optimizer=true + +set batch_sizes=1 16 +set sequence_length=8 128 + +REM Number of inputs (input_ids, token_type_ids, attention_mask) for ONNX model. +REM Note that different input count might lead to different performance +set input_counts=1 + +REM Pretrained transformers models can be a subset of: bert-base-cased roberta-base gpt2 distilgpt2 distilbert-base-uncased +set models_to_test=bert-base-cased gpt2 + +REM If you have mutliple GPUs, you can choose one GPU for test. Here is an example to use the second GPU: +REM set CUDA_VISIBLE_DEVICES=1 + +REM This script will generate a logs file with a list of commands used in tests. +>benchmark.log echo echo ort=%run_ort% torch=%run_torch% torchscript=%run_torchscript% gpu_fp32=%run_gpu_fp32% gpu_fp16=%run_gpu_fp16% cpu=%run_cpu% optimizer=%use_optimizer% batch="%batch_sizes%" sequence="%sequence_length%" models="%models_to_test%" input_counts="%input_counts%" + +REM Set it to false to skip testing. You can use it to dry run this script with the benchmark.log file. +set run_tests=false + +REM ------------------------------------------- +if %run_cpu% == true if %run_gpu_fp32% == true echo cannot test cpu and gpu at same time & goto :EOF +if %run_cpu% == true if %run_gpu_fp16% == true echo cannot test cpu and gpu at same time & goto :EOF + +if %run_install% == true ( + pip uninstall --yes ort_nightly + pip uninstall --yes onnxruntime + pip uninstall --yes onnxruntime-gpu + if %run_cpu% == true ( + pip install onnxruntime + ) else ( + pip install --upgrade onnxruntime-gpu + ) + + pip install --upgrade onnxruntime-tools + pip install --upgrade git+https://github.com/huggingface/transformers +) + +if %run_cli% == true ( + echo Use onnxruntime_tools.transformers.benchmark + set optimizer_script=-m onnxruntime_tools.transformers.benchmark +) else ( + set optimizer_script=benchmark.py +) + +REM remove --overwrite can save some time if you did not update any of these: transformers, PyTorch or fusion logic in optimizer. +set onnx_export_options=-i %input_counts% -v -b 0 -f fusion.csv --overwrite +set benchmark_options=-b %batch_sizes% -s %sequence_length% -t %average_over% -f fusion.csv -r result.csv -d detail.csv + +if %use_optimizer% == true ( + set onnx_export_options=%onnx_export_options% -o + set benchmark_options=%benchmark_options% -o +) + +if %run_gpu_fp32% == true ( + for %%m in (%models_to_test%) DO ( + echo Run GPU FP32 Benchmark on model %%m + call :RunOneTest %%m -g + ) +) + +if %run_gpu_fp16% == true ( + for %%m in (%models_to_test%) DO ( + echo Run GPU FP16 Benchmark on model %%m + call :RunOneTest %%m -g --fp16 + ) +) + +if %run_cpu% == true ( + for %%m in (%models_to_test%) DO ( + echo Run CPU Benchmark on model %%m + call :RunOneTest %%m + ) +) + +if %run_tests%==false more benchmark.log + +call :RemoveDuplicateLines result.csv +call :RemoveDuplicateLines fusion.csv +call :RemoveDuplicateLines detail.csv + +echo Done! + +goto :EOF + +REM ----------------------------- +:RunOneTest + +if %run_ort% == true ( + >>benchmark.log echo python %optimizer_script% -m %1 %onnx_export_options% %2 %3 + >>benchmark.log echo python %optimizer_script% -m %1 %benchmark_options% %2 %3 -i %input_counts% + if %run_tests%==true ( + python %optimizer_script% -m %1 %onnx_export_options% %2 %3 + python %optimizer_script% -m %1 %benchmark_options% %2 %3 -i %input_counts% + ) +) + +if %run_torch% == true ( + >>benchmark.log echo python %optimizer_script% -e torch -m %1 %benchmark_options% %2 %3 + if %run_tests%==true python %optimizer_script% -e torch -m %1 %benchmark_options% %2 %3 +) + +if %run_torchscript% == true ( + >>benchmark.log echo python %optimizer_script% -e torchscript -m %1 %benchmark_options% %2 %3 + if %run_tests%==true python %optimizer_script% -e torchscript -m %1 %benchmark_options% %2 %3 +) + +goto :EOF + + +REM ----------------------------- +:RemoveDuplicateLines +SET FileSize=%~z1 +IF %FileSize% LSS 10 goto :EOF +python -c "import sys; lines=sys.stdin.readlines(); h=lines[0]; print(''.join([h]+list(sorted(set(lines)-set([h])))))" < %1 > sort_%1 +FindStr "[^,]" sort_%1 > summary_%1 +DEL sort_%1 +goto :EOF \ No newline at end of file diff --git a/onnxruntime/python/tools/transformers/run_benchmark.sh b/onnxruntime/python/tools/transformers/run_benchmark.sh new file mode 100644 index 0000000000..7d0557dbb5 --- /dev/null +++ b/onnxruntime/python/tools/transformers/run_benchmark.sh @@ -0,0 +1,153 @@ +# This measures the performance of OnnxRuntime, PyTorch and TorchScript on transformer models. +# Please install PyTorch (see https://pytorch.org/) before running this benchmark. Like the following: +# GPU: conda install pytorch torchvision cudatoolkit=10.1 -c pytorch +# CPU: conda install pytorch torchvision cpuonly -c pytorch + +# When run_cli=true, this script is self-contained and you need not copy other files to run benchmarks +# it will use onnxruntime-tools package. +# If run_cli=false, it depends on other python script (*.py) files in this directory. +run_cli=true + +# only need once +run_install=true + +# Engines to test. +run_ort=true +run_torch=false +run_torchscript=true + +# Devices to test (You can run either CPU or GPU, but not both: gpu need onnxruntime-gpu, and CPU need onnxruntime). +run_gpu_fp32=true +run_gpu_fp16=true +run_cpu=false + +average_over=1000 +# CPU takes longer time to run, only run 100 inferences to get average latency. +if [ "$run_cpu" = true ] ; then + average_over=100 +fi + +# enable optimizer (use script instead of OnnxRuntime for graph optimization) +use_optimizer=true + +# Batch Sizes and Sequence Lengths +batch_sizes="1 4" +sequence_lengths="8 16 32 64 128 256 512 1024" + +# Number of inputs (input_ids, token_type_ids, attention_mask) for ONNX model. +# Not that different input count might lead to different performance +# Here we only test one input (input_ids) for fair comparison with PyTorch. +input_counts=1 + +# Pretrained transformers models can be a subset of: bert-base-cased roberta-base gpt2 distilgpt2 distilbert-base-uncased +models_to_test="bert-base-cased roberta-base gpt2" + +# If you have mutliple GPUs, you can choose one GPU for test. Here is an example to use the second GPU: +# export CUDA_VISIBLE_DEVICES=1 + +#This script will generate a logs file with a list of commands used in tests. +echo echo "ort=$run_ort torch=$run_torch torchscript=$run_torchscript gpu_fp32=$run_gpu_fp32 gpu_fp16=$run_gpu_fp16 cpu=$run_cpu optimizer=$use_optimizer batch=$batch_sizes sequence=$sequence_length models=$models_to_test" >> benchmark.log + +#Set it to false to skip testing. You can use it to dry run this script with the log file. +run_tests=true + +# ------------------------------------------- +if [ "$run_cpu" = true ] ; then + if [ "$run_gpu_fp32" = true ] ; then + echo "cannot test cpu and gpu at same time" + exit 1 + fi + if [ "$run_gpu_fp16" = true ] ; then + echo "cannot test cpu and gpu at same time" + exit 1 + fi +fi + + +if [ "$run_install" = true ] ; then + pip uninstall --yes ort_nightly + pip uninstall --yes onnxruntime + pip uninstall --yes onnxruntime-gpu + if [ "$run_cpu" = true ] ; then + pip install onnxruntime + else + pip install onnxruntime-gpu + fi + pip install --upgrade onnxruntime-tools + pip install --upgrade git+https://github.com/huggingface/transformers +fi + +if [ "$run_cli" = true ] ; then + echo "Use onnxruntime_tools.transformers.benchmark" + optimizer_script="-m onnxruntime_tools.transformers.benchmark" +else + optimizer_script="benchmark.py" +fi + +onnx_export_options="-i $input_counts -v -b 0 --overwrite -f fusion.csv" +benchmark_options="-b $batch_sizes -s $sequence_lengths -t $average_over -f fusion.csv -r result.csv -d detail.csv" + +if [ "$use_optimizer" = true ] ; then + onnx_export_options="$onnx_export_options -o" + benchmark_options="$benchmark_options -o" +fi + +# ------------------------------------------- +run_one_test() { + if [ "$run_ort" = true ] ; then + echo python $optimizer_script -m $1 $onnx_export_options $2 $3 >> benchmark.log + echo python $optimizer_script -m $1 $benchmark_options $2 $3 -i $input_counts >> benchmark.log + if [ "$run_tests" = true ] ; then + python $optimizer_script -m $1 $onnx_export_options $2 $3 + python $optimizer_script -m $1 $benchmark_options $2 $3 -i $input_counts + fi + fi + + if [ "$run_torch" = true ] ; then + echo python $optimizer_script -e torch -m $1 $benchmark_options $2 $3 >> benchmark.log + if [ "$run_tests" = true ] ; then + python $optimizer_script -e torch -m $1 $benchmark_options $2 $3 + fi + fi + + if [ "$run_torchscript" = true ] ; then + echo python $optimizer_script -e torchscript -m $1 $benchmark_options $2 $3 >> benchmark.log + if [ "$run_tests" = true ] ; then + python $optimizer_script -e torchscript -m $1 $benchmark_options $2 $3 + fi + fi +} + +# ------------------------------------------- +if [ "$run_gpu_fp32" = true ] ; then + for m in $models_to_test + do + echo Run GPU FP32 Benchmark on model ${m} + run_one_test "${m}" -g + done +fi + +if [ "$run_gpu_fp16" = true ] ; then + for m in $models_to_test + do + echo Run GPU FP16 Benchmark on model ${m} + run_one_test "${m}" -g --fp16 + done +fi + +if [ "$run_cpu" = true ] ; then + for m in $models_to_test + do + echo Run CPU Benchmark on model ${m} + run_one_test "${m}" + done +fi + +if [ "run_tests" = false ] ; then + more $log_file +fi + +# Remove duplicated lines +awk '!x[$0]++' ./result.csv > summary_result.csv +awk '!x[$0]++' ./fusion.csv > summary_fusion.csv +awk '!x[$0]++' ./detail.csv > summary_detail.csv