mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Add Linux bash and Windows batch scripts for running transformers benchmarks (#3997)
This commit is contained in:
Tianlei Wu
GitHub
parent
212efb6cde
commit
abcd1576c9
2 changed files with 303 additions and 0 deletions
150
onnxruntime/python/tools/transformers/dev_benchmark.cmd
Normal file
150
onnxruntime/python/tools/transformers/dev_benchmark.cmd
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
@echo off
|
||||
|
||||
REM Run benchmark in Windows for developing purpose. For official benchmark, please use run_benchmark.sh.
|
||||
REM Settings are different from run_benchmark.sh: no cli, batch and sequence, input counts, average over 100, no fp16, less models etc.
|
||||
|
||||
REM Please install PyTorch (see https://pytorch.org/) before running this benchmark. Like the following:
|
||||
REM GPU: conda install pytorch torchvision cudatoolkit=10.1 -c pytorch
|
||||
REM CPU: conda install pytorch torchvision cpuonly -c pytorch
|
||||
|
||||
REM When run_cli=true, this script is self-contained and you need not copy other files to run benchmarks
|
||||
REM it will use onnxruntime-tools package.
|
||||
REM If run_cli=false, it depends on other python script (*.py) files in this directory.
|
||||
set run_cli=false
|
||||
|
||||
REM only need once
|
||||
set run_install=false
|
||||
|
||||
REM Engines to test
|
||||
set run_ort=true
|
||||
set run_torch=false
|
||||
set run_torchscript=false
|
||||
|
||||
REM Devices to test.
|
||||
REM Attention: You cannot run both CPU and GPU at the same time: gpu need onnxruntime-gpu, and CPU need onnxruntime.
|
||||
set run_gpu_fp32=true
|
||||
set run_gpu_fp16=false
|
||||
set run_cpu=false
|
||||
|
||||
set average_over=100
|
||||
|
||||
REM Enable optimizer (use script instead of OnnxRuntime for graph optimization)
|
||||
set use_optimizer=true
|
||||
|
||||
set batch_sizes=1 16
|
||||
set sequence_length=8 128
|
||||
|
||||
REM Number of inputs (input_ids, token_type_ids, attention_mask) for ONNX model.
|
||||
REM Note that different input count might lead to different performance
|
||||
set input_counts=1
|
||||
|
||||
REM Pretrained transformers models can be a subset of: bert-base-cased roberta-base gpt2 distilgpt2 distilbert-base-uncased
|
||||
set models_to_test=bert-base-cased gpt2
|
||||
|
||||
REM If you have mutliple GPUs, you can choose one GPU for test. Here is an example to use the second GPU:
|
||||
REM set CUDA_VISIBLE_DEVICES=1
|
||||
|
||||
REM This script will generate a logs file with a list of commands used in tests.
|
||||
>benchmark.log echo echo ort=%run_ort% torch=%run_torch% torchscript=%run_torchscript% gpu_fp32=%run_gpu_fp32% gpu_fp16=%run_gpu_fp16% cpu=%run_cpu% optimizer=%use_optimizer% batch="%batch_sizes%" sequence="%sequence_length%" models="%models_to_test%" input_counts="%input_counts%"
|
||||
|
||||
REM Set it to false to skip testing. You can use it to dry run this script with the benchmark.log file.
|
||||
set run_tests=false
|
||||
|
||||
REM -------------------------------------------
|
||||
if %run_cpu% == true if %run_gpu_fp32% == true echo cannot test cpu and gpu at same time & goto :EOF
|
||||
if %run_cpu% == true if %run_gpu_fp16% == true echo cannot test cpu and gpu at same time & goto :EOF
|
||||
|
||||
if %run_install% == true (
|
||||
pip uninstall --yes ort_nightly
|
||||
pip uninstall --yes onnxruntime
|
||||
pip uninstall --yes onnxruntime-gpu
|
||||
if %run_cpu% == true (
|
||||
pip install onnxruntime
|
||||
) else (
|
||||
pip install --upgrade onnxruntime-gpu
|
||||
)
|
||||
|
||||
pip install --upgrade onnxruntime-tools
|
||||
pip install --upgrade git+https://github.com/huggingface/transformers
|
||||
)
|
||||
|
||||
if %run_cli% == true (
|
||||
echo Use onnxruntime_tools.transformers.benchmark
|
||||
set optimizer_script=-m onnxruntime_tools.transformers.benchmark
|
||||
) else (
|
||||
set optimizer_script=benchmark.py
|
||||
)
|
||||
|
||||
REM remove --overwrite can save some time if you did not update any of these: transformers, PyTorch or fusion logic in optimizer.
|
||||
set onnx_export_options=-i %input_counts% -v -b 0 -f fusion.csv --overwrite
|
||||
set benchmark_options=-b %batch_sizes% -s %sequence_length% -t %average_over% -f fusion.csv -r result.csv -d detail.csv
|
||||
|
||||
if %use_optimizer% == true (
|
||||
set onnx_export_options=%onnx_export_options% -o
|
||||
set benchmark_options=%benchmark_options% -o
|
||||
)
|
||||
|
||||
if %run_gpu_fp32% == true (
|
||||
for %%m in (%models_to_test%) DO (
|
||||
echo Run GPU FP32 Benchmark on model %%m
|
||||
call :RunOneTest %%m -g
|
||||
)
|
||||
)
|
||||
|
||||
if %run_gpu_fp16% == true (
|
||||
for %%m in (%models_to_test%) DO (
|
||||
echo Run GPU FP16 Benchmark on model %%m
|
||||
call :RunOneTest %%m -g --fp16
|
||||
)
|
||||
)
|
||||
|
||||
if %run_cpu% == true (
|
||||
for %%m in (%models_to_test%) DO (
|
||||
echo Run CPU Benchmark on model %%m
|
||||
call :RunOneTest %%m
|
||||
)
|
||||
)
|
||||
|
||||
if %run_tests%==false more benchmark.log
|
||||
|
||||
call :RemoveDuplicateLines result.csv
|
||||
call :RemoveDuplicateLines fusion.csv
|
||||
call :RemoveDuplicateLines detail.csv
|
||||
|
||||
echo Done!
|
||||
|
||||
goto :EOF
|
||||
|
||||
REM -----------------------------
|
||||
:RunOneTest
|
||||
|
||||
if %run_ort% == true (
|
||||
>>benchmark.log echo python %optimizer_script% -m %1 %onnx_export_options% %2 %3
|
||||
>>benchmark.log echo python %optimizer_script% -m %1 %benchmark_options% %2 %3 -i %input_counts%
|
||||
if %run_tests%==true (
|
||||
python %optimizer_script% -m %1 %onnx_export_options% %2 %3
|
||||
python %optimizer_script% -m %1 %benchmark_options% %2 %3 -i %input_counts%
|
||||
)
|
||||
)
|
||||
|
||||
if %run_torch% == true (
|
||||
>>benchmark.log echo python %optimizer_script% -e torch -m %1 %benchmark_options% %2 %3
|
||||
if %run_tests%==true python %optimizer_script% -e torch -m %1 %benchmark_options% %2 %3
|
||||
)
|
||||
|
||||
if %run_torchscript% == true (
|
||||
>>benchmark.log echo python %optimizer_script% -e torchscript -m %1 %benchmark_options% %2 %3
|
||||
if %run_tests%==true python %optimizer_script% -e torchscript -m %1 %benchmark_options% %2 %3
|
||||
)
|
||||
|
||||
goto :EOF
|
||||
|
||||
|
||||
REM -----------------------------
|
||||
:RemoveDuplicateLines
|
||||
SET FileSize=%~z1
|
||||
IF %FileSize% LSS 10 goto :EOF
|
||||
python -c "import sys; lines=sys.stdin.readlines(); h=lines[0]; print(''.join([h]+list(sorted(set(lines)-set([h])))))" < %1 > sort_%1
|
||||
FindStr "[^,]" sort_%1 > summary_%1
|
||||
DEL sort_%1
|
||||
goto :EOF
|
||||
153
onnxruntime/python/tools/transformers/run_benchmark.sh
Normal file
153
onnxruntime/python/tools/transformers/run_benchmark.sh
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
# This measures the performance of OnnxRuntime, PyTorch and TorchScript on transformer models.
|
||||
# Please install PyTorch (see https://pytorch.org/) before running this benchmark. Like the following:
|
||||
# GPU: conda install pytorch torchvision cudatoolkit=10.1 -c pytorch
|
||||
# CPU: conda install pytorch torchvision cpuonly -c pytorch
|
||||
|
||||
# When run_cli=true, this script is self-contained and you need not copy other files to run benchmarks
|
||||
# it will use onnxruntime-tools package.
|
||||
# If run_cli=false, it depends on other python script (*.py) files in this directory.
|
||||
run_cli=true
|
||||
|
||||
# only need once
|
||||
run_install=true
|
||||
|
||||
# Engines to test.
|
||||
run_ort=true
|
||||
run_torch=false
|
||||
run_torchscript=true
|
||||
|
||||
# Devices to test (You can run either CPU or GPU, but not both: gpu need onnxruntime-gpu, and CPU need onnxruntime).
|
||||
run_gpu_fp32=true
|
||||
run_gpu_fp16=true
|
||||
run_cpu=false
|
||||
|
||||
average_over=1000
|
||||
# CPU takes longer time to run, only run 100 inferences to get average latency.
|
||||
if [ "$run_cpu" = true ] ; then
|
||||
average_over=100
|
||||
fi
|
||||
|
||||
# enable optimizer (use script instead of OnnxRuntime for graph optimization)
|
||||
use_optimizer=true
|
||||
|
||||
# Batch Sizes and Sequence Lengths
|
||||
batch_sizes="1 4"
|
||||
sequence_lengths="8 16 32 64 128 256 512 1024"
|
||||
|
||||
# Number of inputs (input_ids, token_type_ids, attention_mask) for ONNX model.
|
||||
# Not that different input count might lead to different performance
|
||||
# Here we only test one input (input_ids) for fair comparison with PyTorch.
|
||||
input_counts=1
|
||||
|
||||
# Pretrained transformers models can be a subset of: bert-base-cased roberta-base gpt2 distilgpt2 distilbert-base-uncased
|
||||
models_to_test="bert-base-cased roberta-base gpt2"
|
||||
|
||||
# If you have mutliple GPUs, you can choose one GPU for test. Here is an example to use the second GPU:
|
||||
# export CUDA_VISIBLE_DEVICES=1
|
||||
|
||||
#This script will generate a logs file with a list of commands used in tests.
|
||||
echo echo "ort=$run_ort torch=$run_torch torchscript=$run_torchscript gpu_fp32=$run_gpu_fp32 gpu_fp16=$run_gpu_fp16 cpu=$run_cpu optimizer=$use_optimizer batch=$batch_sizes sequence=$sequence_length models=$models_to_test" >> benchmark.log
|
||||
|
||||
#Set it to false to skip testing. You can use it to dry run this script with the log file.
|
||||
run_tests=true
|
||||
|
||||
# -------------------------------------------
|
||||
if [ "$run_cpu" = true ] ; then
|
||||
if [ "$run_gpu_fp32" = true ] ; then
|
||||
echo "cannot test cpu and gpu at same time"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$run_gpu_fp16" = true ] ; then
|
||||
echo "cannot test cpu and gpu at same time"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
if [ "$run_install" = true ] ; then
|
||||
pip uninstall --yes ort_nightly
|
||||
pip uninstall --yes onnxruntime
|
||||
pip uninstall --yes onnxruntime-gpu
|
||||
if [ "$run_cpu" = true ] ; then
|
||||
pip install onnxruntime
|
||||
else
|
||||
pip install onnxruntime-gpu
|
||||
fi
|
||||
pip install --upgrade onnxruntime-tools
|
||||
pip install --upgrade git+https://github.com/huggingface/transformers
|
||||
fi
|
||||
|
||||
if [ "$run_cli" = true ] ; then
|
||||
echo "Use onnxruntime_tools.transformers.benchmark"
|
||||
optimizer_script="-m onnxruntime_tools.transformers.benchmark"
|
||||
else
|
||||
optimizer_script="benchmark.py"
|
||||
fi
|
||||
|
||||
onnx_export_options="-i $input_counts -v -b 0 --overwrite -f fusion.csv"
|
||||
benchmark_options="-b $batch_sizes -s $sequence_lengths -t $average_over -f fusion.csv -r result.csv -d detail.csv"
|
||||
|
||||
if [ "$use_optimizer" = true ] ; then
|
||||
onnx_export_options="$onnx_export_options -o"
|
||||
benchmark_options="$benchmark_options -o"
|
||||
fi
|
||||
|
||||
# -------------------------------------------
|
||||
run_one_test() {
|
||||
if [ "$run_ort" = true ] ; then
|
||||
echo python $optimizer_script -m $1 $onnx_export_options $2 $3 >> benchmark.log
|
||||
echo python $optimizer_script -m $1 $benchmark_options $2 $3 -i $input_counts >> benchmark.log
|
||||
if [ "$run_tests" = true ] ; then
|
||||
python $optimizer_script -m $1 $onnx_export_options $2 $3
|
||||
python $optimizer_script -m $1 $benchmark_options $2 $3 -i $input_counts
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$run_torch" = true ] ; then
|
||||
echo python $optimizer_script -e torch -m $1 $benchmark_options $2 $3 >> benchmark.log
|
||||
if [ "$run_tests" = true ] ; then
|
||||
python $optimizer_script -e torch -m $1 $benchmark_options $2 $3
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$run_torchscript" = true ] ; then
|
||||
echo python $optimizer_script -e torchscript -m $1 $benchmark_options $2 $3 >> benchmark.log
|
||||
if [ "$run_tests" = true ] ; then
|
||||
python $optimizer_script -e torchscript -m $1 $benchmark_options $2 $3
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# -------------------------------------------
|
||||
if [ "$run_gpu_fp32" = true ] ; then
|
||||
for m in $models_to_test
|
||||
do
|
||||
echo Run GPU FP32 Benchmark on model ${m}
|
||||
run_one_test "${m}" -g
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$run_gpu_fp16" = true ] ; then
|
||||
for m in $models_to_test
|
||||
do
|
||||
echo Run GPU FP16 Benchmark on model ${m}
|
||||
run_one_test "${m}" -g --fp16
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$run_cpu" = true ] ; then
|
||||
for m in $models_to_test
|
||||
do
|
||||
echo Run CPU Benchmark on model ${m}
|
||||
run_one_test "${m}"
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "run_tests" = false ] ; then
|
||||
more $log_file
|
||||
fi
|
||||
|
||||
# Remove duplicated lines
|
||||
awk '!x[$0]++' ./result.csv > summary_result.csv
|
||||
awk '!x[$0]++' ./fusion.csv > summary_fusion.csv
|
||||
awk '!x[$0]++' ./detail.csv > summary_detail.csv
|
||||
Loading…
Reference in a new issue