mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Improve perf testing (#5760)
* build off a specific commit and archive wheel file * rename to fp32, prefix results w/ commit, add CPU col * rename 99th to 90 percentile * get symbolic_shape from master each time * add install archive wheel, parallel build * shortening hash
This commit is contained in:
Olivia Jain
GitHub
parent
f0142da59c
commit
3738ca7e10
5 changed files with 107 additions and 48 deletions
|
|
@ -884,9 +884,9 @@ def run_onnxruntime(args, models):
|
|||
ep_list.append(args.ep)
|
||||
else:
|
||||
if args.fp16:
|
||||
ep_list = ["CUDAExecutionProvider", "TensorrtExecutionProvider", "CUDAExecutionProvider_fp16", "TensorrtExecutionProvider_fp16"]
|
||||
ep_list = ["CPUExecutionProvider", "CUDAExecutionProvider", "TensorrtExecutionProvider", "CUDAExecutionProvider_fp16", "TensorrtExecutionProvider_fp16"]
|
||||
else:
|
||||
ep_list = ["CUDAExecutionProvider", "TensorrtExecutionProvider"]
|
||||
ep_list = ["CPUExecutionProvider", "CUDAExecutionProvider", "TensorrtExecutionProvider"]
|
||||
|
||||
validation_exemption = ["TensorrtExecutionProvider_fp16"]
|
||||
|
||||
|
|
@ -1005,7 +1005,6 @@ def run_onnxruntime(args, models):
|
|||
for output_meta in sess.get_outputs():
|
||||
logger.info(output_meta)
|
||||
|
||||
|
||||
batch_size = 1
|
||||
result_template = {
|
||||
"engine": "onnxruntime",
|
||||
|
|
@ -1186,12 +1185,14 @@ def output_latency(results, csv_filename):
|
|||
|
||||
with open(csv_filename, mode="a", newline='') as csv_file:
|
||||
column_names = ["Model",
|
||||
"CUDA \nmean (ms)",
|
||||
"CUDA \n90th percentile (ms)",
|
||||
"TRT EP \nmean (ms)",
|
||||
"TRT EP \n90th percentile (ms)",
|
||||
"Standalone TRT \nmean (ms)",
|
||||
"Standalone TRT \n90th percentile (ms)",
|
||||
"CPU \nmean (ms)",
|
||||
"CPU \n 90th percentile (ms)",
|
||||
"CUDA fp32 \nmean (ms)",
|
||||
"CUDA fp32 \n90th percentile (ms)",
|
||||
"TRT EP fp32 \nmean (ms)",
|
||||
"TRT EP fp32 \n90th percentile (ms)",
|
||||
"Standalone TRT fp32 \nmean (ms)",
|
||||
"Standalone TRT fp32 \n90th percentile (ms)",
|
||||
"CUDA fp16 \nmean (ms)",
|
||||
"CUDA fp16 \n90th percentile (ms)",
|
||||
"TRT EP fp16 \nmean (ms)",
|
||||
|
|
@ -1206,69 +1207,77 @@ def output_latency(results, csv_filename):
|
|||
csv_writer.writerow(column_names)
|
||||
|
||||
for key, value in results.items():
|
||||
cpu_average = ""
|
||||
if "CPUExecutionProvider" in value and "average_latency_ms" in value["CPUExecutionProvider"]:
|
||||
cpu_average = value["CPUExecutionProvider"]["average_latency_ms"]
|
||||
|
||||
cpu_90_percentile = ""
|
||||
if "CPUExecutionProvider" in value and "latency_90_percentile" in value["CPUExecutionProvider"]:
|
||||
cpu_90_percentile = value["CPUExecutionProvider"]["latency_90_percentile"]
|
||||
|
||||
cuda_average = ""
|
||||
if 'CUDAExecutionProvider' in value and 'average_latency_ms' in value['CUDAExecutionProvider']:
|
||||
cuda_average = value['CUDAExecutionProvider']['average_latency_ms']
|
||||
|
||||
cuda_99_percentile = ""
|
||||
cuda_90_percentile = ""
|
||||
if 'CUDAExecutionProvider' in value and 'latency_90_percentile' in value['CUDAExecutionProvider']:
|
||||
cuda_99_percentile = value['CUDAExecutionProvider']['latency_90_percentile']
|
||||
cuda_90_percentile = value['CUDAExecutionProvider']['latency_90_percentile']
|
||||
|
||||
trt_average = ""
|
||||
if 'TensorrtExecutionProvider' in value and 'average_latency_ms' in value['TensorrtExecutionProvider']:
|
||||
trt_average = value['TensorrtExecutionProvider']['average_latency_ms']
|
||||
|
||||
trt_99_percentile = ""
|
||||
trt_90_percentile = ""
|
||||
if 'TensorrtExecutionProvider' in value and 'latency_90_percentile' in value['TensorrtExecutionProvider']:
|
||||
trt_99_percentile = value['TensorrtExecutionProvider']['latency_90_percentile']
|
||||
trt_90_percentile = value['TensorrtExecutionProvider']['latency_90_percentile']
|
||||
|
||||
standalone_trt_average = ""
|
||||
if 'Standalone_TRT' in value and 'average_latency_ms' in value['Standalone_TRT']:
|
||||
standalone_trt_average = value['Standalone_TRT']['average_latency_ms']
|
||||
|
||||
standalone_trt_99_percentile = ""
|
||||
standalone_trt_90_percentile = ""
|
||||
if 'Standalone_TRT' in value and 'latency_90_percentile' in value['Standalone_TRT']:
|
||||
standalone_trt_99_percentile = value['Standalone_TRT']['latency_90_percentile']
|
||||
standalone_trt_90_percentile = value['Standalone_TRT']['latency_90_percentile']
|
||||
|
||||
|
||||
cuda_fp16_average = ""
|
||||
if 'CUDAExecutionProvider_fp16' in value and 'average_latency_ms' in value['CUDAExecutionProvider_fp16']:
|
||||
cuda_fp16_average = value['CUDAExecutionProvider_fp16']['average_latency_ms']
|
||||
|
||||
cuda_fp16_99_percentile = ""
|
||||
cuda_fp16_90_percentile = ""
|
||||
if 'CUDAExecutionProvider_fp16' in value and 'latency_90_percentile' in value['CUDAExecutionProvider_fp16']:
|
||||
cuda_fp16_99_percentile = value['CUDAExecutionProvider_fp16']['latency_90_percentile']
|
||||
cuda_fp16_90_percentile = value['CUDAExecutionProvider_fp16']['latency_90_percentile']
|
||||
|
||||
trt_fp16_average = ""
|
||||
if 'TensorrtExecutionProvider_fp16' in value and 'average_latency_ms' in value['TensorrtExecutionProvider_fp16']:
|
||||
trt_fp16_average = value['TensorrtExecutionProvider_fp16']['average_latency_ms']
|
||||
|
||||
trt_fp16_99_percentile = ""
|
||||
trt_fp16_90_percentile = ""
|
||||
if 'TensorrtExecutionProvider_fp16' in value and 'latency_90_percentile' in value['TensorrtExecutionProvider_fp16']:
|
||||
trt_fp16_99_percentile = value['TensorrtExecutionProvider_fp16']['latency_90_percentile']
|
||||
trt_fp16_90_percentile = value['TensorrtExecutionProvider_fp16']['latency_90_percentile']
|
||||
|
||||
standalone_trt_fp16_average = ""
|
||||
if 'Standalone_TRT_fp16' in value and 'average_latency_ms' in value['Standalone_TRT_fp16']:
|
||||
standalone_trt_fp16_average = value['Standalone_TRT_fp16']['average_latency_ms']
|
||||
|
||||
standalone_trt_fp16_99_percentile = ""
|
||||
standalone_trt_fp16_90_percentile = ""
|
||||
if 'Standalone_TRT_fp16' in value and 'latency_90_percentile' in value['Standalone_TRT_fp16']:
|
||||
standalone_trt_fp16_99_percentile = value['Standalone_TRT_fp16']['latency_90_percentile']
|
||||
standalone_trt_fp16_90_percentile = value['Standalone_TRT_fp16']['latency_90_percentile']
|
||||
|
||||
|
||||
row = [key,
|
||||
cuda_average,
|
||||
cuda_99_percentile,
|
||||
cuda_90_percentile,
|
||||
trt_average,
|
||||
trt_99_percentile,
|
||||
trt_90_percentile,
|
||||
standalone_trt_average,
|
||||
standalone_trt_99_percentile,
|
||||
standalone_trt_90_percentile,
|
||||
cuda_fp16_average,
|
||||
cuda_fp16_99_percentile,
|
||||
cuda_fp16_90_percentile,
|
||||
trt_fp16_average,
|
||||
trt_fp16_99_percentile,
|
||||
trt_fp16_90_percentile,
|
||||
standalone_trt_fp16_average,
|
||||
standalone_trt_fp16_99_percentile,
|
||||
standalone_trt_fp16_90_percentile,
|
||||
value['Tensorrt_gain(%)'] if 'Tensorrt_gain(%)' in value else " ",
|
||||
value['Tensorrt_fp16_gain(%)'] if 'Tensorrt_fp16_gain(%)' in value else " "
|
||||
]
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import re
|
|||
import sys
|
||||
import pprint
|
||||
from benchmark import *
|
||||
from perf_utils import get_latest_commit_hash
|
||||
|
||||
def write_model_info_to_file(model, path):
|
||||
with open(path, 'w') as file:
|
||||
|
|
@ -27,10 +28,11 @@ def main():
|
|||
|
||||
model_to_fail_ep = {}
|
||||
|
||||
benchmark_fail_csv = 'fail.csv'
|
||||
benchmark_metrics_csv = 'metrics.csv'
|
||||
benchmark_success_csv = 'success.csv'
|
||||
benchmark_latency_csv = 'latency.csv'
|
||||
commit = get_latest_commit_hash()
|
||||
benchmark_fail_csv = 'fail_' + commit + '.csv'
|
||||
benchmark_metrics_csv = 'metrics_' + commit + '.csv'
|
||||
benchmark_success_csv = 'success_' + commit + '.csv'
|
||||
benchmark_latency_csv = 'latency_' + commit + '.csv'
|
||||
|
||||
for model, model_info in models.items():
|
||||
logger.info("\n" + "="*40 + "="*len(model))
|
||||
|
|
@ -43,7 +45,7 @@ def main():
|
|||
write_model_info_to_file([model_info], model_list_file)
|
||||
|
||||
|
||||
ep_list = ["CUDAExecutionProvider", "TensorrtExecutionProvider", "CUDAExecutionProvider_fp16", "TensorrtExecutionProvider_fp16"]
|
||||
ep_list = ["CPUExecutionProvider", "CUDAExecutionProvider", "TensorrtExecutionProvider", "CUDAExecutionProvider_fp16", "TensorrtExecutionProvider_fp16"]
|
||||
|
||||
for ep in ep_list:
|
||||
if args.running_mode == "validate":
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import os
|
||||
import subprocess
|
||||
import argparse
|
||||
from perf_utils import get_latest_commit_hash
|
||||
|
||||
def parse_arguments():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
|
@ -8,10 +9,28 @@ def parse_arguments():
|
|||
parser.add_argument("-o", "--ort_master_path", required=True, help="ORT master repo")
|
||||
parser.add_argument("-t", "--tensorrt_home", required=True, help="TensorRT home directory")
|
||||
parser.add_argument("-c", "--cuda_home", required=True, help="CUDA home directory")
|
||||
|
||||
parser.add_argument("-v", "--commit_hash", required = False, help="Github commit to test perf off of")
|
||||
parser.add_argument("-s", "--save", required=False, help="Directory to archive wheel file")
|
||||
parser.add_argument("-a", "--use_archived", required=False, help="Archived wheel file")
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
def archive_wheel_file(save_path, ort_wheel_file):
|
||||
if not os.path.exists(save_path):
|
||||
os.mkdir(save_path)
|
||||
p1 = subprocess.Popen(["cp", ort_wheel_file, save_path])
|
||||
p1.wait()
|
||||
|
||||
def install_new_ort_wheel(ort_master_path):
|
||||
ort_wheel_path = os.path.join(ort_master_path, "build", "Linux", "Release", "dist")
|
||||
p1 = subprocess.Popen(["find", ort_wheel_path, "-name", "*.whl"], stdout=subprocess.PIPE)
|
||||
stdout, sterr = p1.communicate()
|
||||
stdout = stdout.decode("utf-8").strip()
|
||||
ort_wheel = stdout.split("\n")[0]
|
||||
p1 = subprocess.Popen(["pip3", "install", "-I", ort_wheel])
|
||||
p1.wait()
|
||||
return ort_wheel
|
||||
|
||||
def main():
|
||||
args = parse_arguments()
|
||||
|
||||
|
|
@ -28,19 +47,28 @@ def main():
|
|||
pwd = os.getcwd()
|
||||
os.chdir(ort_master_path)
|
||||
|
||||
p1 = subprocess.Popen(["git", "pull"])
|
||||
p1.wait()
|
||||
if args.use_archived:
|
||||
ort_wheel_file = args.use_archived
|
||||
p1 = subprocess.Popen(["pip3", "install", "-I", ort_wheel_file])
|
||||
p1.wait()
|
||||
|
||||
else:
|
||||
if args.commit_hash:
|
||||
commit = args.commit_hash
|
||||
p1 = subprocess.Popen(["git", "checkout", commit])
|
||||
else:
|
||||
commit = get_latest_commit_hash()
|
||||
p1 = subprocess.Popen(["git", "pull", "origin", "master"])
|
||||
p1.wait()
|
||||
|
||||
p1 = subprocess.Popen(["./build.sh", "--config", "Release", "--use_tensorrt", "--tensorrt_home", args.tensorrt_home, "--cuda_home", args.cuda_home, "--cudnn", "/usr/lib/x86_64-linux-gnu", "--build_wheel", "--skip_tests"])
|
||||
p1.wait()
|
||||
p1 = subprocess.Popen(["./build.sh", "--config", "Release", "--use_tensorrt", "--tensorrt_home", args.tensorrt_home, "--cuda_home", args.cuda_home, "--cudnn", "/usr/lib/x86_64-linux-gnu", "--build_wheel", "--skip_tests", "--parallel"])
|
||||
p1.wait()
|
||||
|
||||
ort_wheel_path = os.path.join(ort_master_path, "build", "Linux", "Release", "dist")
|
||||
p1 = subprocess.Popen(["find", ort_wheel_path, "-name", "*.whl"], stdout=subprocess.PIPE)
|
||||
stdout, sterr = p1.communicate()
|
||||
stdout = stdout.decode("utf-8").strip()
|
||||
ort_wheel = stdout.split("\n")
|
||||
p1 = subprocess.Popen(["pip3", "install", ort_wheel[0]])
|
||||
p1.wait()
|
||||
ort_wheel_file = install_new_ort_wheel(ort_master_path)
|
||||
|
||||
if args.save:
|
||||
save_path = os.path.join(args.save, commit)
|
||||
archive_wheel_file(save_path, ort_wheel_file)
|
||||
|
||||
os.chdir(pwd)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,16 +5,30 @@ FAIL_MODEL_FILE=".fail_model_map"
|
|||
LATENCY_FILE=".latency_map"
|
||||
METRICS_FILE=".metrics_map"
|
||||
|
||||
cleanup_metadata() {
|
||||
# symbolic shape info
|
||||
SYMBOLIC_SHAPE_INFER="symbolic_shape_infer.py"
|
||||
SYMBOLIC_SHAPE_INFER_LINK="https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/symbolic_shape_infer.py"
|
||||
|
||||
cleanup_files() {
|
||||
rm -f $FAIL_MODEL_FILE
|
||||
rm -f $LATENCY_FILE
|
||||
rm -f $METRICS_FILE
|
||||
rm -f $SYMBOLIC_SHAPE_INFER
|
||||
}
|
||||
|
||||
download_symbolic_shape() {
|
||||
sudo wget -c $SYMBOLIC_SHAPE_INFER_LINK
|
||||
}
|
||||
|
||||
update_files() {
|
||||
cleanup_files
|
||||
download_symbolic_shape
|
||||
}
|
||||
|
||||
# many models
|
||||
if [ "$1" == "many-models" ]
|
||||
then
|
||||
cleanup_metadata
|
||||
update_files
|
||||
python3 benchmark_wrapper.py -r validate -m /home/hcsuser/mount/many-models -o result/"$1"
|
||||
python3 benchmark_wrapper.py -r benchmark -i random -t 10 -m /home/hcsuser/mount/many-models -o result/"$1"
|
||||
fi
|
||||
|
|
@ -23,7 +37,7 @@ fi
|
|||
if [ "$1" == "onnx-zoo-models" ]
|
||||
then
|
||||
MODEL_LIST="model_list.json"
|
||||
cleanup_metadata
|
||||
update_files
|
||||
python3 benchmark_wrapper.py -r validate -m $MODEL_LIST -o result/"$1"
|
||||
python3 benchmark_wrapper.py -r benchmark -i random -t 10 -m $MODEL_LIST -o result/"$1"
|
||||
fi
|
||||
|
|
@ -32,7 +46,7 @@ fi
|
|||
if [ "$1" == "partner-models" ]
|
||||
then
|
||||
MODEL_LIST="/home/hcsuser/perf/partner_model_list.json"
|
||||
cleanup_metadata
|
||||
update_files
|
||||
python3 benchmark_wrapper.py -r validate -m $MODEL_LIST -o result/"$1"
|
||||
python3 benchmark_wrapper.py -r benchmark -i random -t 10 -m $MODEL_LIST -o result/"$1"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -8,6 +8,12 @@ import re
|
|||
debug = False
|
||||
debug_verbose = False
|
||||
|
||||
def get_latest_commit_hash():
|
||||
p1 = subprocess.Popen(["git", "rev-parse", "--short", "HEAD"], stdout = subprocess.PIPE)
|
||||
stdout, sterr = p1.communicate()
|
||||
commit = stdout.decode("utf-8").strip()
|
||||
return commit
|
||||
|
||||
def parse_single_file(f):
|
||||
|
||||
try:
|
||||
|
|
|
|||
Loading…
Reference in a new issue