diff --git a/onnxruntime/python/tools/tensorrt/perf/benchmark.py b/onnxruntime/python/tools/tensorrt/perf/benchmark.py index 06575e81d4..0153ae75c8 100644 --- a/onnxruntime/python/tools/tensorrt/perf/benchmark.py +++ b/onnxruntime/python/tools/tensorrt/perf/benchmark.py @@ -884,9 +884,9 @@ def run_onnxruntime(args, models): ep_list.append(args.ep) else: if args.fp16: - ep_list = ["CUDAExecutionProvider", "TensorrtExecutionProvider", "CUDAExecutionProvider_fp16", "TensorrtExecutionProvider_fp16"] + ep_list = ["CPUExecutionProvider", "CUDAExecutionProvider", "TensorrtExecutionProvider", "CUDAExecutionProvider_fp16", "TensorrtExecutionProvider_fp16"] else: - ep_list = ["CUDAExecutionProvider", "TensorrtExecutionProvider"] + ep_list = ["CPUExecutionProvider", "CUDAExecutionProvider", "TensorrtExecutionProvider"] validation_exemption = ["TensorrtExecutionProvider_fp16"] @@ -1005,7 +1005,6 @@ def run_onnxruntime(args, models): for output_meta in sess.get_outputs(): logger.info(output_meta) - batch_size = 1 result_template = { "engine": "onnxruntime", @@ -1186,12 +1185,14 @@ def output_latency(results, csv_filename): with open(csv_filename, mode="a", newline='') as csv_file: column_names = ["Model", - "CUDA \nmean (ms)", - "CUDA \n90th percentile (ms)", - "TRT EP \nmean (ms)", - "TRT EP \n90th percentile (ms)", - "Standalone TRT \nmean (ms)", - "Standalone TRT \n90th percentile (ms)", + "CPU \nmean (ms)", + "CPU \n 90th percentile (ms)", + "CUDA fp32 \nmean (ms)", + "CUDA fp32 \n90th percentile (ms)", + "TRT EP fp32 \nmean (ms)", + "TRT EP fp32 \n90th percentile (ms)", + "Standalone TRT fp32 \nmean (ms)", + "Standalone TRT fp32 \n90th percentile (ms)", "CUDA fp16 \nmean (ms)", "CUDA fp16 \n90th percentile (ms)", "TRT EP fp16 \nmean (ms)", @@ -1206,69 +1207,77 @@ def output_latency(results, csv_filename): csv_writer.writerow(column_names) for key, value in results.items(): + cpu_average = "" + if "CPUExecutionProvider" in value and "average_latency_ms" in value["CPUExecutionProvider"]: + cpu_average = value["CPUExecutionProvider"]["average_latency_ms"] + + cpu_90_percentile = "" + if "CPUExecutionProvider" in value and "latency_90_percentile" in value["CPUExecutionProvider"]: + cpu_90_percentile = value["CPUExecutionProvider"]["latency_90_percentile"] + cuda_average = "" if 'CUDAExecutionProvider' in value and 'average_latency_ms' in value['CUDAExecutionProvider']: cuda_average = value['CUDAExecutionProvider']['average_latency_ms'] - cuda_99_percentile = "" + cuda_90_percentile = "" if 'CUDAExecutionProvider' in value and 'latency_90_percentile' in value['CUDAExecutionProvider']: - cuda_99_percentile = value['CUDAExecutionProvider']['latency_90_percentile'] + cuda_90_percentile = value['CUDAExecutionProvider']['latency_90_percentile'] trt_average = "" if 'TensorrtExecutionProvider' in value and 'average_latency_ms' in value['TensorrtExecutionProvider']: trt_average = value['TensorrtExecutionProvider']['average_latency_ms'] - trt_99_percentile = "" + trt_90_percentile = "" if 'TensorrtExecutionProvider' in value and 'latency_90_percentile' in value['TensorrtExecutionProvider']: - trt_99_percentile = value['TensorrtExecutionProvider']['latency_90_percentile'] + trt_90_percentile = value['TensorrtExecutionProvider']['latency_90_percentile'] standalone_trt_average = "" if 'Standalone_TRT' in value and 'average_latency_ms' in value['Standalone_TRT']: standalone_trt_average = value['Standalone_TRT']['average_latency_ms'] - standalone_trt_99_percentile = "" + standalone_trt_90_percentile = "" if 'Standalone_TRT' in value and 'latency_90_percentile' in value['Standalone_TRT']: - standalone_trt_99_percentile = value['Standalone_TRT']['latency_90_percentile'] + standalone_trt_90_percentile = value['Standalone_TRT']['latency_90_percentile'] cuda_fp16_average = "" if 'CUDAExecutionProvider_fp16' in value and 'average_latency_ms' in value['CUDAExecutionProvider_fp16']: cuda_fp16_average = value['CUDAExecutionProvider_fp16']['average_latency_ms'] - cuda_fp16_99_percentile = "" + cuda_fp16_90_percentile = "" if 'CUDAExecutionProvider_fp16' in value and 'latency_90_percentile' in value['CUDAExecutionProvider_fp16']: - cuda_fp16_99_percentile = value['CUDAExecutionProvider_fp16']['latency_90_percentile'] + cuda_fp16_90_percentile = value['CUDAExecutionProvider_fp16']['latency_90_percentile'] trt_fp16_average = "" if 'TensorrtExecutionProvider_fp16' in value and 'average_latency_ms' in value['TensorrtExecutionProvider_fp16']: trt_fp16_average = value['TensorrtExecutionProvider_fp16']['average_latency_ms'] - trt_fp16_99_percentile = "" + trt_fp16_90_percentile = "" if 'TensorrtExecutionProvider_fp16' in value and 'latency_90_percentile' in value['TensorrtExecutionProvider_fp16']: - trt_fp16_99_percentile = value['TensorrtExecutionProvider_fp16']['latency_90_percentile'] + trt_fp16_90_percentile = value['TensorrtExecutionProvider_fp16']['latency_90_percentile'] standalone_trt_fp16_average = "" if 'Standalone_TRT_fp16' in value and 'average_latency_ms' in value['Standalone_TRT_fp16']: standalone_trt_fp16_average = value['Standalone_TRT_fp16']['average_latency_ms'] - standalone_trt_fp16_99_percentile = "" + standalone_trt_fp16_90_percentile = "" if 'Standalone_TRT_fp16' in value and 'latency_90_percentile' in value['Standalone_TRT_fp16']: - standalone_trt_fp16_99_percentile = value['Standalone_TRT_fp16']['latency_90_percentile'] + standalone_trt_fp16_90_percentile = value['Standalone_TRT_fp16']['latency_90_percentile'] row = [key, cuda_average, - cuda_99_percentile, + cuda_90_percentile, trt_average, - trt_99_percentile, + trt_90_percentile, standalone_trt_average, - standalone_trt_99_percentile, + standalone_trt_90_percentile, cuda_fp16_average, - cuda_fp16_99_percentile, + cuda_fp16_90_percentile, trt_fp16_average, - trt_fp16_99_percentile, + trt_fp16_90_percentile, standalone_trt_fp16_average, - standalone_trt_fp16_99_percentile, + standalone_trt_fp16_90_percentile, value['Tensorrt_gain(%)'] if 'Tensorrt_gain(%)' in value else " ", value['Tensorrt_fp16_gain(%)'] if 'Tensorrt_fp16_gain(%)' in value else " " ] diff --git a/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py b/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py index c88dfd7de4..49cb17b525 100644 --- a/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py +++ b/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py @@ -9,6 +9,7 @@ import re import sys import pprint from benchmark import * +from perf_utils import get_latest_commit_hash def write_model_info_to_file(model, path): with open(path, 'w') as file: @@ -27,10 +28,11 @@ def main(): model_to_fail_ep = {} - benchmark_fail_csv = 'fail.csv' - benchmark_metrics_csv = 'metrics.csv' - benchmark_success_csv = 'success.csv' - benchmark_latency_csv = 'latency.csv' + commit = get_latest_commit_hash() + benchmark_fail_csv = 'fail_' + commit + '.csv' + benchmark_metrics_csv = 'metrics_' + commit + '.csv' + benchmark_success_csv = 'success_' + commit + '.csv' + benchmark_latency_csv = 'latency_' + commit + '.csv' for model, model_info in models.items(): logger.info("\n" + "="*40 + "="*len(model)) @@ -43,7 +45,7 @@ def main(): write_model_info_to_file([model_info], model_list_file) - ep_list = ["CUDAExecutionProvider", "TensorrtExecutionProvider", "CUDAExecutionProvider_fp16", "TensorrtExecutionProvider_fp16"] + ep_list = ["CPUExecutionProvider", "CUDAExecutionProvider", "TensorrtExecutionProvider", "CUDAExecutionProvider_fp16", "TensorrtExecutionProvider_fp16"] for ep in ep_list: if args.running_mode == "validate": diff --git a/onnxruntime/python/tools/tensorrt/perf/ort_build_latest.py b/onnxruntime/python/tools/tensorrt/perf/ort_build_latest.py index 23b33f4b13..2081bda335 100755 --- a/onnxruntime/python/tools/tensorrt/perf/ort_build_latest.py +++ b/onnxruntime/python/tools/tensorrt/perf/ort_build_latest.py @@ -1,6 +1,7 @@ import os import subprocess import argparse +from perf_utils import get_latest_commit_hash def parse_arguments(): parser = argparse.ArgumentParser() @@ -8,10 +9,28 @@ def parse_arguments(): parser.add_argument("-o", "--ort_master_path", required=True, help="ORT master repo") parser.add_argument("-t", "--tensorrt_home", required=True, help="TensorRT home directory") parser.add_argument("-c", "--cuda_home", required=True, help="CUDA home directory") - + parser.add_argument("-v", "--commit_hash", required = False, help="Github commit to test perf off of") + parser.add_argument("-s", "--save", required=False, help="Directory to archive wheel file") + parser.add_argument("-a", "--use_archived", required=False, help="Archived wheel file") args = parser.parse_args() return args +def archive_wheel_file(save_path, ort_wheel_file): + if not os.path.exists(save_path): + os.mkdir(save_path) + p1 = subprocess.Popen(["cp", ort_wheel_file, save_path]) + p1.wait() + +def install_new_ort_wheel(ort_master_path): + ort_wheel_path = os.path.join(ort_master_path, "build", "Linux", "Release", "dist") + p1 = subprocess.Popen(["find", ort_wheel_path, "-name", "*.whl"], stdout=subprocess.PIPE) + stdout, sterr = p1.communicate() + stdout = stdout.decode("utf-8").strip() + ort_wheel = stdout.split("\n")[0] + p1 = subprocess.Popen(["pip3", "install", "-I", ort_wheel]) + p1.wait() + return ort_wheel + def main(): args = parse_arguments() @@ -28,19 +47,28 @@ def main(): pwd = os.getcwd() os.chdir(ort_master_path) - p1 = subprocess.Popen(["git", "pull"]) - p1.wait() + if args.use_archived: + ort_wheel_file = args.use_archived + p1 = subprocess.Popen(["pip3", "install", "-I", ort_wheel_file]) + p1.wait() + + else: + if args.commit_hash: + commit = args.commit_hash + p1 = subprocess.Popen(["git", "checkout", commit]) + else: + commit = get_latest_commit_hash() + p1 = subprocess.Popen(["git", "pull", "origin", "master"]) + p1.wait() - p1 = subprocess.Popen(["./build.sh", "--config", "Release", "--use_tensorrt", "--tensorrt_home", args.tensorrt_home, "--cuda_home", args.cuda_home, "--cudnn", "/usr/lib/x86_64-linux-gnu", "--build_wheel", "--skip_tests"]) - p1.wait() + p1 = subprocess.Popen(["./build.sh", "--config", "Release", "--use_tensorrt", "--tensorrt_home", args.tensorrt_home, "--cuda_home", args.cuda_home, "--cudnn", "/usr/lib/x86_64-linux-gnu", "--build_wheel", "--skip_tests", "--parallel"]) + p1.wait() - ort_wheel_path = os.path.join(ort_master_path, "build", "Linux", "Release", "dist") - p1 = subprocess.Popen(["find", ort_wheel_path, "-name", "*.whl"], stdout=subprocess.PIPE) - stdout, sterr = p1.communicate() - stdout = stdout.decode("utf-8").strip() - ort_wheel = stdout.split("\n") - p1 = subprocess.Popen(["pip3", "install", ort_wheel[0]]) - p1.wait() + ort_wheel_file = install_new_ort_wheel(ort_master_path) + + if args.save: + save_path = os.path.join(args.save, commit) + archive_wheel_file(save_path, ort_wheel_file) os.chdir(pwd) diff --git a/onnxruntime/python/tools/tensorrt/perf/perf.sh b/onnxruntime/python/tools/tensorrt/perf/perf.sh index 234c2018c7..aa6f29abfa 100755 --- a/onnxruntime/python/tools/tensorrt/perf/perf.sh +++ b/onnxruntime/python/tools/tensorrt/perf/perf.sh @@ -5,16 +5,30 @@ FAIL_MODEL_FILE=".fail_model_map" LATENCY_FILE=".latency_map" METRICS_FILE=".metrics_map" -cleanup_metadata() { +# symbolic shape info +SYMBOLIC_SHAPE_INFER="symbolic_shape_infer.py" +SYMBOLIC_SHAPE_INFER_LINK="https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/symbolic_shape_infer.py" + +cleanup_files() { rm -f $FAIL_MODEL_FILE rm -f $LATENCY_FILE rm -f $METRICS_FILE + rm -f $SYMBOLIC_SHAPE_INFER +} + +download_symbolic_shape() { + sudo wget -c $SYMBOLIC_SHAPE_INFER_LINK +} + +update_files() { + cleanup_files + download_symbolic_shape } # many models if [ "$1" == "many-models" ] then - cleanup_metadata + update_files python3 benchmark_wrapper.py -r validate -m /home/hcsuser/mount/many-models -o result/"$1" python3 benchmark_wrapper.py -r benchmark -i random -t 10 -m /home/hcsuser/mount/many-models -o result/"$1" fi @@ -23,7 +37,7 @@ fi if [ "$1" == "onnx-zoo-models" ] then MODEL_LIST="model_list.json" - cleanup_metadata + update_files python3 benchmark_wrapper.py -r validate -m $MODEL_LIST -o result/"$1" python3 benchmark_wrapper.py -r benchmark -i random -t 10 -m $MODEL_LIST -o result/"$1" fi @@ -32,7 +46,7 @@ fi if [ "$1" == "partner-models" ] then MODEL_LIST="/home/hcsuser/perf/partner_model_list.json" - cleanup_metadata + update_files python3 benchmark_wrapper.py -r validate -m $MODEL_LIST -o result/"$1" python3 benchmark_wrapper.py -r benchmark -i random -t 10 -m $MODEL_LIST -o result/"$1" fi diff --git a/onnxruntime/python/tools/tensorrt/perf/perf_utils.py b/onnxruntime/python/tools/tensorrt/perf/perf_utils.py index 4a583f8168..dcf1958705 100644 --- a/onnxruntime/python/tools/tensorrt/perf/perf_utils.py +++ b/onnxruntime/python/tools/tensorrt/perf/perf_utils.py @@ -8,6 +8,12 @@ import re debug = False debug_verbose = False +def get_latest_commit_hash(): + p1 = subprocess.Popen(["git", "rev-parse", "--short", "HEAD"], stdout = subprocess.PIPE) + stdout, sterr = p1.communicate() + commit = stdout.decode("utf-8").strip() + return commit + def parse_single_file(f): try: