From 234e94b4e1aef7b11f9c939b5bf9f8b1e51d6d64 Mon Sep 17 00:00:00 2001 From: Olivia Jain Date: Mon, 21 Dec 2020 20:23:19 -0800 Subject: [PATCH] Add Status.csv to EP Perf Tool (#6167) * merge master, keep postprocess status commit * download float16.py everytime * removing hardcoded values --- .../python/tools/tensorrt/perf/benchmark.py | 149 ++++++++++++++---- .../tools/tensorrt/perf/benchmark_wrapper.py | 33 ++-- .../python/tools/tensorrt/perf/perf.sh | 35 ++-- 3 files changed, 167 insertions(+), 50 deletions(-) diff --git a/onnxruntime/python/tools/tensorrt/perf/benchmark.py b/onnxruntime/python/tools/tensorrt/perf/benchmark.py index 0153ae75c8..6d17d67f46 100644 --- a/onnxruntime/python/tools/tensorrt/perf/benchmark.py +++ b/onnxruntime/python/tools/tensorrt/perf/benchmark.py @@ -37,7 +37,6 @@ FAIL_MODEL_FILE = ".fail_model_map" LATENCY_FILE = ".latency_map" METRICS_FILE = ".metrics_map" - def run_trt_standalone(trtexec, model_path, ort_inputs, all_inputs_shape, fp16): model_path = "--onnx=" + model_path input_shape = [] @@ -112,7 +111,7 @@ def get_latency_result(runtimes, batch_size): } -def get_ort_session_inputs_and_outptus(name, session, ort_input): +def get_ort_session_inputs_and_outputs(name, session, ort_input): sess_inputs = {} sess_outputs = None @@ -161,7 +160,7 @@ def inference_ort(args, name, session, ep, ort_inputs, result_template, repeat_t repeat_times += 1 # add warn-up run for ort_input in ort_inputs: - sess_inputs, sess_outputs = get_ort_session_inputs_and_outptus(name, session, ort_input) + sess_inputs, sess_outputs = get_ort_session_inputs_and_outputs(name, session, ort_input) if debug: logger.info("ORT session inputs:") logger.info(sess_inputs) @@ -190,7 +189,7 @@ def inference_ort_and_get_prediction(name, session, ort_inputs): ort_outputs = [] for ort_input in ort_inputs: - sess_inputs, sess_outputs = get_ort_session_inputs_and_outptus(name, session, ort_input) + sess_inputs, sess_outputs = get_ort_session_inputs_and_outputs(name, session, ort_input) if debug: logger.info("ORT session inputs:") logger.info(sess_inputs) @@ -405,8 +404,6 @@ def validate(all_ref_outputs, all_outputs, decimal): logger.info('Reference {} results.'.format(len(all_ref_outputs))) logger.info('Predicted {} results.'.format(len(all_outputs))) logger.info('decimal {}'.format(decimal)) - # print(np.array(all_ref_outputs).shape) - # print(np.array(all_outputs).shape) try: for i in range(len(all_outputs)): @@ -416,8 +413,6 @@ def validate(all_ref_outputs, all_outputs, decimal): for j in range(len(outputs)): ref_output = ref_outputs[j] output = outputs[j] - # print(ref_output) - # print(output) # Compare the results with reference outputs up to x decimal places for ref_o, o in zip(ref_output, output): @@ -606,8 +601,6 @@ def update_fail_model_map_ori(model_to_fail_ep, fail_results, model_name, ep, e_ update_fail_report(fail_results, model_name, ep_, e_type, error_message_) model_to_fail_ep[model_name][ep_] = e_type - - def skip_ep(model_name, ep, model_to_fail_ep): if model_name not in model_to_fail_ep: @@ -631,19 +624,19 @@ def read_map_from_file(map_file): return data -def write_map_to_file(model_to_fail_ep, file_name): - existed_model_to_fail_ep = {} +def write_map_to_file(result, file_name): + existed_result = {} if os.path.exists(file_name): - existed_model_to_fail_ep = read_map_from_file(file_name) + existed_result = read_map_from_file(file_name) - for model, ep_list in model_to_fail_ep.items(): - if model in existed_model_to_fail_ep: - existed_model_to_fail_ep[model] = {** existed_model_to_fail_ep[model], ** model_to_fail_ep[model]} + for model, ep_list in result.items(): + if model in existed_result: + existed_result[model] = {** existed_result[model], ** result[model]} else: - existed_model_to_fail_ep[model] = model_to_fail_ep[model] + existed_result[model] = result[model] with open(file_name, 'w') as file: - file.write(json.dumps(existed_model_to_fail_ep)) # use `json.loads` to do the reverse + file.write(json.dumps(existed_result)) # use `json.loads` to do the reverse def get_system_info(): @@ -778,10 +771,10 @@ def parse_models_info_from_directory(path, models): parse_models_info_from_directory(os.path.join(path, dir), models) -def parse_models_info_from_file(path, models): +def parse_models_info_from_file(default_dir, path, models): # default working directory - root_working_directory = "/home/hcsuser/perf/" + root_working_directory = default_dir with open(path) as f: data = json.load(f) @@ -1021,7 +1014,6 @@ def run_onnxruntime(args, models): result = inference_ort(args, name, sess, ep, inputs, result_template, args.test_times, batch_size) if result: success_results.append(result) - logger.info(result) latency_result[ep] = {} latency_result[ep]["average_latency_ms"] = result["average_latency_ms"] @@ -1178,6 +1170,98 @@ def output_fail(model_to_fail_ep, csv_filename): logger.info(f"Failing results are saved to csv file: {csv_filename}") +def read_success_from_file(success_file): + success_results = [] + with open(success_file) as success: + csv_reader = csv.DictReader(success) + for row in csv_reader: + success_results.append(row) + + success_json = json.loads(json.dumps(success_results, indent=4)) + return success_json + +def add_status_dict(status_dict, model_name, ep, status): + if model_name not in status_dict: + status_dict[model_name] = {} + status_dict[model_name][ep] = status + +def build_status(status_dict, results, is_fail): + + if is_fail: + for model, model_info in results.items(): + for ep, ep_info in model_info.items(): + model_name = model + ep = ep + status = 'Fail' + add_status_dict(status_dict, model_name, ep, status) + else: + for result in results: + model_name = result['model_name'] + ep = result['device'] + status = 'Pass' + add_status_dict(status_dict, model_name, ep, status) + + return status_dict + +def output_status(results, csv_filename): + + need_write_header = True + if os.path.exists(csv_filename): + need_write_header = False + + with open(csv_filename, mode="a", newline='') as csv_file: + column_names = ["Model", + "CPU", + "CUDA fp32", + "TRT fp32", + "Standalone TRT fp32", + "CUDA fp16", + "TRT fp16", + "Standalone TRT fp16" + ] + csv_writer = csv.writer(csv_file) + + if need_write_header: + csv_writer.writerow(column_names) + + cpu = "" + cuda_fp32 = "" + trt_fp32 = "" + standalone_fp32 = "" + cuda_fp16 = "" + trt_fp16 = "" + standalone_fp16 = "" + + for model_name, ep_dict in results.items(): + for ep, status in ep_dict.items(): + if ep == "CPUExecutionProvider": + cpu = status + elif ep == "CUDAExecutionProvider": + cuda_fp32 = status + elif ep == "TensorrtExecutionProvider": + trt_fp32 = status + elif ep == "Standalone_TRT": + standalone_fp32 = status + elif ep == "CUDAExecutionProvider_fp16": + cuda_fp16 = status + elif ep == "TensorrtExecutionProvider_fp16": + trt_fp16 = status + elif ep == "Standalone_TRT_fp16": + standalone_fp16 = status + else: + continue + + row = [model_name, + cpu, + cuda_fp32, + trt_fp32, + standalone_fp32, + cuda_fp16, + trt_fp16, + standalone_fp16 + ] + csv_writer.writerow(row) + def output_latency(results, csv_filename): need_write_header = True if os.path.exists(csv_filename): @@ -1391,6 +1475,8 @@ def str2bool(v): def parse_arguments(): parser = argparse.ArgumentParser() + parser.add_argument("-d", "--default_dir", required=False, default="~/", help="Perf folder path") + parser.add_argument("-m", "--model_source", required=False, default="model_list.json", help="Model source: (1) model list file (2) model directory.") parser.add_argument("-r", "--running_mode", required=False, default="benchmark", choices=["validate", "benchmark"], help="Testing mode.") @@ -1429,6 +1515,14 @@ def setup_logger(verbose): coloredlogs.install(fmt='%(message)s') logging.getLogger("transformers").setLevel(logging.WARNING) +def parse_models_helper(args, models): + if ".json" in args.model_source: + logger.info("Parsing model information from file ...") + parse_models_info_from_file(args.default_dir, args.model_source, models) + else: + logger.info("Parsing model information from directory ...") + parse_models_info_from_directory(args.model_source, models) + def main(): args = parse_arguments() setup_logger(False) @@ -1437,13 +1531,7 @@ def main(): logger.info("\n\nStart perf run ...\n") models = {} - - if ".json" in args.model_source: - logger.info("Parsing model information from file ...") - parse_models_info_from_file(args.model_source, models) - else: - logger.info("Parsing model information from directory ...") - parse_models_info_from_directory(args.model_source, models) + parse_models_helper(args, models) if not os.path.exists("symbolic_shape_infer.py"): p1 = subprocess.Popen(["sudo", "wget", "https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/symbolic_shape_infer.py"]) @@ -1473,7 +1561,7 @@ def main(): Path(path).mkdir(parents=True, exist_ok=True) time_stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - + if len(model_to_fail_ep) > 0: logger.info("\n============================================") logger.info("========== Failing Models/EPs ==============") @@ -1493,12 +1581,11 @@ def main(): # add_improvement_information(model_to_latency) pp.pprint(model_to_latency) write_map_to_file(model_to_latency, LATENCY_FILE) - if args.write_test_result: csv_filename = args.benchmark_latency_csv if args.benchmark_latency_csv else f"benchmark_latency_{time_stamp}.csv" csv_filename = os.path.join(path, csv_filename) output_latency(model_to_latency, csv_filename) - + if success_results: csv_filename = args.benchmark_success_csv if args.benchmark_success_csv else f"benchmark_success_{time_stamp}.csv" csv_filename = os.path.join(path, csv_filename) diff --git a/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py b/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py index 49cb17b525..03a8b573b4 100644 --- a/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py +++ b/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py @@ -21,10 +21,7 @@ def main(): pp = pprint.PrettyPrinter(indent=4) models = {} - if ".json" in args.model_source: - parse_models_info_from_file(args.model_source, models) - else: - parse_models_info_from_directory(args.model_source, models) + parse_models_helper(args, models) model_to_fail_ep = {} @@ -33,6 +30,7 @@ def main(): benchmark_metrics_csv = 'metrics_' + commit + '.csv' benchmark_success_csv = 'success_' + commit + '.csv' benchmark_latency_csv = 'latency_' + commit + '.csv' + benchmark_status_csv = 'status_' + commit + '.csv' for model, model_info in models.items(): logger.info("\n" + "="*40 + "="*len(model)) @@ -43,14 +41,13 @@ def main(): model_list_file = os.path.join(os.getcwd(), model +'.json') write_model_info_to_file([model_info], model_list_file) - - ep_list = ["CPUExecutionProvider", "CUDAExecutionProvider", "TensorrtExecutionProvider", "CUDAExecutionProvider_fp16", "TensorrtExecutionProvider_fp16"] for ep in ep_list: if args.running_mode == "validate": p = subprocess.run(["python3", "benchmark.py", + "-d", args.default_dir, "-r", args.running_mode, "-m", model_list_file, "--ep", ep, @@ -61,6 +58,7 @@ def main(): elif args.running_mode == "benchmark": p = subprocess.run(["python3", "benchmark.py", + "-d", args.default_dir, "-r", args.running_mode, "-m", model_list_file, "--ep", ep, @@ -68,14 +66,13 @@ def main(): "-o", args.perf_result_path, "--write_test_result", "false", "--benchmark_latency_csv", benchmark_latency_csv, - "--benchmark_success_csv", benchmark_success_csv]) + "--benchmark_success_csv", benchmark_success_csv]) logger.info(p) if p.returncode != 0: error_type = "runtime error" error_message = "perf script exited with returncode = " + str(p.returncode) logger.error(error_message) - update_fail_model_map(model_to_fail_ep, model, ep, error_type, error_message) write_map_to_file(model_to_fail_ep, FAIL_MODEL_FILE) logger.info(model_to_fail_ep) @@ -98,7 +95,6 @@ def main(): logger.info(model_to_fail_ep) - logger.info("\n=========================================") logger.info("========== Models/EPs metrics ==========") logger.info("=========================================") @@ -108,6 +104,25 @@ def main(): output_metrics(model_to_metrics, os.path.join(path, benchmark_metrics_csv)) elif args.running_mode == "benchmark": + logger.info("\n=======================================================") + logger.info("=========== Models/EPs Status (accumulated) ===========") + logger.info("=======================================================") + + model_status = {} + success_path = os.path.join(path, benchmark_success_csv) + if os.path.exists(success_path): + model_success = read_success_from_file(success_path) + is_fail = False + model_status = build_status(model_status, model_success, is_fail) + if os.path.exists(FAIL_MODEL_FILE): + model_fail = read_map_from_file(FAIL_MODEL_FILE) + is_fail = True + model_status = build_status(model_status, model_fail, is_fail) + + pp.pprint(model_status) + output_status(model_status, os.path.join(path, benchmark_status_csv)) + logger.info("\nSaved model status results to {}".format(benchmark_status_csv)) + logger.info("\n=======================================================") logger.info("=========== Models/EPs latency (accumulated) ===========") logger.info("=======================================================") diff --git a/onnxruntime/python/tools/tensorrt/perf/perf.sh b/onnxruntime/python/tools/tensorrt/perf/perf.sh index aa6f29abfa..bc52eae561 100755 --- a/onnxruntime/python/tools/tensorrt/perf/perf.sh +++ b/onnxruntime/python/tools/tensorrt/perf/perf.sh @@ -5,32 +5,39 @@ FAIL_MODEL_FILE=".fail_model_map" LATENCY_FILE=".latency_map" METRICS_FILE=".metrics_map" -# symbolic shape info +# files to download info SYMBOLIC_SHAPE_INFER="symbolic_shape_infer.py" SYMBOLIC_SHAPE_INFER_LINK="https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/symbolic_shape_infer.py" +FLOAT_16="float16.py" +FLOAT_16_LINK="https://raw.githubusercontent.com/microsoft/onnxconverter-common/master/onnxconverter_common/float16.py" + +# root working directory +DEFAULT_DIR="./" cleanup_files() { rm -f $FAIL_MODEL_FILE rm -f $LATENCY_FILE rm -f $METRICS_FILE rm -f $SYMBOLIC_SHAPE_INFER + rm -f $FLOAT_16 } -download_symbolic_shape() { +download_files() { sudo wget -c $SYMBOLIC_SHAPE_INFER_LINK + sudo wget -c $FLOAT_16_LINK } update_files() { cleanup_files - download_symbolic_shape + download_files } # many models if [ "$1" == "many-models" ] then update_files - python3 benchmark_wrapper.py -r validate -m /home/hcsuser/mount/many-models -o result/"$1" - python3 benchmark_wrapper.py -r benchmark -i random -t 10 -m /home/hcsuser/mount/many-models -o result/"$1" + python3 benchmark_wrapper.py -d $DEFAULT_DIR -r validate -m /home/hcsuser/mount/many-models -o result/"$1" + python3 benchmark_wrapper.py -d $DEFAULT_DIR -r benchmark -i random -t 10 -m /home/hcsuser/mount/many-models -o result/"$1" fi # ONNX model zoo @@ -38,16 +45,24 @@ if [ "$1" == "onnx-zoo-models" ] then MODEL_LIST="model_list.json" update_files - python3 benchmark_wrapper.py -r validate -m $MODEL_LIST -o result/"$1" - python3 benchmark_wrapper.py -r benchmark -i random -t 10 -m $MODEL_LIST -o result/"$1" + python3 benchmark_wrapper.py -d $DEFAULT_DIR -r validate -m $MODEL_LIST -o result/"$1" + python3 benchmark_wrapper.py -d $DEFAULT_DIR -r benchmark -i random -t 10 -m $MODEL_LIST -o result/"$1" fi # 1P models if [ "$1" == "partner-models" ] then - MODEL_LIST="/home/hcsuser/perf/partner_model_list.json" + MODEL_LIST="partner_model_list.json" update_files - python3 benchmark_wrapper.py -r validate -m $MODEL_LIST -o result/"$1" - python3 benchmark_wrapper.py -r benchmark -i random -t 10 -m $MODEL_LIST -o result/"$1" + python3 benchmark_wrapper.py -d $DEFAULT_DIR -r validate -m $MODEL_LIST -o result/"$1" + python3 benchmark_wrapper.py -d $DEFAULT_DIR -r benchmark -i random -t 10 -m $MODEL_LIST -o result/"$1" fi +# Test models +if [ "$1" == "selected-models" ] +then + MODEL_LIST="selected_models.json" + update_files + python3 benchmark_wrapper.py -d $DEFAULT_DIR -r validate -m $MODEL_LIST -o result/"$1" + python3 benchmark_wrapper.py -d $DEFAULT_DIR -r benchmark -i random -t 1 -m $MODEL_LIST -o result/"$1" +fi