diff --git a/dockerfiles/Dockerfile.tensorrt b/dockerfiles/Dockerfile.tensorrt index 9b4db6b5f5..9947c54a82 100644 --- a/dockerfiles/Dockerfile.tensorrt +++ b/dockerfiles/Dockerfile.tensorrt @@ -6,7 +6,6 @@ # nVidia TensorRT Base Image ARG TRT_CONTAINER_VERSION=21.12 -ARG TRT_VERSION=8.2.1.8 FROM nvcr.io/nvidia/tensorrt:${TRT_CONTAINER_VERSION}-py3 ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime @@ -26,8 +25,8 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR # Checkout appropriate TRT_VERSION and build RUN cd onnxruntime &&\ - trt_version=${TRT_VERSION%.*.*} &&\ + trt_version=${TRT_VERSION:0:3} &&\ ./onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh ${trt_version} &&\ - /bin/sh build.sh --parallel --build_shared_lib --skip_submodule_sync --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /usr/lib/x86_64-linux-gnu/ --config Release --build_wheel --skip_tests --skip_submodule_sync --cmake_extra_defines '"CMAKE_CUDA_ARCHITECTURES='${CMAKE_CUDA_ARCHITECTURES}'"' &&\ + /bin/sh build.sh --parallel --build_shared_lib --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /usr/lib/x86_64-linux-gnu/ --config Release --build_wheel --skip_tests --skip_submodule_sync --cmake_extra_defines '"CMAKE_CUDA_ARCHITECTURES='${CMAKE_CUDA_ARCHITECTURES}'"' &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\ - cd .. \ No newline at end of file + cd .. diff --git a/onnxruntime/python/tools/tensorrt/perf/benchmark.py b/onnxruntime/python/tools/tensorrt/perf/benchmark.py index dedaa5d960..9510125b9f 100644 --- a/onnxruntime/python/tools/tensorrt/perf/benchmark.py +++ b/onnxruntime/python/tools/tensorrt/perf/benchmark.py @@ -42,6 +42,7 @@ trt_native_fp16_gain = 'TRT_Standalone_fp16_gain(%)' FAIL_MODEL_FILE = ".fail_model_map" LATENCY_FILE = ".latency_map" METRICS_FILE = ".metrics_map" +SESSION_FILE = ".session_map" MEMORY_FILE = './temp_memory.csv' def split_and_sort_output(string_list): @@ -612,7 +613,6 @@ def update_metrics_map_ori(model_to_metrics, name, ep_to_operator): logger.info('TRT FP16 operator map:') pp.pprint(trt_fp16_op_map) - ################################################################################################### # # model: {ep1: {error_type: xxx, error_message: xxx}, ep2: {error_type: xx, error_message: xx}} @@ -910,13 +910,18 @@ def run_symbolic_shape_inference(model_path, new_model_path): logger.error(e) return False, "Symbolic shape inference error" +def time_and_create_session(model_path, providers, session_options): + start = datetime.now() + session = onnxruntime.InferenceSession(model_path, providers=providers, sess_options=session_options) + end = datetime.now() + creation_time = (end - start).total_seconds() + return session, creation_time + def create_session(model_path, providers, session_options): logger.info(model_path) try: - session = onnxruntime.InferenceSession(model_path, providers=providers, sess_options=session_options) - return session - + return time_and_create_session(model_path, providers, session_options) except Exception as e: # shape inference required on model if "shape inference" in str(e): @@ -926,9 +931,8 @@ def create_session(model_path, providers, session_options): status = run_symbolic_shape_inference(model_path, new_model_path) if not status[0]: # symbolic shape inference error e = status[1] - raise Exception(e) - session = onnxruntime.InferenceSession(new_model_path, providers=providers, sess_options=session_options) - return session + raise Exception(e) + return time_and_create_session(new_model_path, providers, session_options) else: raise Exception(e) @@ -938,7 +942,8 @@ def run_onnxruntime(args, models): model_to_latency = {} # model -> cuda and tensorrt latency model_to_metrics = {} # model -> metrics from profiling file model_to_fail_ep = {} # model -> failing ep - + model_to_session = {} # models -> session creation time + ep_list = [] if args.ep: ep_list.append(args.ep) @@ -1021,7 +1026,6 @@ def run_onnxruntime(args, models): if standalone_trt_fp16 == ep: fp16 = True - print(fp16) inputs, ref_outputs = get_test_data(fp16, test_data_dir, all_inputs_shape) # generate random input data if args.input_data == "random": @@ -1045,7 +1049,7 @@ def run_onnxruntime(args, models): # create onnxruntime inference session try: - sess = create_session(model_path, providers, options) + sess, _ = create_session(model_path, providers, options) except Exception as e: logger.error(e) @@ -1133,12 +1137,16 @@ def run_onnxruntime(args, models): # create onnxruntime inference session try: - sess = create_session(model_path, ep_to_provider_list[ep], options) + sess, creation_time = create_session(model_path, ep_to_provider_list[ep], options) + except Exception as e: logger.error(e) update_fail_model_map(model_to_fail_ep, name, ep, 'runtime error', e) continue + if creation_time: + model_to_session[name] = copy.deepcopy({ep: creation_time}) + sess.disable_fallback() logger.info("start to inference {} with {} ...".format(name, ep)) @@ -1199,7 +1207,7 @@ def run_onnxruntime(args, models): # end of model - return success_results, model_to_latency, model_to_fail_ep, model_to_metrics + return success_results, model_to_latency, model_to_fail_ep, model_to_metrics, model_to_session def calculate_gain(value, ep1, ep2): ep1_latency = float(value[ep1]['average_latency_ms']) @@ -1356,6 +1364,54 @@ def output_specs(info, csv_filename): 'Version': [cpu_version, gpu_version, tensorrt_version, cuda_version, cudnn_version]}) table.to_csv(csv_filename, index=False) +def output_session_creation(results, csv_filename): + need_write_header = True + if os.path.exists(csv_filename): + need_write_header = False + + with open(csv_filename, mode="a", newline='') as csv_file: + column_names = [model_title] + for provider in ort_provider_list: + column_names.append(provider + session_ending) + + csv_writer = csv.writer(csv_file) + + + csv_writer = csv.writer(csv_file) + + if need_write_header: + csv_writer.writerow(column_names) + + cpu_time = "" + cuda_fp32_time = "" + trt_fp32_time = "" + cuda_fp16_time = "" + trt_fp16_time = "" + + for model_name, ep_dict in results.items(): + for ep, time in ep_dict.items(): + if ep == cpu: + cpu_time = time + elif ep == cuda: + cuda_fp32_time = time + elif ep == trt: + trt_fp32_time = time + elif ep == cuda_fp16: + cuda_fp16_time = time + elif ep == trt_fp16: + trt_fp16_time = time + else: + continue + + row = [model_name, + cpu_time, + cuda_fp32_time, + trt_fp32_time, + cuda_fp16_time, + trt_fp16_time] + csv_writer.writerow(row) + + def output_latency(results, csv_filename): need_write_header = True if os.path.exists(csv_filename): @@ -1657,7 +1713,7 @@ def main(): parse_models_helper(args, models) perf_start_time = datetime.now() - success_results, model_to_latency, model_to_fail_ep, model_to_metrics = run_onnxruntime(args, models) + success_results, model_to_latency, model_to_fail_ep, model_to_metrics, model_to_session = run_onnxruntime(args, models) perf_end_time = datetime.now() logger.info("Done running the perf.") @@ -1720,5 +1776,8 @@ def main(): csv_filename = os.path.join(path, csv_filename) output_metrics(model_to_metrics, csv_filename) + if len(model_to_session) > 0: + write_map_to_file(model_to_session, SESSION_FILE) + if __name__ == "__main__": main() diff --git a/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py b/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py index d2542c645c..29d5a0d42e 100644 --- a/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py +++ b/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py @@ -52,6 +52,7 @@ def main(): benchmark_success_csv = 'success.csv' benchmark_latency_csv = 'latency.csv' benchmark_status_csv = 'status.csv' + benchmark_session_csv = 'session.csv' specs_csv = 'specs.csv' for model, model_info in models.items(): @@ -119,6 +120,16 @@ def main(): output_metrics(model_to_metrics, os.path.join(path, benchmark_metrics_csv)) logger.info("\nSaved model metrics results to {}".format(benchmark_metrics_csv)) + logger.info("\n=========================================") + logger.info("======= Models/EPs session creation =======") + logger.info("=========================================") + + if os.path.exists(SESSION_FILE): + model_to_session = read_map_from_file(SESSION_FILE) + pretty_print(pp, model_to_session) + output_session_creation(model_to_session, os.path.join(path, benchmark_session_csv)) + logger.info("\nSaved session creation results to {}".format(benchmark_session_csv)) + elif args.running_mode == "benchmark": logger.info("\n=========================================================") logger.info("========== Failing Models/EPs (accumulated) ==============") diff --git a/onnxruntime/python/tools/tensorrt/perf/build/build_image.sh b/onnxruntime/python/tools/tensorrt/perf/build/build_image.sh index 88a0119e07..8d4bfb49be 100755 --- a/onnxruntime/python/tools/tensorrt/perf/build/build_image.sh +++ b/onnxruntime/python/tools/tensorrt/perf/build/build_image.sh @@ -7,7 +7,7 @@ o) TRT_DOCKERFILE_PATH=${OPTARG};; p) PERF_DOCKERFILE_PATH=${OPTARG};; b) ORT_BRANCH=${OPTARG};; i) IMAGE_NAME=${OPTARG};; -t) TRT=${OPTARG};; +t) TRT_CONTAINER=${OPTARG};; v) TRT_VERSION=${OPTARG};; c) CMAKE_CUDA_ARCHITECTURES=${OPTARG};; esac @@ -15,5 +15,5 @@ done IMAGE=onnxruntime -docker build --no-cache -t $IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg TRT=$TRT --build-arg TRT_VERSION=$TRT_VERSION --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH -f $TRT_DOCKERFILE_PATH . -docker build --no-cache --build-arg IMAGE=$IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH --build-arg TRT_VERSION=$TRT_VERSION -t $IMAGE_NAME -f $PERF_DOCKERFILE_PATH . \ No newline at end of file +docker build --no-cache -t $IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg TRT_CONTAINER_VERSION=$TRT_CONTAINER --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH -f $TRT_DOCKERFILE_PATH . +docker build --no-cache --build-arg IMAGE=$IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH --build-arg TRT_VERSION=$TRT_VERSION -t $IMAGE_NAME -f $PERF_DOCKERFILE_PATH . diff --git a/onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh b/onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh index 20e51fba2e..8634eb41f8 100755 --- a/onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh +++ b/onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh @@ -1,5 +1,6 @@ #!/bin/bash +echo "checking out onnx-tensorrt for correct version" echo "$1" if [ ! "$1" = "8.2" ] then @@ -15,4 +16,4 @@ then git checkout "$1"'.1' fi cd $CUR_PWD -fi \ No newline at end of file +fi diff --git a/onnxruntime/python/tools/tensorrt/perf/perf_utils.py b/onnxruntime/python/tools/tensorrt/perf/perf_utils.py index 08528147fe..5182147fa3 100644 --- a/onnxruntime/python/tools/tensorrt/perf/perf_utils.py +++ b/onnxruntime/python/tools/tensorrt/perf/perf_utils.py @@ -33,7 +33,8 @@ group_title = 'Group' avg_ending = ' \nmean (ms)' percentile_ending = ' \n90th percentile (ms)' memory_ending = ' \npeak memory usage (MiB)' - +session_ending = ' \n session creation time (s)' +ort_provider_list = [cpu, cuda, trt, cuda_fp16, trt_fp16] provider_list = [cpu, cuda, trt, standalone_trt, cuda_fp16, trt_fp16, standalone_trt_fp16] table_headers = [model_title] + provider_list @@ -260,4 +261,4 @@ def get_profile_metrics(path, profile_already_parsed, logger=None): logger.info("No profile metrics got.") return None - return data[-1] \ No newline at end of file + return data[-1] diff --git a/onnxruntime/python/tools/tensorrt/perf/post.py b/onnxruntime/python/tools/tensorrt/perf/post.py index 6285a64772..037be08571 100644 --- a/onnxruntime/python/tools/tensorrt/perf/post.py +++ b/onnxruntime/python/tools/tensorrt/perf/post.py @@ -25,6 +25,7 @@ latency = 'latency' status = 'status' latency_over_time = 'latency_over_time' specs = 'specs' +session = 'session' time_string_format = '%Y-%m-%d %H:%M:%S' @@ -92,16 +93,23 @@ def get_status(status, model_group): status = adjust_columns(status, status_columns, status_db_columns, model_group) return status -def get_specs(specs, branch, commit_id): +def get_specs(specs, branch, commit_id, upload_time): specs = specs.append({'.': 6, 'Spec': 'Branch', 'Version' : branch}, ignore_index=True) specs = specs.append({'.': 7, 'Spec': 'CommitId', 'Version' : commit_id}, ignore_index=True) + specs = specs.append({'.': 8, 'Spec': 'UploadTime', 'Version' : upload_time}, ignore_index=True) return specs -def write_table(ingest_client, table, table_name, trt_version, upload_time): +def get_session(session, model_group): + session_columns = session.keys() + session_db_columns = [model_title] + ort_provider_list + session = adjust_columns(session, session_columns, session_db_columns, model_group) + return session + +def write_table(ingest_client, table, table_name, upload_time, identifier): if table.empty: return - table = table.assign(TrtVersion=trt_version) # add TrtVersion table = table.assign(UploadTime=upload_time) # add UploadTime + table = table.assign(Identifier=identifier) # add Identifier ingestion_props = IngestionProperties( database=database, table=table_name, @@ -115,6 +123,9 @@ def get_time(): date_time = time.strftime(time_string_format) return date_time +def get_identifier(commit_id, trt_version, branch): + return commit_id + '_' + trt_version + '_' + branch + def main(): args = parse_arguments() @@ -123,14 +134,15 @@ def main(): kcsb_ingest = KustoConnectionStringBuilder.with_az_cli_authentication(cluster_ingest) ingest_client = QueuedIngestClient(kcsb_ingest) date_time = get_time() - + identifier = get_identifier(args.commit_hash, args.trt_version, args.branch) + try: result_file = args.report_folder folders = os.listdir(result_file) os.chdir(result_file) - tables = [fail, memory, latency, status, latency_over_time, specs] + tables = [fail, memory, latency, status, latency_over_time, specs, session] table_results = {} for table_name in tables: table_results[table_name] = pd.DataFrame() @@ -140,8 +152,10 @@ def main(): csv_filenames = os.listdir() for csv in csv_filenames: table = parse_csv(csv) + if session in csv: + table_results[session] = table_results[session].append(get_session(table, model_group), ignore_index=True) if specs in csv: - table_results[specs] = table_results[specs].append(get_specs(table, args.branch, args.commit_hash), ignore_index=True) + table_results[specs] = table_results[specs].append(get_specs(table, args.branch, args.commit_hash, date_time), ignore_index=True) if fail in csv: table_results[fail] = table_results[fail].append(get_failures(table, model_group), ignore_index=True) if latency in csv: @@ -154,7 +168,7 @@ def main(): for table in tables: print('writing ' + table + ' to database') db_table_name = 'ep_model_' + table - write_table(ingest_client, table_results[table], db_table_name, args.trt_version, date_time) + write_table(ingest_client, table_results[table], db_table_name, date_time, identifier) except BaseException as e: print(str(e)) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml index 6b18a83e93..1282b6e669 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml @@ -171,6 +171,12 @@ jobs: scriptLocation: inlineScript scriptType: bash inlineScript: | - python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory)/Artifact/result -c $(Build.SourceVersion) -u "https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" -t ${{ parameters.TrtVersion }} -b $(branch) + short_hash=$(git rev-parse --short HEAD^) && + python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory)/Artifact/result -c $short_hash -u "https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" -t ${{ parameters.TrtVersion }} -b $(branch) + + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - template: templates/clean-agent-build-directory-step.yml \ No newline at end of file