mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-27 20:02:15 +00:00
Track Session Creation Time (#10281)
* add back previous changes lost in merge * post session to dashboard * post session creation time to dashboard * fix trt 8 functionality: * add component governance * Remove hardcoded values * Update linux-gpu-tensorrt-daily-perf-pipeline.yml for Azure Pipelines * cleanup errors * post results only once * checkout 8.0 GA * try build 8.0 without building shared lib * add back build_shared_lib, not the problem * add upload_time to table * use identifier to post * Shorten to TRT x.x * shorten commit hash using rev_parse * use shortened commit hash * use nvidia's default TRT_VERSION
This commit is contained in:
Olivia Jain
GitHub
parent
d2b1424968
commit
eee627fde9
8 changed files with 122 additions and 31 deletions
|
|
@ -6,7 +6,6 @@
|
|||
|
||||
# nVidia TensorRT Base Image
|
||||
ARG TRT_CONTAINER_VERSION=21.12
|
||||
ARG TRT_VERSION=8.2.1.8
|
||||
FROM nvcr.io/nvidia/tensorrt:${TRT_CONTAINER_VERSION}-py3
|
||||
|
||||
ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime
|
||||
|
|
@ -26,8 +25,8 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR
|
|||
|
||||
# Checkout appropriate TRT_VERSION and build
|
||||
RUN cd onnxruntime &&\
|
||||
trt_version=${TRT_VERSION%.*.*} &&\
|
||||
trt_version=${TRT_VERSION:0:3} &&\
|
||||
./onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh ${trt_version} &&\
|
||||
/bin/sh build.sh --parallel --build_shared_lib --skip_submodule_sync --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /usr/lib/x86_64-linux-gnu/ --config Release --build_wheel --skip_tests --skip_submodule_sync --cmake_extra_defines '"CMAKE_CUDA_ARCHITECTURES='${CMAKE_CUDA_ARCHITECTURES}'"' &&\
|
||||
/bin/sh build.sh --parallel --build_shared_lib --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /usr/lib/x86_64-linux-gnu/ --config Release --build_wheel --skip_tests --skip_submodule_sync --cmake_extra_defines '"CMAKE_CUDA_ARCHITECTURES='${CMAKE_CUDA_ARCHITECTURES}'"' &&\
|
||||
pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\
|
||||
cd ..
|
||||
cd ..
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ trt_native_fp16_gain = 'TRT_Standalone_fp16_gain(%)'
|
|||
FAIL_MODEL_FILE = ".fail_model_map"
|
||||
LATENCY_FILE = ".latency_map"
|
||||
METRICS_FILE = ".metrics_map"
|
||||
SESSION_FILE = ".session_map"
|
||||
MEMORY_FILE = './temp_memory.csv'
|
||||
|
||||
def split_and_sort_output(string_list):
|
||||
|
|
@ -612,7 +613,6 @@ def update_metrics_map_ori(model_to_metrics, name, ep_to_operator):
|
|||
logger.info('TRT FP16 operator map:')
|
||||
pp.pprint(trt_fp16_op_map)
|
||||
|
||||
|
||||
###################################################################################################
|
||||
#
|
||||
# model: {ep1: {error_type: xxx, error_message: xxx}, ep2: {error_type: xx, error_message: xx}}
|
||||
|
|
@ -910,13 +910,18 @@ def run_symbolic_shape_inference(model_path, new_model_path):
|
|||
logger.error(e)
|
||||
return False, "Symbolic shape inference error"
|
||||
|
||||
def time_and_create_session(model_path, providers, session_options):
|
||||
start = datetime.now()
|
||||
session = onnxruntime.InferenceSession(model_path, providers=providers, sess_options=session_options)
|
||||
end = datetime.now()
|
||||
creation_time = (end - start).total_seconds()
|
||||
return session, creation_time
|
||||
|
||||
def create_session(model_path, providers, session_options):
|
||||
logger.info(model_path)
|
||||
|
||||
try:
|
||||
session = onnxruntime.InferenceSession(model_path, providers=providers, sess_options=session_options)
|
||||
return session
|
||||
|
||||
return time_and_create_session(model_path, providers, session_options)
|
||||
except Exception as e:
|
||||
# shape inference required on model
|
||||
if "shape inference" in str(e):
|
||||
|
|
@ -926,9 +931,8 @@ def create_session(model_path, providers, session_options):
|
|||
status = run_symbolic_shape_inference(model_path, new_model_path)
|
||||
if not status[0]: # symbolic shape inference error
|
||||
e = status[1]
|
||||
raise Exception(e)
|
||||
session = onnxruntime.InferenceSession(new_model_path, providers=providers, sess_options=session_options)
|
||||
return session
|
||||
raise Exception(e)
|
||||
return time_and_create_session(new_model_path, providers, session_options)
|
||||
else:
|
||||
raise Exception(e)
|
||||
|
||||
|
|
@ -938,7 +942,8 @@ def run_onnxruntime(args, models):
|
|||
model_to_latency = {} # model -> cuda and tensorrt latency
|
||||
model_to_metrics = {} # model -> metrics from profiling file
|
||||
model_to_fail_ep = {} # model -> failing ep
|
||||
|
||||
model_to_session = {} # models -> session creation time
|
||||
|
||||
ep_list = []
|
||||
if args.ep:
|
||||
ep_list.append(args.ep)
|
||||
|
|
@ -1021,7 +1026,6 @@ def run_onnxruntime(args, models):
|
|||
if standalone_trt_fp16 == ep:
|
||||
fp16 = True
|
||||
|
||||
print(fp16)
|
||||
inputs, ref_outputs = get_test_data(fp16, test_data_dir, all_inputs_shape)
|
||||
# generate random input data
|
||||
if args.input_data == "random":
|
||||
|
|
@ -1045,7 +1049,7 @@ def run_onnxruntime(args, models):
|
|||
|
||||
# create onnxruntime inference session
|
||||
try:
|
||||
sess = create_session(model_path, providers, options)
|
||||
sess, _ = create_session(model_path, providers, options)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
|
|
@ -1133,12 +1137,16 @@ def run_onnxruntime(args, models):
|
|||
|
||||
# create onnxruntime inference session
|
||||
try:
|
||||
sess = create_session(model_path, ep_to_provider_list[ep], options)
|
||||
sess, creation_time = create_session(model_path, ep_to_provider_list[ep], options)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
update_fail_model_map(model_to_fail_ep, name, ep, 'runtime error', e)
|
||||
continue
|
||||
|
||||
if creation_time:
|
||||
model_to_session[name] = copy.deepcopy({ep: creation_time})
|
||||
|
||||
sess.disable_fallback()
|
||||
|
||||
logger.info("start to inference {} with {} ...".format(name, ep))
|
||||
|
|
@ -1199,7 +1207,7 @@ def run_onnxruntime(args, models):
|
|||
|
||||
# end of model
|
||||
|
||||
return success_results, model_to_latency, model_to_fail_ep, model_to_metrics
|
||||
return success_results, model_to_latency, model_to_fail_ep, model_to_metrics, model_to_session
|
||||
|
||||
def calculate_gain(value, ep1, ep2):
|
||||
ep1_latency = float(value[ep1]['average_latency_ms'])
|
||||
|
|
@ -1356,6 +1364,54 @@ def output_specs(info, csv_filename):
|
|||
'Version': [cpu_version, gpu_version, tensorrt_version, cuda_version, cudnn_version]})
|
||||
table.to_csv(csv_filename, index=False)
|
||||
|
||||
def output_session_creation(results, csv_filename):
|
||||
need_write_header = True
|
||||
if os.path.exists(csv_filename):
|
||||
need_write_header = False
|
||||
|
||||
with open(csv_filename, mode="a", newline='') as csv_file:
|
||||
column_names = [model_title]
|
||||
for provider in ort_provider_list:
|
||||
column_names.append(provider + session_ending)
|
||||
|
||||
csv_writer = csv.writer(csv_file)
|
||||
|
||||
|
||||
csv_writer = csv.writer(csv_file)
|
||||
|
||||
if need_write_header:
|
||||
csv_writer.writerow(column_names)
|
||||
|
||||
cpu_time = ""
|
||||
cuda_fp32_time = ""
|
||||
trt_fp32_time = ""
|
||||
cuda_fp16_time = ""
|
||||
trt_fp16_time = ""
|
||||
|
||||
for model_name, ep_dict in results.items():
|
||||
for ep, time in ep_dict.items():
|
||||
if ep == cpu:
|
||||
cpu_time = time
|
||||
elif ep == cuda:
|
||||
cuda_fp32_time = time
|
||||
elif ep == trt:
|
||||
trt_fp32_time = time
|
||||
elif ep == cuda_fp16:
|
||||
cuda_fp16_time = time
|
||||
elif ep == trt_fp16:
|
||||
trt_fp16_time = time
|
||||
else:
|
||||
continue
|
||||
|
||||
row = [model_name,
|
||||
cpu_time,
|
||||
cuda_fp32_time,
|
||||
trt_fp32_time,
|
||||
cuda_fp16_time,
|
||||
trt_fp16_time]
|
||||
csv_writer.writerow(row)
|
||||
|
||||
|
||||
def output_latency(results, csv_filename):
|
||||
need_write_header = True
|
||||
if os.path.exists(csv_filename):
|
||||
|
|
@ -1657,7 +1713,7 @@ def main():
|
|||
parse_models_helper(args, models)
|
||||
|
||||
perf_start_time = datetime.now()
|
||||
success_results, model_to_latency, model_to_fail_ep, model_to_metrics = run_onnxruntime(args, models)
|
||||
success_results, model_to_latency, model_to_fail_ep, model_to_metrics, model_to_session = run_onnxruntime(args, models)
|
||||
perf_end_time = datetime.now()
|
||||
|
||||
logger.info("Done running the perf.")
|
||||
|
|
@ -1720,5 +1776,8 @@ def main():
|
|||
csv_filename = os.path.join(path, csv_filename)
|
||||
output_metrics(model_to_metrics, csv_filename)
|
||||
|
||||
if len(model_to_session) > 0:
|
||||
write_map_to_file(model_to_session, SESSION_FILE)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ def main():
|
|||
benchmark_success_csv = 'success.csv'
|
||||
benchmark_latency_csv = 'latency.csv'
|
||||
benchmark_status_csv = 'status.csv'
|
||||
benchmark_session_csv = 'session.csv'
|
||||
specs_csv = 'specs.csv'
|
||||
|
||||
for model, model_info in models.items():
|
||||
|
|
@ -119,6 +120,16 @@ def main():
|
|||
output_metrics(model_to_metrics, os.path.join(path, benchmark_metrics_csv))
|
||||
logger.info("\nSaved model metrics results to {}".format(benchmark_metrics_csv))
|
||||
|
||||
logger.info("\n=========================================")
|
||||
logger.info("======= Models/EPs session creation =======")
|
||||
logger.info("=========================================")
|
||||
|
||||
if os.path.exists(SESSION_FILE):
|
||||
model_to_session = read_map_from_file(SESSION_FILE)
|
||||
pretty_print(pp, model_to_session)
|
||||
output_session_creation(model_to_session, os.path.join(path, benchmark_session_csv))
|
||||
logger.info("\nSaved session creation results to {}".format(benchmark_session_csv))
|
||||
|
||||
elif args.running_mode == "benchmark":
|
||||
logger.info("\n=========================================================")
|
||||
logger.info("========== Failing Models/EPs (accumulated) ==============")
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ o) TRT_DOCKERFILE_PATH=${OPTARG};;
|
|||
p) PERF_DOCKERFILE_PATH=${OPTARG};;
|
||||
b) ORT_BRANCH=${OPTARG};;
|
||||
i) IMAGE_NAME=${OPTARG};;
|
||||
t) TRT=${OPTARG};;
|
||||
t) TRT_CONTAINER=${OPTARG};;
|
||||
v) TRT_VERSION=${OPTARG};;
|
||||
c) CMAKE_CUDA_ARCHITECTURES=${OPTARG};;
|
||||
esac
|
||||
|
|
@ -15,5 +15,5 @@ done
|
|||
|
||||
IMAGE=onnxruntime
|
||||
|
||||
docker build --no-cache -t $IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg TRT=$TRT --build-arg TRT_VERSION=$TRT_VERSION --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH -f $TRT_DOCKERFILE_PATH .
|
||||
docker build --no-cache --build-arg IMAGE=$IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH --build-arg TRT_VERSION=$TRT_VERSION -t $IMAGE_NAME -f $PERF_DOCKERFILE_PATH .
|
||||
docker build --no-cache -t $IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg TRT_CONTAINER_VERSION=$TRT_CONTAINER --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH -f $TRT_DOCKERFILE_PATH .
|
||||
docker build --no-cache --build-arg IMAGE=$IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH --build-arg TRT_VERSION=$TRT_VERSION -t $IMAGE_NAME -f $PERF_DOCKERFILE_PATH .
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo "checking out onnx-tensorrt for correct version"
|
||||
echo "$1"
|
||||
if [ ! "$1" = "8.2" ]
|
||||
then
|
||||
|
|
@ -15,4 +16,4 @@ then
|
|||
git checkout "$1"'.1'
|
||||
fi
|
||||
cd $CUR_PWD
|
||||
fi
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -33,7 +33,8 @@ group_title = 'Group'
|
|||
avg_ending = ' \nmean (ms)'
|
||||
percentile_ending = ' \n90th percentile (ms)'
|
||||
memory_ending = ' \npeak memory usage (MiB)'
|
||||
|
||||
session_ending = ' \n session creation time (s)'
|
||||
ort_provider_list = [cpu, cuda, trt, cuda_fp16, trt_fp16]
|
||||
provider_list = [cpu, cuda, trt, standalone_trt, cuda_fp16, trt_fp16, standalone_trt_fp16]
|
||||
table_headers = [model_title] + provider_list
|
||||
|
||||
|
|
@ -260,4 +261,4 @@ def get_profile_metrics(path, profile_already_parsed, logger=None):
|
|||
logger.info("No profile metrics got.")
|
||||
return None
|
||||
|
||||
return data[-1]
|
||||
return data[-1]
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ latency = 'latency'
|
|||
status = 'status'
|
||||
latency_over_time = 'latency_over_time'
|
||||
specs = 'specs'
|
||||
session = 'session'
|
||||
|
||||
time_string_format = '%Y-%m-%d %H:%M:%S'
|
||||
|
||||
|
|
@ -92,16 +93,23 @@ def get_status(status, model_group):
|
|||
status = adjust_columns(status, status_columns, status_db_columns, model_group)
|
||||
return status
|
||||
|
||||
def get_specs(specs, branch, commit_id):
|
||||
def get_specs(specs, branch, commit_id, upload_time):
|
||||
specs = specs.append({'.': 6, 'Spec': 'Branch', 'Version' : branch}, ignore_index=True)
|
||||
specs = specs.append({'.': 7, 'Spec': 'CommitId', 'Version' : commit_id}, ignore_index=True)
|
||||
specs = specs.append({'.': 8, 'Spec': 'UploadTime', 'Version' : upload_time}, ignore_index=True)
|
||||
return specs
|
||||
|
||||
def write_table(ingest_client, table, table_name, trt_version, upload_time):
|
||||
def get_session(session, model_group):
|
||||
session_columns = session.keys()
|
||||
session_db_columns = [model_title] + ort_provider_list
|
||||
session = adjust_columns(session, session_columns, session_db_columns, model_group)
|
||||
return session
|
||||
|
||||
def write_table(ingest_client, table, table_name, upload_time, identifier):
|
||||
if table.empty:
|
||||
return
|
||||
table = table.assign(TrtVersion=trt_version) # add TrtVersion
|
||||
table = table.assign(UploadTime=upload_time) # add UploadTime
|
||||
table = table.assign(Identifier=identifier) # add Identifier
|
||||
ingestion_props = IngestionProperties(
|
||||
database=database,
|
||||
table=table_name,
|
||||
|
|
@ -115,6 +123,9 @@ def get_time():
|
|||
date_time = time.strftime(time_string_format)
|
||||
return date_time
|
||||
|
||||
def get_identifier(commit_id, trt_version, branch):
|
||||
return commit_id + '_' + trt_version + '_' + branch
|
||||
|
||||
def main():
|
||||
|
||||
args = parse_arguments()
|
||||
|
|
@ -123,14 +134,15 @@ def main():
|
|||
kcsb_ingest = KustoConnectionStringBuilder.with_az_cli_authentication(cluster_ingest)
|
||||
ingest_client = QueuedIngestClient(kcsb_ingest)
|
||||
date_time = get_time()
|
||||
|
||||
identifier = get_identifier(args.commit_hash, args.trt_version, args.branch)
|
||||
|
||||
try:
|
||||
result_file = args.report_folder
|
||||
|
||||
folders = os.listdir(result_file)
|
||||
os.chdir(result_file)
|
||||
|
||||
tables = [fail, memory, latency, status, latency_over_time, specs]
|
||||
tables = [fail, memory, latency, status, latency_over_time, specs, session]
|
||||
table_results = {}
|
||||
for table_name in tables:
|
||||
table_results[table_name] = pd.DataFrame()
|
||||
|
|
@ -140,8 +152,10 @@ def main():
|
|||
csv_filenames = os.listdir()
|
||||
for csv in csv_filenames:
|
||||
table = parse_csv(csv)
|
||||
if session in csv:
|
||||
table_results[session] = table_results[session].append(get_session(table, model_group), ignore_index=True)
|
||||
if specs in csv:
|
||||
table_results[specs] = table_results[specs].append(get_specs(table, args.branch, args.commit_hash), ignore_index=True)
|
||||
table_results[specs] = table_results[specs].append(get_specs(table, args.branch, args.commit_hash, date_time), ignore_index=True)
|
||||
if fail in csv:
|
||||
table_results[fail] = table_results[fail].append(get_failures(table, model_group), ignore_index=True)
|
||||
if latency in csv:
|
||||
|
|
@ -154,7 +168,7 @@ def main():
|
|||
for table in tables:
|
||||
print('writing ' + table + ' to database')
|
||||
db_table_name = 'ep_model_' + table
|
||||
write_table(ingest_client, table_results[table], db_table_name, args.trt_version, date_time)
|
||||
write_table(ingest_client, table_results[table], db_table_name, date_time, identifier)
|
||||
|
||||
except BaseException as e:
|
||||
print(str(e))
|
||||
|
|
|
|||
|
|
@ -171,6 +171,12 @@ jobs:
|
|||
scriptLocation: inlineScript
|
||||
scriptType: bash
|
||||
inlineScript: |
|
||||
python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory)/Artifact/result -c $(Build.SourceVersion) -u "https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" -t ${{ parameters.TrtVersion }} -b $(branch)
|
||||
short_hash=$(git rev-parse --short HEAD^) &&
|
||||
python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory)/Artifact/result -c $short_hash -u "https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" -t ${{ parameters.TrtVersion }} -b $(branch)
|
||||
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
|
||||
- template: templates/clean-agent-build-directory-step.yml
|
||||
Loading…
Reference in a new issue