[EP Perf] Customize onnx-tensorrt commit id when init CI tasks (#20175)

### Description
<!-- Describe your changes. -->
Customize commit id of onnx-tensorrt in EP Perf CI variables when
testing OSS parsers in different versions

### To Verify

![image](https://github.com/microsoft/onnxruntime/assets/109183385/9dc650d8-377d-4223-8951-f0849b1fe984)

After assigning `onnxTensorrtCommitId` in EP Perf CI Variables, 
CI would prompt during the step of **[Build latest ORT Image with
TensorRT OSS
parser](https://aiinfra.visualstudio.com/Lotus/_build/results?buildId=438217&view=logs&j=b6bfa4e2-8141-507f-8ca1-59b3f929fa71&t=fc64e110-ab59-54e4-1c37-853e84a52a7e&l=396450)**:
```
Updated deps.txt with new commit id a43ce67187bab219520fd80f21af8bbd4354bc8c and hash 572535aefef477050f86744dfab1fef840198035
```
And CI would [overwrite the line of onnx_tensorrt in
deps.txt](https://aiinfra.visualstudio.com/Lotus/_build/results?buildId=438217&view=logs&j=b6bfa4e2-8141-507f-8ca1-59b3f929fa71&t=fc64e110-ab59-54e4-1c37-853e84a52a7e&l=396451)
which was assigned as:
```
onnx_tensorrt;a43ce67187.zip;572535aefef477050f86744dfab1fef840198035

```


### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
To save time of modifying deps.txt and manually calculating zip hash
This commit is contained in:
Yifan Li 2024-04-10 09:46:05 -07:00 committed by GitHub
parent 471e969e2f
commit 9577fe454d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 56 additions and 2 deletions

View file

@ -10,6 +10,7 @@ import argparse
import os
import pty
import shlex
import subprocess
import sys
from typing import List, Optional
@ -172,6 +173,55 @@ def docker_build_trt_bin(args: argparse.Namespace):
sys.exit(1)
def overwrite_onnx_tensorrt_commit_id(commit_id):
"""
Overwrite onnx-tensorrt commit id in cmake/deps.txt.
"""
deps_file_path = "../../../../../../cmake/deps.txt"
line_index = None
zip_url = None
with open(deps_file_path) as file:
lines = file.readlines()
for i, line in enumerate(lines):
if line.startswith("onnx_tensorrt"):
parts = line.split(";")
zip_url = ";".join([parts[0], f"https://github.com/onnx/onnx-tensorrt/archive/{commit_id}.zip", parts[2]])
line_index = i
break
if line_index and zip_url:
wget_command = f"wget {zip_url.split(';')[1]} -O temp.zip"
subprocess.run(wget_command, shell=True, check=True)
sha1sum_command = "sha1sum temp.zip"
result = subprocess.run(sha1sum_command, shell=True, capture_output=True, text=True, check=True)
hash_value = result.stdout.split()[0]
lines[line_index] = zip_url.split(";")[0] + ";" + zip_url.split(";")[1] + ";" + hash_value + "\n"
with open(deps_file_path, "w") as file:
file.writelines(lines)
print(f"Updated deps.txt with new commit id {commit_id} and hash {hash_value}")
# Verify updated deps.txt
try:
with open(deps_file_path) as file:
lines = file.readlines()
for line in lines:
if line.startswith("onnx_tensorrt"):
print(line.strip())
break
except Exception as e:
print(f"Failed to read the file: {e}")
os.remove("temp.zip")
else:
print("onnx_tensorrt commit id overwrite failed, entry not found in deps.txt")
def parse_arguments() -> argparse.Namespace:
"""
Parses command-line arguments and returns an object with each argument as a field.
@ -187,6 +237,7 @@ def parse_arguments() -> argparse.Namespace:
"-t", "--trt_version", default="8.6.cuda_11_8_cudnn_8", help="TensorRT version (e.g., 8.6.cuda_11_8_cudnn_8)"
)
parser.add_argument("-a", "--cuda_arch", default="75", help="CUDA architecture (e.g., 75)")
parser.add_argument("-o", "--oss_parser_commit_id", default="", help="commit id of onnx-tensorrt")
# Command-line options for installing TensorRT from binaries.
parser.add_argument(
@ -223,6 +274,8 @@ def main() -> int:
if args.install_bin:
docker_build_trt_bin(args)
else:
if args.oss_parser_commit_id != "":
overwrite_onnx_tensorrt_commit_id(args.oss_parser_commit_id)
docker_build_trt(args)
return 0

View file

@ -311,7 +311,7 @@ def get_specs(specs, branch, commit_hash, commit_datetime):
:return: The updated table.
"""
init_id = int(specs.tail(1).get(".", 0)) + 1
init_id = int(specs.tail(1).get(".", 0).iloc[0]) + 1
specs_additional = pd.DataFrame(
{
".": [init_id, init_id + 1, init_id + 2],
@ -417,6 +417,7 @@ def main():
identifier = get_identifier(
args.commit_datetime, args.commit_hash, args.trt_version, args.branch, args.use_tensorrt_oss_parser
)
print(f"DB record identifier: {identifier}")
upload_time = datetime.datetime.now(tz=datetime.timezone.utc).replace(microsecond=0)
try:

View file

@ -107,7 +107,7 @@ jobs:
# Build ORT with TensorRT OSS parser
- ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, true)) }}:
- script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 --use_tensorrt_oss_parser'
- script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 -o $(onnxTensorrtCommitId) --use_tensorrt_oss_parser'
displayName: 'Build latest ORT Image with TensorRT OSS parser'
workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build'