mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
57dfd15d7b
### Description
1. Remove 'dnf update' from docker build scripts, because it upgrades TRT
packages from CUDA 11.x to CUDA 12.x.
To reproduce it, you can run the following commands in a CentOS CUDA
11.x docker image such as nvidia/cuda:11.8.0-cudnn8-devel-ubi8.
```
export v=8.6.1.6-1.cuda11.8
dnf install -y libnvinfer8-${v} libnvparsers8-${v} libnvonnxparsers8-${v} libnvinfer-plugin8-${v} libnvinfer-vc-plugin8-${v} libnvinfer-devel-${v} libnvparsers-devel-${v} libnvonnxparsers-devel-${v} libnvinfer-plugin-devel-${v} libnvinfer-vc-plugin-devel-${v} libnvinfer-headers-devel-${v} libnvinfer-headers-plugin-devel-${v}
dnf update -y
```
The last command will generate the following outputs:
```
========================================================================================================================
Package Architecture Version Repository Size
========================================================================================================================
Upgrading:
libnvinfer-devel x86_64 8.6.1.6-1.cuda12.0 cuda 542 M
libnvinfer-headers-devel x86_64 8.6.1.6-1.cuda12.0 cuda 118 k
libnvinfer-headers-plugin-devel x86_64 8.6.1.6-1.cuda12.0 cuda 14 k
libnvinfer-plugin-devel x86_64 8.6.1.6-1.cuda12.0 cuda 13 M
libnvinfer-plugin8 x86_64 8.6.1.6-1.cuda12.0 cuda 13 M
libnvinfer-vc-plugin-devel x86_64 8.6.1.6-1.cuda12.0 cuda 107 k
libnvinfer-vc-plugin8 x86_64 8.6.1.6-1.cuda12.0 cuda 251 k
libnvinfer8 x86_64 8.6.1.6-1.cuda12.0 cuda 543 M
libnvonnxparsers-devel x86_64 8.6.1.6-1.cuda12.0 cuda 467 k
libnvonnxparsers8 x86_64 8.6.1.6-1.cuda12.0 cuda 757 k
libnvparsers-devel x86_64 8.6.1.6-1.cuda12.0 cuda 2.0 M
libnvparsers8 x86_64 8.6.1.6-1.cuda12.0 cuda 854 k
Installing dependencies:
cuda-toolkit-12-0-config-common noarch 12.0.146-1 cuda 7.7 k
cuda-toolkit-12-config-common noarch 12.2.140-1 cuda 7.9 k
libcublas-12-0 x86_64 12.0.2.224-1 cuda 361 M
libcublas-devel-12-0 x86_64 12.0.2.224-1 cuda 397 M
Transaction Summary
========================================================================================================================
```
As you can see from the output, they are CUDA 12 packages.
The problem can also be solved by lock the packages' versions by using
"dnf versionlock" command right after installing the CUDA/TRT packages.
However, going forward, to get the better reproducibility, I suggest
manually fix dnf package versions in the installation scripts like we do
for TRT now.
```bash
v="8.6.1.6-1.cuda11.8" &&\
yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo &&\
yum -y install libnvinfer8-${v} libnvparsers8-${v} libnvonnxparsers8-${v} libnvinfer-plugin8-${v} libnvinfer-vc-plugin8-${v}\
libnvinfer-devel-${v} libnvparsers-devel-${v} libnvonnxparsers-devel-${v} libnvinfer-plugin-devel-${v} libnvinfer-vc-plugin-devel-${v} libnvinfer-headers-devel-${v} libnvinfer-headers-plugin-devel-${v}
```
When we have a need to upgrade a package due to security alert or some
other reasons, we manually change the version string instead of relying
on "dnf update". Though this approach increases efforts, it can make our
pipeines more stable.
2. Move python test to docker
### Motivation and Context
Right now the nightly gpu package mixes using CUDA 11.x and CUDA 12.x
and the result package is totally not usable(crashes every time)
100 lines
3.8 KiB
YAML
100 lines
3.8 KiB
YAML
resources:
|
|
pipelines:
|
|
- pipeline: build
|
|
source: 'Python packaging pipeline'
|
|
trigger: true
|
|
branch: main # branch to pick the artifact, Used only for manual triggered pipeline runs for testing the pipeline itself
|
|
#TODO: Remove the following dependency. Running python tests should not need to use manylinux.
|
|
repositories:
|
|
- repository: manylinux # The name used to reference this repository in the checkout step
|
|
type: Github
|
|
endpoint: Microsoft
|
|
name: pypa/manylinux
|
|
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
|
|
|
stages:
|
|
- stage: Linux_Test_CPU_x86_64_stage
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test-cpu.yml
|
|
parameters:
|
|
arch: 'x86_64'
|
|
machine_pool: 'onnxruntime-Ubuntu2004-AMD-CPU'
|
|
base_image: 'registry.access.redhat.com/ubi8/ubi'
|
|
devtoolset_rootpath: /opt/rh/gcc-toolset-12/root
|
|
ld_library_path_arg: /opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64
|
|
prepend_path: '/opt/rh/gcc-toolset-12/root/usr/bin:'
|
|
|
|
- stage: Linux_Test_CPU_aarch64_stage
|
|
dependsOn: []
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test-cpu.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'aiinfra-linux-ARM64-CPU-2019'
|
|
base_image: 'arm64v8/almalinux:8'
|
|
devtoolset_rootpath: /opt/rh/gcc-toolset-12/root
|
|
ld_library_path_arg: /opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64
|
|
prepend_path: '/opt/rh/gcc-toolset-12/root/usr/bin:'
|
|
|
|
- stage: Packages_Somking_Test
|
|
dependsOn: []
|
|
jobs:
|
|
- template: templates/py-package-smoking-test.yml
|
|
parameters:
|
|
job_name: Test_MAC_Wheels
|
|
machine_pool:
|
|
vmImage: 'macOS-13'
|
|
itemPattern: '*/*mac*x86_64.whl'
|
|
- template: templates/py-package-smoking-test.yml
|
|
parameters:
|
|
job_name: Test_LINUX_x86_64_Wheels
|
|
itemPattern: '*/*manylinux*x86_64.whl'
|
|
machine_pool:
|
|
vmImage: 'ubuntu-22.04'
|
|
|
|
# ****The following Stage depend on all previous tags. ***
|
|
|
|
# GPU resources are very limited,
|
|
# To utilize gpu resource more efficiently, run GPU job only after all cpus jobs succeed
|
|
- stage: Linux_Test_GPU_x86_64_stage
|
|
dependsOn:
|
|
- Linux_Test_CPU_x86_64_stage
|
|
- Linux_Test_CPU_aarch64_stage
|
|
- Packages_Somking_Test
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test-cuda.yml
|
|
parameters:
|
|
arch: 'x86_64'
|
|
machine_pool: 'Onnxruntime-Linux-GPU'
|
|
device: 'GPU'
|
|
python_wheel_suffix: '_gpu'
|
|
timeout: 480
|
|
|
|
|
|
# if final job not extecuted, it will not run nightlly build
|
|
- stage: Final
|
|
dependsOn:
|
|
- Linux_Test_GPU_x86_64_stage
|
|
jobs:
|
|
- job: Final
|
|
# Run this step only if all previous steps are succeeded and (this build was triggered by a resource trigger or it was triggered by another build).
|
|
condition: and(succeeded(), eq(variables['Build.Reason'], 'ResourceTrigger'))
|
|
timeoutInMinutes: 60
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
workspace:
|
|
clean: all
|
|
pool: Azure-Pipelines-EO-Windows2022-aiinfra
|
|
steps:
|
|
- task: PowerShell@2
|
|
displayName: 'Add Build Tag'
|
|
env:
|
|
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
|
|
inputs:
|
|
targetType: inline
|
|
script: |
|
|
$headers = New-Object "System.Collections.Generic.Dictionary[[String],[String]]"
|
|
$headers.Add("Authorization", "Bearer $env:SYSTEM_ACCESSTOKEN")
|
|
$headers.Add("Content-Type", "application/json")
|
|
$uri = "https://dev.azure.com/aiinfra/Lotus/_apis/build/builds/$(resources.pipeline.build.runID)/tags/test%20pass?api-version=6.0"
|
|
Invoke-RestMethod -Uri $uri -Headers $headers -Method PUT
|