mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-24 19:43:35 +00:00
d13cabf7f9
### Description This PR upgrades CUDA 11 build pipelines' GCC version from 8 to 11. ### Motivation and Context GCC8 has an experimental std::filesystem implementation which is not ABI compatible with the formal one in later GCC releases. It didn't cause trouble for us, however, ONNX community has encountered this issue much. For example, https://github.com/onnx/onnx/issues/6047 . So this PR increases the minimum supported GCC version from 8 to 9, and removes the references to GCC's "stdc++fs" library. Please note we compile our code on RHEL8 and RHEL8's libstdc++ doesn't have the fs library, which means the binaries in ONNX Runtime's official packages always static link to the fs library. It is just a matter of which version of the library, an experimental one or a more mature one. And it is an implementation detail that is not visible from outside. Anyway, a newer GCC is better. It will give us the chance to use many C++20 features. #### Why we were using GCC 8? It is because all our Linux packages were built on RHEL8 or its equivalents. The default GCC version in RHEL8 is 8. RHEL also provides additional GCC versions from RH devtoolset. UBI8 is the abbreviation of Red Hat Universal Base Image 8, which is the containerized RHEL8. UBI8 is free, which means it doesn't require a subscription(while RHEL does). The only devtoolset that UBI8 provides is GCC 12, which is too new for being used with CUDA 11.8. And our CUDA 11.8's build env is a docker image from Nvidia that is based on UBI8. #### How the problem is solved Almalinux is an alternative to RHEL. Almalinux 8 provides GCC 11. And the CUDA 11.8 docker image from Nvidia is open source, which means we can rebuild the image based on Almalinux 8 to get GCC 11. I've done this, but I cannot republish the new image due to various complicated license restrictions. Therefore I put them at an internal location in onnxruntimebuildcache.azurecr.io.
88 lines
2.9 KiB
YAML
88 lines
2.9 KiB
YAML
resources:
|
|
pipelines:
|
|
- pipeline: build
|
|
source: 'Python packaging pipeline'
|
|
trigger: true
|
|
branch: main # branch to pick the artifact, Used only for manual triggered pipeline runs for testing the pipeline itself
|
|
|
|
stages:
|
|
- stage: Linux_Test_CPU_x86_64_stage
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test-cpu.yml
|
|
parameters:
|
|
arch: 'x86_64'
|
|
machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
|
|
|
|
|
|
- stage: Linux_Test_CPU_aarch64_stage
|
|
dependsOn: []
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test-cpu.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
|
|
- stage: Packages_Somking_Test
|
|
dependsOn: []
|
|
jobs:
|
|
- template: templates/py-package-smoking-test.yml
|
|
parameters:
|
|
job_name: Test_MAC_Wheels
|
|
machine_pool:
|
|
vmImage: 'macOS-latest'
|
|
itemPattern: '*/*mac*x86_64.whl'
|
|
- template: templates/py-package-smoking-test.yml
|
|
parameters:
|
|
job_name: Test_LINUX_x86_64_Wheels
|
|
itemPattern: '*/*manylinux*x86_64.whl'
|
|
machine_pool:
|
|
name: 'onnxruntime-Ubuntu2204-AMD-CPU'
|
|
|
|
# ****The following Stage depend on all previous tags. ***
|
|
|
|
# GPU resources are very limited,
|
|
# To utilize gpu resource more efficiently, run GPU job only after all cpus jobs succeed
|
|
- stage: Linux_Test_GPU_x86_64_stage
|
|
dependsOn:
|
|
- Linux_Test_CPU_x86_64_stage
|
|
- Linux_Test_CPU_aarch64_stage
|
|
- Packages_Somking_Test
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test-cuda.yml
|
|
parameters:
|
|
arch: 'x86_64'
|
|
machine_pool: 'Onnxruntime-Linux-GPU'
|
|
python_wheel_suffix: '_gpu'
|
|
timeout: 480
|
|
docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240531.1
|
|
trt_version: '10.0.1.6-1.cuda11.8'
|
|
cuda_version: '11.8'
|
|
|
|
|
|
# if final job not extecuted, it will not run nightlly build
|
|
- stage: Final
|
|
dependsOn:
|
|
- Linux_Test_GPU_x86_64_stage
|
|
jobs:
|
|
- job: Final
|
|
# Run this step only if all previous steps are succeeded and (this build was triggered by a resource trigger or it was triggered by another build).
|
|
condition: and(succeeded(), eq(variables['Build.Reason'], 'ResourceTrigger'))
|
|
timeoutInMinutes: 60
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
workspace:
|
|
clean: all
|
|
pool: Onnxruntime-Win-CPU-2022
|
|
steps:
|
|
- task: PowerShell@2
|
|
displayName: 'Add Build Tag'
|
|
env:
|
|
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
|
|
inputs:
|
|
targetType: inline
|
|
script: |
|
|
$headers = New-Object "System.Collections.Generic.Dictionary[[String],[String]]"
|
|
$headers.Add("Authorization", "Bearer $env:SYSTEM_ACCESSTOKEN")
|
|
$headers.Add("Content-Type", "application/json")
|
|
$uri = "https://dev.azure.com/aiinfra/Lotus/_apis/build/builds/$(resources.pipeline.build.runID)/tags/test%20pass?api-version=6.0"
|
|
Invoke-RestMethod -Uri $uri -Headers $headers -Method PUT
|