mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-21 19:18:55 +00:00
1dce5e1732
### Description Some test thresholds that previously worked in T4 GPU does not work anymore. The reason is current pipeline uses A10, and TF32 is enabled by default. Disable TF32 in Linux GPU CI Pipeline in testing to avoid such random test failure. ### Motivation and Context Linux Test has random failure at tests: ProviderOptionsTest > testCUDAOptions() FAILED org.opentest4j.AssertionFailedError: array contents differ at index [446], expected: <0.0419757> but was: <0.041948937> at app//org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151) at app//org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132) at app//org.junit.jupiter.api.AssertArrayEquals.failArraysNotEqual(AssertArrayEquals.java:440) at app//org.junit.jupiter.api.AssertArrayEquals.assertArrayEquals(AssertArrayEquals.java:290) at app//org.junit.jupiter.api.AssertArrayEquals.assertArrayEquals(AssertArrayEquals.java:123) at app//org.junit.jupiter.api.AssertArrayEquals.assertArrayEquals(AssertArrayEquals.java:119) at app//org.junit.jupiter.api.Assertions.assertArrayEquals(Assertions.java:1360) at app//ai.onnxruntime.providers.ProviderOptionsTest.runProvider(ProviderOptionsTest.java:99) at app//ai.onnxruntime.providers.ProviderOptionsTest.testCUDAOptions(ProviderOptionsTest.java:43) org.opentest4j.AssertionFailedError: array contents differ at index [6], expected: <0.0225981> but was: <0.022587791> at app//org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151) at app//org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132) at app//org.junit.jupiter.api.AssertArrayEquals.failArraysNotEqual(AssertArrayEquals.java:440) at app//org.junit.jupiter.api.AssertArrayEquals.assertArrayEquals(AssertArrayEquals.java:290) at app//org.junit.jupiter.api.AssertArrayEquals.assertArrayEquals(AssertArrayEquals.java:123) at app//org.junit.jupiter.api.AssertArrayEquals.assertArrayEquals(AssertArrayEquals.java:119) at app//org.junit.jupiter.api.Assertions.assertArrayEquals(Assertions.java:1360) at app//ai.onnxruntime.InferenceTest.runProvider(InferenceTest.java:676) at app//ai.onnxruntime.InferenceTest.testCUDA(InferenceTest.java:615)
223 lines
7.6 KiB
YAML
223 lines
7.6 KiB
YAML
##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
|
|
trigger:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
pr:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
#### end trigger ####
|
|
parameters:
|
|
- name: CudaVersion
|
|
displayName: CUDA version
|
|
type: string
|
|
default: '11.8'
|
|
values:
|
|
- 11.8
|
|
- 12.2
|
|
resources:
|
|
repositories:
|
|
- repository: manylinux
|
|
type: Github
|
|
endpoint: Microsoft
|
|
name: pypa/manylinux
|
|
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
|
|
|
variables:
|
|
- name: docker_base_image
|
|
${{ if eq(parameters.CudaVersion, '11.8') }}:
|
|
value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
|
|
${{ if eq(parameters.CudaVersion, '12.2') }}:
|
|
value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8
|
|
|
|
- name: linux_trt_version
|
|
${{ if eq(parameters.CudaVersion, '11.8') }}:
|
|
value: 8.6.1.6-1.cuda11.8
|
|
${{ if eq(parameters.CudaVersion, '12.2') }}:
|
|
value: 8.6.1.6-1.cuda12.0
|
|
|
|
- name: Repository
|
|
${{ if eq(parameters.CudaVersion, '11.8') }}:
|
|
value: 'onnxruntimecuda11build'
|
|
${{ if eq(parameters.CudaVersion, '12.2') }}:
|
|
value: 'onnxruntimecuda12build'
|
|
|
|
jobs:
|
|
- job: Linux_Build
|
|
timeoutInMinutes: 120
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
CCACHE_DIR: $(Pipeline.Workspace)/ccache
|
|
workspace:
|
|
clean: all
|
|
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
|
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "
|
|
--network=host
|
|
--build-arg BASEIMAGE=$(docker_base_image)
|
|
--build-arg TRT_VERSION=$(linux_trt_version)
|
|
--build-arg BUILD_UID=$( id -u )
|
|
"
|
|
Repository: $(Repository)
|
|
|
|
- task: Cache@2
|
|
inputs:
|
|
key: '"ccache" | "${{parameters.CudaVersion}}" |"$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
|
|
path: $(CCACHE_DIR)
|
|
restoreKeys: |
|
|
"ccache" | "${{parameters.CudaVersion}}" | "$(Build.SourceBranch)"
|
|
"ccache"
|
|
cacheHitVar: CACHE_RESTORED
|
|
displayName: Cach Task
|
|
|
|
- script: |
|
|
sudo mkdir -p $(Pipeline.Workspace)/ccache
|
|
condition: ne(variables.CACHE_RESTORED, 'true')
|
|
displayName: Create Cache Dir
|
|
|
|
- script: |
|
|
set -e -x
|
|
mkdir -p $HOME/.onnx
|
|
docker run -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume $(Pipeline.Workspace)/ccache:/cache \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
-e CCACHE_DIR=/cache \
|
|
$(Repository) \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
env; \
|
|
ccache -s; \
|
|
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator Ninja \
|
|
--config Release --update --build \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel --use_binskim_compliant_compile_flags \
|
|
--build_wheel \
|
|
--enable_onnx_tests --use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda-${{parameters.CudaVersion}} --cudnn_home=/usr/local/cuda-${{parameters.CudaVersion}} \
|
|
--enable_cuda_profiling --enable_cuda_nhwc_ops \
|
|
--enable_pybind --build_java \
|
|
--use_cache \
|
|
--cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86; \
|
|
ccache -sv; \
|
|
ccache -z"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
displayName: Build Onnxruntime
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11
|
|
rm -f $(Build.BinariesDirectory)/Release/models
|
|
find $(Build.BinariesDirectory)/Release/_deps -mindepth 1 ! -regex '^$(Build.BinariesDirectory)/Release/_deps/onnx-src\(/.*\)?' -delete
|
|
cd $(Build.BinariesDirectory)/Release
|
|
find -executable -type f > $(Build.BinariesDirectory)/Release/perms.txt
|
|
|
|
- task: PublishPipelineArtifact@0
|
|
displayName: 'Publish Pipeline Artifact'
|
|
inputs:
|
|
artifactName: 'drop-linux'
|
|
targetPath: '$(Build.BinariesDirectory)/Release'
|
|
|
|
- template: templates/explicitly-defined-final-tasks.yml
|
|
|
|
- job: Linux_Test
|
|
timeoutInMinutes: 180
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
workspace:
|
|
clean: all
|
|
pool: onnxruntime-Linux-GPU-A10
|
|
dependsOn:
|
|
- Linux_Build
|
|
steps:
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Pipeline Artifact'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'drop-linux'
|
|
targetPath: '$(Build.BinariesDirectory)/Release'
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "
|
|
--network=host
|
|
--build-arg BASEIMAGE=$(docker_base_image)
|
|
--build-arg TRT_VERSION=$(linux_trt_version)
|
|
--build-arg BUILD_UID=$( id -u )
|
|
"
|
|
Repository: $(Repository)
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
set -e -x
|
|
mkdir -p $HOME/.onnx
|
|
docker run --gpus all --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory)/Release:/build/Release \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume /data/onnx:/data/onnx \
|
|
-e NVIDIA_TF32_OVERRIDE=0 \
|
|
$(Repository) \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
cp /onnxruntime_src/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt /tmp/requirements.txt; \
|
|
ln -s /opt/python/cp38-cp38/bin/python3 /tmp/python3; \
|
|
/tmp/python3 -m pip install -r /tmp/requirements.txt; \
|
|
/tmp/python3 -m pip install /build/Release/dist/*.whl; \
|
|
cd /build/Release && xargs -a /build/Release/perms.txt chmod a+x; \
|
|
cd /onnxruntime_src/java && /onnxruntime_src/java/gradlew cmakeCheck -DcmakeBuildDir=/build/Release -DUSE_CUDA=1; \
|
|
cd /tmp; \
|
|
/tmp/python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests \
|
|
--use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda --cudnn_home=/usr/local/cuda \
|
|
--enable_pybind --build_java --ctest_path '' "
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|