mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-24 19:43:35 +00:00
f70215d4e6
1. Update google benchmark from 1.8.3 to 1.8.5 2. Update google test from commit in main branch to tag 1.15.0 3. Update pybind11 from 2.12.0 to 2.13.1 4. Update pytorch cpuinfo to include the support for Arm Neoverse V2, Cortex X4, A720 and A520. 5. Update re2 from 2024-05-01 to 2024-07-02 6. Update cmake to 3.30.1 7. Update Linux docker images 8. Fix a warning in test/perftest/ort_test_session.cc:826:37: error: implicit conversion loses integer precision: 'streamoff' (aka 'long long') to 'const std::streamsize' (aka 'const long') [-Werror,-Wshorten-64-to-32]
223 lines
8.1 KiB
YAML
223 lines
8.1 KiB
YAML
##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
|
|
trigger:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
pr:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
#### end trigger ####
|
|
parameters:
|
|
- name: CudaVersion
|
|
displayName: CUDA version
|
|
type: string
|
|
default: '11.8'
|
|
values:
|
|
- 11.8
|
|
- 12.2
|
|
|
|
- name: SpecificArtifact
|
|
displayName: Use Specific Artifact
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: BuildId
|
|
displayName: Specific Artifact's BuildId
|
|
type: string
|
|
default: '0'
|
|
|
|
variables:
|
|
- name: docker_base_image
|
|
${{ if eq(parameters.CudaVersion, '11.8') }}:
|
|
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240531.1
|
|
${{ if eq(parameters.CudaVersion, '12.2') }}:
|
|
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20240719.1
|
|
|
|
- name: Repository
|
|
${{ if eq(parameters.CudaVersion, '11.8') }}:
|
|
value: 'onnxruntimecuda11manylinuxbuild'
|
|
${{ if eq(parameters.CudaVersion, '12.2') }}:
|
|
value: 'onnxruntimecuda12manylinuxbuild'
|
|
|
|
stages:
|
|
- stage: Linux_Build
|
|
jobs:
|
|
- job: Linux_Build
|
|
timeoutInMinutes: 120
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
CCACHE_DIR: $(Pipeline.Workspace)/ccache
|
|
workspace:
|
|
clean: all
|
|
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
|
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "--build-arg BASEIMAGE=$(docker_base_image) --build-arg BUILD_UID=$( id -u )"
|
|
Repository: $(Repository)
|
|
|
|
- task: Cache@2
|
|
inputs:
|
|
key: '"ccache" | "${{parameters.CudaVersion}}" |"$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
|
|
path: $(CCACHE_DIR)
|
|
restoreKeys: |
|
|
"ccache" | "${{parameters.CudaVersion}}" | "$(Build.SourceBranch)"
|
|
"ccache"
|
|
cacheHitVar: CACHE_RESTORED
|
|
displayName: Cach Task
|
|
|
|
- script: |
|
|
sudo mkdir -p $(Pipeline.Workspace)/ccache
|
|
condition: ne(variables.CACHE_RESTORED, 'true')
|
|
displayName: Create Cache Dir
|
|
|
|
- script: |
|
|
set -e -x
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume $(Pipeline.Workspace)/ccache:/cache \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
-e CCACHE_DIR=/cache -w /onnxruntime_src \
|
|
$(Repository) tools/ci_build/github/linux/build_cuda_ci.sh
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
displayName: Build Onnxruntime
|
|
|
|
- script: $(Build.SourcesDirectory)/tools/ci_build/github/linux/delete_unused_files_before_upload.sh
|
|
|
|
- task: PublishPipelineArtifact@0
|
|
displayName: 'Publish Pipeline Artifact'
|
|
inputs:
|
|
artifactName: 'drop-linux'
|
|
targetPath: '$(Build.BinariesDirectory)/Release'
|
|
|
|
- template: templates/explicitly-defined-final-tasks.yml
|
|
|
|
- stage: Linux_Test
|
|
dependsOn:
|
|
- Linux_Build
|
|
jobs:
|
|
- job: Linux_Test
|
|
timeoutInMinutes: 180
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
workspace:
|
|
clean: all
|
|
pool: onnxruntime-Linux-GPU-T4
|
|
steps:
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: templates/flex-downloadPipelineArtifact.yml
|
|
parameters:
|
|
ArtifactName: 'drop-linux'
|
|
StepName: 'Download Pipeline Artifact - Linux Build'
|
|
TargetPath: '$(Build.BinariesDirectory)/Release'
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "--build-arg BASEIMAGE=$(docker_base_image) --build-arg BUILD_UID=$( id -u )"
|
|
Repository: $(Repository)
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
set -e -x
|
|
mkdir -p $HOME/.onnx
|
|
docker run --gpus all --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory)/Release:/build/Release \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume /data/onnx:/data/onnx \
|
|
-e NVIDIA_TF32_OVERRIDE=0 \
|
|
$(Repository) \
|
|
/bin/bash -c '
|
|
nvidia-smi; \
|
|
/sbin/ldconfig -N -v $(sed "s/:/ /" <<< $LD_LIBRARY_PATH) 2>/dev/null | grep -E "libcudart.so|libcudnn.so|libnvinfer.so"; \
|
|
cat /usr/local/cuda/include/cuda.h | grep -m1 CUDA_VERSION; \
|
|
cat /usr/include/cudnn_version.h | grep CUDNN_MAJOR -m1 -A 2; \
|
|
ln -s /opt/python/cp38-cp38/bin/python3 /tmp/python3; \
|
|
/tmp/python3 -m pip install /build/Release/dist/*.whl; \
|
|
/tmp/python3 -u -c "from onnxruntime.capi._pybind_state import (OrtDevice as C_OrtDevice) ; \
|
|
ort_device = C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0); \
|
|
print(ort_device); print(ort_device.device_type(), C_OrtDevice.cuda()); \
|
|
assert(ort_device.device_type()==1); assert(C_OrtDevice.cuda()==1);" \
|
|
'
|
|
displayName: 'Check GPU'
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
set -e -x
|
|
mkdir -p $HOME/.onnx
|
|
docker run --gpus all --shm-size=1g --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory)/Release:/build/Release \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume /data/onnx:/data/onnx \
|
|
-e NVIDIA_TF32_OVERRIDE=0 \
|
|
$(Repository) \
|
|
/bin/bash -c '
|
|
set -ex; \
|
|
cp /onnxruntime_src/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt /tmp/requirements.txt; \
|
|
ln -s /opt/python/cp38-cp38/bin/python3 /tmp/python3; \
|
|
/tmp/python3 -m pip install -r /tmp/requirements.txt; \
|
|
/tmp/python3 -m pip install /build/Release/dist/*.whl; \
|
|
cd /build/Release && xargs -a /build/Release/perms.txt chmod a+x; \
|
|
cd /onnxruntime_src/java && /onnxruntime_src/java/gradlew cmakeCheck -DcmakeBuildDir=/build/Release -DUSE_CUDA=1; \
|
|
cd /tmp; \
|
|
/tmp/python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests \
|
|
--enable_transformers_tool_test --use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda --cudnn_home=/usr/local/cuda \
|
|
--enable_pybind --build_java --ctest_path "" ; \
|
|
'
|
|
displayName: 'Run Tests'
|
|
|
|
- template: templates/check_test_result.yml
|
|
parameters:
|
|
FileName: '$(Build.BinariesDirectory)/Release/onnxruntime_test_all.Release.results.xml'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|