mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-18 01:54:05 +00:00
4af593a722
1. Add python 3.13 to our python packaging pipelines 2. Because numpy 2.0.0 doesn't support thread free python, this PR also upgrades numpy to the latest 3. Delete some unused files.
225 lines
8.1 KiB
YAML
225 lines
8.1 KiB
YAML
##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
|
|
### please do rerun set-trigger-rules.py ###
|
|
trigger:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
pr:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
#### end trigger ####
|
|
parameters:
|
|
- name: CudaVersion
|
|
displayName: CUDA version
|
|
type: string
|
|
default: '12.2'
|
|
values:
|
|
- 11.8
|
|
- 12.2
|
|
|
|
- name: SpecificArtifact
|
|
displayName: Use Specific Artifact
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: BuildId
|
|
displayName: Specific Artifact's BuildId
|
|
type: string
|
|
default: '0'
|
|
|
|
variables:
|
|
- name: docker_base_image
|
|
${{ if eq(parameters.CudaVersion, '11.8') }}:
|
|
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241010.2
|
|
${{ if eq(parameters.CudaVersion, '12.2') }}:
|
|
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241010.2
|
|
|
|
- name: Repository
|
|
${{ if eq(parameters.CudaVersion, '11.8') }}:
|
|
value: 'onnxruntimecuda11manylinuxbuild'
|
|
${{ if eq(parameters.CudaVersion, '12.2') }}:
|
|
value: 'onnxruntimecuda12manylinuxbuild'
|
|
|
|
stages:
|
|
- stage: Linux_Build
|
|
jobs:
|
|
- job: Linux_Build
|
|
timeoutInMinutes: 120
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
CCACHE_DIR: $(Pipeline.Workspace)/ccache
|
|
workspace:
|
|
clean: all
|
|
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
|
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "--build-arg BASEIMAGE=$(docker_base_image) --build-arg BUILD_UID=$( id -u )"
|
|
Repository: $(Repository)
|
|
|
|
- task: Cache@2
|
|
inputs:
|
|
key: '"ccache" | "${{parameters.CudaVersion}}" |"$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
|
|
path: $(CCACHE_DIR)
|
|
restoreKeys: |
|
|
"ccache" | "${{parameters.CudaVersion}}" | "$(Build.SourceBranch)"
|
|
"ccache"
|
|
cacheHitVar: CACHE_RESTORED
|
|
displayName: Cach Task
|
|
|
|
- script: |
|
|
sudo mkdir -p $(Pipeline.Workspace)/ccache
|
|
condition: ne(variables.CACHE_RESTORED, 'true')
|
|
displayName: Create Cache Dir
|
|
|
|
- script: |
|
|
set -e -x
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume $(Pipeline.Workspace)/ccache:/cache \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
-e CCACHE_DIR=/cache -w /onnxruntime_src \
|
|
$(Repository) tools/ci_build/github/linux/build_cuda_ci.sh
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
displayName: Build Onnxruntime
|
|
|
|
- script: $(Build.SourcesDirectory)/tools/ci_build/github/linux/delete_unused_files_before_upload.sh
|
|
|
|
- task: PublishPipelineArtifact@0
|
|
displayName: 'Publish Pipeline Artifact'
|
|
inputs:
|
|
artifactName: 'drop-linux'
|
|
targetPath: '$(Build.BinariesDirectory)/Release'
|
|
|
|
- template: templates/explicitly-defined-final-tasks.yml
|
|
|
|
- stage: Linux_Test
|
|
dependsOn:
|
|
- Linux_Build
|
|
jobs:
|
|
- job: Linux_Test
|
|
timeoutInMinutes: 180
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
workspace:
|
|
clean: all
|
|
pool: onnxruntime-Linux-GPU-T4
|
|
steps:
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: templates/flex-downloadPipelineArtifact.yml
|
|
parameters:
|
|
ArtifactName: 'drop-linux'
|
|
StepName: 'Download Pipeline Artifact - Linux Build'
|
|
TargetPath: '$(Build.BinariesDirectory)/Release'
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "--build-arg BASEIMAGE=$(docker_base_image) --build-arg BUILD_UID=$( id -u )"
|
|
Repository: $(Repository)
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
set -e -x
|
|
mkdir -p $HOME/.onnx
|
|
docker run --gpus all --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory)/Release:/build/Release \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume /data/onnx:/data/onnx \
|
|
-e NVIDIA_TF32_OVERRIDE=0 \
|
|
$(Repository) \
|
|
/bin/bash -c '
|
|
set -e
|
|
nvidia-smi; \
|
|
/sbin/ldconfig -N -v $(sed "s/:/ /" <<< $LD_LIBRARY_PATH) 2>/dev/null | grep -E "libcudart.so|libcudnn.so|libnvinfer.so"; \
|
|
cat /usr/local/cuda/include/cuda.h | grep -m1 CUDA_VERSION; \
|
|
cat /usr/include/cudnn_version.h | grep CUDNN_MAJOR -m1 -A 2; \
|
|
export PATH=/opt/python/cp312-cp312/bin:$PATH; \
|
|
python3 -m pip install /build/Release/dist/*.whl; \
|
|
python3 -u -c "from onnxruntime.capi._pybind_state import (OrtDevice as C_OrtDevice) ; \
|
|
ort_device = C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0); \
|
|
print(ort_device); print(ort_device.device_type(), C_OrtDevice.cuda()); \
|
|
assert(ort_device.device_type()==1); assert(C_OrtDevice.cuda()==1);" \
|
|
'
|
|
displayName: 'Check GPU'
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
set -e -x
|
|
mkdir -p $HOME/.onnx
|
|
docker run --gpus all --shm-size=1g --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory)/Release:/build/Release \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume /data/onnx:/data/onnx \
|
|
-e NVIDIA_TF32_OVERRIDE=0 \
|
|
$(Repository) \
|
|
/bin/bash -c '
|
|
set -ex; \
|
|
cp /onnxruntime_src/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt /tmp/requirements.txt; \
|
|
export PATH=/opt/python/cp312-cp312/bin:$PATH; \
|
|
python3 -m pip install -r /tmp/requirements.txt; \
|
|
python3 -m pip install /build/Release/dist/*.whl; \
|
|
cd /build/Release && xargs -a /build/Release/perms.txt chmod a+x; \
|
|
cd /onnxruntime_src/java && /onnxruntime_src/java/gradlew cmakeCheck -DcmakeBuildDir=/build/Release -DUSE_CUDA=1; \
|
|
cd /tmp; \
|
|
python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests \
|
|
--enable_transformers_tool_test --use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda --cudnn_home=/usr/local/cuda \
|
|
--enable_pybind --build_java --ctest_path "" ; \
|
|
'
|
|
displayName: 'Run Tests'
|
|
|
|
- template: templates/check_test_result.yml
|
|
parameters:
|
|
FileName: '$(Build.BinariesDirectory)/Release/onnxruntime_test_all.Release.results.xml'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|