mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-24 02:47:54 +00:00
4af593a722
1. Add python 3.13 to our python packaging pipelines 2. Because numpy 2.0.0 doesn't support thread free python, this PR also upgrades numpy to the latest 3. Delete some unused files.
309 lines
12 KiB
YAML
309 lines
12 KiB
YAML
##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
|
|
### please do rerun set-trigger-rules.py ###
|
|
trigger:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
pr:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
#### end trigger ####
|
|
|
|
stages:
|
|
- stage: x64
|
|
dependsOn: []
|
|
jobs:
|
|
- job: Linux_Debug
|
|
timeoutInMinutes: 180
|
|
workspace:
|
|
clean: all
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
|
|
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
|
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
|
|
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=registry.access.redhat.com/ubi8/ubi"
|
|
Repository: onnxruntimecpubuildcentos8x64
|
|
|
|
- template: templates/linux-build-step-with-cache.yml
|
|
parameters:
|
|
WithCache: false
|
|
Today: $(TODAY)
|
|
AdditionalKey: onnxruntime_linux_debug_with_address_sanitizer
|
|
CacheDir: $(ORT_CACHE_DIR)
|
|
ChangeEveryCommit: true
|
|
BuildStep:
|
|
- task: CmdLine@2
|
|
displayName: 'build'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume /data/models:/data/models:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuildcentos8x64 \
|
|
/bin/bash -c '
|
|
set -ex; \
|
|
python3.12 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator 'Ninja' \
|
|
--config Debug \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel --use_binskim_compliant_compile_flags \
|
|
--enable_onnx_tests --enable_address_sanitizer \
|
|
--update --build;
|
|
python3.12 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator 'Ninja' \
|
|
--config Debug \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel --use_binskim_compliant_compile_flags \
|
|
--enable_onnx_tests --enable_address_sanitizer \
|
|
--test;
|
|
'
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: PublishTestResults@2
|
|
displayName: 'Publish unit test results'
|
|
inputs:
|
|
testResultsFiles: '**/*.results.xml'
|
|
searchFolder: '$(Build.BinariesDirectory)'
|
|
testRunTitle: 'Unit Test Run'
|
|
condition: succeededOrFailed()
|
|
|
|
- job: Linux_Release
|
|
timeoutInMinutes: 180
|
|
workspace:
|
|
clean: all
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
|
|
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
|
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- task: DownloadPackage@1
|
|
displayName: 'Download ARM64 GCC'
|
|
inputs:
|
|
packageType: upack
|
|
feed: '/7424c8e4-5c62-490e-95c4-79446f31017c'
|
|
definition: 'gcc_aarch64_linux_gnu_host_x86_64'
|
|
version: 13.2.1
|
|
downloadPath: $(Build.BinariesDirectory)/gcc
|
|
|
|
- task: DownloadPackage@1
|
|
displayName: 'Download ARM32 GCC'
|
|
inputs:
|
|
packageType: upack
|
|
feed: '/7424c8e4-5c62-490e-95c4-79446f31017c'
|
|
definition: 'gcc_aarch32_linux_gnu_host_x86_64'
|
|
version: 13.2.1
|
|
downloadPath: $(Build.BinariesDirectory)/gcc
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
|
|
Context: tools/ci_build/github/linux/docker/
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=registry.access.redhat.com/ubi8/ubi"
|
|
Repository: onnxruntimecpubuild
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Update deps.txt'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
|
|
arguments: --new_dir $(Build.BinariesDirectory)/deps
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
pythonInterpreter: /usr/bin/python3
|
|
|
|
- script: |
|
|
set -e -x
|
|
# ARM64 build
|
|
mkdir -p $(Build.BinariesDirectory)/gccbin
|
|
tar -Jxf $(Build.BinariesDirectory)/gcc/arm-gnu-toolchain-13.2.rel1-x86_64-aarch64-none-linux-gnu.tar.xz --strip=1 -C $(Build.BinariesDirectory)/gccbin
|
|
export PATH=$(Build.BinariesDirectory)/gccbin/bin:$PATH
|
|
mkdir $(Build.BinariesDirectory)/aarch64build
|
|
cd $(Build.BinariesDirectory)/aarch64build
|
|
cmake $(Build.SourcesDirectory)/cmake -Donnxruntime_ENABLE_CPUINFO=OFF -DPython_EXECUTABLE=/usr/bin/python3 -DPYTHON_EXECUTABLE=/usr/bin/python3 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=$(Build.SourcesDirectory)/cmake/linux_arm64_crosscompile_toolchain.cmake -G Ninja
|
|
ninja
|
|
rm -rf $(Build.BinariesDirectory)/aarch64build $(Build.BinariesDirectory)/gccbin
|
|
# ARM32 build
|
|
mkdir -p $(Build.BinariesDirectory)/gccbin
|
|
tar -Jxf $(Build.BinariesDirectory)/gcc/arm-gnu-toolchain-13.2.rel1-x86_64-arm-none-linux-gnueabihf.tar.xz --strip=1 -C $(Build.BinariesDirectory)/gccbin
|
|
ls $(Build.BinariesDirectory)/gccbin/bin
|
|
mkdir $(Build.BinariesDirectory)/arm32build
|
|
cd $(Build.BinariesDirectory)/arm32build
|
|
cmake $(Build.SourcesDirectory)/cmake -Donnxruntime_ENABLE_CPUINFO=OFF -DPython_EXECUTABLE=/usr/bin/python3 -DPYTHON_EXECUTABLE=/usr/bin/python3 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=$(Build.SourcesDirectory)/cmake/linux_arm32_crosscompile_toolchain.cmake -G Ninja
|
|
ninja
|
|
rm -rf $(Build.BinariesDirectory)/arm32build $(Build.BinariesDirectory)/gccbin
|
|
displayName: Cross-compile for Linux ARM32 and ARM64
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Update deps.txt'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
|
|
arguments: --new_dir /build/deps
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
pythonInterpreter: /usr/bin/python3
|
|
|
|
- template: templates/linux-build-step-with-cache.yml
|
|
parameters:
|
|
WithCache: true
|
|
Today: $(TODAY)
|
|
AdditionalKey: onnxruntime
|
|
CacheDir: $(ORT_CACHE_DIR)
|
|
ChangeEveryCommit: true
|
|
BuildStep:
|
|
- task: CmdLine@2
|
|
displayName: 'build'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume $(ORT_CACHE_DIR):/cache \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
-e CCACHE_DIR=/cache \
|
|
onnxruntimecpubuild \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
ccache -s; \
|
|
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator 'Ninja' \
|
|
--config Release \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel --use_binskim_compliant_compile_flags \
|
|
--build_wheel \
|
|
--build_csharp \
|
|
--enable_onnx_tests \
|
|
--enable_transformers_tool_test \
|
|
--use_cache \
|
|
--update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \
|
|
ccache -sv; \
|
|
ccache -z"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- script: |
|
|
ln -s /data/models $(Build.BinariesDirectory)/models
|
|
displayName: link model dir
|
|
|
|
- bash: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
pushd /onnxruntime_src/csharp; \
|
|
dotnet restore /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln; \
|
|
dotnet build /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln -c Release; \
|
|
dotnet test /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln -c Release -f net8.0 --no-build -l \"console;verbosity=normal\"; \
|
|
popd
|
|
"
|
|
displayName: 'Dotnet build C# sln and Test'
|
|
|
|
- bash: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
/bin/bash /onnxruntime_src/tools/scripts/python_test.sh /onnxruntime_src /build Release && \
|
|
/bin/bash /onnxruntime_src/tools/scripts/symbolic_shape_infer_test.sh /build
|
|
"
|
|
displayName: 'Run Release tests and symbolic shape infer test'
|
|
|
|
- template: templates/check_test_result.yml
|
|
parameters:
|
|
FileName: '$(Build.BinariesDirectory)/Release/onnxruntime_test_all.Release.results.xml'
|
|
|
|
- task: PublishTestResults@2
|
|
displayName: 'Publish unit test results'
|
|
inputs:
|
|
testResultsFiles: '**/*.results.xml'
|
|
searchFolder: '$(Build.BinariesDirectory)'
|
|
testRunTitle: 'Unit Test Run'
|
|
condition: succeededOrFailed()
|
|
|
|
- stage: arm64_build
|
|
dependsOn: []
|
|
jobs:
|
|
- template: templates/py-linux.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
with_cache: true
|
|
cmake_build_type: Release
|
|
|
|
- stage: arm64_test
|
|
dependsOn: ['arm64_build']
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test-cpu.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|