mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
5f6a3158f8
### Description 1. Enable VCPKG flag in Windows CPU CI build pipelines. 2. Increased the min supported cmake version from 3.26 to 3.28. Because of it, drop the support for the old way of finding python by "find_package(PythonLibs)". Therefore, in build.py we no longer set "PYTHON_EXECUTABLE" cmake var when doing cmake configure. 3. Added "xnnpack-ep" as a feature for ORT's vcpkg config. 4. Added asset cache support for ORT's vcpkg build 5. Added VCPKG triplet files for Android build. 6. Set VCPKG triplet to "universal2-osx" if CMAKE_OSX_ARCHITECTURES was found in cmake extra defines. 7. Removed a small piece of code in build.py, which was for support CUDA version < 11.8. 8. Fixed an issue that CMAKE_OSX_ARCHITECTURES sometimes got specified twice when build.py invoked cmake. 9. Added more model tests to Android build. After this change, we will test all ONNX versions instead of just the latest one. 10. Fixed issues that are related to build.py's "--build_nuget" parameter. Also, enable the flag in most Windows CPU CI build jobs. 11. Removed a restriction in build.py that disallowed cross-compiling Windows ARM64 nuget package on Windows x86. ### Motivation and Context Adopt vcpkg.
310 lines
12 KiB
YAML
310 lines
12 KiB
YAML
##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
|
|
### please do rerun set-trigger-rules.py ###
|
|
trigger:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
pr:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
#### end trigger ####
|
|
|
|
stages:
|
|
- stage: x64
|
|
dependsOn: []
|
|
jobs:
|
|
- job: Linux_Debug
|
|
timeoutInMinutes: 180
|
|
workspace:
|
|
clean: all
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
|
|
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
|
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
|
steps:
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
|
|
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
|
Repository: onnxruntimecpubuildcentos8x64
|
|
|
|
- template: templates/linux-build-step-with-cache.yml
|
|
parameters:
|
|
WithCache: false
|
|
Today: $(TODAY)
|
|
AdditionalKey: onnxruntime_linux_debug_with_address_sanitizer
|
|
CacheDir: $(ORT_CACHE_DIR)
|
|
ChangeEveryCommit: true
|
|
BuildStep:
|
|
- task: CmdLine@2
|
|
displayName: 'build'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume /data/models:/data/models:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuildcentos8x64 \
|
|
/bin/bash -c 'set -ex; \
|
|
python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator Ninja \
|
|
--config Debug \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel \
|
|
--enable_onnx_tests --enable_address_sanitizer \
|
|
--update --build;
|
|
python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator Ninja \
|
|
--config Debug \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel \
|
|
--enable_onnx_tests --enable_address_sanitizer \
|
|
--test;
|
|
'
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: PublishTestResults@2
|
|
displayName: 'Publish unit test results'
|
|
inputs:
|
|
testResultsFiles: '**/*.results.xml'
|
|
searchFolder: '$(Build.BinariesDirectory)'
|
|
testRunTitle: 'Unit Test Run'
|
|
condition: succeededOrFailed()
|
|
|
|
- job: Linux_Release
|
|
timeoutInMinutes: 180
|
|
workspace:
|
|
clean: all
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
|
|
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
|
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
|
steps:
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- task: DownloadPackage@1
|
|
displayName: 'Download ARM64 GCC'
|
|
inputs:
|
|
packageType: upack
|
|
feed: '/7424c8e4-5c62-490e-95c4-79446f31017c'
|
|
definition: 'gcc_aarch64_linux_gnu_host_x86_64'
|
|
version: 13.2.1
|
|
downloadPath: $(Build.BinariesDirectory)/gcc
|
|
|
|
- task: DownloadPackage@1
|
|
displayName: 'Download ARM32 GCC'
|
|
inputs:
|
|
packageType: upack
|
|
feed: '/7424c8e4-5c62-490e-95c4-79446f31017c'
|
|
definition: 'gcc_aarch32_linux_gnu_host_x86_64'
|
|
version: 13.2.1
|
|
downloadPath: $(Build.BinariesDirectory)/gcc
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
|
|
Context: tools/ci_build/github/linux/docker/
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
|
Repository: onnxruntimecpubuild
|
|
|
|
- script: |
|
|
set -e -x
|
|
# ARM64 build
|
|
mkdir -p $(Build.BinariesDirectory)/gccbin
|
|
tar -Jxf $(Build.BinariesDirectory)/gcc/arm-gnu-toolchain-13.2.rel1-x86_64-aarch64-none-linux-gnu.tar.xz --strip=1 -C $(Build.BinariesDirectory)/gccbin
|
|
export PATH=$(Build.BinariesDirectory)/gccbin/bin:$PATH
|
|
mkdir $(Build.BinariesDirectory)/aarch64build
|
|
cd $(Build.BinariesDirectory)/aarch64build
|
|
cmake $(Build.SourcesDirectory)/cmake -Donnxruntime_ENABLE_CPUINFO=OFF -DPython_EXECUTABLE=/usr/bin/python3 -DPYTHON_EXECUTABLE=/usr/bin/python3 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=$(Build.SourcesDirectory)/cmake/linux_arm64_crosscompile_toolchain.cmake -G Ninja
|
|
ninja
|
|
rm -rf $(Build.BinariesDirectory)/aarch64build $(Build.BinariesDirectory)/gccbin
|
|
# ARM32 build
|
|
mkdir -p $(Build.BinariesDirectory)/gccbin
|
|
tar -Jxf $(Build.BinariesDirectory)/gcc/arm-gnu-toolchain-13.2.rel1-x86_64-arm-none-linux-gnueabihf.tar.xz --strip=1 -C $(Build.BinariesDirectory)/gccbin
|
|
ls $(Build.BinariesDirectory)/gccbin/bin
|
|
mkdir $(Build.BinariesDirectory)/arm32build
|
|
cd $(Build.BinariesDirectory)/arm32build
|
|
cmake $(Build.SourcesDirectory)/cmake -Donnxruntime_ENABLE_CPUINFO=OFF -DPython_EXECUTABLE=/usr/bin/python3 -DPYTHON_EXECUTABLE=/usr/bin/python3 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=$(Build.SourcesDirectory)/cmake/linux_arm32_crosscompile_toolchain.cmake -G Ninja
|
|
ninja
|
|
rm -rf $(Build.BinariesDirectory)/arm32build $(Build.BinariesDirectory)/gccbin
|
|
displayName: Cross-compile for Linux ARM32 and ARM64
|
|
|
|
- template: templates/linux-build-step-with-cache.yml
|
|
parameters:
|
|
WithCache: true
|
|
Today: $(TODAY)
|
|
AdditionalKey: onnxruntime
|
|
CacheDir: $(ORT_CACHE_DIR)
|
|
ChangeEveryCommit: true
|
|
BuildStep:
|
|
- task: CmdLine@2
|
|
displayName: 'build'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume $(ORT_CACHE_DIR):/cache \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
-e CCACHE_DIR=/cache \
|
|
onnxruntimecpubuild \
|
|
/bin/bash -c '
|
|
set -ex; \
|
|
ccache -s; \
|
|
PATH=/opt/python/cp310-cp310/bin:$PATH python /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator Ninja \
|
|
--config Release \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel --use_binskim_compliant_compile_flags \
|
|
--build_wheel \
|
|
--build_csharp \
|
|
--enable_onnx_tests \
|
|
--enable_transformers_tool_test \
|
|
--use_cache \
|
|
--update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \
|
|
ccache -sv; \
|
|
ccache -z'
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- script: |
|
|
ln -s /data/models $(Build.BinariesDirectory)/models
|
|
displayName: link model dir
|
|
|
|
- bash: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
pushd /onnxruntime_src/csharp; \
|
|
dotnet restore /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln; \
|
|
dotnet build /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln -c Release; \
|
|
dotnet test /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln -c Release -f net8.0 --no-build -l \"console;verbosity=normal\"; \
|
|
popd
|
|
"
|
|
displayName: 'Dotnet build C# sln and Test'
|
|
|
|
- bash: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
/bin/bash /onnxruntime_src/tools/scripts/python_test.sh /onnxruntime_src /build Release && \
|
|
/bin/bash /onnxruntime_src/tools/scripts/symbolic_shape_infer_test.sh /build
|
|
"
|
|
displayName: 'Run Release tests and symbolic shape infer test'
|
|
|
|
- template: templates/check_test_result.yml
|
|
parameters:
|
|
FileName: '$(Build.BinariesDirectory)/Release/onnxruntime_test_all.Release.results.xml'
|
|
|
|
- task: PublishTestResults@2
|
|
displayName: 'Publish unit test results'
|
|
inputs:
|
|
testResultsFiles: '**/*.results.xml'
|
|
searchFolder: '$(Build.BinariesDirectory)'
|
|
testRunTitle: 'Unit Test Run'
|
|
condition: succeededOrFailed()
|
|
|
|
- stage: arm64_build
|
|
dependsOn: []
|
|
jobs:
|
|
- template: templates/py-linux.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
with_cache: true
|
|
cmake_build_type: Release
|
|
python_exe_path: '/opt/python/cp310-cp310/bin/python3.10'
|
|
|
|
- stage: arm64_test
|
|
dependsOn: ['arm64_build']
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test-cpu.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
ep: 'cpu'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
|
|
- stage: arm64_build_xnnpack
|
|
dependsOn: []
|
|
jobs:
|
|
- template: templates/py-linux.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
with_cache: true
|
|
cmake_build_type: Release
|
|
ep: 'XNNPack'
|
|
extra_build_arg: '--use_xnnpack'
|
|
python_exe_path: '/opt/python/cp310-cp310/bin/python3.10'
|
|
|
|
- stage: arm64_test_xnnpack
|
|
dependsOn: ['arm64_build_xnnpack']
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test-cpu.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
ep: 'XNNPack'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|