mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-13 18:08:13 +00:00
0e8d4c3d21
### Description 1. Add two build jobs for enabling Address Sanitizer in CI. One for Windows CPU, One for Linux CPU. 2. Set default compiler flags/linker flags in build.py for normal Windows/Linux/MacOS build. This can help control compiler flags in a more centralized way. 3. All Windows binaries in our official packages will be built with "/PROFILE" flag. Symbols of onnxruntime.dll can be found at [Microsoft public symbol server](https://learn.microsoft.com/en-us/windows-hardware/drivers/debugger/microsoft-public-symbols). Limitations: 1. On Linux Address Sanitizer ignores RPATH settings in ELF binaries. Therefore once Address Sanitizer is enabled, before running tests we need to manually set LD_LIBRARY_PATH properly otherwise libonnxruntime.so may not be able to find custom ops and shared EPs. 4. On Linux we also need to set LD_PRELOAD before running some tests(if the main executable, like python, is not built with address sanitizer. On Windows we do not need to. 5. On Windows before running python tests we should manually copy address sanitizer DLL to the onnxruntime/capi directory, because python 3.8 and above has enabled "Safe DLL Search Mode" that wouldn't use the information provided by PATH env. 6. On Linux Address Sanitizer found a lot of memory leaks from our python binding code. Therefore right now we cannot enable Address Sanitizer when building ONNX Runtime with python binding. 7. Address Sanitizer itself uses a lot of memory address space and delays memory deallocations, which is easy to cause OOM issues in 32-bit applications. We cannot run all the tests in onnxruntime_test_all in 32-bit mode with Address Sanitizer due to this reason. However, we still can run individual tests in such a way. We just cannot run all of them in one process. ### Motivation and Context To catch memory issues.
280 lines
11 KiB
YAML
280 lines
11 KiB
YAML
##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
|
|
trigger:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
pr:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
#### end trigger ####
|
|
|
|
resources:
|
|
repositories:
|
|
- repository: manylinux # The name used to reference this repository in the checkout step
|
|
type: Github
|
|
endpoint: Microsoft
|
|
name: pypa/manylinux
|
|
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
|
stages:
|
|
- stage: x64
|
|
dependsOn: []
|
|
jobs:
|
|
- job: Linux_Debug
|
|
timeoutInMinutes: 180
|
|
workspace:
|
|
clean: all
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
|
|
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
|
pool: onnxruntime-Ubuntu2004-AMD-CPU
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/inference/x64/default/cpu/Dockerfile
|
|
Context: tools/ci_build/github/linux/docker/inference/x64/default/cpu
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=registry.access.redhat.com/ubi8/ubi"
|
|
Repository: onnxruntimecpubuildcentos8x64
|
|
|
|
- template: templates/linux-build-step-with-cache.yml
|
|
parameters:
|
|
WithCache: false
|
|
Today: $(TODAY)
|
|
AdditionalKey: onnxruntime_linux_debug_with_address_sanitizer
|
|
CacheDir: $(ORT_CACHE_DIR)
|
|
ChangeEveryCommit: true
|
|
BuildStep:
|
|
- task: CmdLine@2
|
|
displayName: 'build'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume /data/models:/data/models:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuildcentos8x64 \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
python3.9 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator 'Ninja' \
|
|
--config Debug \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel \
|
|
--build_csharp \
|
|
--enable_onnx_tests --enable_address_sanitizer \
|
|
--update --build;
|
|
LD_PRELOAD=/usr/lib64/libasan.so.6 python3.9 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator 'Ninja' \
|
|
--config Debug \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel \
|
|
--build_csharp \
|
|
--enable_onnx_tests --enable_address_sanitizer \
|
|
--test;"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: PublishTestResults@2
|
|
displayName: 'Publish unit test results'
|
|
inputs:
|
|
testResultsFiles: '**/*.results.xml'
|
|
searchFolder: '$(Build.BinariesDirectory)'
|
|
testRunTitle: 'Unit Test Run'
|
|
condition: succeededOrFailed()
|
|
- job: Linux_Release
|
|
timeoutInMinutes: 180
|
|
workspace:
|
|
clean: all
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
|
|
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
|
pool: onnxruntime-Ubuntu2004-AMD-CPU
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- task: DownloadPackage@1
|
|
displayName: 'Download ARM64 GCC'
|
|
inputs:
|
|
packageType: upack
|
|
feed: '/7424c8e4-5c62-490e-95c4-79446f31017c'
|
|
definition: 'gcc_aarch64_linux_gnu_host_x86_64'
|
|
version: 13.2.1
|
|
downloadPath: $(Build.BinariesDirectory)/gcc
|
|
|
|
- task: DownloadPackage@1
|
|
displayName: 'Download ARM32 GCC'
|
|
inputs:
|
|
packageType: upack
|
|
feed: '/7424c8e4-5c62-490e-95c4-79446f31017c'
|
|
definition: 'gcc_aarch32_linux_gnu_host_x86_64'
|
|
version: 13.2.1
|
|
downloadPath: $(Build.BinariesDirectory)/gcc
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
|
|
Context: tools/ci_build/github/linux/docker/
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=registry.access.redhat.com/ubi8/ubi"
|
|
Repository: onnxruntimecpubuild
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Update deps.txt'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
|
|
arguments: --new_dir $(Build.BinariesDirectory)/deps
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
pythonInterpreter: /usr/bin/python3
|
|
|
|
- script: |
|
|
set -e -x
|
|
# ARM64 build
|
|
mkdir -p $(Build.BinariesDirectory)/gccbin
|
|
tar -Jxf $(Build.BinariesDirectory)/gcc/arm-gnu-toolchain-13.2.rel1-x86_64-aarch64-none-linux-gnu.tar.xz --strip=1 -C $(Build.BinariesDirectory)/gccbin
|
|
export PATH=$(Build.BinariesDirectory)/gccbin/bin:$PATH
|
|
mkdir $(Build.BinariesDirectory)/aarch64build
|
|
cd $(Build.BinariesDirectory)/aarch64build
|
|
cmake $(Build.SourcesDirectory)/cmake -Donnxruntime_ENABLE_CPUINFO=OFF -DPython_EXECUTABLE=/usr/bin/python3 -DPYTHON_EXECUTABLE=/usr/bin/python3 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=$(Build.SourcesDirectory)/cmake/linux_arm64_crosscompile_toolchain.cmake -G Ninja
|
|
ninja
|
|
rm -rf $(Build.BinariesDirectory)/aarch64build $(Build.BinariesDirectory)/gccbin
|
|
# ARM32 build
|
|
mkdir -p $(Build.BinariesDirectory)/gccbin
|
|
tar -Jxf $(Build.BinariesDirectory)/gcc/arm-gnu-toolchain-13.2.rel1-x86_64-arm-none-linux-gnueabihf.tar.xz --strip=1 -C $(Build.BinariesDirectory)/gccbin
|
|
ls $(Build.BinariesDirectory)/gccbin/bin
|
|
mkdir $(Build.BinariesDirectory)/arm32build
|
|
cd $(Build.BinariesDirectory)/arm32build
|
|
cmake $(Build.SourcesDirectory)/cmake -Donnxruntime_ENABLE_CPUINFO=OFF -DPython_EXECUTABLE=/usr/bin/python3 -DPYTHON_EXECUTABLE=/usr/bin/python3 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=$(Build.SourcesDirectory)/cmake/linux_arm32_crosscompile_toolchain.cmake -G Ninja
|
|
ninja
|
|
rm -rf $(Build.BinariesDirectory)/arm32build $(Build.BinariesDirectory)/gccbin
|
|
displayName: Cross-compile for Linux ARM32 and ARM64
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Update deps.txt'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
|
|
arguments: --new_dir /build/deps
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
pythonInterpreter: /usr/bin/python3
|
|
|
|
- template: templates/linux-build-step-with-cache.yml
|
|
parameters:
|
|
WithCache: true
|
|
Today: $(TODAY)
|
|
AdditionalKey: onnxruntime
|
|
CacheDir: $(ORT_CACHE_DIR)
|
|
ChangeEveryCommit: true
|
|
BuildStep:
|
|
- task: CmdLine@2
|
|
displayName: 'build'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume $(ORT_CACHE_DIR):/cache \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
-e CCACHE_DIR=/cache \
|
|
onnxruntimecpubuild \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
ccache -s; \
|
|
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator 'Ninja' \
|
|
--config Release \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel \
|
|
--build_wheel \
|
|
--build_csharp \
|
|
--enable_onnx_tests \
|
|
--enable_transformers_tool_test \
|
|
--use_cache \
|
|
--update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \
|
|
ccache -sv; \
|
|
ccache -z"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- script: |
|
|
ln -s /data/models $(Build.BinariesDirectory)/models
|
|
displayName: link model dir
|
|
|
|
|
|
|
|
- task: PublishTestResults@2
|
|
displayName: 'Publish unit test results'
|
|
inputs:
|
|
testResultsFiles: '**/*.results.xml'
|
|
searchFolder: '$(Build.BinariesDirectory)'
|
|
testRunTitle: 'Unit Test Run'
|
|
condition: succeededOrFailed()
|
|
|
|
- stage: arm64_build
|
|
dependsOn: []
|
|
jobs:
|
|
- template: templates/py-linux.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
base_image: 'arm64v8/almalinux:8'
|
|
devtoolset_rootpath: /opt/rh/gcc-toolset-12/root
|
|
ld_library_path_arg: /opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64
|
|
prepend_path: '/opt/rh/gcc-toolset-12/root/usr/bin:'
|
|
with_cache: true
|
|
cmake_build_type: Release
|
|
|
|
- stage: arm64_test
|
|
dependsOn: ['arm64_build']
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test-cpu.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
base_image: 'arm64v8/almalinux:8'
|
|
devtoolset_rootpath: /opt/rh/gcc-toolset-12/root
|
|
ld_library_path_arg: /opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64
|
|
prepend_path: '/opt/rh/gcc-toolset-12/root/usr/bin:'
|