mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-25 02:50:42 +00:00
6db72165eb
### Description 1. Fix python packaging test pipeline. There was an error in tools/ci_build/github/linux/run_python_tests.sh that it installed a released version of onnxruntime python package from pypi.org to run the test. Supposedly it should pick one from the current build. 2. Refactor the pipeline to allow choosing cmake build type from the web UI when manually trigger a build. Now this feature is for Linux only. Because I don't want to change too much when we are about to cut a release branch. After that I will expand it to all platforms. This feature is useful for debugging pipeline issues, also, we may consider having a nightly pipeline to run all tests in Debug mode which may catch extra bugs because in debug mode we can enforce range check. Test run: https://aiinfra.visualstudio.com/Lotus/_build/results?buildId=342674&view=results ### Motivation and Context Currently the pipeline has a crash error. AB#18580
252 lines
9.9 KiB
YAML
252 lines
9.9 KiB
YAML
trigger:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
pr:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
resources:
|
|
repositories:
|
|
- repository: manylinux # The name used to reference this repository in the checkout step
|
|
type: Github
|
|
endpoint: Microsoft
|
|
name: pypa/manylinux
|
|
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
|
stages:
|
|
- stage: x64
|
|
dependsOn: []
|
|
jobs:
|
|
- job: Linux_Build
|
|
timeoutInMinutes: 180
|
|
workspace:
|
|
clean: all
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
|
|
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
|
pool: onnxruntime-Ubuntu2004-AMD-CPU
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- task: NodeTool@0
|
|
inputs:
|
|
versionSpec: '16.x'
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: '3.8'
|
|
addToPath: true
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
|
Repository: onnxruntimecpubuild
|
|
|
|
- template: templates/linux-build-step-with-cache.yml
|
|
parameters:
|
|
WithCache: true
|
|
Today: $(TODAY)
|
|
AdditionalKey: onnxruntime
|
|
CacheDir: $(ORT_CACHE_DIR)
|
|
ChangeEveryCommit: true
|
|
BuildStep:
|
|
- task: CmdLine@2
|
|
displayName: 'build'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
mkdir -p $(Pipeline.Workspace)/ccache
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume $(ORT_CACHE_DIR):/cache \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
-e CCACHE_DIR=/cache \
|
|
onnxruntimecpubuild \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
ccache -s; \
|
|
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator Ninja \
|
|
--config Debug Release \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel \
|
|
--build_wheel \
|
|
--build_csharp \
|
|
--enable_onnx_tests \
|
|
--enable_transformers_tool_test \
|
|
--use_cache \
|
|
--build_java --build_nodejs --update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \
|
|
ccache -sv; \
|
|
ccache -z"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: UseDotNet@2
|
|
displayName: "Setup dotnet"
|
|
inputs:
|
|
version: '6.0.408'
|
|
|
|
- task: DotNetCoreCLI@2
|
|
displayName: "Restore C# packages"
|
|
inputs:
|
|
command: 'restore'
|
|
projects: '$(Build.SourcesDirectory)/csharp/OnnxRuntime.DesktopOnly.CSharp.sln'
|
|
|
|
# the props file was generated with docker container paths. convert to the 'real' path by replacing the
|
|
# the container path of '/build'. The '>' prefix is to match the closing angle bracket of the tag.
|
|
# e.g. <OnnxRuntimeBuildDirectory>/build/... so we only match the start of a path.
|
|
# We use powershell so we don't need extra escaping of the '/' chars in the path.
|
|
- task: CmdLine@2
|
|
displayName: 'Update props from docker path to local and create models link'
|
|
inputs:
|
|
script: |
|
|
pwsh -Command '(Get-Content $(Build.SourcesDirectory)/csharp/Directory.Build.props) -replace ">/build", ">$(Build.BinariesDirectory)" | Set-Content $(Build.SourcesDirectory)/csharp/Directory.Build.props'
|
|
cat $(Build.SourcesDirectory)/csharp/Directory.Build.props
|
|
ln -s /data/models $(Build.BinariesDirectory)/models
|
|
|
|
- task: DotNetCoreCLI@2
|
|
displayName: 'dotnet build C# sln'
|
|
inputs:
|
|
command: 'build'
|
|
projects: '$(Build.SourcesDirectory)/csharp/OnnxRuntime.DesktopOnly.CSharp.sln'
|
|
|
|
- task: DotNetCoreCLI@2
|
|
displayName: 'dotnet test C#'
|
|
inputs:
|
|
command: 'test'
|
|
projects: '$(Build.SourcesDirectory)/csharp/OnnxRuntime.DesktopOnly.CSharp.sln'
|
|
# extra logging so all tests are listed in output to validate what's actually run
|
|
arguments: '-f net6.0 --no-build -l "console;verbosity=normal"'
|
|
workingDirectory: $(Build.SourcesDirectory)/csharp
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install python deps and run java tests'
|
|
inputs:
|
|
script: |
|
|
set -e -x
|
|
python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml ort-nightly-directml onnx -qq
|
|
cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt $(Build.BinariesDirectory)/requirements.txt
|
|
# Test ORT with the latest ONNX release.
|
|
sed -i "s/git+http:\/\/github\.com\/onnx\/onnx.*/onnx/" $(Build.BinariesDirectory)/requirements.txt
|
|
python3 -m pip install -r $(Build.BinariesDirectory)/requirements.txt
|
|
mkdir $(Build.BinariesDirectory)/requirements_torch_cpu/
|
|
cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt $(Build.BinariesDirectory)/requirements_torch_cpu/requirements.txt
|
|
python3 -m pip install -r $(Build.BinariesDirectory)/requirements_torch_cpu/requirements.txt
|
|
cd $(Build.SourcesDirectory)/java
|
|
$(Build.SourcesDirectory)/java/gradlew "cmakeCheck" "-DcmakeBuildDir=$(Build.BinariesDirectory)/Release"
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install Release python package'
|
|
inputs:
|
|
script: |
|
|
rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11
|
|
python3 -m pip install $(Build.BinariesDirectory)/Release/dist/*.whl
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Run Release unit tests'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/build.py
|
|
workingDirectory: $(Build.BinariesDirectory)/Release
|
|
arguments: >-
|
|
--build_dir $(Build.BinariesDirectory)
|
|
--cmake_generator Ninja
|
|
--config Release
|
|
--test
|
|
--skip_submodule_sync
|
|
--build_shared_lib
|
|
--parallel
|
|
--build_wheel
|
|
--enable_onnx_tests
|
|
--enable_transformers_tool_test
|
|
--build_nodejs
|
|
--ctest_path ""
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install Debug python package'
|
|
inputs:
|
|
script: |
|
|
set -e -x
|
|
rm -rf $(Build.BinariesDirectory)/Debug/onnxruntime $(Build.BinariesDirectory)/Debug/pybind11
|
|
python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml ort-nightly-directml -qq
|
|
python3 -m pip install $(Build.BinariesDirectory)/Debug/dist/*.whl
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Run Debug unit tests'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/build.py
|
|
workingDirectory: $(Build.BinariesDirectory)/Debug
|
|
arguments: >-
|
|
--build_dir $(Build.BinariesDirectory)
|
|
--cmake_generator Ninja
|
|
--config Debug
|
|
--test
|
|
--skip_submodule_sync
|
|
--build_shared_lib
|
|
--parallel
|
|
--build_wheel
|
|
--enable_onnx_tests
|
|
--enable_transformers_tool_test
|
|
--build_nodejs
|
|
--ctest_path ""
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Symbolic shape infer'
|
|
inputs:
|
|
scriptPath: $(Build.BinariesDirectory)/Release/onnxruntime_test_python_symbolic_shape_infer.py
|
|
workingDirectory: $(Build.BinariesDirectory)/Release
|
|
|
|
- task: PublishTestResults@2
|
|
displayName: 'Publish unit test results'
|
|
inputs:
|
|
testResultsFiles: '**/*.results.xml'
|
|
searchFolder: '$(Build.BinariesDirectory)'
|
|
testRunTitle: 'Unit Test Run'
|
|
condition: succeededOrFailed()
|
|
|
|
|
|
- stage: arm64_build
|
|
dependsOn: []
|
|
jobs:
|
|
- template: templates/py-linux.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
base_image: 'arm64v8/centos:7'
|
|
devtoolset_rootpath: /opt/rh/devtoolset-10/root
|
|
ld_library_path_arg: /opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64
|
|
prepend_path: '/opt/rh/devtoolset-10/root/usr/bin:'
|
|
with_cache: true
|
|
cmake_build_type: Release
|
|
|
|
- stage: arm64_test
|
|
dependsOn: ['arm64_build']
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
device: 'CPU'
|