mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-22 22:01:08 +00:00
d1b2b35cd2
### Description <!-- Describe your changes. --> Various fixes to the CSharp setup - fix warnings - fix invalid tests - update test sdk nuget package - enables testing on linux - fixes issue with some unit tests not running in CI - run unit tests in linux pipeline using dotnet ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. --> Unit tests weren't breaking in CIs for both Windows and Linux builds and should have been.
233 lines
9.6 KiB
YAML
233 lines
9.6 KiB
YAML
resources:
|
|
repositories:
|
|
- repository: manylinux # The name used to reference this repository in the checkout step
|
|
type: Github
|
|
endpoint: Microsoft
|
|
name: pypa/manylinux
|
|
ref: aead4d751c2101e23336aa73f2380df83e7a13f3
|
|
stages:
|
|
- stage: x64
|
|
dependsOn: []
|
|
jobs:
|
|
- job: Linux_Build
|
|
timeoutInMinutes: 180
|
|
workspace:
|
|
clean: all
|
|
variables:
|
|
skipComponentGovernanceDetection: true
|
|
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
|
|
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
|
pool: onnxruntime-Ubuntu2004-AMD-CPU
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- task: NodeTool@0
|
|
inputs:
|
|
versionSpec: '16.x'
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: '3.8'
|
|
addToPath: true
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
|
Repository: onnxruntimecpubuild
|
|
|
|
- template: templates/linux-build-step-with-cache.yml
|
|
parameters:
|
|
WithCache: true
|
|
Today: $(TODAY)
|
|
AdditionalKey: onnxruntime
|
|
CacheDir: $(ORT_CACHE_DIR)
|
|
ChangeEveryCommit: true
|
|
BuildStep:
|
|
- task: CmdLine@2
|
|
displayName: 'build'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
mkdir -p $(Pipeline.Workspace)/ccache
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
--volume $(ORT_CACHE_DIR):/cache \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
-e CCACHE_DIR=/cache \
|
|
onnxruntimecpubuild \
|
|
/bin/bash -c "
|
|
set -ex; \
|
|
ccache -s; \
|
|
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator Ninja \
|
|
--config Debug Release \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel \
|
|
--build_wheel \
|
|
--build_csharp \
|
|
--enable_onnx_tests \
|
|
--enable_transformers_tool_test \
|
|
--use_cache \
|
|
--build_java --build_nodejs --update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \
|
|
ccache -sv; \
|
|
ccache -z"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: UseDotNet@2
|
|
displayName: "Setup dotnet"
|
|
inputs:
|
|
version: '6.0.408'
|
|
|
|
- task: DotNetCoreCLI@2
|
|
displayName: "Restore C# packages"
|
|
inputs:
|
|
command: 'restore'
|
|
projects: '$(Build.SourcesDirectory)/csharp/OnnxRuntime.DesktopOnly.CSharp.sln'
|
|
|
|
# the props file was generated with docker container paths. convert to the 'real' path by replacing the
|
|
# the container path of '/build'. The '>' prefix is to match the closing angle bracket of the tag.
|
|
# e.g. <OnnxRuntimeBuildDirectory>/build/... so we only match the start of a path.
|
|
# We use powershell so we don't need extra escaping of the '/' chars in the path.
|
|
- task: CmdLine@2
|
|
displayName: 'Update props from docker path to local and create models link'
|
|
inputs:
|
|
script: |
|
|
pwsh -Command '(Get-Content $(Build.SourcesDirectory)/csharp/Directory.Build.props) -replace ">/build", ">$(Build.BinariesDirectory)" | Set-Content $(Build.SourcesDirectory)/csharp/Directory.Build.props'
|
|
cat $(Build.SourcesDirectory)/csharp/Directory.Build.props
|
|
ln -s /data/models $(Build.BinariesDirectory)/models
|
|
|
|
- task: DotNetCoreCLI@2
|
|
displayName: 'dotnet build C# sln'
|
|
inputs:
|
|
command: 'build'
|
|
projects: '$(Build.SourcesDirectory)/csharp/OnnxRuntime.DesktopOnly.CSharp.sln'
|
|
|
|
- task: DotNetCoreCLI@2
|
|
displayName: 'dotnet test C#'
|
|
inputs:
|
|
command: 'test'
|
|
projects: '$(Build.SourcesDirectory)/csharp/OnnxRuntime.DesktopOnly.CSharp.sln'
|
|
# extra logging so all tests are listed in output to validate what's actually run
|
|
arguments: '-f net6.0 --no-build -l "console;verbosity=normal"'
|
|
workingDirectory: $(Build.SourcesDirectory)/csharp
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install python deps and run java tests'
|
|
inputs:
|
|
script: |
|
|
set -e -x
|
|
python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml ort-nightly-directml onnx -qq
|
|
cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt $(Build.BinariesDirectory)/requirements.txt
|
|
# Test ORT with the latest ONNX release.
|
|
sed -i "s/git+http:\/\/github\.com\/onnx\/onnx.*/onnx/" $(Build.BinariesDirectory)/requirements.txt
|
|
python3 -m pip install -r $(Build.BinariesDirectory)/requirements.txt
|
|
mkdir $(Build.BinariesDirectory)/requirements_torch_cpu/
|
|
cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt $(Build.BinariesDirectory)/requirements_torch_cpu/requirements.txt
|
|
python3 -m pip install -r $(Build.BinariesDirectory)/requirements_torch_cpu/requirements.txt
|
|
cd $(Build.SourcesDirectory)/java
|
|
$(Build.SourcesDirectory)/java/gradlew "cmakeCheck" "-DcmakeBuildDir=$(Build.BinariesDirectory)/Release"
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install Release python package'
|
|
inputs:
|
|
script: |
|
|
rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11
|
|
python3 -m pip install $(Build.BinariesDirectory)/Release/dist/*.whl
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Run Release unit tests'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/build.py
|
|
workingDirectory: $(Build.BinariesDirectory)/Release
|
|
arguments: >-
|
|
--build_dir $(Build.BinariesDirectory)
|
|
--cmake_generator Ninja
|
|
--config Release
|
|
--test
|
|
--skip_submodule_sync
|
|
--build_shared_lib
|
|
--parallel
|
|
--build_wheel
|
|
--enable_onnx_tests
|
|
--enable_transformers_tool_test
|
|
--build_nodejs
|
|
--ctest_path ""
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install Debug python package'
|
|
inputs:
|
|
script: |
|
|
set -e -x
|
|
rm -rf $(Build.BinariesDirectory)/Debug/onnxruntime $(Build.BinariesDirectory)/Debug/pybind11
|
|
python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml ort-nightly-directml -qq
|
|
python3 -m pip install $(Build.BinariesDirectory)/Debug/dist/*.whl
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Run Debug unit tests'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/build.py
|
|
workingDirectory: $(Build.BinariesDirectory)/Debug
|
|
arguments: >-
|
|
--build_dir $(Build.BinariesDirectory)
|
|
--cmake_generator Ninja
|
|
--config Debug
|
|
--test
|
|
--skip_submodule_sync
|
|
--build_shared_lib
|
|
--parallel
|
|
--build_wheel
|
|
--enable_onnx_tests
|
|
--enable_transformers_tool_test
|
|
--build_nodejs
|
|
--ctest_path ""
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Symbolic shape infer'
|
|
inputs:
|
|
scriptPath: $(Build.BinariesDirectory)/Release/onnxruntime_test_python_symbolic_shape_infer.py
|
|
workingDirectory: $(Build.BinariesDirectory)/Release
|
|
|
|
- task: PublishTestResults@2
|
|
displayName: 'Publish unit test results'
|
|
inputs:
|
|
testResultsFiles: '**/*.results.xml'
|
|
searchFolder: '$(Build.BinariesDirectory)'
|
|
testRunTitle: 'Unit Test Run'
|
|
condition: succeededOrFailed()
|
|
|
|
|
|
- stage: arm64_build
|
|
dependsOn: []
|
|
jobs:
|
|
- template: templates/py-linux.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
base_image: 'arm64v8/centos:7'
|
|
devtoolset_rootpath: /opt/rh/devtoolset-10/root
|
|
ld_library_path_arg: /opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64
|
|
prepend_path: '/opt/rh/devtoolset-10/root/usr/bin:'
|
|
with_cache: true
|
|
|
|
- stage: arm64_test
|
|
dependsOn: ['arm64_build']
|
|
jobs:
|
|
- template: templates/py-packaging-linux-test.yml
|
|
parameters:
|
|
arch: 'aarch64'
|
|
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
|
|
device: 'CPU'
|