mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-06 00:03:22 +00:00
e7987a6b0b
### Description This PR will set default python to 3.10 except tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml. This is needed because we are no longer using python 3.8 This PR excludes changes for Big Models CI, because it will require additional changes. Which will be track in USER STORY 52729 ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. -->
339 lines
14 KiB
YAML
339 lines
14 KiB
YAML
# This CI has the following steps:
|
|
# 1. Build full ORT, install the full ORT python wheel and use it to generate ort format test models
|
|
# and include ops config file for step 3.
|
|
# 2. Build minimal ORT including all the kernels and disable exceptions.
|
|
# This step is build only to safe-guard the --disable_exceptions option.
|
|
# 3. Build minimal ORT include only the kernels using the include ops config file from step 1,
|
|
# and the models from <repo root>/onnxruntime/test/testdata/, run UT, and use onnx_test_runner to
|
|
# test the ort format models generated in step 1.
|
|
# Exceptions are enabled in this step to help debugging in case of CI failure.
|
|
# This step builds and tests ORT with (3a) and without (3b) type reduction enabled.
|
|
# 4. Build minimal ORT with type reduction from a globally allowed types list.
|
|
# This step uses a hard-coded list of types which may not include the types needed by the models
|
|
# in <repo root>/onnxruntime/test/testdata/, so the tests for those models are skipped.
|
|
# 5. Build extended minimal ORT and run tests.
|
|
# 6. Build with all optional features disabled and no kernels.
|
|
# 6a: regular build with python enabled checks that the exclusions don't break code paths in a full build.
|
|
# 6b: minimal build with exceptions and python disabled checks that the exclusions don't break code paths in a
|
|
# minimal build.
|
|
# 6c: extended minimal build with exceptions and python disabled checks that the exclusions don't break code paths
|
|
# in an extended minimal build.
|
|
# 7. Build extended minimal ORT with NNAPI, with exceptions/RTTI/ml_ops disabled, for Android(arm64-v8a),
|
|
# this safe-guards the extended minimal build with NNAPI EP.
|
|
trigger:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
pr:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
jobs:
|
|
- job: Linux_CPU_Minimal_Build_E2E
|
|
timeoutInMinutes: 120
|
|
workspace:
|
|
clean: all
|
|
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
|
variables:
|
|
ORT_CACHE_DIR: $(Pipeline.Workspace)/ort_ccache
|
|
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
|
test_data_directory: $(Build.SourcesDirectory)/.test_data
|
|
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: "templates/use-android-ndk.yml"
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
|
Repository: onnxruntimecpubuild
|
|
|
|
- task: CmdLine@2
|
|
displayName: Create test data directory
|
|
inputs:
|
|
script: |
|
|
# Create a folder for all test data
|
|
mkdir -p $(test_data_directory)
|
|
# create empty config used in some parts
|
|
touch $(test_data_directory)/include_no_operators.config
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- template: templates/linux-build-step-with-cache.yml
|
|
parameters:
|
|
WithCache: true
|
|
Today: $(TODAY)
|
|
AdditionalKey: onnxruntime__full
|
|
CacheDir: $(ORT_CACHE_DIR)
|
|
ChangeEveryCommit: true
|
|
BuildStep:
|
|
- task: CmdLine@2
|
|
displayName: 1. Build full onnxruntime and generate ORT format test files
|
|
inputs:
|
|
script: |
|
|
docker run --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $(test_data_directory):/home/onnxruntimedev/.test_data \
|
|
--volume $(ORT_CACHE_DIR):/cache \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
-e CCACHE_DIR=/cache \
|
|
-e ORT_BUILD_WITH_CACHE=1 \
|
|
onnxruntimecpubuild \
|
|
/bin/bash -c "
|
|
set -e -x;
|
|
/onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh /build/1; \
|
|
ccache -sv; \
|
|
ccache -z;"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 2. Build minimal onnxruntime [exceptions DISABLED, type reduction DISABLED, training ops ENABLED]
|
|
inputs:
|
|
script: |
|
|
# We will try to build minimal ORT with exceptions disabled and training ops enabled
|
|
# Only the building process is verified here, no test will be performed
|
|
docker run --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build/2 --cmake_generator Ninja \
|
|
--config Debug \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel --use_binskim_compliant_compile_flags \
|
|
--skip_tests \
|
|
--minimal_build \
|
|
--disable_exceptions \
|
|
--enable_training_ops
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 3a. Build minimal onnxruntime [exceptions ENABLED, type reduction DISABLED, custom ops ENABLED] and run tests
|
|
inputs:
|
|
script: |
|
|
docker run --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $(test_data_directory):/home/onnxruntimedev/.test_data \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
|
|
--build-directory /build/3a \
|
|
--reduced-ops-config /home/onnxruntimedev/.test_data/required_ops.ort_models.config \
|
|
--enable-custom-ops
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 3b. Build minimal onnxruntime [exceptions ENABLED, type reduction ENABLED] and run tests
|
|
inputs:
|
|
script: |
|
|
docker run --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $(test_data_directory):/home/onnxruntimedev/.test_data \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
|
|
--build-directory /build/3b \
|
|
--reduced-ops-config /home/onnxruntimedev/.test_data/required_ops_and_types.ort_models.config \
|
|
--enable-type-reduction
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 4. Build minimal onnxruntime [exceptions ENABLED, type reduction ENABLED (globally allowed types)] and run tests
|
|
inputs:
|
|
script: |
|
|
printf "%s\n%s\n" \
|
|
"!globally_allowed_types;bool,float,int8_t,uint8_t" \
|
|
"!no_ops_specified_means_all_ops_are_required" \
|
|
> $(test_data_directory)/globally_allowed_types.config && \
|
|
docker run --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $(test_data_directory):/home/onnxruntimedev/.test_data \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
|
|
--build-directory /build/4 \
|
|
--reduced-ops-config /home/onnxruntimedev/.test_data/globally_allowed_types.config \
|
|
--enable-type-reduction \
|
|
--skip-model-tests
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 5. Build extended minimal onnxruntime and run tests
|
|
inputs:
|
|
script: |
|
|
docker run --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build/5 --cmake_generator Ninja \
|
|
--config Debug \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib --use_binskim_compliant_compile_flags \
|
|
--parallel \
|
|
--minimal_build extended
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 6a. Regular build with python and all optional features disabled.
|
|
inputs:
|
|
script: |
|
|
docker run --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $(test_data_directory):/home/onnxruntimedev/.test_data \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build/6a \
|
|
--cmake_generator Ninja \
|
|
--config MinSizeRel \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--build_wheel \
|
|
--parallel --use_binskim_compliant_compile_flags \
|
|
--skip_tests \
|
|
--disable_ml_ops \
|
|
--disable_types sparsetensor float8 optional \
|
|
--include_ops_by_config /home/onnxruntimedev/.test_data/include_no_operators.config \
|
|
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 6b. Minimal build with all optional features disabled.
|
|
inputs:
|
|
script: |
|
|
docker run --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $(test_data_directory):/home/onnxruntimedev/.test_data \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build/6b \
|
|
--cmake_generator Ninja \
|
|
--config MinSizeRel \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel --use_binskim_compliant_compile_flags \
|
|
--minimal_build \
|
|
--disable_exceptions \
|
|
--disable_ml_ops \
|
|
--skip_tests \
|
|
--enable_reduced_operator_type_support \
|
|
--disable_types sparsetensor optional float8 \
|
|
--include_ops_by_config /home/onnxruntimedev/.test_data/include_no_operators.config \
|
|
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 6c. Extended minimal build with all optional features disabled.
|
|
inputs:
|
|
script: |
|
|
docker run --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $(test_data_directory):/home/onnxruntimedev/.test_data \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build/6c \
|
|
--cmake_generator Ninja \
|
|
--config MinSizeRel \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib --use_binskim_compliant_compile_flags \
|
|
--parallel \
|
|
--minimal_build extended \
|
|
--disable_exceptions \
|
|
--disable_ml_ops \
|
|
--skip_tests \
|
|
--enable_reduced_operator_type_support \
|
|
--disable_types sparsetensor optional float8 \
|
|
--include_ops_by_config /home/onnxruntimedev/.test_data/include_no_operators.config \
|
|
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 7. Extended minimal build with NNAPI EP for Android(arm64-v8a) and skip tests.
|
|
inputs:
|
|
script: |
|
|
NDK_HOME=$(realpath $ANDROID_NDK_HOME)
|
|
docker run --rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $ANDROID_HOME:/android_home \
|
|
--volume $NDK_HOME:/ndk_home \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
|
-e NIGHTLY_BUILD \
|
|
onnxruntimecpubuild \
|
|
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build/7 \
|
|
--cmake_generator Ninja \
|
|
--config MinSizeRel \
|
|
--skip_submodule_sync \
|
|
--parallel --use_binskim_compliant_compile_flags \
|
|
--android \
|
|
--android_sdk_path /android_home \
|
|
--android_ndk_path /ndk_home \
|
|
--android_abi=arm64-v8a \
|
|
--android_api=29 \
|
|
--use_nnapi \
|
|
--minimal_build extended \
|
|
--build_shared_lib \
|
|
--disable_ml_ops \
|
|
--disable_exceptions \
|
|
--skip_tests --path_to_protoc_exe /usr/bin/protoc
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- template: templates/explicitly-defined-final-tasks.yml
|