mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-23 02:38:28 +00:00
Update onnxruntime binary size checks ci pipeline's docker image (#23405)
1. Update onnxruntime binary size checks ci pipeline's docker image. Use a different docker image that is not manylinux based. The new one is smaller. 2. Add flatbuffers tools/ci_build/requirements/pybind/requirements.txt 3. Delete tools/ci_build/github/azure-pipelines/py-package-build-pipeline.yml. The pipeline was for generating packages for Olive, but it went unused. And the content is highly duplicated with our official python packaging pipeline. 4. A lot of YAML files reference pypa/manylinux git repo but do not use it. This PR removes the references.
This commit is contained in:
parent
db8e10b0b9
commit
d461ca9dcd
18 changed files with 51 additions and 678 deletions
|
|
@ -6,7 +6,6 @@
|
|||
|
||||
set -e
|
||||
set -x
|
||||
export PATH=/opt/python/cp312-cp312/bin:$PATH
|
||||
|
||||
ls /build
|
||||
ls /build/deps
|
||||
|
|
@ -25,7 +24,7 @@ ANDROID_SDK_HOME="/android_home"
|
|||
ANDROID_NDK_HOME="/ndk_home"
|
||||
QNN_HOME="/qnn_home"
|
||||
|
||||
|
||||
python3 -m pip install -r /onnxruntime_src/tools/ci_build/requirements/pybind/requirements.txt
|
||||
# Base command for building the AAR package
|
||||
COMMAND="python3 $BUILD_SCRIPT --build_dir /build --config $BUILD_CONFIG --android_sdk_path $ANDROID_SDK_HOME --android_ndk_path $ANDROID_NDK_HOME $BUILD_SETTINGS"
|
||||
|
||||
|
|
|
|||
|
|
@ -4,14 +4,6 @@ parameters:
|
|||
type: boolean
|
||||
default: false
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: manylinux
|
||||
type: Github
|
||||
endpoint: Microsoft
|
||||
name: pypa/manylinux
|
||||
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
||||
|
||||
stages:
|
||||
- template: templates/android-binary-size-check-stage.yml
|
||||
parameters:
|
||||
|
|
|
|||
|
|
@ -6,14 +6,6 @@ parameters:
|
|||
type: boolean
|
||||
default: true
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: manylinux
|
||||
type: Github
|
||||
endpoint: Microsoft
|
||||
name: pypa/manylinux
|
||||
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
||||
|
||||
stages:
|
||||
|
||||
# build binaries for Android
|
||||
|
|
|
|||
|
|
@ -70,11 +70,6 @@ resources:
|
|||
type: github
|
||||
endpoint: ort-examples
|
||||
name: microsoft/onnxruntime-inference-examples
|
||||
- repository: manylinux
|
||||
type: Github
|
||||
endpoint: Microsoft
|
||||
name: pypa/manylinux
|
||||
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
||||
|
||||
variables:
|
||||
- template: templates/common-variables.yml
|
||||
|
|
|
|||
|
|
@ -78,11 +78,7 @@ resources:
|
|||
type: github
|
||||
endpoint: ort-examples
|
||||
name: microsoft/onnxruntime-inference-examples
|
||||
- repository: manylinux
|
||||
type: Github
|
||||
endpoint: Microsoft
|
||||
name: pypa/manylinux
|
||||
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
||||
|
||||
|
||||
stages:
|
||||
# Set ReleaseVersionSuffix
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ stages:
|
|||
onnxruntimecpubuildcentos8x64 \
|
||||
/bin/bash -c '
|
||||
set -ex; \
|
||||
python3.12 /onnxruntime_src/tools/ci_build/build.py \
|
||||
python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir /build --cmake_generator 'Ninja' \
|
||||
--config Debug \
|
||||
--skip_submodule_sync \
|
||||
|
|
@ -87,7 +87,7 @@ stages:
|
|||
--parallel --use_binskim_compliant_compile_flags \
|
||||
--enable_onnx_tests --enable_address_sanitizer \
|
||||
--update --build;
|
||||
python3.12 /onnxruntime_src/tools/ci_build/build.py \
|
||||
python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir /build --cmake_generator 'Ninja' \
|
||||
--config Debug \
|
||||
--skip_submodule_sync \
|
||||
|
|
|
|||
|
|
@ -67,10 +67,10 @@ jobs:
|
|||
|
||||
- template: templates/get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
|
||||
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu
|
||||
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
||||
Repository: onnxruntimecpubuild
|
||||
Repository: onnxruntimecpubuildcentos8x64_packaging
|
||||
|
||||
- task: CmdLine@2
|
||||
displayName: Create test data directory
|
||||
|
|
@ -104,7 +104,7 @@ jobs:
|
|||
-e BUILD_BUILDNUMBER \
|
||||
-e CCACHE_DIR=/cache \
|
||||
-e ORT_BUILD_WITH_CACHE=1 \
|
||||
onnxruntimecpubuild \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
/bin/bash -c "
|
||||
set -e -x;
|
||||
/onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh /build/1; \
|
||||
|
|
@ -124,8 +124,8 @@ jobs:
|
|||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
onnxruntimecpubuild \
|
||||
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
bash -c "python3 -m pip install -r /onnxruntime_src/tools/ci_build/requirements/pybind/requirements.txt && python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir /build/2 --cmake_generator Ninja \
|
||||
--config Debug \
|
||||
--skip_submodule_sync \
|
||||
|
|
@ -134,7 +134,7 @@ jobs:
|
|||
--skip_tests \
|
||||
--minimal_build \
|
||||
--disable_exceptions \
|
||||
--enable_training_ops
|
||||
--enable_training_ops"
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: CmdLine@2
|
||||
|
|
@ -148,7 +148,7 @@ jobs:
|
|||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
onnxruntimecpubuild \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
|
||||
--build-directory /build/3a \
|
||||
--reduced-ops-config /home/onnxruntimedev/.test_data/required_ops.ort_models.config \
|
||||
|
|
@ -166,7 +166,7 @@ jobs:
|
|||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
onnxruntimecpubuild \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
|
||||
--build-directory /build/3b \
|
||||
--reduced-ops-config /home/onnxruntimedev/.test_data/required_ops_and_types.ort_models.config \
|
||||
|
|
@ -188,7 +188,7 @@ jobs:
|
|||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
onnxruntimecpubuild \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
|
||||
--build-directory /build/4 \
|
||||
--reduced-ops-config /home/onnxruntimedev/.test_data/globally_allowed_types.config \
|
||||
|
|
@ -206,14 +206,14 @@ jobs:
|
|||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
onnxruntimecpubuild \
|
||||
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
bash -c "python3 -m pip install -r /onnxruntime_src/tools/ci_build/requirements/pybind/requirements.txt && python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir /build/5 --cmake_generator Ninja \
|
||||
--config Debug \
|
||||
--skip_submodule_sync \
|
||||
--build_shared_lib --use_binskim_compliant_compile_flags \
|
||||
--parallel \
|
||||
--minimal_build extended
|
||||
--minimal_build extended"
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: CmdLine@2
|
||||
|
|
@ -227,8 +227,8 @@ jobs:
|
|||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
onnxruntimecpubuild \
|
||||
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
bash -c "python3 -m pip install -r /onnxruntime_src/tools/ci_build/requirements/pybind/requirements.txt && python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir /build/6a \
|
||||
--cmake_generator Ninja \
|
||||
--config MinSizeRel \
|
||||
|
|
@ -240,7 +240,7 @@ jobs:
|
|||
--disable_ml_ops \
|
||||
--disable_types sparsetensor float8 optional \
|
||||
--include_ops_by_config /home/onnxruntimedev/.test_data/include_no_operators.config \
|
||||
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF
|
||||
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF"
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: CmdLine@2
|
||||
|
|
@ -254,8 +254,8 @@ jobs:
|
|||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
onnxruntimecpubuild \
|
||||
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
bash -c "python3 -m pip install -r /onnxruntime_src/tools/ci_build/requirements/pybind/requirements.txt && python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir /build/6b \
|
||||
--cmake_generator Ninja \
|
||||
--config MinSizeRel \
|
||||
|
|
@ -269,7 +269,7 @@ jobs:
|
|||
--enable_reduced_operator_type_support \
|
||||
--disable_types sparsetensor optional float8 \
|
||||
--include_ops_by_config /home/onnxruntimedev/.test_data/include_no_operators.config \
|
||||
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF
|
||||
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF"
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: CmdLine@2
|
||||
|
|
@ -283,8 +283,8 @@ jobs:
|
|||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
onnxruntimecpubuild \
|
||||
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
bash -c "python3 -m pip install -r /onnxruntime_src/tools/ci_build/requirements/pybind/requirements.txt && python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir /build/6c \
|
||||
--cmake_generator Ninja \
|
||||
--config MinSizeRel \
|
||||
|
|
@ -298,7 +298,7 @@ jobs:
|
|||
--enable_reduced_operator_type_support \
|
||||
--disable_types sparsetensor optional float8 \
|
||||
--include_ops_by_config /home/onnxruntimedev/.test_data/include_no_operators.config \
|
||||
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF
|
||||
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF"
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: CmdLine@2
|
||||
|
|
@ -313,8 +313,8 @@ jobs:
|
|||
--volume $NDK_HOME:/ndk_home \
|
||||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
onnxruntimecpubuild \
|
||||
/opt/python/cp310-cp310/bin/python3.10 /onnxruntime_src/tools/ci_build/build.py \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
bash -c "python3 -m pip install -r /onnxruntime_src/tools/ci_build/requirements/pybind/requirements.txt && python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir /build/7 \
|
||||
--cmake_generator Ninja \
|
||||
--config MinSizeRel \
|
||||
|
|
@ -330,7 +330,7 @@ jobs:
|
|||
--build_shared_lib \
|
||||
--disable_ml_ops \
|
||||
--disable_exceptions \
|
||||
--skip_tests --path_to_protoc_exe /usr/bin/protoc
|
||||
--skip_tests"
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- template: templates/explicitly-defined-final-tasks.yml
|
||||
|
|
|
|||
|
|
@ -39,14 +39,6 @@ parameters:
|
|||
- 'custom'
|
||||
default: 'nightly (@dev)'
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: manylinux # The name used to reference this repository in the checkout step
|
||||
type: Github
|
||||
endpoint: Microsoft
|
||||
name: pypa/manylinux
|
||||
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
||||
|
||||
variables:
|
||||
skipComponentGovernanceDetection: true
|
||||
${{ if eq(parameters.NpmPublish, 'nightly (@dev)') }}:
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
parameters:
|
||||
- name: enable_linux_cpu
|
||||
displayName: 'Whether Linux CPU package is built.'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
- name: enable_linux_gpu
|
||||
displayName: 'Whether Linux GPU package is built.'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
- name: enable_windows_cpu
|
||||
displayName: 'Whether Windows CPU package is built.'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
- name: enable_windows_gpu
|
||||
displayName: 'Whether Windows GPU package is built.'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
- name: python_version
|
||||
displayName: 'Python version used for build'
|
||||
type: object
|
||||
default: ["'3.8'"]
|
||||
|
||||
- name: cpu_build_py_parameters
|
||||
displayName: 'Extra parameters to pass to build.py for CPU package.'
|
||||
type: string
|
||||
default: '--use_openvino CPU'
|
||||
|
||||
- name: gpu_build_py_parameters
|
||||
displayName: 'Extra parameters to pass to build.py for GPU package.'
|
||||
type: string
|
||||
default: ' '
|
||||
|
||||
- name: ubuntu_version_linux_cpu
|
||||
displayName: 'Ubuntu Version for Linux CPU package.'
|
||||
type: string
|
||||
default: '20.04'
|
||||
|
||||
trigger: none
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: manylinux
|
||||
type: Github
|
||||
endpoint: Microsoft
|
||||
name: pypa/manylinux
|
||||
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
||||
|
||||
stages:
|
||||
- template: templates/py-packaging-selectable-stage.yml
|
||||
parameters:
|
||||
enable_linux_gpu: ${{ parameters.enable_linux_gpu }}
|
||||
enable_linux_cpu: ${{ parameters.enable_linux_cpu }}
|
||||
enable_windows_cpu: ${{ parameters.enable_windows_cpu }}
|
||||
enable_windows_gpu: ${{ parameters.enable_windows_gpu }}
|
||||
python_version: ${{ parameters.python_version }}
|
||||
cpu_build_py_parameters: ${{ parameters.cpu_build_py_parameters }}
|
||||
gpu_build_py_parameters: ${{ parameters.gpu_build_py_parameters }}
|
||||
ubuntu_version_linux_cpu: ${{ parameters.ubuntu_version_linux_cpu }}
|
||||
|
|
@ -38,13 +38,12 @@ stages:
|
|||
submodules: none
|
||||
|
||||
- template: use-android-ndk.yml
|
||||
#TODO: use a different docker file since this job doesn't need to rely on manylinux
|
||||
- template: get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
|
||||
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu
|
||||
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
||||
Repository: onnxruntimecpubuild
|
||||
Repository: onnxruntimecpubuildcentos8x64_packaging
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Set variables from config file "${{ parameters.BuildConfigFile }}"'
|
||||
|
|
@ -83,6 +82,7 @@ stages:
|
|||
--volume $(Build.BinariesDirectory):/build \
|
||||
--volume $ANDROID_HOME:/android_home \
|
||||
--volume $NDK_HOME:/ndk_home \
|
||||
-w /onnxruntime_src \
|
||||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
|
|
@ -90,11 +90,10 @@ stages:
|
|||
-e BUILD_ID=$(Build.BuildId) \
|
||||
-e BUILD_REASON=$(Build.Reason) \
|
||||
-e BUILD_BRANCH=$(Build.SourceBranch) \
|
||||
onnxruntimecpubuild \
|
||||
/opt/python/cp310-cp310/bin/python3 /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
bash -c "python3 -m pip install -r tools/ci_build/requirements/pybind/requirements.txt && python3 tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py \
|
||||
--build_dir /build/1a \
|
||||
${BINARY_SIZE_THRESHOLD_ARGS} \
|
||||
"/onnxruntime_src/${{ parameters.BuildConfigFile }}"
|
||||
${BINARY_SIZE_THRESHOLD_ARGS} ${{ parameters.BuildConfigFile }} "
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: AzureCLI@2
|
||||
|
|
@ -139,6 +138,7 @@ stages:
|
|||
--volume $(Build.BinariesDirectory):/build \
|
||||
--volume $ANDROID_HOME:/android_home \
|
||||
--volume $NDK_HOME:/ndk_home \
|
||||
-w /onnxruntime_src \
|
||||
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
|
||||
-e NIGHTLY_BUILD \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
|
|
@ -146,11 +146,10 @@ stages:
|
|||
-e BUILD_ID=$(Build.BuildId) \
|
||||
-e BUILD_REASON=$(Build.Reason) \
|
||||
-e BUILD_BRANCH=$(Build.SourceBranch) \
|
||||
onnxruntimecpubuild \
|
||||
/opt/python/cp310-cp310/bin/python3 /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
bash -c "python3 -m pip install -r tools/ci_build/requirements/pybind/requirements.txt && python3 tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py \
|
||||
--build_dir /build/1b \
|
||||
--with_debug_info \
|
||||
"/onnxruntime_src/${{ parameters.BuildConfigFile }}"
|
||||
--with_debug_info ${{ parameters.BuildConfigFile }}"
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: PublishPipelineArtifact@1
|
||||
|
|
|
|||
|
|
@ -76,13 +76,12 @@ jobs:
|
|||
mkdir -p $(artifacts_directory)
|
||||
workingDirectory: $(Build.BinariesDirectory)
|
||||
|
||||
#TODO: use a different docker file since this job doesn't need to rely on manylinux
|
||||
- template: get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
|
||||
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu
|
||||
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
||||
Repository: onnxruntimecpubuild
|
||||
Repository: onnxruntimecpubuildcentos8x64_packaging
|
||||
|
||||
- template: set-version-number-variables-step.yml
|
||||
|
||||
|
|
@ -127,7 +126,7 @@ jobs:
|
|||
-e PUBLISH_EXECUTABLES=${{parameters.publish_executables}} \
|
||||
-e PACKAGE_NAME=${{parameters.packageName}} \
|
||||
-e RELEASE_VERSION_SUFFIX=${{parameters.ReleaseVersionSuffix}} \
|
||||
onnxruntimecpubuild \
|
||||
onnxruntimecpubuildcentos8x64_packaging \
|
||||
/bin/bash /onnxruntime_src/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh $USE_QNN
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ jobs:
|
|||
set -e -x
|
||||
mkdir -p $HOME/.onnx
|
||||
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
|
||||
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3.12 \
|
||||
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3 \
|
||||
/onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \
|
||||
--skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed"
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
|
|
|||
|
|
@ -1,529 +0,0 @@
|
|||
parameters:
|
||||
- name: enable_linux_cpu
|
||||
displayName: 'Whether Linux CPU package is built.'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
- name: enable_windows_cpu
|
||||
displayName: 'Whether Windows CPU package is built.'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
- name: enable_linux_gpu
|
||||
displayName: 'Whether Linux GPU package is built.'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
- name: enable_windows_gpu
|
||||
displayName: 'Whether Windows GPU package is built.'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
- name: python_version
|
||||
displayName: 'Python version used for build'
|
||||
type: object
|
||||
default: ["'3.8'"]
|
||||
|
||||
- name: cpu_build_py_parameters
|
||||
displayName: 'Extra parameters to pass to build.py for CPU package.'
|
||||
type: string
|
||||
default: '--use_openvino CPU'
|
||||
|
||||
- name: gpu_build_py_parameters
|
||||
displayName: 'Extra parameters to pass to build.py for GPU package.'
|
||||
type: string
|
||||
default: ' '
|
||||
|
||||
- name: ubuntu_version_linux_cpu
|
||||
displayName: 'Ubuntu Version for Linux CPU package.'
|
||||
type: string
|
||||
default: '20.04'
|
||||
|
||||
stages:
|
||||
- stage: Python_Packaging
|
||||
|
||||
jobs:
|
||||
- ${{ if eq(parameters.enable_linux_cpu, true) }}:
|
||||
- job: Linux_CPU_py_Wheels
|
||||
timeoutInMinutes: 90
|
||||
workspace:
|
||||
clean: all
|
||||
pool: onnxruntime-Ubuntu2204-AMD-CPU
|
||||
strategy:
|
||||
matrix:
|
||||
${{ each PythonVersion in parameters.python_version }}:
|
||||
'Python${{ PythonVersion }}':
|
||||
PythonVersion: ${{ PythonVersion }}
|
||||
variables:
|
||||
UbuntuVersion: ${{ parameters.ubuntu_version_linux_cpu }}
|
||||
steps:
|
||||
- checkout: self
|
||||
clean: true
|
||||
submodules: recursive
|
||||
|
||||
- template: get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
DockerBuildArgs: "--build-arg PYTHON_VERSION=$(PythonVersion) --build-arg UBUNTU_VERSION=$(UbuntuVersion)"
|
||||
Repository: onnxruntimeubuntupython$(PythonVersion)cpubuild
|
||||
|
||||
- task: CmdLine@2
|
||||
displayName: 'Build Python Wheel'
|
||||
inputs:
|
||||
script: |
|
||||
mkdir -p $HOME/.onnx
|
||||
docker run --rm \
|
||||
--volume /data/onnx:/data/onnx:ro \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--volume /data/models:/build/models:ro \
|
||||
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
onnxruntimeubuntupython$(PythonVersion)cpubuild \
|
||||
python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir /build \
|
||||
--config Release --update --build \
|
||||
--skip_submodule_sync \
|
||||
--parallel \
|
||||
--enable_lto \
|
||||
--build_wheel \
|
||||
--enable_onnx_tests \
|
||||
--test \
|
||||
--ctest_path '' \
|
||||
${{ parameters.cpu_build_py_parameters }}
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)/Release/dist'
|
||||
Contents: '*.whl'
|
||||
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Publish Artifact: ONNXRuntime python wheel and documentation'
|
||||
inputs:
|
||||
ArtifactName: onnxruntime
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters:
|
||||
condition: 'succeeded'
|
||||
|
||||
- template: clean-agent-build-directory-step.yml
|
||||
|
||||
- ${{ if eq(parameters.enable_windows_cpu, true) }}:
|
||||
- job: Windows_CPU_py_Wheels
|
||||
pool: 'onnxruntime-cpu-openvino-winbuild'
|
||||
strategy:
|
||||
matrix:
|
||||
${{ each PythonVersion in parameters.python_version }}:
|
||||
'Python${{ PythonVersion }}':
|
||||
PythonVersion: ${{ PythonVersion }}
|
||||
variables:
|
||||
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
|
||||
EnvSetupScript: setup_env.bat
|
||||
setVcvars: true
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
timeoutInMinutes: 120
|
||||
workspace:
|
||||
clean: all
|
||||
|
||||
steps:
|
||||
- checkout: self
|
||||
clean: true
|
||||
submodules: recursive
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: $(PythonVersion)
|
||||
addToPath: true
|
||||
|
||||
- task: BatchScript@1
|
||||
displayName: 'setup env'
|
||||
inputs:
|
||||
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
|
||||
modifyEnvironment: true
|
||||
workingFolder: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: BatchScript@1
|
||||
displayName: 'setup OpenVino env'
|
||||
inputs:
|
||||
filename: 'C:\Program Files\Intel\openvino_2021.4.752\bin\setupvars.bat'
|
||||
modifyEnvironment: true
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Install ONNX'
|
||||
inputs:
|
||||
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig)
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: >
|
||||
--config $(BuildConfig)
|
||||
--enable_lto
|
||||
--build_dir $(Build.BinariesDirectory)
|
||||
--skip_submodule_sync
|
||||
--cmake_generator "Visual Studio 17 2022"
|
||||
--enable_pybind
|
||||
--enable_onnx_tests
|
||||
${{ parameters.cpu_build_py_parameters }}
|
||||
--parallel --update
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build'
|
||||
inputs:
|
||||
solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln'
|
||||
platform: x64
|
||||
configuration: $(BuildConfig)
|
||||
msbuildArchitecture: x64
|
||||
maximumCpuCount: true
|
||||
logProjectEvents: true
|
||||
workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
|
||||
createLogFile: true
|
||||
|
||||
# Esrp signing
|
||||
- template: win-esrp-dll.yml
|
||||
parameters:
|
||||
FolderPath: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)\onnxruntime\capi'
|
||||
DisplayName: 'ESRP - Sign Native dlls'
|
||||
DoEsrp: true
|
||||
Pattern: '*.pyd,*.dll'
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Build wheel'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\setup.py'
|
||||
arguments: 'bdist_wheel'
|
||||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)\dist'
|
||||
Contents: '*.whl'
|
||||
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Publish Artifact: ONNXRuntime python wheel'
|
||||
inputs:
|
||||
ArtifactName: onnxruntime
|
||||
|
||||
- script: |
|
||||
7z x *.whl
|
||||
workingDirectory: '$(Build.ArtifactStagingDirectory)'
|
||||
displayName: 'unzip the package'
|
||||
- task: CredScan@3
|
||||
displayName: 'Run CredScan'
|
||||
inputs:
|
||||
debugMode: false
|
||||
continueOnError: true
|
||||
|
||||
- task: BinSkim@4
|
||||
displayName: 'Run BinSkim'
|
||||
inputs:
|
||||
AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'
|
||||
continueOnError: true
|
||||
|
||||
- task: DeleteFiles@1
|
||||
displayName: 'Delete files from $(Build.BinariesDirectory)\$(BuildConfig)'
|
||||
condition: and (succeeded(), eq(variables['PythonVersion'], '3.8'))
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
|
||||
Contents: |
|
||||
**/*.obj
|
||||
**/*.pdb
|
||||
**/*.dll
|
||||
|
||||
- powershell: |
|
||||
python -m pip uninstall -y onnxruntime onnxruntime-gpu -qq
|
||||
Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.whl | foreach {pip --disable-pip-version-check install --force-reinstall --upgrade $_.fullname tabulate}
|
||||
python -m pip install protobuf==3.18.1
|
||||
Remove-Item -Recurse -Force onnxruntime
|
||||
python onnx_backend_test_series.py
|
||||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
displayName: 'Run Python Tests'
|
||||
|
||||
#Skip it for 32 bits x86 build. Currently the scan tool has a bug: it doesn't allow me use 64 bits link.exe
|
||||
#in 32 bits Win32 build. I tried all the settings but they all don't work.
|
||||
- task: SDLNativeRules@3
|
||||
displayName: 'Run the PREfast SDL Native Rules for MSBuild'
|
||||
condition: and (succeeded(), eq(variables['PythonVersion'], '3.8'))
|
||||
inputs:
|
||||
msBuildArchitecture: amd64
|
||||
setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --enable_pybind --enable_onnx_tests --parallel --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON'
|
||||
msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\Debug\onnxruntime.sln" /p:platform="x64" /p:configuration=Debug /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
|
||||
excludedPaths: '$(Build.BinariesDirectory)#$(Build.SourcesDirectory)\cmake#C:\program files (x86)'
|
||||
|
||||
- task: TSAUpload@2
|
||||
displayName: 'TSA upload'
|
||||
condition: and(and (succeeded(), eq(variables['PythonVersion'], '3.8')), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
|
||||
inputs:
|
||||
GdnPublishTsaOnboard: false
|
||||
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
|
||||
continueOnError: true
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters:
|
||||
condition: 'succeeded'
|
||||
|
||||
|
||||
- ${{ if eq(parameters.enable_linux_gpu, true) }}:
|
||||
- job: Linux_py_GPU_Wheels
|
||||
timeoutInMinutes: 300
|
||||
workspace:
|
||||
clean: all
|
||||
pool: Onnxruntime-Linux-GPU
|
||||
strategy:
|
||||
matrix:
|
||||
${{ each PythonVersion in parameters.python_version }}:
|
||||
'Python${{ PythonVersion }}':
|
||||
PythonVersion: ${{ PythonVersion }}
|
||||
steps:
|
||||
- checkout: self
|
||||
clean: true
|
||||
submodules: recursive
|
||||
|
||||
- template: set-python-manylinux-variables-step.yml
|
||||
|
||||
- template: get-docker-image-steps.yml
|
||||
parameters:
|
||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_8_tensorrt8_6
|
||||
Context: tools/ci_build/github/linux/docker
|
||||
DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-11/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-11/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-11/root/usr/lib64:/opt/rh/devtoolset-11/root/usr/lib:/opt/rh/devtoolset-11/root/usr/lib64/dyninst:/opt/rh/devtoolset-11/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )"
|
||||
Repository: onnxruntimecuda118xtrt86build
|
||||
|
||||
- task: CmdLine@2
|
||||
displayName: 'Build Python Wheel'
|
||||
inputs:
|
||||
script: |
|
||||
mkdir -p $HOME/.onnx
|
||||
docker run --gpus all -e CC=/opt/rh/devtoolset-11/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-11/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \
|
||||
--volume /data/onnx:/data/onnx:ro \
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
||||
--volume $(Build.BinariesDirectory):/build \
|
||||
--volume /data/models:/build/models:ro \
|
||||
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
||||
-e BUILD_BUILDNUMBER \
|
||||
onnxruntimecuda118xtrt86build \
|
||||
$(PythonManylinuxDir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir /build --cmake_generator Ninja \
|
||||
--config Release --update --build \
|
||||
--skip_submodule_sync \
|
||||
--parallel \
|
||||
--build_wheel \
|
||||
--enable_onnx_tests --use_tensorrt --cuda_version=11.8 --tensorrt_home=/usr --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \
|
||||
${{ parameters.gpu_build_py_parameters }} \
|
||||
--cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc 'CMAKE_CUDA_ARCHITECTURES=75;80;90'
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- task: CmdLine@2
|
||||
displayName: 'Running tests'
|
||||
condition: and(succeeded(), eq(variables['PythonVersion'], '3.8'))
|
||||
inputs:
|
||||
script: |
|
||||
set -e -x
|
||||
rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11
|
||||
sudo rm -f /build /onnxruntime_src
|
||||
sudo ln -s $(Build.SourcesDirectory) /onnxruntime_src
|
||||
python3 -m pip uninstall -y onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml -qq
|
||||
cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt $(Build.BinariesDirectory)/requirements.txt
|
||||
# Test ORT with the latest ONNX release.
|
||||
sed -i "s/git+http:\/\/github\.com\/onnx\/onnx.*/onnx/" $(Build.BinariesDirectory)/requirements.txt
|
||||
python3 -m pip install -r $(Build.BinariesDirectory)/requirements.txt
|
||||
python3 -m pip install $(Build.BinariesDirectory)/Release/dist/*.whl
|
||||
cd $(Build.BinariesDirectory)/Release
|
||||
ls $(Build.BinariesDirectory)/models
|
||||
rmdir $(Build.BinariesDirectory)/models
|
||||
ln -sf /data/models $(Build.BinariesDirectory)
|
||||
python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir $(Build.BinariesDirectory) --cmake_generator Ninja \
|
||||
--config Release --test \
|
||||
--skip_submodule_sync \
|
||||
--parallel \
|
||||
--build_wheel \
|
||||
--enable_onnx_tests --use_tensorrt --cuda_version=11.8 --tensorrt_home=/usr --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \
|
||||
${{ parameters.gpu_build_py_parameters }} --ctest_path '' \
|
||||
--cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc 'CMAKE_CUDA_ARCHITECTURES=75;80;90'
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)/Release/dist'
|
||||
Contents: '*.whl'
|
||||
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Publish Artifact: ONNXRuntime python wheel'
|
||||
inputs:
|
||||
ArtifactName: onnxruntime_gpu
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters:
|
||||
condition: 'succeeded'
|
||||
|
||||
- template: clean-agent-build-directory-step.yml
|
||||
|
||||
- ${{ if eq(parameters.enable_windows_gpu, true) }}:
|
||||
- job: Windows_py_GPU_Wheels
|
||||
workspace:
|
||||
clean: all
|
||||
pool: 'onnxruntime-Win2022-GPU-A10'
|
||||
timeoutInMinutes: 300
|
||||
variables:
|
||||
- template: common-variables.yml
|
||||
CUDA_VERSION: '11.8'
|
||||
buildArch: x64
|
||||
EpBuildFlags: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda11 }}" --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=75;80;90"
|
||||
EnvSetupScript: setup_env_gpu.bat
|
||||
EP_NAME: gpu
|
||||
VSGenerator: 'Visual Studio 17 2022'
|
||||
strategy:
|
||||
matrix:
|
||||
${{ each PythonVersion in parameters.python_version }}:
|
||||
'Python${{ PythonVersion }}':
|
||||
PythonVersion: ${{ PythonVersion }}
|
||||
steps:
|
||||
- checkout: self
|
||||
clean: true
|
||||
submodules: recursive
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: $(PythonVersion)
|
||||
addToPath: true
|
||||
architecture: 'x64'
|
||||
|
||||
- task: BatchScript@1
|
||||
displayName: 'setup env'
|
||||
inputs:
|
||||
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
|
||||
modifyEnvironment: true
|
||||
workingFolder: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Install ONNX'
|
||||
inputs:
|
||||
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\RelWithDebInfo\installed -build_config RelWithDebInfo
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: >
|
||||
--config RelWithDebInfo
|
||||
--build_dir $(Build.BinariesDirectory)
|
||||
--skip_submodule_sync
|
||||
--cmake_generator "$(VSGenerator)"
|
||||
--enable_pybind
|
||||
--enable_onnx_tests
|
||||
${{ parameters.gpu_build_py_parameters }}
|
||||
--parallel --update
|
||||
$(EpBuildFlags)
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build'
|
||||
inputs:
|
||||
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
|
||||
platform: x64
|
||||
configuration: RelWithDebInfo
|
||||
msbuildArchitecture: $(buildArch)
|
||||
maximumCpuCount: true
|
||||
logProjectEvents: true
|
||||
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
|
||||
createLogFile: true
|
||||
|
||||
# Esrp signing
|
||||
- template: win-esrp-dll.yml
|
||||
parameters:
|
||||
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi'
|
||||
DisplayName: 'ESRP - Sign Native dlls'
|
||||
DoEsrp: true
|
||||
Pattern: '*.pyd,*.dll'
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Build wheel'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\setup.py'
|
||||
arguments: 'bdist_wheel ${{ parameters.gpu_build_py_parameters }} --wheel_name_suffix=$(EP_NAME)'
|
||||
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
|
||||
|
||||
- task: CopyFiles@2
|
||||
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
|
||||
Contents: '*.whl'
|
||||
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Publish Artifact: ONNXRuntime python wheel'
|
||||
inputs:
|
||||
ArtifactName: onnxruntime_gpu
|
||||
|
||||
- script: |
|
||||
7z x *.whl
|
||||
workingDirectory: '$(Build.ArtifactStagingDirectory)'
|
||||
displayName: 'unzip the package'
|
||||
|
||||
- task: CredScan@3
|
||||
displayName: 'Run CredScan'
|
||||
inputs:
|
||||
debugMode: false
|
||||
continueOnError: true
|
||||
|
||||
- task: BinSkim@4
|
||||
displayName: 'Run BinSkim'
|
||||
inputs:
|
||||
AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'
|
||||
|
||||
- task: DeleteFiles@1
|
||||
displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo'
|
||||
condition: and (succeeded(), eq(variables['PythonVersion'], '3.8'))
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
|
||||
Contents: |
|
||||
**/*.obj
|
||||
**/*.pdb
|
||||
**/*.dll
|
||||
|
||||
- powershell: |
|
||||
python -m pip uninstall -y onnxruntime onnxruntime-gpu -qq
|
||||
Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate}
|
||||
Remove-Item -Recurse -Force onnxruntime
|
||||
python onnx_backend_test_series.py
|
||||
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
|
||||
displayName: 'Run Python Tests'
|
||||
|
||||
#Manually set msBuildCommandline so that we can also set CAExcludePath
|
||||
- task: SDLNativeRules@3
|
||||
displayName: 'Run the PREfast SDL Native Rules for MSBuild'
|
||||
condition: and (succeeded(), eq(variables['PythonVersion'], '3.8'))
|
||||
inputs:
|
||||
msBuildArchitecture: amd64
|
||||
setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "$(VSGenerator)" --enable_pybind --enable_onnx_tests ${{ parameters.gpu_build_py_parameters }} --parallel $(EpBuildFlags) --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON'
|
||||
${{if eq(variables.VSGenerator, 'Visual Studio 16 2019')}}:
|
||||
msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform=x64 /p:configuration="RelWithDebInfo" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64'
|
||||
${{else}}:
|
||||
msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform=x64 /p:configuration="RelWithDebInfo" /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
|
||||
excludedPaths: '$(Build.BinariesDirectory)#$(Build.SourcesDirectory)\cmake#C:\program files (x86)'
|
||||
|
||||
- task: TSAUpload@2
|
||||
displayName: 'TSA upload'
|
||||
condition: and(and (succeeded(), eq(variables['PythonVersion'], '3.8')), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
|
||||
inputs:
|
||||
GdnPublishTsaOnboard: false
|
||||
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
|
||||
continueOnError: true
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters:
|
||||
condition: 'succeeded'
|
||||
|
||||
|
|
@ -2,4 +2,4 @@
|
|||
set -e -x
|
||||
docker run --rm --volume \
|
||||
$BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}build \
|
||||
/bin/bash -c "/usr/bin/python3.12 /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr/local/cuda-$CUDA_VERSION --skip_tests --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=75;80;90' && cd /build/Release && make install DESTDIR=/build/installed"
|
||||
/bin/bash -c "/usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr/local/cuda-$CUDA_VERSION --skip_tests --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=75;80;90' && cd /build/Release && make install DESTDIR=/build/installed"
|
||||
|
|
|
|||
|
|
@ -3,4 +3,4 @@ set -e -x
|
|||
mkdir -p $HOME/.onnx
|
||||
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \
|
||||
--volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}xtrt86build \
|
||||
/bin/bash -c "/usr/bin/python3.12 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_tests --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_java --build_nodejs --use_tensorrt --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr --tensorrt_home=/usr --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=75;80;90' && cd /build/Release && make install DESTDIR=/build/installed"
|
||||
/bin/bash -c "/usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_tests --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_java --build_nodejs --use_tensorrt --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr --tensorrt_home=/usr --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=75;80;90' && cd /build/Release && make install DESTDIR=/build/installed"
|
||||
|
|
|
|||
|
|
@ -5,10 +5,10 @@
|
|||
|
||||
set -e
|
||||
set -x
|
||||
export PATH=/opt/python/cp310-cp310/bin:$PATH
|
||||
|
||||
BUILD_DIR=${1:?"usage: $0 <build directory>"}
|
||||
|
||||
python3 -m pip install -r /onnxruntime_src/tools/ci_build/github/linux/python/requirements.txt
|
||||
# Validate the operator kernel registrations, as the ORT model uses hashes of the kernel registration details
|
||||
# to find kernels. If the hashes from the registration details are incorrect we will produce a model that will break
|
||||
# when the registration is fixed in the future.
|
||||
|
|
@ -26,7 +26,6 @@ python3 /onnxruntime_src/tools/ci_build/build.py \
|
|||
--build_wheel \
|
||||
--skip_tests \
|
||||
--enable_training_ops \
|
||||
--enable_pybind --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp310-cp310/include/python3.10 PYTHON_LIBRARY=/usr/lib64/librt.so \
|
||||
--use_nnapi \
|
||||
--use_coreml
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
set -e
|
||||
set -x
|
||||
export PATH=/opt/python/cp310-cp310/bin:$PATH
|
||||
|
||||
USAGE_TEXT="Usage:
|
||||
-b|--build-directory <build directory>
|
||||
Specifies the build directory. Required.
|
||||
|
|
@ -65,7 +65,7 @@ if [[ -z "${BUILD_DIR}" || -z "${REDUCED_OPS_CONFIG_FILE}" ]]; then
|
|||
echo "$USAGE_TEXT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 -m pip install -r /onnxruntime_src/tools/ci_build/github/linux/python/requirements.txt
|
||||
# Perform a minimal build with required ops and run ORT minimal build UTs
|
||||
python3 /onnxruntime_src/tools/ci_build/build.py \
|
||||
--build_dir ${BUILD_DIR} --cmake_generator Ninja \
|
||||
|
|
|
|||
|
|
@ -6,3 +6,4 @@ sympy>=1.10
|
|||
packaging
|
||||
cerberus
|
||||
psutil
|
||||
flatbuffers
|
||||
|
|
|
|||
Loading…
Reference in a new issue