saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-06-29 03:30:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'upstream/master' into DmlDev

Browse source
This commit is contained in:
ISS Build Account 2020-06-23 12:17:45 +00:00
commit 028b75e19f
15 changed files with 476 additions and 782 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -313,37 +313,7 @@ const std::vector<std::string>& GetAllProviders() {
const std::vector<std::string>& GetAvailableProviders() {
auto InitializeProviders = []() {
std::vector<std::string> available_providers = {kCpuExecutionProvider};
#ifdef USE_TENSORRT
available_providers.push_back(kTensorrtExecutionProvider);
#endif
#ifdef USE_MIGRAPHX
available_providers.push_back(kMIGraphXExecutionProvider);
#endif
#ifdef USE_CUDA
available_providers.push_back(kCudaExecutionProvider);
#endif
#ifdef USE_DNNL
available_providers.push_back(kDnnlExecutionProvider);
#endif
#ifdef USE_NGRAPH
available_providers.push_back(kNGraphExecutionProvider);
#endif
#ifdef USE_OPENVINO
available_providers.push_back(kOpenVINOExecutionProvider);
#endif
#ifdef USE_NUPHAR
available_providers.push_back(kNupharExecutionProvider);
#endif
#ifdef USE_VITISAI
available_providers.push_back(kVitisAIExecutionProvider);
#endif
#ifdef USE_ACL
available_providers.push_back(kAclExecutionProvider);
#endif
#ifdef USE_ARMNN
available_providers.push_back(kArmNNExecutionProvider);
#endif
std::vector<std::string> available_providers(std::begin(providers_available), std::end(providers_available));
return available_providers;
};
static std::vector<std::string> available_providers = InitializeProviders();

479
tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
View file

@ -1,479 +0,0 @@
# Note that any variable set in this file overrides a value set within the Azure pipeline UX.
# Therefore, do not set default values here, but instead do it within a template script.
#
# parameters:
# - name: is_featurizers_build
# displayName: "Is Featurizers Build"
# type: boolean
# default: false
jobs:
- job: Manylinux2010_py_Wheels
timeoutInMinutes: 90
workspace:
clean: all
pool: Linux-CPU
strategy:
matrix:
Python35:
python.version: '3.5'
python.dir: '/opt/python/cp35-cp35m'
python.include.dir: '/opt/python/cp35-cp35m/include/python3.5m'
Python36:
python.version: '3.6'
python.dir: '/opt/python/cp36-cp36m'
python.include.dir: '/opt/python/cp36-cp36m/include/python3.6m'
Python37:
python.version: '3.7'
python.dir: '/opt/python/cp37-cp37m'
python.include.dir: '/opt/python/cp37-cp37m/include/python3.7m'
Python38:
python.version: '3.8'
python.dir: '/opt/python/cp38-cp38'
python.include.dir: '/opt/python/cp38-cp38/include/python3.8'
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/set-test-data-variables-step.yml
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
is_nightly_build: $(NIGHTLY_BUILD)
- task: CmdLine@2
inputs:
script: |
sudo docker build --pull -t onnxruntime-manylinux-$(python.version) --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.manylinux1 .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD -e BUILD_BUILDNUMBER onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp --enable_onnx_tests $(FeaturizerBuildFlag) --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)'
Contents: 'Release/dist/*-manylinux1_x86_64.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: templates/component-governance-component-detection-steps.yml
- template: templates/clean-agent-build-directory-step.yml
- job: Manylinux2010_py_GPU_Wheels
timeoutInMinutes: 90
workspace:
clean: all
pool: Linux-GPU-CUDA10
strategy:
matrix:
Python35:
python.version: '3.5'
python.dir: '/opt/python/cp35-cp35m'
python.include.dir: '/opt/python/cp35-cp35m/include/python3.5m'
Python36:
python.version: '3.6'
python.dir: '/opt/python/cp36-cp36m'
python.include.dir: '/opt/python/cp36-cp36m/include/python3.6m'
Python37:
python.version: '3.7'
python.dir: '/opt/python/cp37-cp37m'
python.include.dir: '/opt/python/cp37-cp37m/include/python3.7m'
Python38:
python.version: '3.8'
python.dir: '/opt/python/cp38-cp38'
python.include.dir: '/opt/python/cp38-cp38/include/python3.8'
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/set-test-data-variables-step.yml
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
is_nightly_build: $(NIGHTLY_BUILD)
- task: CmdLine@2
inputs:
script: |
sudo docker build --pull -t onnxruntime-manylinux-gpu-$(python.version) --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.manylinux2010_gpu .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
sudo --preserve-env docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD -e BUILD_BUILDNUMBER onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_wheel --enable_onnx_tests $(FeaturizerBuildFlag) --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)'
Contents: 'Release/dist/*-manylinux2010_x86_64.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_gpu
- template: templates/component-governance-component-detection-steps.yml
- template: templates/clean-agent-build-directory-step.yml
- job: Windows_py_Wheels
pool: 'Win-CPU-2019'
strategy:
matrix:
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
variables:
OrtPackageId: 'Microsoft.ML.OnnxRuntime'
MsbuildArguments: '-maxcpucount'
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
EnvSetupScript: setup_env.bat
buildArch: x64
setVcvars: true
BuildConfig: 'Release'
GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
timeoutInMinutes: 60
workspace:
clean: all
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/enable-telemetry.yml
- task: UsePythonVersion@0
inputs:
versionSpec: $(python.version)
addToPath: true
architecture: 'x64'
- template: templates/set-test-data-variables-step.yml
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
is_nightly_build: $(NIGHTLY_BUILD)
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy==1.16.6
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- powershell: |
$Env:USE_MSVC_STATIC_RUNTIME=1
$Env:ONNX_ML=1
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
python setup.py bdist_wheel
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
displayName: 'Install ONNX'
- task: PythonScript@0
displayName: 'BUILD'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --enable_pybind --use_openmp --enable_onnx_tests $(FeaturizerBuildFlag) --parallel --use_telemetry'
workingDirectory: '$(Build.BinariesDirectory)'
# Esrp signing
- template: templates/win-esrp-dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: true
Pattern: '*.pyd,*.dll'
- task: PythonScript@0
displayName: 'Build wheel'
inputs:
scriptPath: '$(Build.SourcesDirectory)\setup.py'
arguments: bdist_wheel $(WheelBuildFlag)
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
Contents: '**\dist\*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- script: |
rename *.whl *.zip
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
displayName: 'Rename file extension for codesign validation'
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- job: Windows_py_GPU_Wheels
workspace:
clean: all
pool: 'Win-GPU-2019'
timeoutInMinutes: 60
variables:
CUDA_VERSION: '10.1'
buildArch: x64
EnvSetupScript: setup_env_cuda.bat
GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
strategy:
matrix:
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/enable-telemetry.yml
- task: UsePythonVersion@0
inputs:
versionSpec: $(python.version)
addToPath: true
architecture: 'x64'
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy==1.16.6
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- powershell: |
$Env:USE_MSVC_STATIC_RUNTIME=1
$Env:ONNX_ML=1
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
python setup.py bdist_wheel
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
displayName: 'Install ONNX'
- template: templates/set-test-data-variables-step.yml
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
is_nightly_build: $(NIGHTLY_BUILD)
- task: PythonScript@0
displayName: 'build'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: --config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --enable_pybind --enable_onnx_tests $(FeaturizerBuildFlag) --parallel --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows10-x64-v7.6.5.32\cuda" --use_telemetry
workingDirectory: '$(Build.BinariesDirectory)'
# Esrp signing
- template: templates/win-esrp-dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: true
Pattern: '*.pyd,*.dll'
- task: PythonScript@0
displayName: 'Build wheel'
inputs:
scriptPath: '$(Build.SourcesDirectory)\setup.py'
arguments: bdist_wheel --use_cuda $(WheelBuildFlag)
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
Contents: '**\dist\*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- script: |
rename *.whl *.zip
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
displayName: 'Rename file extension for codesign validation'
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_gpu
- template: templates/component-governance-component-detection-steps.yml
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- job: MacOS_py_Wheels
workspace:
clean: all
pool:
vmImage: 'macOS-10.14'
strategy:
matrix:
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
is_nightly_build: $(NIGHTLY_BUILD)
- task: UsePythonVersion@0
displayName: 'Use Python'
inputs:
versionSpec: $(python.version)
- script: |
sudo python -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer
./build.sh --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp $(FeaturizerBuildFlag)
displayName: 'Command Line Script'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.SourcesDirectory)/build/Linux/Release/dist'
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: templates/component-governance-component-detection-steps.yml
- job: Linux_ARM_py_Wheels
timeoutInMinutes: 60
workspace:
clean: all
pool: 'Linux-CPU'
condition: ne(variables['is_featurizers_build'], 'true')
strategy:
matrix:
Py37:
python.include: '3.7m'
cp.tag: 'cp37-cp37m'
Py36:
python.include: '3.6m'
cp.tag: 'cp36-cp36m'
Py35:
python.include: '3.5m'
cp.tag: 'cp35-cp35m'
steps:
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
is_nightly_build: $(NIGHTLY_BUILD)
- task: CmdLine@2
inputs:
script: |
set -e -x
sudo apt-get install -y qemu-user-static
cat << EOF > tool-chain.cmake
SET(CMAKE_SYSTEM_NAME Linux)
SET(CMAKE_SYSTEM_VERSION 1)
SET(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)
SET(CMAKE_C_FLAGS "-march=armv8-a -mtune=generic -Wno-unused-parameter -Wno-type-limits")
SET(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)
SET(CMAKE_CXX_FLAGS "-march=armv8-a -mtune=generic -Wno-unused-parameter -Wno-type-limits")
SET(CMAKE_FIND_ROOT_PATH /mnt/toolchains/manylinux2014_aarch64)
SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
SET(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
EOF
export PATH=/mnt/toolchains/gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu/bin:$PATH
azcopy cp https://onnxruntimetestdata.blob.core.windows.net/models/toolchains.tar.xz $(Build.BinariesDirectory)/toolchains.tar.xz
sudo rm -rf /mnt/toolchains
sudo mkdir /mnt/toolchains
sudo tar -Jxf $(Build.BinariesDirectory)/toolchains.tar.xz -C /mnt/toolchains
aria2c -q https://github.com/protocolbuffers/protobuf/releases/download/v3.11.1/protoc-3.11.1-linux-x86_64.zip
unzip protoc-3.11.1-linux-x86_64.zip
sudo cp /mnt/toolchains/manylinux2014_aarch64/usr/include/stdlib.h /mnt/toolchains/gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu/aarch64-linux-gnu/libc/usr/include/
cmake -Donnxruntime_GCC_STATIC_CPP_RUNTIME=ON -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB=OFF -DCMAKE_TOOLCHAIN_FILE=tool-chain.cmake -Donnxruntime_ENABLE_PYTHON=ON -DPYTHON_LIBRARY=dl -DPYTHON_EXECUTABLE=/mnt/toolchains/manylinux2014_aarch64/opt/python/'$(cp.tag)'/bin/python3 -Donnxruntime_BUILD_SHARED_LIB=OFF -Donnxruntime_RUN_ONNX_TESTS=OFF -Donnxruntime_DEV_MODE=ON -DONNX_CUSTOM_PROTOC_EXECUTABLE=$(Build.BinariesDirectory)/bin/protoc "-DPYTHON_INCLUDE_DIR=/mnt/toolchains/manylinux2014_aarch64/usr/include;/mnt/toolchains/manylinux2014_aarch64/opt/python/$(cp.tag)/include/python$(python.include)" -DNUMPY_INCLUDE_DIR=/mnt/toolchains $(Build.SourcesDirectory)/cmake
make -j$(getconf _NPROCESSORS_ONLN)
sudo --preserve-env docker run -e BUILD_BUILDNUMBER -v /usr/bin/qemu-aarch64-static:/usr/bin/qemu-aarch64-static -v $(Build.BinariesDirectory):/tmp/a -v $(Build.SourcesDirectory):/tmp/b -w /tmp/a -u $(id -u ${USER}):$(id -g ${USER}) --rm quay.io/pypa/manylinux2014_aarch64 /opt/python/'$(cp.tag)'/bin/python3 /tmp/b/setup.py bdist_wheel $(WheelBuildFlag)
workingDirectory: $(Build.BinariesDirectory)
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
PathtoPublish: '$(Build.BinariesDirectory)/dist'
ArtifactName: onnxruntime
- template: templates/clean-agent-build-directory-step.yml

12
tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
View file

@ -105,7 +105,7 @@ jobs:
buildparameter:
steps:
- template: templates/enable-telemetry.yml
- template: templates/telemetry-steps.yml
- task: UsePythonVersion@0
inputs:
@ -141,7 +141,7 @@ jobs:
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests --use_telemetry $(buildparameter)'
arguments: '--config RelWithDebInfo --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)'
workingDirectory: '$(Build.BinariesDirectory)'
@ -161,7 +161,7 @@ jobs:
displayName: 'test'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests --use_telemetry $(buildparameter)'
arguments: '--config RelWithDebInfo --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)'
workingDirectory: '$(Build.BinariesDirectory)'
- template: templates/c-api-artifacts-package-and-publish-steps-windows.yml
@ -190,7 +190,7 @@ jobs:
CUDA_VERSION: '10.1'
buildparameter: --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows10-x64-v7.6.5.32\cuda"
steps:
- template: templates/enable-telemetry.yml
- template: templates/telemetry-steps.yml
- task: UsePythonVersion@0
inputs:
@ -225,7 +225,7 @@ jobs:
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests --use_telemetry $(buildparameter)'
arguments: '--config RelWithDebInfo --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)'
workingDirectory: '$(Build.BinariesDirectory)'
@ -245,7 +245,7 @@ jobs:
displayName: 'test'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests --use_telemetry $(buildparameter)'
arguments: '--config RelWithDebInfo --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)'
workingDirectory: '$(Build.BinariesDirectory)'
- template: templates/c-api-artifacts-package-and-publish-steps-windows.yml

7
tools/ci_build/github/azure-pipelines/featurizers-py-packaging-pipeline.yml Normal file
View file

@ -0,0 +1,7 @@
trigger: none
stages:
- template: templates/py-packaging-stage.yml
parameters:
build_py_parameters: --use_featurizers --wheel_name_suffix=featurizer
enable_linux_arm: false

2
tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml
View file

@ -90,7 +90,7 @@ jobs:
buildparameter:
steps:
- template: templates/enable-telemetry.yml
- template: templates/telemetry-steps.yml
- task: UsePythonVersion@0
inputs:

10
tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline.yml
View file

@ -3,12 +3,14 @@ trigger: none
stages:
- template: templates/py-packaging-stage.yml
parameters:
enable_training: true
build_py_parameters: --enable_training --wheel_name_suffix=training
enable_linux_cpu: false
enable_linux_gpu: true
# Python 3.5 build has issues with nightly torch and torchvision dependencies
enable_linux_gpu_py35: false
enable_windows_cpu: false
enable_windows_gpu: false
enable_mac_cpu: false
python_version_strategy_matrix:
Python36:
python.version: '3.6'
enable_linux_arm: false
docker_image_prefix: onnxruntime-training
linux_gpu_dockerfile: Dockerfile.manylinux2014_gpu

4
tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml Normal file
View file

@ -0,0 +1,4 @@
trigger: none
stages:
- template: templates/py-packaging-stage.yml

13
tools/ci_build/github/azure-pipelines/templates/enable-telemetry.yml
View file

@ -1,13 +0,0 @@
steps:
- powershell: |
if($env:TELEMETRYGUID)
{
$length = $env:TELEMETRYGUID.length
$fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \
TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")"
New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force
Write-Output "Enabling TELEMETRY"
}
displayName: 'Create TraceLoggingConfigPrivate.h For WinML Telemetry'
env:
TELEMETRYGUID: $(TELEMETRYGUID)

610
tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
View file

@ -1,14 +1,10 @@
parameters:
- name: build_py_parameters
displayName: 'Extra parameters to pass to build.py.'
displayName: >
Extra parameters to pass to build.py. Don't put newlines in here.
type: string
default: ''
- name: enable_training
displayName: 'Whether training functionality is enabled.'
type: boolean
default: false
- name: enable_linux_cpu
displayName: 'Whether Linux CPU package is built.'
type: boolean
@ -19,6 +15,13 @@ parameters:
type: boolean
default: true
- name: enable_linux_gpu_py35
displayName: >
Whether Linux GPU package is built for Python 3.5.
enable_linux_gpu must be true for this to have an effect.
type: boolean
default: true
- name: enable_windows_cpu
displayName: 'Whether Windows CPU package is built.'
type: boolean
@ -34,37 +37,26 @@ parameters:
type: boolean
default: true
- name: python_version_strategy_matrix
displayName: 'Strategy matrix specifying the possible values of the python.version variable.'
type: object
default:
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
- name: enable_linux_arm
displayName: 'Whether Linux ARM package is built.'
type: boolean
default: true
- name: docker_image_prefix
displayName: 'Prefix to use for docker image names.'
type: string
default: 'onnxruntime'
- name: linux_gpu_dockerfile
displayName: >
Name of the Dockerfile to use for the Linux GPU build.
This should refer to a Dockerfile at tools/ci_build/github/linux/docker.
type: string
default: 'Dockerfile.manylinux2010_gpu'
stages:
- stage: Python_Packaging
variables:
${{ if eq(parameters.enable_training, true) }}:
actual_extra_build_py_parameters: >
${{ parameters.build_py_parameters }}
--enable_training
--wheel_name_suffix training
docker_image_prefix: onnxruntime-training
linux_gpu_dockerfile: Dockerfile.manylinux2014_gpu
${{ if ne(parameters.enable_training, true) }}:
actual_extra_build_py_parameters: '${{ parameters.build_py_parameters }}'
docker_image_prefix: onnxruntime
linux_gpu_dockerfile: Dockerfile.manylinux2010_gpu
jobs:
- ${{ if eq(parameters.enable_linux_cpu, true) }}:
- job: Linux_py_Wheels
@ -74,63 +66,70 @@ stages:
pool: Linux-CPU
strategy:
matrix:
${{ parameters.python_version_strategy_matrix }}
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
steps:
- checkout: self
clean: true
submodules: recursive
- checkout: self
clean: true
submodules: recursive
- template: set-py-packaging-variables-step.yml
- template: set-py-packaging-variables-step.yml
- task: CmdLine@2
inputs:
script: |
sudo docker build \
--pull \
-t ${{ variables.docker_image_prefix }}-manylinux-$(python.version) \
--build-arg BUILD_USER=onnxruntimedev \
--build-arg BUILD_UID=$(id -u) \
--build-arg PYTHON_VERSION=$(python.version) \
-f Dockerfile.manylinux1 .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
sudo --preserve-env docker run \
--rm \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
${{ variables.docker_image_prefix }}-manylinux-$(python.version) \
- task: CmdLine@2
inputs:
script: |
sudo docker build --pull \
-t ${{ parameters.docker_image_prefix }}-manylinux-$(python.version) \
--build-arg BUILD_USER=onnxruntimedev \
--build-arg BUILD_UID=$(id -u) \
--build-arg PYTHON_VERSION=$(python.version) \
-f Dockerfile.manylinux1 .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
sudo --preserve-env docker run --rm \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
${{ parameters.docker_image_prefix }}-manylinux-$(python.version) \
$(python.manylinux.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build \
--config Release \
--cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.manylinux.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so \
--skip_submodule_sync \
--parallel \
--build_wheel \
--use_openmp \
--enable_onnx_tests
workingDirectory: $(Build.SourcesDirectory)
--enable_onnx_tests \
${{ parameters.build_py_parameters }} \
--cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.manylinux.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)'
Contents: 'Release/dist/*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)'
Contents: 'Release/dist/*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: component-governance-component-detection-steps.yml
- template: component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- template: clean-agent-build-directory-step.yml
- template: clean-agent-build-directory-step.yml
- ${{ if eq(parameters.enable_linux_gpu, true) }}:
- job: Linux_py_GPU_Wheels
@ -140,74 +139,87 @@ stages:
pool: Linux-GPU-CUDA10
strategy:
matrix:
${{ parameters.python_version_strategy_matrix }}
${{ if eq(parameters.enable_linux_gpu_py35, true) }}:
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
steps:
- checkout: self
clean: true
submodules: recursive
- checkout: self
clean: true
submodules: recursive
- template: set-py-packaging-variables-step.yml
- template: set-py-packaging-variables-step.yml
- task: CmdLine@2
inputs:
script: |
sudo docker build \
--pull \
-t ${{ variables.docker_image_prefix }}-manylinux-gpu-$(python.version) \
--build-arg BUILD_USER=onnxruntimedev \
--build-arg BUILD_UID=$(id -u) \
--build-arg PYTHON_VERSION=$(python.version) \
--build-arg BUILD_EXTR_PAR="${{ variables.actual_extra_build_py_parameters }}" \
-f ${{ variables.linux_gpu_dockerfile }} .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
sudo --preserve-env docker run \
--gpus all \
--rm \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro \
-e NVIDIA_VISIBLE_DEVICES=all \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
${{ variables.docker_image_prefix }}-manylinux-gpu-$(python.version) \
- task: CmdLine@2
inputs:
script: |
sudo docker build --pull \
-t ${{ parameters.docker_image_prefix }}-manylinux-gpu-$(python.version) \
--build-arg BUILD_USER=onnxruntimedev \
--build-arg BUILD_UID=$(id -u) \
--build-arg PYTHON_VERSION=$(python.version) \
--build-arg BUILD_EXTR_PAR="${{ parameters.build_py_parameters }}" \
-f ${{ parameters.linux_gpu_dockerfile }} .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
sudo --preserve-env docker run --rm --gpus all \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro \
-e NVIDIA_VISIBLE_DEVICES=all \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
${{ parameters.docker_image_prefix }}-manylinux-gpu-$(python.version) \
$(python.manylinux.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build \
--config Release \
--cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.manylinux.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so \
--skip_submodule_sync \
--parallel \
--build_wheel \
--enable_onnx_tests \
--use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 \
${{ variables.actual_extra_build_py_parameters }}
workingDirectory: $(Build.SourcesDirectory)
${{ parameters.build_py_parameters }} \
--cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.manylinux.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so \
--use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)'
Contents: 'Release/dist/*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)'
Contents: 'Release/dist/*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_gpu
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_gpu
- template: component-governance-component-detection-steps.yml
- template: component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- template: clean-agent-build-directory-step.yml
- template: clean-agent-build-directory-step.yml
- ${{ if eq(parameters.enable_windows_cpu, true) }}:
- job: Windows_py_Wheels
pool: 'Win-CPU-2019'
strategy:
matrix:
${{ parameters.python_version_strategy_matrix }}
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
variables:
OrtPackageId: 'Microsoft.ML.OnnxRuntime'
MsbuildArguments: '-maxcpucount'
@ -216,6 +228,7 @@ stages:
buildArch: x64
setVcvars: true
BuildConfig: 'Release'
GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
timeoutInMinutes: 60
workspace:
clean: all
@ -225,12 +238,16 @@ stages:
clean: true
submodules: recursive
- template: telemetry-steps.yml
- task: UsePythonVersion@0
inputs:
versionSpec: $(python.version)
addToPath: true
architecture: 'x64'
- template: set-nightly-build-option-variable-step.yml
- task: BatchScript@1
displayName: 'setup env'
inputs:
@ -240,7 +257,6 @@ stages:
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy==1.16.6
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
@ -263,13 +279,29 @@ stages:
--build_dir $(Build.BinariesDirectory)
--skip_submodule_sync
--cmake_generator "Visual Studio 16 2019"
--build_wheel
--enable_pybind
--use_openmp
--enable_onnx_tests
${{ parameters.build_py_parameters }}
--parallel
${{ variables.actual_extra_build_py_parameters }}
$(TelemetryOption)
workingDirectory: '$(Build.BinariesDirectory)'
# Esrp signing
- template: win-esrp-dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: true
Pattern: '*.pyd,*.dll'
- task: PythonScript@0
displayName: 'Build wheel'
inputs:
scriptPath: '$(Build.SourcesDirectory)\setup.py'
arguments: 'bdist_wheel ${{ parameters.build_py_parameters }} $(NightlyBuildOption)'
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
@ -277,6 +309,19 @@ stages:
Contents: '**\dist\*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- script: |
rename *.whl *.zip
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
displayName: 'Rename file extension for codesign validation'
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
@ -284,12 +329,14 @@ stages:
- template: component-governance-component-detection-steps.yml
parameters:
condition : 'succeeded'
condition: 'succeeded'
- template: clean-agent-build-directory-step.yml
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- ${{ if eq(parameters.enable_windows_gpu, true) }}:
- job: Windows_py_GPU_Wheels
- job: Windows_py_GPU_Wheels
workspace:
clean: all
pool: 'Win-GPU-2019'
@ -298,72 +345,117 @@ stages:
CUDA_VERSION: '10.1'
buildArch: x64
EnvSetupScript: setup_env_cuda.bat
GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
strategy:
matrix:
${{ parameters.python_version_strategy_matrix }}
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
steps:
- checkout: self
clean: true
submodules: recursive
- checkout: self
clean: true
submodules: recursive
- task: UsePythonVersion@0
inputs:
versionSpec: $(python.version)
addToPath: true
architecture: 'x64'
- template: telemetry-steps.yml
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- task: UsePythonVersion@0
inputs:
versionSpec: $(python.version)
addToPath: true
architecture: 'x64'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy==1.16.6
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy==1.16.6
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- powershell: |
$Env:USE_MSVC_STATIC_RUNTIME=1
$Env:ONNX_ML=1
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
python setup.py bdist_wheel
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
displayName: 'Install ONNX'
- template: set-nightly-build-option-variable-step.yml
- task: PythonScript@0
displayName: 'build'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config RelWithDebInfo
--build_dir $(Build.BinariesDirectory)
--skip_submodule_sync
--cmake_generator "Visual Studio 16 2019"
--enable_pybind
--enable_onnx_tests
${{ parameters.build_py_parameters }}
--parallel
--use_cuda --cuda_version=$(CUDA_VERSION)
--cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)"
--cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows10-x64-v7.6.5.32\cuda"
$(TelemetryOption)
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- powershell: |
$Env:USE_MSVC_STATIC_RUNTIME=1
$Env:ONNX_ML=1
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
python setup.py bdist_wheel
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
displayName: 'Install ONNX'
# Esrp signing
- template: win-esrp-dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: true
Pattern: '*.pyd,*.dll'
- task: PythonScript@0
displayName: 'build'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config RelWithDebInfo
--build_dir $(Build.BinariesDirectory)
--skip_submodule_sync
--cmake_generator "Visual Studio 16 2019"
--build_wheel
--enable_onnx_tests
--parallel
--use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows10-x64-v7.6.5.32\cuda"
${{ variables.actual_extra_build_py_parameters }}
workingDirectory: '$(Build.BinariesDirectory)'
- task: PythonScript@0
displayName: 'Build wheel'
inputs:
scriptPath: '$(Build.SourcesDirectory)\setup.py'
arguments: 'bdist_wheel --use_cuda ${{ parameters.build_py_parameters }} $(NightlyBuildOption)'
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
Contents: '**\dist\*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
Contents: '**\dist\*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_gpu
- script: |
rename *.whl *.zip
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
displayName: 'Rename file extension for codesign validation'
- template: component-governance-component-detection-steps.yml
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- template: clean-agent-build-directory-step.yml
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_gpu
- template: component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- ${{ if eq(parameters.enable_mac_cpu, true) }}:
- job: MacOS_py_Wheels
@ -373,39 +465,133 @@ stages:
vmImage: 'macOS-10.14'
strategy:
matrix:
${{ parameters.python_version_strategy_matrix }}
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
steps:
- checkout: self
clean: true
submodules: recursive
- checkout: self
clean: true
submodules: recursive
- task: UsePythonVersion@0
displayName: 'Use Python'
inputs:
versionSpec: $(python.version)
- task: UsePythonVersion@0
displayName: 'Use Python'
inputs:
versionSpec: $(python.version)
- script: |
sudo python -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer
./build.sh \
--config Release \
--skip_submodule_sync \
--parallel \
--build_wheel \
--use_openmp \
${{ variables.actual_extra_build_py_parameters }}
displayName: 'Command Line Script'
- script: |
sudo python -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer
./build.sh --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp ${{ parameters.build_py_parameters }}
displayName: 'Command Line Script'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.SourcesDirectory)/build/Linux/Release/dist'
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.SourcesDirectory)/build/Linux/Release/dist'
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: component-governance-component-detection-steps.yml
- template: component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- ${{ if eq(parameters.enable_linux_arm, true) }}:
- job: Linux_ARM_py_Wheels
timeoutInMinutes: 60
workspace:
clean: all
pool: 'Linux-CPU'
strategy:
matrix:
Py37:
python.version: '3.7'
Py36:
python.version: '3.6'
Py35:
python.version: '3.5'
steps:
- checkout: self
clean: true
submodules: recursive
- template: set-py-packaging-variables-step.yml
- template: set-nightly-build-option-variable-step.yml
- task: CmdLine@2
inputs:
script: |
set -e -x
sudo apt-get install -y qemu-user-static
cat << EOF > tool-chain.cmake
SET(CMAKE_SYSTEM_NAME Linux)
SET(CMAKE_SYSTEM_VERSION 1)
SET(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)
SET(CMAKE_C_FLAGS "-march=armv8-a -mtune=generic -Wno-unused-parameter -Wno-type-limits")
SET(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)
SET(CMAKE_CXX_FLAGS "-march=armv8-a -mtune=generic -Wno-unused-parameter -Wno-type-limits")
SET(CMAKE_FIND_ROOT_PATH /mnt/toolchains/manylinux2014_aarch64)
SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
SET(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
EOF
export PATH=/mnt/toolchains/gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu/bin:$PATH
azcopy cp https://onnxruntimetestdata.blob.core.windows.net/models/toolchains.tar.xz $(Build.BinariesDirectory)/toolchains.tar.xz
sudo rm -rf /mnt/toolchains
sudo mkdir /mnt/toolchains
sudo tar -Jxf $(Build.BinariesDirectory)/toolchains.tar.xz -C /mnt/toolchains
aria2c -q https://github.com/protocolbuffers/protobuf/releases/download/v3.11.1/protoc-3.11.1-linux-x86_64.zip
unzip protoc-3.11.1-linux-x86_64.zip
sudo cp /mnt/toolchains/manylinux2014_aarch64/usr/include/stdlib.h /mnt/toolchains/gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu/aarch64-linux-gnu/libc/usr/include/
cmake \
-Donnxruntime_GCC_STATIC_CPP_RUNTIME=ON \
-DCMAKE_BUILD_TYPE=Release \
-Dprotobuf_WITH_ZLIB=OFF \
-DCMAKE_TOOLCHAIN_FILE=tool-chain.cmake \
-Donnxruntime_ENABLE_PYTHON=ON \
-DPYTHON_LIBRARY=dl \
-DPYTHON_EXECUTABLE=/mnt/toolchains/manylinux2014_aarch64/'$(python.manylinux.dir)'/bin/python3 \
-Donnxruntime_BUILD_SHARED_LIB=OFF \
-Donnxruntime_RUN_ONNX_TESTS=OFF \
-Donnxruntime_DEV_MODE=ON \
-DONNX_CUSTOM_PROTOC_EXECUTABLE=$(Build.BinariesDirectory)/bin/protoc \
"-DPYTHON_INCLUDE_DIR=/mnt/toolchains/manylinux2014_aarch64/usr/include;/mnt/toolchains/manylinux2014_aarch64/$(python.manylinux.include.dir)" \
-DNUMPY_INCLUDE_DIR=/mnt/toolchains \
$(Build.SourcesDirectory)/cmake
make -j$(getconf _NPROCESSORS_ONLN)
sudo --preserve-env docker run --rm \
-e BUILD_BUILDNUMBER \
-v /usr/bin/qemu-aarch64-static:/usr/bin/qemu-aarch64-static \
-v $(Build.BinariesDirectory):/tmp/a \
-v $(Build.SourcesDirectory):/tmp/b \
-w /tmp/a \
-u $(id -u ${USER}):$(id -g ${USER}) \
quay.io/pypa/manylinux2014_aarch64 \
'$(python.manylinux.dir)'/bin/python3 /tmp/b/setup.py \
bdist_wheel ${{ parameters.build_py_parameters }} $(NightlyBuildOption)
workingDirectory: $(Build.BinariesDirectory)
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
PathtoPublish: '$(Build.BinariesDirectory)/dist'
ArtifactName: onnxruntime
- template: component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- template: clean-agent-build-directory-step.yml

22
tools/ci_build/github/azure-pipelines/templates/set-featurizer-build-flag-step.yml
View file

@ -1,22 +0,0 @@
parameters:
is_featurizers_build: false
is_nightly_build: 0
steps:
- task: PythonScript@0
displayName: "Set FeaturizerBuildFlag variable"
inputs:
scriptSource: inline
script: |-
import os
if "${{ parameters.is_featurizers_build }}".lower() == "true":
flags = "--use_featurizers --wheel_name_suffix=featurizer"
else:
flags = ""
print("##vso[task.setvariable variable=FeaturizerBuildFlag]%s" % flags)
if "${{ parameters.is_nightly_build }}" == "1":
flags = flags + " --nightly_build"
print("##vso[task.setvariable variable=WheelBuildFlag]%s" % flags)

15
tools/ci_build/github/azure-pipelines/templates/set-nightly-build-option-variable-step.yml Normal file
View file

@ -0,0 +1,15 @@
# sets the NightlyBuildOption variable if the NIGHTLY_BUILD environment variable is appropriately set
steps:
- task: PythonScript@0
displayName: 'Set NightlyBuildOption variable'
inputs:
scriptSource: inline
script: |
import os
name = "NightlyBuildOption"
value = "--nightly_build" if os.environ.get("NIGHTLY_BUILD") == "1" else ""
print("Setting variable: {} = '{}'".format(name, value))
print("##vso[task.setvariable variable={}]{}".format(name, value))

4
tools/ci_build/github/azure-pipelines/templates/set-py-packaging-variables-step.yml
View file

@ -12,25 +12,21 @@ steps:
variables = {
"python.manylinux.dir": "/opt/python/cp35-cp35m",
"python.manylinux.include.dir": "/opt/python/cp35-cp35m/include/python3.5m",
"python.whl.impl.abi.tags": "cp35-cp35m",
}
elif version == "3.6":
variables = {
"python.manylinux.dir": "/opt/python/cp36-cp36m",
"python.manylinux.include.dir": "/opt/python/cp36-cp36m/include/python3.6m",
"python.whl.impl.abi.tags": "cp36-cp36m",
}
elif version == "3.7":
variables = {
"python.manylinux.dir": "/opt/python/cp37-cp37m",
"python.manylinux.include.dir": "/opt/python/cp37-cp37m/include/python3.7m",
"python.whl.impl.abi.tags": "cp37-cp37m",
}
elif version == "3.8":
variables = {
"python.manylinux.dir": "/opt/python/cp38-cp38",
"python.manylinux.include.dir": "/opt/python/cp38-cp38/include/python3.8",
"python.whl.impl.abi.tags": "cp38-cp38",
}
else:
raise ValueError("Unsupported Python version: '{}'".format(version))

28
tools/ci_build/github/azure-pipelines/templates/telemetry-steps.yml Normal file
View file

@ -0,0 +1,28 @@
# sets the TelemetryOption variable and optionally generates TraceLoggingConfigPrivate.h based on the TELEMETRYGUID
# variable
steps:
- ${{ if ne(variables.TELEMETRYGUID, '') }}:
- task: PythonScript@0
displayName: 'Set TelemetryOption variable [telemetry enabled]'
inputs:
scriptSource: inline
script: |
print("##vso[task.setvariable variable=TelemetryOption]--use_telemetry")
- powershell: |
$length = $env:TELEMETRYGUID.length
$fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \
TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")"
New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force
displayName: 'Create TraceLoggingConfigPrivate.h For WinML Telemetry'
env:
TELEMETRYGUID: $(TELEMETRYGUID)
- ${{ if eq(variables.TELEMETRYGUID, '') }}:
- task: PythonScript@0
displayName: 'Set TelemetryOption variable [telemetry disabled]'
inputs:
scriptSource: inline
script: |
print("##vso[task.setvariable variable=TelemetryOption]")

18
tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml
View file

@ -4,7 +4,7 @@ parameters:
BuildForStore: 'false'
steps:
- template: enable-telemetry.yml
- template: telemetry-steps.yml
- task: UsePythonVersion@0
inputs:
@ -38,21 +38,21 @@ steps:
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
${{ if and(eq(parameters.BuildArch, 'x64'), eq(parameters.BuildForStore, 'false')) }}:
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto'
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos $(TelemetryOption) --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto'
${{ if and(eq(parameters.BuildArch, 'x64'), eq(parameters.BuildForStore, 'true')) }}:
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --enable_windows_store'
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos $(TelemetryOption) --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --enable_windows_store'
${{ if and(eq(parameters.BuildArch, 'x86'), eq(parameters.BuildForStore, 'false')) }}:
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --x86'
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos $(TelemetryOption) --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --x86'
${{ if and(eq(parameters.BuildArch, 'x86'), eq(parameters.BuildForStore, 'true')) }}:
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --x86 --enable_windows_store'
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos $(TelemetryOption) --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --x86 --enable_windows_store'
${{ if and(eq(parameters.BuildArch, 'arm64'), eq(parameters.BuildForStore, 'false')) }}:
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --arm64'
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos $(TelemetryOption) --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --arm64'
${{ if and(eq(parameters.BuildArch, 'arm64'), eq(parameters.BuildForStore, 'true')) }}:
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --arm64 --enable_windows_store'
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos $(TelemetryOption) --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --arm64 --enable_windows_store'
${{ if and(eq(parameters.BuildArch, 'arm'), eq(parameters.BuildForStore, 'false')) }}:
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --arm'
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos $(TelemetryOption) --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --arm'
${{ if and(eq(parameters.BuildArch, 'arm'), eq(parameters.BuildForStore, 'true')) }}:
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --arm --enable_windows_store'
arguments: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos $(TelemetryOption) --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --arm --enable_windows_store'
workingDirectory: '$(Build.BinariesDirectory)'
- ${{ if notIn(parameters['sln_platform'], 'Win32', 'x64') }}:

2
tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_gpu
View file

@ -1,4 +1,4 @@
FROM quay.io/pypa/manylinux2014_x86_64:2020-05-03-c2ce11e
FROM quay.io/pypa/manylinux2014_x86_64:2020-04-04-c0949b6
ARG PYTHON_VERSION=3.5
ARG BUILD_EXTR_PAR