saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
onnxruntime/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
Changming Sun d68245853e
Disable downloading test data on Linux (#3581)
2020-04-18 15:54:58 -07:00

345 lines
13 KiB
YAML
Raw Blame History

# Note that any variable set in this file overrides a value set within the Azure pipeline UX.
# Therefore, do not set default values here, but instead do it within a template script.
#
# parameters:
# - name: is_featurizers_build
# displayName: "Is Featurizers Build"
# type: boolean
# default: false
jobs:
- job: Manylinux2010_py_Wheels
timeoutInMinutes: 90
workspace:
clean: all
pool: Linux-CPU
strategy:
matrix:
Python35:
python.version: '3.5'
python.dir: '/opt/python/cp35-cp35m'
python.include.dir: '/opt/python/cp35-cp35m/include/python3.5m'
Python36:
python.version: '3.6'
python.dir: '/opt/python/cp36-cp36m'
python.include.dir: '/opt/python/cp36-cp36m/include/python3.6m'
Python37:
python.version: '3.7'
python.dir: '/opt/python/cp37-cp37m'
python.include.dir: '/opt/python/cp37-cp37m/include/python3.7m'
Python38:
python.version: '3.8'
python.dir: '/opt/python/cp38-cp38'
python.include.dir: '/opt/python/cp38-cp38/include/python3.8'
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/set-test-data-variables-step.yml
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
- task: CmdLine@2
inputs:
script: |
docker build --pull -t onnxruntime-manylinux-$(python.version) --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.manylinux1 .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD -e BUILD_BUILDNUMBER onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp --enable_onnx_tests $(FeaturizerBuildFlag) --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)'
Contents: 'Release/dist/*-manylinux1_x86_64.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: templates/component-governance-component-detection-steps.yml
- template: templates/clean-agent-build-directory-step.yml
- job: Manylinux2010_py_GPU_Wheels
timeoutInMinutes: 90
workspace:
clean: all
pool: Linux-GPU-CUDA10
strategy:
matrix:
Python35:
python.version: '3.5'
python.dir: '/opt/python/cp35-cp35m'
python.include.dir: '/opt/python/cp35-cp35m/include/python3.5m'
Python36:
python.version: '3.6'
python.dir: '/opt/python/cp36-cp36m'
python.include.dir: '/opt/python/cp36-cp36m/include/python3.6m'
Python37:
python.version: '3.7'
python.dir: '/opt/python/cp37-cp37m'
python.include.dir: '/opt/python/cp37-cp37m/include/python3.7m'
Python38:
python.version: '3.8'
python.dir: '/opt/python/cp38-cp38'
python.include.dir: '/opt/python/cp38-cp38/include/python3.8'
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/set-test-data-variables-step.yml
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
- task: CmdLine@2
inputs:
script: |
docker build --pull -t onnxruntime-manylinux-gpu-$(python.version) --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.manylinux2010_gpu .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD -e BUILD_BUILDNUMBER onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp --enable_onnx_tests $(FeaturizerBuildFlag) --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)'
Contents: 'Release/dist/*-manylinux2010_x86_64.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_gpu
- template: templates/component-governance-component-detection-steps.yml
- template: templates/clean-agent-build-directory-step.yml
- job: Windows_py_Wheels
pool: 'Win-CPU-2019'
strategy:
matrix:
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
variables:
OrtPackageId: 'Microsoft.ML.OnnxRuntime'
MsbuildArguments: '-maxcpucount'
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
EnvSetupScript: setup_env.bat
buildArch: x64
setVcvars: true
BuildConfig: 'Release'
timeoutInMinutes: 60
workspace:
clean: all
steps:
- checkout: self
clean: true
submodules: recursive
- task: UsePythonVersion@0
inputs:
versionSpec: $(python.version)
addToPath: true
architecture: 'x64'
- template: templates/set-test-data-variables-step.yml
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy==1.18
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- powershell: |
$Env:USE_MSVC_STATIC_RUNTIME=1
$Env:ONNX_ML=1
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
python setup.py bdist_wheel
Get-ChildItem -Path dist/*.whl | foreach {pip install --upgrade $_.fullname}
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
displayName: 'Install ONNX'
- task: PythonScript@0
displayName: 'BUILD'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_openmp --enable_onnx_tests $(FeaturizerBuildFlag) --parallel'
workingDirectory: '$(Build.BinariesDirectory)'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
Contents: '**\dist\*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- template: templates/clean-agent-build-directory-step.yml
- job: Windows_py_GPU_Wheels
workspace:
clean: all
pool: 'Win-GPU-2019'
timeoutInMinutes: 60
variables:
CUDA_VERSION: '10.1'
buildArch: x64
EnvSetupScript: setup_env_cuda.bat
strategy:
matrix:
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
steps:
- checkout: self
clean: true
submodules: recursive
- task: UsePythonVersion@0
inputs:
versionSpec: $(python.version)
addToPath: true
architecture: 'x64'
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy==1.18
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- powershell: |
$Env:USE_MSVC_STATIC_RUNTIME=1
$Env:ONNX_ML=1
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
python setup.py bdist_wheel
Get-ChildItem -Path dist/*.whl | foreach {pip install --upgrade $_.fullname}
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
displayName: 'Install ONNX'
- template: templates/set-test-data-variables-step.yml
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
- task: PythonScript@0
displayName: 'build'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: --config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_openmp --enable_onnx_tests $(FeaturizerBuildFlag) --parallel --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows10-x64-v7.6.5.32\cuda"
workingDirectory: '$(Build.BinariesDirectory)'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
Contents: '**\dist\*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_gpu
- template: templates/component-governance-component-detection-steps.yml
- template: templates/clean-agent-build-directory-step.yml
- job: MacOS_py_Wheels
workspace:
clean: all
pool:
vmImage: 'macOS-10.14'
strategy:
matrix:
Python35:
python.version: '3.5'
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/set-featurizer-build-flag-step.yml
parameters:
is_featurizers_build: $(is_featurizers_build)
- task: UsePythonVersion@0
displayName: 'Use Python'
inputs:
versionSpec: $(python.version)
- script: |
sudo python -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer
./build.sh --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp $(FeaturizerBuildFlag)
displayName: 'Command Line Script'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.SourcesDirectory)/build/Linux/Release/dist'
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- template: templates/component-governance-component-detection-steps.yml
Reference in a new issue View git blame Copy permalink