saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-05-31 23:27:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
onnxruntime/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
Changming Sun 5eec4f66ed
Refactor manylinux docker image and the related pipelines (#4751) 
1. Publish the image ACR, instead of building it every time for every PR
2. Make USE_MKLML and USE_OPENMP be able to co-exist. Currently both of them are enabled in our Linux CI build but indeed only one of them is taking effect.
3. Split nuphar and DNNL to separated pipelines.
4. Fix two warnings in onnxruntime/core/optimizer/matmul_scale_fusion.cc and onnxruntime/test/tvm/tvm_basic_test.cc.
5. Update the manylinux2010_x86_64 image to the latest.
2020-08-17 09:40:31 -07:00

275 lines
12 KiB
YAML
Raw Blame History

jobs:
- job: Linux_C_API_Packaging_CPU_x64
workspace:
clean: all
timeoutInMinutes: 60
pool:
vmImage: 'ubuntu-latest'
steps:
- template: templates/set-version-number-variables-step.yml
- task: Docker@2
displayName: login
inputs:
containerRegistry: onnxruntimeregistry
command: login
addPipelineData: false
- task: CmdLine@2
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn /opt/python/cp37-cp37m/bin/python3 \
/onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto \
--skip_submodule_sync --parallel --build_shared_lib --use_openmp
workingDirectory: $(Build.SourcesDirectory)
- task: Docker@2
displayName: logout
inputs:
containerRegistry: onnxruntimeregistry
command: logout
addPipelineData: false
- template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
buildConfig: 'Release'
artifactName: 'onnxruntime-linux-x64-$(OnnxRuntimeVersion)'
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
commitId: $(OnnxRuntimeGitCommitHash)
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- template: templates/clean-agent-build-directory-step.yml
- job: Linux_C_API_Packaging_GPU_x64
workspace:
clean: all
timeoutInMinutes: 120
pool: 'Linux-GPU-CUDA10'
steps:
- template: templates/set-version-number-variables-step.yml
- task: CmdLine@2
inputs:
script: |
docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 \
--cudnn_home=/usr/local/cuda-10.1
workingDirectory: $(Build.SourcesDirectory)
- template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
buildConfig: 'Release'
artifactName: 'onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion)'
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
commitId: $(OnnxRuntimeGitCommitHash)
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- template: templates/clean-agent-build-directory-step.yml
- job: MacOS_C_API_Packaging_CPU_x64
workspace:
clean: all
pool:
vmImage: 'macOS-10.14'
timeoutInMinutes: 120
steps:
- template: templates/set-version-number-variables-step.yml
- script: |
sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config RelWithDebInfo
displayName: 'Build and Test MacOS'
- template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
buildConfig: 'RelWithDebInfo'
artifactName: 'onnxruntime-osx-x64-$(OnnxRuntimeVersion)'
libraryName: 'libonnxruntime.$(OnnxRuntimeVersion).dylib'
commitId: $(OnnxRuntimeGitCommitHash)
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- template: templates/clean-agent-build-directory-step.yml
- job: Windows_Packaging_CPU
workspace:
clean: all
pool: 'Win-CPU-2019'
timeoutInMinutes: 160
strategy:
maxParallel: 2
matrix:
x86:
EnvSetupScript: setup_env_x86.bat
buildArch: x86
msbuildArch: x86
msbuildPlatform: Win32
buildparameter: --x86
x64:
EnvSetupScript: setup_env.bat
buildArch: x64
msbuildArch: amd64
msbuildPlatform: x64
buildparameter:
steps:
- template: templates/telemetry-steps.yml
- task: UsePythonVersion@0
inputs:
versionSpec: '3.7'
addToPath: true
architecture: $(buildArch)
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- powershell: |
$Env:USE_MSVC_STATIC_RUNTIME=1
$Env:ONNX_ML=1
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
python setup.py bdist_wheel
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
displayName: 'Install ONNX'
- template: templates/set-test-data-variables-step.yml
- template: templates/set-version-number-variables-step.yml
- task: PythonScript@0
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)'
workingDirectory: '$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build'
inputs:
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
platform: $(msbuildPlatform)
configuration: RelWithDebInfo
msbuildArchitecture: $(buildArch)
maximumCpuCount: true
logProjectEvents: true
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
createLogFile: true
- task: PythonScript@0
displayName: 'test'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --use_openmp --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)'
workingDirectory: '$(Build.BinariesDirectory)'
- template: templates/c-api-artifacts-package-and-publish-steps-windows.yml
parameters:
buildConfig: RelWithDebInfo
artifactName: 'onnxruntime-win-$(buildArch)-$(OnnxRuntimeVersion)'
commitId: $(OnnxRuntimeGitCommitHash)
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- job: Windows_Packaging_GPU
workspace:
clean: all
pool: 'Win-GPU-2019'
timeoutInMinutes: 120
variables:
EnvSetupScript: setup_env_cuda.bat
buildArch: x64
msbuildArch: amd64
msbuildPlatform: x64
CUDA_VERSION: '10.1'
buildparameter: --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows10-x64-v7.6.5.32\cuda"
steps:
- template: templates/telemetry-steps.yml
- task: UsePythonVersion@0
inputs:
versionSpec: '3.7'
addToPath: true
architecture: $(buildArch)
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- powershell: |
$Env:USE_MSVC_STATIC_RUNTIME=1
$Env:ONNX_ML=1
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
python setup.py bdist_wheel
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
displayName: 'Install ONNX'
- template: templates/set-test-data-variables-step.yml
- template: templates/set-version-number-variables-step.yml
- task: PythonScript@0
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)'
workingDirectory: '$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build'
inputs:
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
platform: $(msbuildPlatform)
configuration: RelWithDebInfo
msbuildArchitecture: $(buildArch)
maximumCpuCount: true
logProjectEvents: true
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
createLogFile: true
- task: PythonScript@0
displayName: 'test'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) $(buildparameter)'
workingDirectory: '$(Build.BinariesDirectory)'
- template: templates/c-api-artifacts-package-and-publish-steps-windows.yml
parameters:
buildConfig: RelWithDebInfo
artifactName: 'onnxruntime-win-$(buildArch)-gpu-$(OnnxRuntimeVersion)'
commitId: $(OnnxRuntimeGitCommitHash)
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
Reference in a new issue View git blame Copy permalink