mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-27 20:02:15 +00:00
eafd67b8fd
1. Update CUDA version from 11.4 to 11.6.
2. Update Manylinux version
3. Upgrade GCC version from 10 to 11 for most x86_64 pipelines. CentOS 7 ARM64 doesn't have GCC 11 yet.
4. Refactor python packaging pipeline:
a. Split Linux GPU build job to two parts, build and test, so that the
build part doesn't need to use a GPU machine
b. Make the Linux GPU build job and Linux CPU build job more similar: share the same bash script and yaml file.
5. Temporarily disable Attention_Mask1D_Fp16_B2_FusedNoPadding because it is causing one of our packaging pipeline to fail. I have created an ADO task for this.
116 lines
3.8 KiB
YAML
116 lines
3.8 KiB
YAML
trigger: none
|
|
|
|
variables:
|
|
- name: isMain
|
|
value: ${{ eq(variables['Build.SourceBranch'], 'refs/heads/main') }}
|
|
- name: finalStorage
|
|
${{ if eq(variables['isMain'], 'true') }}:
|
|
value: '--final_storage'
|
|
${{ else }}:
|
|
value: ''
|
|
|
|
resources:
|
|
repositories:
|
|
- repository: manylinux
|
|
type: Github
|
|
endpoint: Microsoft
|
|
name: pypa/manylinux
|
|
ref: 1a61614cabfd6b91c6afd6d9e172cc5b838c65fe
|
|
|
|
stages:
|
|
- stage: Python_Packaging
|
|
|
|
jobs:
|
|
- job: Linux_Training_CPU_Wheels
|
|
timeoutInMinutes: 120
|
|
workspace:
|
|
clean: all
|
|
pool: Linux-CPU
|
|
|
|
strategy:
|
|
matrix:
|
|
Python37:
|
|
PythonVersion: '3.7'
|
|
Python38:
|
|
PythonVersion: '3.8'
|
|
Python39:
|
|
PythonVersion: '3.9'
|
|
|
|
steps:
|
|
- task: CmdLine@2
|
|
displayName: 'check variables'
|
|
inputs:
|
|
script: |
|
|
echo "Branch is "${{ variables['Build.SourceBranch'] }} && \
|
|
echo "isMain is "${{ variables['isMain'] }} && \
|
|
echo "final_storage is "${{ variables['finalStorage'] }}
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
|
|
- template: templates/set-python-manylinux-variables-step.yml
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: >-
|
|
--build-arg PYTHON_VERSION=$(PythonVersion)
|
|
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tu
|
|
--build-arg BUILD_UID=$(id -u)
|
|
Repository: onnxruntimetrainingcpubuild
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'build onnxruntime'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume /data/models:/build/models:ro \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimetrainingcpubuild \
|
|
$(PythonManylinuxDir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator Ninja \
|
|
--config Debug Release \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel \
|
|
--build_wheel \
|
|
--enable_onnx_tests \
|
|
--enable_pybind --build_java --build_nodejs --enable_training
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CopyFiles@2
|
|
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
|
|
inputs:
|
|
SourceFolder: '$(Build.BinariesDirectory)'
|
|
Contents: 'Release/dist/*.whl'
|
|
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
|
|
|
- task: PublishBuildArtifacts@1
|
|
displayName: 'Publish Artifact: ONNXRuntime python wheel and documentation'
|
|
inputs:
|
|
ArtifactName: onnxruntime_training_cpu
|
|
|
|
- task: CmdLine@2
|
|
condition: succeeded()
|
|
displayName: 'Upload wheel'
|
|
inputs:
|
|
script: |
|
|
files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
|
|
echo ${files[0]} && \
|
|
echo ${{ variables['finalStorage'] }} && \
|
|
tools/ci_build/upload_python_package_to_azure_storage.py \
|
|
--python_wheel_path ${files[0]} ${{ variables['finalStorage'] }}
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|