mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-18 21:21:17 +00:00
eafd67b8fd
1. Update CUDA version from 11.4 to 11.6.
2. Update Manylinux version
3. Upgrade GCC version from 10 to 11 for most x86_64 pipelines. CentOS 7 ARM64 doesn't have GCC 11 yet.
4. Refactor python packaging pipeline:
a. Split Linux GPU build job to two parts, build and test, so that the
build part doesn't need to use a GPU machine
b. Make the Linux GPU build job and Linux CPU build job more similar: share the same bash script and yaml file.
5. Temporarily disable Attention_Mask1D_Fp16_B2_FusedNoPadding because it is causing one of our packaging pipeline to fail. I have created an ADO task for this.
137 lines
No EOL
4.5 KiB
YAML
137 lines
No EOL
4.5 KiB
YAML
resources:
|
|
repositories:
|
|
- repository: manylinux
|
|
type: Github
|
|
endpoint: Microsoft
|
|
name: pypa/manylinux
|
|
ref: 1a61614cabfd6b91c6afd6d9e172cc5b838c65fe
|
|
|
|
jobs:
|
|
- job: Linux_Build
|
|
timeoutInMinutes: 120
|
|
workspace:
|
|
clean: all
|
|
pool: onnxruntime-training-linux-ext-custom-ops
|
|
steps:
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_eager_cpu
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
|
Repository: onnxruntimecpubuild
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Create and start docker container'
|
|
inputs:
|
|
script: |
|
|
docker run -it -d \
|
|
--name external_custom_ops_container \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
onnxruntimecpubuild \
|
|
/bin/bash
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'ONNXRuntime build'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker exec -t \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
external_custom_ops_container \
|
|
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build \
|
|
--config Debug Release \
|
|
--parallel \
|
|
--build_wheel \
|
|
--skip_tests \
|
|
--enable_training \
|
|
--build_eager_mode \
|
|
--enable_pybind \
|
|
--enable_external_custom_op_schemas
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install Release python package'
|
|
inputs:
|
|
script: |
|
|
rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11
|
|
docker exec -t external_custom_ops_container \
|
|
/bin/bash -c "/opt/python/cp38-cp38/bin/python3 -m pip install /build/Release/dist/*.whl"
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install Pybind11'
|
|
inputs:
|
|
script: |
|
|
docker exec -t \
|
|
-w /build/Debug/external_custom_ops \
|
|
external_custom_ops_container \
|
|
/opt/python/cp38-cp38/bin/python3 -m pip install pybind11
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Build and Install custom ops python package'
|
|
inputs:
|
|
script: |
|
|
docker exec -t \
|
|
-w /build/Debug/external_custom_ops \
|
|
external_custom_ops_container \
|
|
/opt/python/cp38-cp38/bin/python3 -m pip install .
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Test using external custom op module'
|
|
inputs:
|
|
script: |
|
|
docker exec -t \
|
|
-w /build/Debug/external_custom_ops \
|
|
external_custom_ops_container \
|
|
/opt/python/cp38-cp38/bin/python3 test.py
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install Debug python package'
|
|
inputs:
|
|
script: |
|
|
rm -rf $(Build.BinariesDirectory)/Debug/onnxruntime $(Build.BinariesDirectory)/Debug/pybind11
|
|
docker exec -t external_custom_ops_container \
|
|
/bin/bash -c "/opt/python/cp38-cp38/bin/python3 -m pip install /build/Debug/dist/*.whl"
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Build and Install custom ops python package'
|
|
inputs:
|
|
script: |
|
|
docker exec -t \
|
|
-w /build/Debug/external_custom_ops \
|
|
external_custom_ops_container \
|
|
/opt/python/cp38-cp38/bin/python3 -m pip install .
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Test using external custom op module'
|
|
inputs:
|
|
script: |
|
|
docker exec -t \
|
|
-w /build/Debug/external_custom_ops \
|
|
external_custom_ops_container \
|
|
/opt/python/cp38-cp38/bin/python3 test.py
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Stop and remove docker container'
|
|
inputs:
|
|
script: |
|
|
docker stop external_custom_ops_container
|
|
docker rm external_custom_ops_container |