mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
83be3759bc
ORTModule requires two PyTorch CPP extensions that are currently JIT compiled. The runtime compilation can cause issues in some environments without all build requirements or in environments with multiple instances of ORTModule running in parallel This PR creates a custom command to compile such extensions that must be manually executed before ORTModule is executed for the first time. When users try to use ORTModule before the extensions are compiled, an error with instructions are raised PyTorch CPP Extensions for ORTModule can be compiled by running: python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install Full build environment is needed for this
114 lines
4.7 KiB
YAML
114 lines
4.7 KiB
YAML
jobs:
|
|
- job: Linux_Build
|
|
timeoutInMinutes: 120
|
|
workspace:
|
|
clean: all
|
|
pool: Linux-CPU-2019
|
|
steps:
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
|
|
- task: NodeTool@0
|
|
inputs:
|
|
versionSpec: '12.16.3'
|
|
|
|
- template: templates/get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
|
Repository: onnxruntimecpubuild
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'build'
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm \
|
|
--volume /data/onnx:/data/onnx:ro \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
|
|
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
|
|
-e NIGHTLY_BUILD \
|
|
-e BUILD_BUILDNUMBER \
|
|
onnxruntimecpubuild \
|
|
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
|
|
--build_dir /build --cmake_generator Ninja \
|
|
--config Debug Release \
|
|
--skip_submodule_sync \
|
|
--build_shared_lib \
|
|
--parallel \
|
|
--build_wheel \
|
|
--enable_onnx_tests --enable_transformers_tool_test\
|
|
--build_java --build_nodejs --update --build
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install python deps and run java tests'
|
|
inputs:
|
|
script: |
|
|
set -e -x
|
|
python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq
|
|
cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt $(Build.BinariesDirectory)/requirements.txt
|
|
#Test ORT with the latest ONNX release.
|
|
sed -i 's/git+http:\/\/github\.com\/onnx\/onnx.*/onnx==1.9.0/' $(Build.BinariesDirectory)/requirements.txt
|
|
python3 -m pip install -r $(Build.BinariesDirectory)/requirements.txt
|
|
cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu.txt $(Build.BinariesDirectory)/requirements_torch_cpu.txt
|
|
python3 -m pip install -r $(Build.BinariesDirectory)/requirements_torch_cpu.txt
|
|
ln -sf /data/models $(Build.BinariesDirectory)
|
|
cd $(Build.SourcesDirectory)/java
|
|
/usr/local/gradle/bin/gradle "cmakeCheck" "-DcmakeBuildDir=$(Build.BinariesDirectory)/Release"
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install Release python package'
|
|
inputs:
|
|
script: |
|
|
rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11
|
|
python3 -m pip install $(Build.BinariesDirectory)/Release/dist/*.whl
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Run Release unit tests'
|
|
inputs:
|
|
script: |
|
|
cd /tmp
|
|
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --cmake_generator Ninja --config Release --test --skip_submodule_sync --build_shared_lib --parallel --build_wheel --enable_onnx_tests --enable_transformers_tool_test --build_nodejs --ctest_path ""
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Install Debug python package'
|
|
inputs:
|
|
script: |
|
|
rm -rf $(Build.BinariesDirectory)/Debug/onnxruntime $(Build.BinariesDirectory)/Debug/pybind11
|
|
python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq
|
|
python3 -m pip install $(Build.BinariesDirectory)/Debug/dist/*.whl
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Run Debug unit tests'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/build.py
|
|
arguments: --build_dir $(Build.BinariesDirectory) --cmake_generator Ninja --config Debug --test --skip_submodule_sync --build_shared_lib --parallel --build_wheel --enable_onnx_tests --enable_transformers_tool_test --build_nodejs --ctest_path ""
|
|
workingDirectory: /tmp
|
|
|
|
- task: CmdLine@2
|
|
displayName: 'Symbolic shape infer'
|
|
inputs:
|
|
script: |
|
|
cd $(Build.BinariesDirectory)/Release
|
|
python3 $(Build.BinariesDirectory)/Release/onnxruntime_test_python_symbolic_shape_infer.py
|
|
|
|
- task: PublishTestResults@2
|
|
displayName: 'Publish unit test results'
|
|
inputs:
|
|
testResultsFiles: '**/*.results.xml'
|
|
searchFolder: '$(Build.BinariesDirectory)'
|
|
testRunTitle: 'Unit Test Run'
|
|
condition: succeededOrFailed()
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|