mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-06 04:28:32 +00:00
83be3759bc
ORTModule requires two PyTorch CPP extensions that are currently JIT compiled. The runtime compilation can cause issues in some environments without all build requirements or in environments with multiple instances of ORTModule running in parallel This PR creates a custom command to compile such extensions that must be manually executed before ORTModule is executed for the first time. When users try to use ORTModule before the extensions are compiled, an error with instructions are raised PyTorch CPP Extensions for ORTModule can be compiled by running: python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install Full build environment is needed for this
50 lines
1.6 KiB
YAML
50 lines
1.6 KiB
YAML
trigger: none
|
|
|
|
jobs:
|
|
- job: Onnxruntime_Linux_GPU_Distributed_Test
|
|
|
|
timeoutInMinutes: 120
|
|
pool: 'Linux-Multi-GPU-V100'
|
|
|
|
steps:
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
|
|
- template: templates/run-docker-build-steps.yml
|
|
parameters:
|
|
RunDockerBuildArgs: |
|
|
-o ubuntu20.04 -d gpu \
|
|
-t onnxruntime_distributed_tests_image \
|
|
-x " \
|
|
--use_cuda --cuda_version=11.1 --cuda_home=/usr/local/cuda-11.1 --cudnn_home=/usr/local/cuda-11.1 \
|
|
--config RelWithDebInfo \
|
|
--enable_training \
|
|
--update --build \
|
|
--build_wheel \
|
|
" \
|
|
-m
|
|
DisplayName: 'Build'
|
|
|
|
# Entry point for all distributed CI tests.
|
|
# Refer to orttraining/orttraining/test/python/how_to_add_distributed_ci_pipeline_tests.md for guidelines on how to add new tests to this pipeline.
|
|
- script: |
|
|
docker run \
|
|
--gpus all \
|
|
--shm-size=1024m \
|
|
--rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
onnxruntime_distributed_tests_image \
|
|
/build/RelWithDebInfo/launch_test.py \
|
|
--cmd_line_with_args "python orttraining_distributed_tests.py" \
|
|
--cwd /build/RelWithDebInfo \
|
|
displayName: 'Run orttraining_distributed_tests.py'
|
|
condition: succeededOrFailed()
|
|
timeoutInMinutes: 120
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|