mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-20 19:12:24 +00:00
83be3759bc
ORTModule requires two PyTorch CPP extensions that are currently JIT compiled. The runtime compilation can cause issues in some environments without all build requirements or in environments with multiple instances of ORTModule running in parallel This PR creates a custom command to compile such extensions that must be manually executed before ORTModule is executed for the first time. When users try to use ORTModule before the extensions are compiled, an error with instructions are raised PyTorch CPP Extensions for ORTModule can be compiled by running: python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install Full build environment is needed for this
55 lines
2.2 KiB
YAML
55 lines
2.2 KiB
YAML
trigger: none
|
|
|
|
jobs:
|
|
- job: Onnxruntime_Linux_GPU_ORTModule_Distributed_Test
|
|
|
|
timeoutInMinutes: 120
|
|
pool: 'Linux-Multi-GPU-V100'
|
|
|
|
steps:
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
|
|
- template: templates/run-docker-build-steps.yml
|
|
parameters:
|
|
RunDockerBuildArgs: |
|
|
-o ubuntu20.04 -d gpu \
|
|
-t onnxruntime_ortmodule_distributed_tests_image \
|
|
-x " \
|
|
--config RelWithDebInfo \
|
|
--use_cuda --cuda_version=11.1 --cuda_home=/usr/local/cuda-11.1 --cudnn_home=/usr/local/cuda-11.1 \
|
|
--enable_training \
|
|
--update --build \
|
|
--build_wheel --cmake_extra_defines onnxruntime_ENABLE_TRAINING_TORCH_INTEROP=ON \
|
|
" \
|
|
-m \
|
|
-u \
|
|
-e
|
|
DisplayName: 'Build'
|
|
|
|
- bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdata-storage-key) -s "//orttrainingtestdata.file.core.windows.net/mnist" -d "/mnist"
|
|
displayName: 'Mount MNIST'
|
|
condition: succeededOrFailed()
|
|
|
|
# Entry point for all ORTModule distributed tests
|
|
# Refer to orttraining/orttraining/test/python/how_to_add_ortmodule_distributed_ci_pipeline_tests.md for guidelines on how to add new tests to this pipeline.
|
|
- script: |
|
|
docker run \
|
|
--gpus all \
|
|
--shm-size=1024m \
|
|
--rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume /mnist:/mnist \
|
|
onnxruntime_ortmodule_distributed_tests_image \
|
|
bash -c "rm -rf /build/RelWithDebInfo/onnxruntime/ && python3 -m pip install /build/RelWithDebInfo/dist/onnxruntime*.whl && python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install && /build/RelWithDebInfo/launch_test.py --cmd_line_with_args 'python orttraining_ortmodule_distributed_tests.py --mnist /mnist' --cwd /build/RelWithDebInfo" \
|
|
displayName: 'Run orttraining_ortmodule_distributed_tests.py'
|
|
condition: succeededOrFailed()
|
|
timeoutInMinutes: 30
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|