mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-09 17:28:58 +00:00
a46e49b439
### Description * Fix migraphx build error caused by https://github.com/microsoft/onnxruntime/pull/21598: Add a conditional compile on code block that depends on ROCm >= 6.2. Note that the pipeline uses ROCm 6.0. Unblock orttraining-linux-gpu-ci-pipeline and orttraining-ortmodule-distributed and orttraining-amd-gpu-ci-pipeline pipelines: * Disable a model test in linux GPU training ci pipelines caused by https://github.com/microsoft/onnxruntime/pull/19470: Sometime, cudnn frontend throws exception that cudnn graph does not support a Conv node of keras_lotus_resnet3D model on V100 GPU. Note that same test does not throw exception in other GPU pipelines. The failure might be related to cudnn 8.9 and V100 GPU used in the pipeline (Amper GPUs and cuDNN 9.x do not have the issue). The actual fix requires fallback logic, which will take time to implement, so we temporarily disable the test in training pipelines. * Force install torch for cuda 11.8. (The docker has torch 2.4.0 for cuda 12.1 to build torch extension, which it is not compatible cuda 11.8). Note that this is temporary walkround. More elegant fix is to make sure right torch version in docker build step, that might need update install_python_deps.sh and corresponding requirements.txt. * Skip test_gradient_correctness_conv1d since it causes segment fault. Root cause need more investigation (maybe due to cudnn frontend as well). * Skip test_aten_attention since it causes assert failure. Root cause need more investigation (maybe due to torch version). * Skip orttraining_ortmodule_distributed_tests.py since it has error that compiler for torch extension does not support c++17. One possible fix it to set the following compile argument inside setup.py of extension fused_adam: extra_compile_args['cxx'] = ['-std=c++17']. However, due to the urgency of unblocking the pipelines, just disable the test for now. * skip test_softmax_bf16_large. For some reason, torch.cuda.is_bf16_supported() returns True in V100 with torch 2.3.1, so the test was run in CI, but V100 does not support bf16 natively. * Fix typo of deterministic ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. -->
134 lines
4.7 KiB
YAML
134 lines
4.7 KiB
YAML
##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
|
|
trigger:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
pr:
|
|
branches:
|
|
include:
|
|
- main
|
|
- rel-*
|
|
paths:
|
|
exclude:
|
|
- docs/**
|
|
- README.md
|
|
- CONTRIBUTING.md
|
|
- BUILD.md
|
|
- 'js/web'
|
|
- 'onnxruntime/core/providers/js'
|
|
#### end trigger ####
|
|
|
|
stages:
|
|
- stage: ORTModuleDistributedTest
|
|
dependsOn: []
|
|
jobs:
|
|
- job: Onnxruntime_Linux_GPU_ORTModule_Distributed_Test
|
|
|
|
timeoutInMinutes: 120
|
|
pool: 'Onnxruntime-Linux-GPU-NC24sv3'
|
|
|
|
steps:
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
|
|
- template: templates/jobs/download_training_test_data.yml
|
|
|
|
- template: templates/run-docker-build-steps.yml
|
|
parameters:
|
|
RunDockerBuildArgs: |
|
|
-o ubuntu20.04 -d gpu \
|
|
-t onnxruntime_ortmodule_distributed_tests_image \
|
|
-x " \
|
|
--config RelWithDebInfo \
|
|
--use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \
|
|
--enable_training \
|
|
--update --build \
|
|
--build_wheel \
|
|
" \
|
|
-m \
|
|
-u \
|
|
-e
|
|
DisplayName: 'Build'
|
|
|
|
# Entry point for all ORTModule distributed tests
|
|
# Refer to orttraining/orttraining/test/python/how_to_add_ortmodule_distributed_ci_pipeline_tests.md for guidelines on how to add new tests to this pipeline.
|
|
- script: |
|
|
docker run \
|
|
--gpus all \
|
|
--shm-size=1024m \
|
|
--rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume $(Agent.TempDirectory)/mnist:/mnist \
|
|
onnxruntime_ortmodule_distributed_tests_image \
|
|
bash -c "rm -rf /build/RelWithDebInfo/onnxruntime/ && python3 -m pip install /build/RelWithDebInfo/dist/onnxruntime*.whl && python3 -m pip install torch==2.3.1+cu118 --index-url https://download.pytorch.org/whl/cu118 && python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install && echo temporarily skip /build/RelWithDebInfo/launch_test.py --cmd_line_with_args 'python orttraining_ortmodule_distributed_tests.py --mnist /mnist' --cwd /build/RelWithDebInfo" \
|
|
displayName: 'Run orttraining_ortmodule_distributed_tests.py'
|
|
condition: succeededOrFailed()
|
|
timeoutInMinutes: 30
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|
|
|
|
- stage: DistributedInferenceTest
|
|
dependsOn: []
|
|
jobs:
|
|
- job: Onnxruntime_Linux_GPU_Inference_Distributed_Test
|
|
|
|
timeoutInMinutes: 120
|
|
pool: 'Onnxruntime-Linux-GPU-NC24sv3'
|
|
|
|
steps:
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
|
|
- template: templates/run-docker-build-steps.yml
|
|
parameters:
|
|
RunDockerBuildArgs: |
|
|
-o ubuntu20.04 -d gpu \
|
|
-t onnxruntime_ortmodule_distributed_tests_image \
|
|
-x " \
|
|
--config RelWithDebInfo \
|
|
--use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \
|
|
--update --build \
|
|
--build_wheel \
|
|
--use_mpi \
|
|
--enable_nccl \
|
|
" \
|
|
-m \
|
|
-u \
|
|
-e
|
|
DisplayName: 'Build'
|
|
|
|
- script: |
|
|
docker run \
|
|
--gpus all \
|
|
--shm-size=1024m \
|
|
--rm \
|
|
--volume $(Build.SourcesDirectory):/onnxruntime_src \
|
|
--volume $(Build.BinariesDirectory):/build \
|
|
--volume /mnist:/mnist \
|
|
onnxruntime_ortmodule_distributed_tests_image \
|
|
bash -c "rm -rf /build/RelWithDebInfo/onnxruntime/ && python3 -m pip install mpi4py onnxscript && python3 -m pip install /build/RelWithDebInfo/dist/onnxruntime*.whl && mpirun -n 4 -x NCCL_DEBUG=INFO python /onnxruntime_src/onnxruntime/test/python/onnxruntime_test_collective.py && mpirun -n 2 -x NCCL_DEBUG=INFO python /onnxruntime_src/onnxruntime/test/python/onnxruntime_test_distributed.py && mpirun -n 2 -x NCCL_DEBUG=INFO python /onnxruntime_src/onnxruntime/test/python/transformers/sharded_moe/test_sharded_moe.py" \
|
|
displayName: 'Run onnxruntime_test_collective.py, onnxruntime_test_distributed.py and test_sharded_moe.py'
|
|
condition: succeededOrFailed()
|
|
timeoutInMinutes: 30
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|