update the amd ci pipeline (#10723)

* [TEST] test to get amd pipeline information * [FIX] lower the threshold * [UPDATE] add retry task * [UPDATE] add retry task * [ERROR] error to occur retry * [FIX] error * [UPDATE] update retryCountOnTaskFailure to 1 time * [UPDATE] add showmeminfo
2026-07-22 19:23:30 +00:00 · 2022-03-07 18:39:42 +08:00 · 2022-03-07 18:39:42 +08:00 · 55af7a96a7
commit 55af7a96a7
parent 60acfd3dd8
3 changed files with 182 additions and 117 deletions
--- a/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json
+++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json
@ -53,5 +53,5 @@
            "loss": 1.5503
        }
    ],
-    "samples_per_second": 25.587
+    "samples_per_second": 24.756
 }
--- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml
@ -33,90 +33,110 @@ jobs:
  #    sed -i 's|session_options.use_deterministic_compute = False|session_options.use_deterministic_compute = True|g' \
  #       orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py
  #  displayName: 'Toggle ON deterministic compute mode for ORTModule'
-  
-  - script: |-
-      export ROCM_HOME=/opt/rocm
-      python tools/ci_build/build.py \
-      --config RelWithDebInfo \
-      --enable_training \
-      --mpi_home /opt/ompi \
-      --use_rocm \
-      --rocm_version=4.3.1 \
-      --rocm_home /opt/rocm \
-      --nccl_home /opt/rocm \
-      --update \
-      --build_dir ./build \
-      --build \
-      --parallel 8 \
-      --build_wheel \
-      --skip_tests
+
+  - task: CmdLine@2
+    displayName: 'Check ROCm Environment'
+    inputs:
+      script: |-
+        echo $(Agent.Name)
+        bash ./tools/ci_build/github/pai/pai_get_thread.sh $(Agent.Name)
+    target: host
+
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        export ROCM_HOME=/opt/rocm
+        python tools/ci_build/build.py \
+        --config RelWithDebInfo \
+        --enable_training \
+        --mpi_home /opt/ompi \
+        --use_rocm \
+        --rocm_version=4.3.1 \
+        --rocm_home /opt/rocm \
+        --nccl_home /opt/rocm \
+        --update \
+        --build_dir ./build \
+        --build \
+        --parallel 8 \
+        --build_wheel \
+        --skip_tests
    displayName: 'Build onnxruntime'
-  
+    
  - bash: |-
      echo "##vso[task.setvariable variable=onnxruntimeBuildSucceeded]true"
    displayName: 'Set Onnxruntime Build Succeeded'
-
-  - script: |-
-      cd ./build/RelWithDebInfo &&\
-      ../../tools/ci_build/github/pai/pai_test_launcher.sh
+  
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        cd ./build/RelWithDebInfo &&\
+        ../../tools/ci_build/github/pai/pai_test_launcher.sh
    displayName: 'Run onnxruntime unit tests'
-
-  - script: |-
-      cd ./build/RelWithDebInfo
-      export PYTHONPATH=$PWD
-      python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install
+    
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        cd ./build/RelWithDebInfo
+        export PYTHONPATH=$PWD
+        python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install
    displayName: 'Compile torch extensions into build directory'
    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
  
-  - script: |-
-      cd ./build/RelWithDebInfo
-      export PYTHONPATH=$PWD
-      export ORTMODULE_ONNX_OPSET_VERSION=12
-      python \
-        /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \
-        --model_name_or_path bert-large-uncased \
-        --dataset_name wikitext \
-        --dataset_config_name wikitext-2-raw-v1 \
-        --do_train \
-        --max_steps 260 \
-        --logging_steps 20 \
-        --output_dir ./test-mlm-bbu \
-        --overwrite_output_dir \
-        --per_device_train_batch_size 8 \
-        --fp16 \
-        --dataloader_num_workers 1 \
-        --ort \
-        --skip_memory_metrics
-      python ../../orttraining/tools/ci_test/compare_huggingface.py \
-        ci-pipeline-actual.json \
-        ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1.json
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        cd ./build/RelWithDebInfo
+        export PYTHONPATH=$PWD
+        export ORTMODULE_ONNX_OPSET_VERSION=12
+        python \
+          /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \
+          --model_name_or_path bert-large-uncased \
+          --dataset_name wikitext \
+          --dataset_config_name wikitext-2-raw-v1 \
+          --do_train \
+          --max_steps 260 \
+          --logging_steps 20 \
+          --output_dir ./test-mlm-bbu \
+          --overwrite_output_dir \
+          --per_device_train_batch_size 8 \
+          --fp16 \
+          --dataloader_num_workers 1 \
+          --ort \
+          --skip_memory_metrics
+        python ../../orttraining/tools/ci_test/compare_huggingface.py \
+          ci-pipeline-actual.json \
+          ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1.json
    displayName: 'Run Python Hugging-Face BERT-L test'
+    retryCountOnTaskFailure: 1
    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
  
-  - script: |-
-      cd ./build/RelWithDebInfo
-      export PYTHONPATH=$PWD
-      export ORTMODULE_ONNX_OPSET_VERSION=12
-      python \
-        /stage/huggingface-transformers/examples/pytorch/language-modeling/run_clm.py \
-        --model_name_or_path gpt2 \
-        --dataset_name wikitext \
-        --dataset_config_name wikitext-2-raw-v1 \
-        --do_train \
-        --label_smoothing 0.1 \
-        --max_steps 260 \
-        --logging_steps 20 \
-        --overwrite_output_dir \
-        --output_dir ./test-clm \
-        --per_device_train_batch_size 8 \
-        --fp16 \
-        --dataloader_num_workers 1 \
-        --ort \
-        --skip_memory_metrics
-      python ../../orttraining/tools/ci_test/compare_huggingface.py \
-        ci-pipeline-actual.json \
-        ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        cd ./build/RelWithDebInfo
+        export PYTHONPATH=$PWD
+        export ORTMODULE_ONNX_OPSET_VERSION=12
+        python \
+          /stage/huggingface-transformers/examples/pytorch/language-modeling/run_clm.py \
+          --model_name_or_path gpt2 \
+          --dataset_name wikitext \
+          --dataset_config_name wikitext-2-raw-v1 \
+          --do_train \
+          --label_smoothing 0.1 \
+          --max_steps 260 \
+          --logging_steps 20 \
+          --overwrite_output_dir \
+          --output_dir ./test-clm \
+          --per_device_train_batch_size 8 \
+          --fp16 \
+          --dataloader_num_workers 1 \
+          --ort \
+          --skip_memory_metrics
+        python ../../orttraining/tools/ci_test/compare_huggingface.py \
+          ci-pipeline-actual.json \
+          ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json
    displayName: 'Run Python Hugging-Face GPT2 test'
+    retryCountOnTaskFailure: 1
    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
  
 #  - script: |-
@ -174,29 +194,32 @@ jobs:
  #  displayName: 'Run Python Hugging-Face RoBERTa-L test'
  #  condition: succeededOrFailed() # ensure all tests are run
  
-  - script: |-
-      cd ./build/RelWithDebInfo
-      export PYTHONPATH=$PWD
-      export ORTMODULE_ONNX_OPSET_VERSION=12
-      python \
-        /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \
-        --model_name_or_path distilbert-base-uncased \
-        --dataset_name wikitext \
-        --dataset_config_name wikitext-2-raw-v1 \
-        --do_train \
-        --max_steps 260 \
-        --logging_steps 20 \
-        --output_dir ./test-mlm-bbu \
-        --overwrite_output_dir \
-        --per_device_train_batch_size 32 \
-        --fp16 \
-        --dataloader_num_workers 1 \
-        --ort \
-        --skip_memory_metrics
-      python ../../orttraining/tools/ci_test/compare_huggingface.py \
-        ci-pipeline-actual.json \
-        ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1.json
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        cd ./build/RelWithDebInfo
+        export PYTHONPATH=$PWD
+        export ORTMODULE_ONNX_OPSET_VERSION=12
+        python \
+          /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \
+          --model_name_or_path distilbert-base-uncased \
+          --dataset_name wikitext \
+          --dataset_config_name wikitext-2-raw-v1 \
+          --do_train \
+          --max_steps 260 \
+          --logging_steps 20 \
+          --output_dir ./test-mlm-bbu \
+          --overwrite_output_dir \
+          --per_device_train_batch_size 32 \
+          --fp16 \
+          --dataloader_num_workers 1 \
+          --ort \
+          --skip_memory_metrics
+        python ../../orttraining/tools/ci_test/compare_huggingface.py \
+          ci-pipeline-actual.json \
+          ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1.json
    displayName: 'Run Python Hugging-Face DistilBERT test'
+    retryCountOnTaskFailure: 1
    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
  
  #- script: |-
@ -251,36 +274,47 @@ jobs:
  #  displayName: 'Run Python Hugging-Face T5-L test'
  #  condition: succeededOrFailed() # ensure all tests are run
  
-  - script: |-
-      python orttraining/tools/ci_test/download_azure_blob_archive.py \
-        --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \
-        --target_dir training_e2e_test_data \
-        --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        python orttraining/tools/ci_test/download_azure_blob_archive.py \
+          --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \
+          --target_dir training_e2e_test_data \
+          --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9
    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
    displayName: 'Download onnxruntime_training_data.zip data'
  
-  - script: |-
-     python orttraining/tools/ci_test/run_batch_size_test.py \
-        --binary_dir build/RelWithDebInfo \
-        --model_root training_e2e_test_data/models \
-       --gpu_sku MI100_32G
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        python orttraining/tools/ci_test/run_batch_size_test.py \
+            --binary_dir build/RelWithDebInfo \
+            --model_root training_e2e_test_data/models \
+          --gpu_sku MI100_32G
    displayName: 'Run C++ BERT-L batch size test'
+    retryCountOnTaskFailure: 1
    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
  
-  - script: |-
-      python orttraining/tools/ci_test/run_bert_perf_test.py \
-        --binary_dir build/RelWithDebInfo \
-        --model_root training_e2e_test_data/models \
-        --training_data_root training_e2e_test_data/data \
-        --gpu_sku MI100_32G
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        python orttraining/tools/ci_test/run_bert_perf_test.py \
+          --binary_dir build/RelWithDebInfo \
+          --model_root training_e2e_test_data/models \
+          --training_data_root training_e2e_test_data/data \
+          --gpu_sku MI100_32G
    displayName: 'Run C++ BERT-L performance test'
+    retryCountOnTaskFailure: 1
    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
  
-  - script: |-
-      python orttraining/tools/ci_test/run_convergence_test.py \
-        --binary_dir build/RelWithDebInfo \
-        --model_root training_e2e_test_data/models \
-        --training_data_root training_e2e_test_data/data \
-        --gpu_sku MI100_32G
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        python orttraining/tools/ci_test/run_convergence_test.py \
+          --binary_dir build/RelWithDebInfo \
+          --model_root training_e2e_test_data/models \
+          --training_data_root training_e2e_test_data/data \
+          --gpu_sku MI100_32G
    displayName: 'Run C++ BERT-L convergence test'
+    retryCountOnTaskFailure: 1
    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
--- a/tools/ci_build/github/pai/pai_get_thread.sh
+++ b/tools/ci_build/github/pai/pai_get_thread.sh
@ -0,0 +1,31 @@
+#!/bin/bash 
+
+agentName=$1
+finalCharacter=${agentName: -1}
+echo "agent name $agentName"
+echo "agent name final character : $finalCharacter"
+targetRender=$((finalCharacter+128))
+
+echo -e "\n ---- rocm-smi"
+rocm-smi
+
+echo -e "\n ---- rocm-smi --showpids"
+rocm-smi --showpids
+
+echo -e "\n ---- rocm-smi --showpidgpus"
+rocm-smi --showpidgpus 
+
+echo -e "\n ---- rocm-smi --showpids detail"
+rocm-smi --showpids | awk '$1 ~/[0-9]+/{if((NR>6)) {print $1}}' | xargs -I {} ps {}
+
+echo -e "\n ---- rocm-smi --showmeminfo"
+rocm-smi  --showmeminfo vram vis_vram gtt
+
+echo -e "\n ---- show all renders"
+lsof /dev/dri/renderD*
+
+echo -e "\n ---- show specific render"
+lsof /dev/dri/renderD${targetRender}
+
+echo -e "\n ---- show specific render pids detail"
+lsof /dev/dri/renderD${targetRender} | grep "mem" | awk '{print $2}' | xargs -I {} ps {}