diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json index 7e8459eb4d..a14b4f5f33 100644 --- a/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json @@ -53,5 +53,5 @@ "loss": 1.5503 } ], - "samples_per_second": 25.587 + "samples_per_second": 24.756 } \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml index be6938a3ea..13b5c9df66 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml @@ -33,90 +33,110 @@ jobs: # sed -i 's|session_options.use_deterministic_compute = False|session_options.use_deterministic_compute = True|g' \ # orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py # displayName: 'Toggle ON deterministic compute mode for ORTModule' - - - script: |- - export ROCM_HOME=/opt/rocm - python tools/ci_build/build.py \ - --config RelWithDebInfo \ - --enable_training \ - --mpi_home /opt/ompi \ - --use_rocm \ - --rocm_version=4.3.1 \ - --rocm_home /opt/rocm \ - --nccl_home /opt/rocm \ - --update \ - --build_dir ./build \ - --build \ - --parallel 8 \ - --build_wheel \ - --skip_tests + + - task: CmdLine@2 + displayName: 'Check ROCm Environment' + inputs: + script: |- + echo $(Agent.Name) + bash ./tools/ci_build/github/pai/pai_get_thread.sh $(Agent.Name) + target: host + + - task: CmdLine@2 + inputs: + script: |- + export ROCM_HOME=/opt/rocm + python tools/ci_build/build.py \ + --config RelWithDebInfo \ + --enable_training \ + --mpi_home /opt/ompi \ + --use_rocm \ + --rocm_version=4.3.1 \ + --rocm_home /opt/rocm \ + --nccl_home /opt/rocm \ + --update \ + --build_dir ./build \ + --build \ + --parallel 8 \ + --build_wheel \ + --skip_tests displayName: 'Build onnxruntime' - + - bash: |- echo "##vso[task.setvariable variable=onnxruntimeBuildSucceeded]true" displayName: 'Set Onnxruntime Build Succeeded' - - - script: |- - cd ./build/RelWithDebInfo &&\ - ../../tools/ci_build/github/pai/pai_test_launcher.sh + + - task: CmdLine@2 + inputs: + script: |- + cd ./build/RelWithDebInfo &&\ + ../../tools/ci_build/github/pai/pai_test_launcher.sh displayName: 'Run onnxruntime unit tests' - - - script: |- - cd ./build/RelWithDebInfo - export PYTHONPATH=$PWD - python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install + + - task: CmdLine@2 + inputs: + script: |- + cd ./build/RelWithDebInfo + export PYTHONPATH=$PWD + python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install displayName: 'Compile torch extensions into build directory' condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed - - script: |- - cd ./build/RelWithDebInfo - export PYTHONPATH=$PWD - export ORTMODULE_ONNX_OPSET_VERSION=12 - python \ - /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \ - --model_name_or_path bert-large-uncased \ - --dataset_name wikitext \ - --dataset_config_name wikitext-2-raw-v1 \ - --do_train \ - --max_steps 260 \ - --logging_steps 20 \ - --output_dir ./test-mlm-bbu \ - --overwrite_output_dir \ - --per_device_train_batch_size 8 \ - --fp16 \ - --dataloader_num_workers 1 \ - --ort \ - --skip_memory_metrics - python ../../orttraining/tools/ci_test/compare_huggingface.py \ - ci-pipeline-actual.json \ - ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1.json + - task: CmdLine@2 + inputs: + script: |- + cd ./build/RelWithDebInfo + export PYTHONPATH=$PWD + export ORTMODULE_ONNX_OPSET_VERSION=12 + python \ + /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \ + --model_name_or_path bert-large-uncased \ + --dataset_name wikitext \ + --dataset_config_name wikitext-2-raw-v1 \ + --do_train \ + --max_steps 260 \ + --logging_steps 20 \ + --output_dir ./test-mlm-bbu \ + --overwrite_output_dir \ + --per_device_train_batch_size 8 \ + --fp16 \ + --dataloader_num_workers 1 \ + --ort \ + --skip_memory_metrics + python ../../orttraining/tools/ci_test/compare_huggingface.py \ + ci-pipeline-actual.json \ + ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1.json displayName: 'Run Python Hugging-Face BERT-L test' + retryCountOnTaskFailure: 1 condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed - - script: |- - cd ./build/RelWithDebInfo - export PYTHONPATH=$PWD - export ORTMODULE_ONNX_OPSET_VERSION=12 - python \ - /stage/huggingface-transformers/examples/pytorch/language-modeling/run_clm.py \ - --model_name_or_path gpt2 \ - --dataset_name wikitext \ - --dataset_config_name wikitext-2-raw-v1 \ - --do_train \ - --label_smoothing 0.1 \ - --max_steps 260 \ - --logging_steps 20 \ - --overwrite_output_dir \ - --output_dir ./test-clm \ - --per_device_train_batch_size 8 \ - --fp16 \ - --dataloader_num_workers 1 \ - --ort \ - --skip_memory_metrics - python ../../orttraining/tools/ci_test/compare_huggingface.py \ - ci-pipeline-actual.json \ - ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json + - task: CmdLine@2 + inputs: + script: |- + cd ./build/RelWithDebInfo + export PYTHONPATH=$PWD + export ORTMODULE_ONNX_OPSET_VERSION=12 + python \ + /stage/huggingface-transformers/examples/pytorch/language-modeling/run_clm.py \ + --model_name_or_path gpt2 \ + --dataset_name wikitext \ + --dataset_config_name wikitext-2-raw-v1 \ + --do_train \ + --label_smoothing 0.1 \ + --max_steps 260 \ + --logging_steps 20 \ + --overwrite_output_dir \ + --output_dir ./test-clm \ + --per_device_train_batch_size 8 \ + --fp16 \ + --dataloader_num_workers 1 \ + --ort \ + --skip_memory_metrics + python ../../orttraining/tools/ci_test/compare_huggingface.py \ + ci-pipeline-actual.json \ + ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json displayName: 'Run Python Hugging-Face GPT2 test' + retryCountOnTaskFailure: 1 condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed # - script: |- @@ -174,29 +194,32 @@ jobs: # displayName: 'Run Python Hugging-Face RoBERTa-L test' # condition: succeededOrFailed() # ensure all tests are run - - script: |- - cd ./build/RelWithDebInfo - export PYTHONPATH=$PWD - export ORTMODULE_ONNX_OPSET_VERSION=12 - python \ - /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \ - --model_name_or_path distilbert-base-uncased \ - --dataset_name wikitext \ - --dataset_config_name wikitext-2-raw-v1 \ - --do_train \ - --max_steps 260 \ - --logging_steps 20 \ - --output_dir ./test-mlm-bbu \ - --overwrite_output_dir \ - --per_device_train_batch_size 32 \ - --fp16 \ - --dataloader_num_workers 1 \ - --ort \ - --skip_memory_metrics - python ../../orttraining/tools/ci_test/compare_huggingface.py \ - ci-pipeline-actual.json \ - ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1.json + - task: CmdLine@2 + inputs: + script: |- + cd ./build/RelWithDebInfo + export PYTHONPATH=$PWD + export ORTMODULE_ONNX_OPSET_VERSION=12 + python \ + /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \ + --model_name_or_path distilbert-base-uncased \ + --dataset_name wikitext \ + --dataset_config_name wikitext-2-raw-v1 \ + --do_train \ + --max_steps 260 \ + --logging_steps 20 \ + --output_dir ./test-mlm-bbu \ + --overwrite_output_dir \ + --per_device_train_batch_size 32 \ + --fp16 \ + --dataloader_num_workers 1 \ + --ort \ + --skip_memory_metrics + python ../../orttraining/tools/ci_test/compare_huggingface.py \ + ci-pipeline-actual.json \ + ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1.json displayName: 'Run Python Hugging-Face DistilBERT test' + retryCountOnTaskFailure: 1 condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed #- script: |- @@ -251,36 +274,47 @@ jobs: # displayName: 'Run Python Hugging-Face T5-L test' # condition: succeededOrFailed() # ensure all tests are run - - script: |- - python orttraining/tools/ci_test/download_azure_blob_archive.py \ - --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ - --target_dir training_e2e_test_data \ - --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 + - task: CmdLine@2 + inputs: + script: |- + python orttraining/tools/ci_test/download_azure_blob_archive.py \ + --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ + --target_dir training_e2e_test_data \ + --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed displayName: 'Download onnxruntime_training_data.zip data' - - script: |- - python orttraining/tools/ci_test/run_batch_size_test.py \ - --binary_dir build/RelWithDebInfo \ - --model_root training_e2e_test_data/models \ - --gpu_sku MI100_32G + - task: CmdLine@2 + inputs: + script: |- + python orttraining/tools/ci_test/run_batch_size_test.py \ + --binary_dir build/RelWithDebInfo \ + --model_root training_e2e_test_data/models \ + --gpu_sku MI100_32G displayName: 'Run C++ BERT-L batch size test' + retryCountOnTaskFailure: 1 condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed - - script: |- - python orttraining/tools/ci_test/run_bert_perf_test.py \ - --binary_dir build/RelWithDebInfo \ - --model_root training_e2e_test_data/models \ - --training_data_root training_e2e_test_data/data \ - --gpu_sku MI100_32G + - task: CmdLine@2 + inputs: + script: |- + python orttraining/tools/ci_test/run_bert_perf_test.py \ + --binary_dir build/RelWithDebInfo \ + --model_root training_e2e_test_data/models \ + --training_data_root training_e2e_test_data/data \ + --gpu_sku MI100_32G displayName: 'Run C++ BERT-L performance test' + retryCountOnTaskFailure: 1 condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed - - script: |- - python orttraining/tools/ci_test/run_convergence_test.py \ - --binary_dir build/RelWithDebInfo \ - --model_root training_e2e_test_data/models \ - --training_data_root training_e2e_test_data/data \ - --gpu_sku MI100_32G + - task: CmdLine@2 + inputs: + script: |- + python orttraining/tools/ci_test/run_convergence_test.py \ + --binary_dir build/RelWithDebInfo \ + --model_root training_e2e_test_data/models \ + --training_data_root training_e2e_test_data/data \ + --gpu_sku MI100_32G displayName: 'Run C++ BERT-L convergence test' + retryCountOnTaskFailure: 1 condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed diff --git a/tools/ci_build/github/pai/pai_get_thread.sh b/tools/ci_build/github/pai/pai_get_thread.sh new file mode 100644 index 0000000000..5e2811d579 --- /dev/null +++ b/tools/ci_build/github/pai/pai_get_thread.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +agentName=$1 +finalCharacter=${agentName: -1} +echo "agent name $agentName" +echo "agent name final character : $finalCharacter" +targetRender=$((finalCharacter+128)) + +echo -e "\n ---- rocm-smi" +rocm-smi + +echo -e "\n ---- rocm-smi --showpids" +rocm-smi --showpids + +echo -e "\n ---- rocm-smi --showpidgpus" +rocm-smi --showpidgpus + +echo -e "\n ---- rocm-smi --showpids detail" +rocm-smi --showpids | awk '$1 ~/[0-9]+/{if((NR>6)) {print $1}}' | xargs -I {} ps {} + +echo -e "\n ---- rocm-smi --showmeminfo" +rocm-smi --showmeminfo vram vis_vram gtt + +echo -e "\n ---- show all renders" +lsof /dev/dri/renderD* + +echo -e "\n ---- show specific render" +lsof /dev/dri/renderD${targetRender} + +echo -e "\n ---- show specific render pids detail" +lsof /dev/dri/renderD${targetRender} | grep "mem" | awk '{print $2}' | xargs -I {} ps {} \ No newline at end of file