From e758870b181ae63a10e3d282807459f4bb6d7836 Mon Sep 17 00:00:00 2001 From: Suffian Khan Date: Wed, 15 Sep 2021 12:32:02 -0700 Subject: [PATCH] Upgrade ROCm CI pipeline for ROCm 4.3.1 and permit run inside container (#9070) * try to run inside 4.3.1 container * no \ in container run command * remove networking options * try with adding video render groups * add job to build docker image * try without 1st stage * change alpha, beta to float * try adding service connection * retain huggingface directory * static video and render gid * use runtime expression for variables * install torch-ort * pin sacrebleu==1.5.1 * update curves for rocm 4.3.1 * try again * disable determinism and only check tail of loss curve and with a much larger threshold of 0.05 * disable RoBERTa due to high run variablity on ROCm 4.3.1 * put reduction unit tests back in --- .../providers/rocm/reduction/reduction_ops.cc | 8 +- .../tools/ci_test/compare_huggingface.py | 34 ++++-- ....huggingface.bart-large-rocm4.3.1-det.json | 57 ++++++++++ ...i100.huggingface.bart-large-rocm4.3.1.json | 57 ++++++++++ ....huggingface.bert-large-rocm4.3.1-det.json | 57 ++++++++++ ...i100.huggingface.bert-large-rocm4.3.1.json | 57 ++++++++++ ...ingface.distilbert-base-rocm4.3.1-det.json | 57 ++++++++++ ...huggingface.distilbert-base-rocm4.3.1.json | 57 ++++++++++ ...-mi100.huggingface.gpt2-rocm4.3.1-det.json | 57 ++++++++++ .../ci-mi100.huggingface.gpt2-rocm4.3.1.json | 57 ++++++++++ ...ggingface.roberta-large-rocm4.3.1-det.json | 57 ++++++++++ ...0.huggingface.roberta-large-rocm4.3.1.json | 57 ++++++++++ .../orttraining-pai-ci-pipeline.yml | 103 ++++++++++-------- .../pai/rocm-ci-pipeline-env.Dockerfile | 27 +++++ 14 files changed, 682 insertions(+), 60 deletions(-) create mode 100644 orttraining/tools/ci_test/results/ci-mi100.huggingface.bart-large-rocm4.3.1-det.json create mode 100644 orttraining/tools/ci_test/results/ci-mi100.huggingface.bart-large-rocm4.3.1.json create mode 100644 orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1-det.json create mode 100644 orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1.json create mode 100644 orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1-det.json create mode 100644 orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1.json create mode 100644 orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1-det.json create mode 100644 orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json create mode 100644 orttraining/tools/ci_test/results/ci-mi100.huggingface.roberta-large-rocm4.3.1-det.json create mode 100644 orttraining/tools/ci_test/results/ci-mi100.huggingface.roberta-large-rocm4.3.1.json create mode 100644 tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile diff --git a/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc b/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc index f1fcbcc635..901c18132f 100644 --- a/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc +++ b/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc @@ -192,8 +192,8 @@ Status ReduceKernel::ReduceKernelShared( // // NOTE: this workaround can be removed in ROCm 4.3: // https://github.com/ROCmSoftwarePlatform/MIOpen/pull/914 - const auto one = ReduceConsts::One; - const auto zero = ReduceConsts::Zero; + const auto one = Consts::One; + const auto zero = Consts::Zero; MiopenTensor input_tensor; MiopenTensor output_tensor; ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_miopen, miopen_type_X)); @@ -521,8 +521,8 @@ Status ReduceComputeCore(ROCMExecutionProvider& rocm_ep, const Tensor& input, Pr // // NOTE: this workaround can be removed in ROCm 4.3: // https://github.com/ROCmSoftwarePlatform/MIOpen/pull/914 - const auto one = ReduceConsts::One; - const auto zero = ReduceConsts::Zero; + const float one = Consts::One; + const float zero = Consts::Zero; MiopenTensor input_tensor; MiopenTensor output_tensor; ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_miopen, miopen_type_X)); diff --git a/orttraining/tools/ci_test/compare_huggingface.py b/orttraining/tools/ci_test/compare_huggingface.py index 4950485221..dafa4632b9 100755 --- a/orttraining/tools/ci_test/compare_huggingface.py +++ b/orttraining/tools/ci_test/compare_huggingface.py @@ -1,5 +1,6 @@ import sys import json +import collections actual = sys.argv[1] expect = sys.argv[2] @@ -10,17 +11,32 @@ with open(actual) as file_actual: with open(expect) as file_expect: json_expect = json.loads(file_expect.read()) -# loss curve match -def almost_equal(x, y, threshold=0.0001): +def almost_equal(x, y, threshold=0.05): return abs(x-y) < threshold -for i in range(len(json_actual['steps'])): +# loss curve tail match +loss_tail_length = 4 +loss_tail_matches = collections.deque(maxlen=loss_tail_length) +logged_steps = len(json_actual['steps']) +for i in range(logged_steps): step_actual = json_actual['steps'][i] step_expect = json_expect['steps'][i] - print('step {} loss actual {:.6f} expected {:.6f}'.format(step_actual['step'], step_actual['loss'], step_expect['loss'])) - assert(step_actual['step'] == step_expect['step']) - assert(almost_equal(step_actual['loss'], step_expect['loss'])) -# perf match -print('samples_per_second actual {:.3f} expected {:.3f}'.format(json_actual['samples_per_second'], json_expect['samples_per_second'])) -assert(json_actual['samples_per_second'] >= 0.95*json_expect['samples_per_second']) + is_match = step_actual['step'] == step_expect['step'] + is_match = is_match if almost_equal(step_actual['loss'], step_expect['loss']) else False + loss_tail_matches.append(is_match) + + print('step {} loss actual {:.6f} expected {:.6f} match {}'.format( + step_actual['step'], step_actual['loss'], step_expect['loss'], + is_match if logged_steps - i <= loss_tail_length else 'n/a')) + +success = all(loss_tail_matches) + +# performance match +threshold = 0.97 +is_performant = json_actual['samples_per_second'] >= threshold*json_expect['samples_per_second'] +success = success if is_performant else False +print('samples_per_second actual {:.3f} expected {:.3f} in-range {}'.format( + json_actual['samples_per_second'], json_expect['samples_per_second'], is_performant)) + +assert(success) diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.bart-large-rocm4.3.1-det.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.bart-large-rocm4.3.1-det.json new file mode 100644 index 0000000000..a2ec795a6f --- /dev/null +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.bart-large-rocm4.3.1-det.json @@ -0,0 +1,57 @@ +{ + "steps": [ + { + "step": 20, + "loss": 5.9986 + }, + { + "step": 40, + "loss": 3.7825 + }, + { + "step": 60, + "loss": 3.4456 + }, + { + "step": 80, + "loss": 3.3592 + }, + { + "step": 100, + "loss": 3.2141 + }, + { + "step": 120, + "loss": 3.1794 + }, + { + "step": 140, + "loss": 3.1514 + }, + { + "step": 160, + "loss": 3.0671 + }, + { + "step": 180, + "loss": 3.0059 + }, + { + "step": 200, + "loss": 3.0095 + }, + { + "step": 220, + "loss": 2.9664 + }, + { + "step": 240, + "loss": 2.9331 + }, + { + "step": 260, + "loss": 2.9197 + } + ], + "samples_per_second": 21.725 +} \ No newline at end of file diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.bart-large-rocm4.3.1.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.bart-large-rocm4.3.1.json new file mode 100644 index 0000000000..550fc28752 --- /dev/null +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.bart-large-rocm4.3.1.json @@ -0,0 +1,57 @@ +{ + "steps": [ + { + "step": 20, + "loss": 5.9983 + }, + { + "step": 40, + "loss": 3.7951 + }, + { + "step": 60, + "loss": 3.4942 + }, + { + "step": 80, + "loss": 3.3776 + }, + { + "step": 100, + "loss": 3.2268 + }, + { + "step": 120, + "loss": 3.1892 + }, + { + "step": 140, + "loss": 3.148 + }, + { + "step": 160, + "loss": 3.0686 + }, + { + "step": 180, + "loss": 3.0081 + }, + { + "step": 200, + "loss": 3.0125 + }, + { + "step": 220, + "loss": 2.9611 + }, + { + "step": 240, + "loss": 2.9391 + }, + { + "step": 260, + "loss": 2.9259 + } + ], + "samples_per_second": 62.209 +} \ No newline at end of file diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1-det.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1-det.json new file mode 100644 index 0000000000..99bb9c9b2f --- /dev/null +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1-det.json @@ -0,0 +1,57 @@ +{ + "steps": [ + { + "step": 20, + "loss": 1.9885 + }, + { + "step": 40, + "loss": 1.8471 + }, + { + "step": 60, + "loss": 1.7499 + }, + { + "step": 80, + "loss": 1.6867 + }, + { + "step": 100, + "loss": 1.6936 + }, + { + "step": 120, + "loss": 1.7 + }, + { + "step": 140, + "loss": 1.6409 + }, + { + "step": 160, + "loss": 1.6912 + }, + { + "step": 180, + "loss": 1.6118 + }, + { + "step": 200, + "loss": 1.6115 + }, + { + "step": 220, + "loss": 1.6632 + }, + { + "step": 240, + "loss": 1.5335 + }, + { + "step": 260, + "loss": 1.543 + } + ], + "samples_per_second": 21.976 +} \ No newline at end of file diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1.json new file mode 100644 index 0000000000..3892728a3b --- /dev/null +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1.json @@ -0,0 +1,57 @@ +{ + "steps": [ + { + "step": 20, + "loss": 1.9926 + }, + { + "step": 40, + "loss": 1.8427 + }, + { + "step": 60, + "loss": 1.7542 + }, + { + "step": 80, + "loss": 1.6775 + }, + { + "step": 100, + "loss": 1.6865 + }, + { + "step": 120, + "loss": 1.6837 + }, + { + "step": 140, + "loss": 1.632 + }, + { + "step": 160, + "loss": 1.6826 + }, + { + "step": 180, + "loss": 1.6048 + }, + { + "step": 200, + "loss": 1.6049 + }, + { + "step": 220, + "loss": 1.6554 + }, + { + "step": 240, + "loss": 1.5331 + }, + { + "step": 260, + "loss": 1.5406 + } + ], + "samples_per_second": 23.986 +} \ No newline at end of file diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1-det.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1-det.json new file mode 100644 index 0000000000..869fe768e0 --- /dev/null +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1-det.json @@ -0,0 +1,57 @@ +{ + "steps": [ + { + "step": 20, + "loss": 2.4661 + }, + { + "step": 40, + "loss": 2.1771 + }, + { + "step": 60, + "loss": 2.0779 + }, + { + "step": 80, + "loss": 2.0619 + }, + { + "step": 100, + "loss": 2.0197 + }, + { + "step": 120, + "loss": 2.0521 + }, + { + "step": 140, + "loss": 2.061 + }, + { + "step": 160, + "loss": 1.9941 + }, + { + "step": 180, + "loss": 1.9846 + }, + { + "step": 200, + "loss": 1.9379 + }, + { + "step": 220, + "loss": 1.9671 + }, + { + "step": 240, + "loss": 1.9789 + }, + { + "step": 260, + "loss": 1.9761 + } + ], + "samples_per_second": 100.505 +} \ No newline at end of file diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1.json new file mode 100644 index 0000000000..b516145100 --- /dev/null +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1.json @@ -0,0 +1,57 @@ +{ + "steps": [ + { + "step": 20, + "loss": 2.4661 + }, + { + "step": 40, + "loss": 2.1771 + }, + { + "step": 60, + "loss": 2.078 + }, + { + "step": 80, + "loss": 2.0619 + }, + { + "step": 100, + "loss": 2.0197 + }, + { + "step": 120, + "loss": 2.0522 + }, + { + "step": 140, + "loss": 2.0608 + }, + { + "step": 160, + "loss": 1.9941 + }, + { + "step": 180, + "loss": 1.9846 + }, + { + "step": 200, + "loss": 1.9379 + }, + { + "step": 220, + "loss": 1.9671 + }, + { + "step": 240, + "loss": 1.9789 + }, + { + "step": 260, + "loss": 1.9761 + } + ], + "samples_per_second": 143.195 +} \ No newline at end of file diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1-det.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1-det.json new file mode 100644 index 0000000000..4d44ed331b --- /dev/null +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1-det.json @@ -0,0 +1,57 @@ +{ + "steps": [ + { + "step": 20, + "loss": 4.5012 + }, + { + "step": 40, + "loss": 1.7472 + }, + { + "step": 60, + "loss": 1.6405 + }, + { + "step": 80, + "loss": 1.605 + }, + { + "step": 100, + "loss": 1.5867 + }, + { + "step": 120, + "loss": 1.5764 + }, + { + "step": 140, + "loss": 1.5689 + }, + { + "step": 160, + "loss": 1.5624 + }, + { + "step": 180, + "loss": 1.558 + }, + { + "step": 200, + "loss": 1.5549 + }, + { + "step": 220, + "loss": 1.5532 + }, + { + "step": 240, + "loss": 1.5518 + }, + { + "step": 260, + "loss": 1.5503 + } + ], + "samples_per_second": 24.26 +} \ No newline at end of file diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json new file mode 100644 index 0000000000..7e8459eb4d --- /dev/null +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json @@ -0,0 +1,57 @@ +{ + "steps": [ + { + "step": 20, + "loss": 4.5007 + }, + { + "step": 40, + "loss": 1.7472 + }, + { + "step": 60, + "loss": 1.6405 + }, + { + "step": 80, + "loss": 1.605 + }, + { + "step": 100, + "loss": 1.5866 + }, + { + "step": 120, + "loss": 1.5763 + }, + { + "step": 140, + "loss": 1.5688 + }, + { + "step": 160, + "loss": 1.5624 + }, + { + "step": 180, + "loss": 1.5579 + }, + { + "step": 200, + "loss": 1.5548 + }, + { + "step": 220, + "loss": 1.5531 + }, + { + "step": 240, + "loss": 1.5517 + }, + { + "step": 260, + "loss": 1.5503 + } + ], + "samples_per_second": 25.587 +} \ No newline at end of file diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.roberta-large-rocm4.3.1-det.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.roberta-large-rocm4.3.1-det.json new file mode 100644 index 0000000000..612b35e74f --- /dev/null +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.roberta-large-rocm4.3.1-det.json @@ -0,0 +1,57 @@ +{ + "steps": [ + { + "step": 20, + "loss": 4.8469 + }, + { + "step": 40, + "loss": 3.9548 + }, + { + "step": 60, + "loss": 3.6641 + }, + { + "step": 80, + "loss": 2.2794 + }, + { + "step": 100, + "loss": 1.991 + }, + { + "step": 120, + "loss": 1.6128 + }, + { + "step": 140, + "loss": 1.5511 + }, + { + "step": 160, + "loss": 1.4651 + }, + { + "step": 180, + "loss": 1.4358 + }, + { + "step": 200, + "loss": 1.2693 + }, + { + "step": 220, + "loss": 1.3146 + }, + { + "step": 240, + "loss": 1.2687 + }, + { + "step": 260, + "loss": 1.3037 + } + ], + "samples_per_second": 37.771 +} \ No newline at end of file diff --git a/orttraining/tools/ci_test/results/ci-mi100.huggingface.roberta-large-rocm4.3.1.json b/orttraining/tools/ci_test/results/ci-mi100.huggingface.roberta-large-rocm4.3.1.json new file mode 100644 index 0000000000..1bb3755dc3 --- /dev/null +++ b/orttraining/tools/ci_test/results/ci-mi100.huggingface.roberta-large-rocm4.3.1.json @@ -0,0 +1,57 @@ +{ + "steps": [ + { + "step": 20, + "loss": 4.9194 + }, + { + "step": 40, + "loss": 3.9688 + }, + { + "step": 60, + "loss": 3.9312 + }, + { + "step": 80, + "loss": 3.6313 + }, + { + "step": 100, + "loss": 3.6505 + }, + { + "step": 120, + "loss": 3.5074 + }, + { + "step": 140, + "loss": 3.0419 + }, + { + "step": 160, + "loss": 2.2601 + }, + { + "step": 180, + "loss": 1.8854 + }, + { + "step": 200, + "loss": 1.6407 + }, + { + "step": 220, + "loss": 1.5782 + }, + { + "step": 240, + "loss": 1.3746 + }, + { + "step": 260, + "loss": 1.3685 + } + ], + "samples_per_second": 41.34 +} \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml index f692834a6c..b0f3fafc37 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml @@ -3,32 +3,40 @@ trigger: none name: 'orttraining_ci_$(Date:yyyyMMdd)_$(Rev:r)' pool: 'AMD-GPU' +# gid of video and render group on gcr-openpai-35 and -36 +variables: + - name: video + value: 44 + - name: render + value: 109 + +# generated from tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile +container: + image: onnxruntimebuildcache.azurecr.io/rocm-ci-pipeline-env + endpoint: onnxruntimebuildcache + options: --privileged -e HIP_VISIBLE_DEVICES --security-opt seccomp=unconfined --device=/dev/kfd --device=/dev/dri --group-add $(video) --group-add $(render) + steps: - checkout: self clean: true - script: |- - echo "##vso[task.prependpath]/home/ciagent/conda/bin/" - echo "##vso[task.prependpath]/home/ciagent/pkg/openmpi-4.0.5/bin/" - echo '##vso[task.setvariable variable=LD_LIBRARY_PATH]/home/ciagent/pkg/openmpi-4.0.5/lib/' - eval "$('/home/ciagent/conda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" echo "Selecting GPU based on HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES" displayName: 'Initialize environment' -# update these if the E2E test data changes -- script: |- - sed -i 's|session_options.use_deterministic_compute = False|session_options.use_deterministic_compute = True|g' \ - orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py - displayName: 'Toggle ON deterministic compute mode for ORTModule' +#- script: |- +# sed -i 's|session_options.use_deterministic_compute = False|session_options.use_deterministic_compute = True|g' \ +# orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py +# displayName: 'Toggle ON deterministic compute mode for ORTModule' - script: |- export ROCM_HOME=/opt/rocm python tools/ci_build/build.py \ --config RelWithDebInfo \ --enable_training \ - --mpi_home /home/ciagent/pkg/openmpi-4.0.5 \ + --mpi_home /opt/ompi \ --use_rocm \ - --rocm_version=4.2 \ + --rocm_version=4.3.1 \ --rocm_home /opt/rocm \ --nccl_home /opt/rocm \ --update \ @@ -55,7 +63,7 @@ steps: cd ./build/RelWithDebInfo export PYTHONPATH=$PWD python \ - /home/ciagent/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \ + /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \ --model_name_or_path bert-large-uncased \ --dataset_name wikitext \ --dataset_config_name wikitext-2-raw-v1 \ @@ -71,7 +79,7 @@ steps: --skip_memory_metrics python ../../orttraining/tools/ci_test/compare_huggingface.py \ ci-pipeline-actual.json \ - ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large.json + ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1.json displayName: 'Run Python Hugging-Face BERT-L test' condition: succeededOrFailed() # ensure all tests are run @@ -79,7 +87,7 @@ steps: cd ./build/RelWithDebInfo export PYTHONPATH=$PWD python \ - /home/ciagent/huggingface-transformers/examples/pytorch/language-modeling/run_clm.py \ + /stage/huggingface-transformers/examples/pytorch/language-modeling/run_clm.py \ --model_name_or_path gpt2 \ --dataset_name wikitext \ --dataset_config_name wikitext-2-raw-v1 \ @@ -96,7 +104,7 @@ steps: --skip_memory_metrics python ../../orttraining/tools/ci_test/compare_huggingface.py \ ci-pipeline-actual.json \ - ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2.json + ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json displayName: 'Run Python Hugging-Face GPT2 test' condition: succeededOrFailed() # ensure all tests are run @@ -104,7 +112,7 @@ steps: cd ./build/RelWithDebInfo export PYTHONPATH=$PWD python \ - /home/ciagent/huggingface-transformers/examples/pytorch/translation/run_translation.py \ + /stage/huggingface-transformers/examples/pytorch/translation/run_translation.py \ --dataset_name wmt16 \ --dataset_config ro-en \ --model_name_or_path facebook/bart-large \ @@ -124,41 +132,42 @@ steps: --skip_memory_metrics python ../../orttraining/tools/ci_test/compare_huggingface.py \ ci-pipeline-actual.json \ - ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bart-large.json + ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bart-large-rocm4.3.1.json displayName: 'Run Python Hugging-Face BART-L test' condition: succeededOrFailed() # ensure all tests are run -- script: |- - cd ./build/RelWithDebInfo - export PYTHONPATH=$PWD - python \ - /home/ciagent/huggingface-transformers/examples/pytorch/question-answering/run_qa.py \ - --model_name_or_path roberta-large \ - --dataset_name squad \ - --do_train \ - --per_device_train_batch_size 16 \ - --learning_rate 3e-5 \ - --max_steps 260 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir ./roberta_res \ - --overwrite_output_dir \ - --logging_steps 20 \ - --fp16 \ - --dataloader_num_workers 1 \ - --ort \ - --skip_memory_metrics - python ../../orttraining/tools/ci_test/compare_huggingface.py \ - ci-pipeline-actual.json \ - ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.roberta-large.json - displayName: 'Run Python Hugging-Face RoBERTa-L test' - condition: succeededOrFailed() # ensure all tests are run +# todo: investigate RoBERTa high run variability on ROCm 4.3.1 +#- script: |- +# cd ./build/RelWithDebInfo +# export PYTHONPATH=$PWD +# python \ +# /stage/huggingface-transformers/examples/pytorch/question-answering/run_qa.py \ +# --model_name_or_path roberta-large \ +# --dataset_name squad \ +# --do_train \ +# --per_device_train_batch_size 16 \ +# --learning_rate 3e-5 \ +# --max_steps 260 \ +# --max_seq_length 384 \ +# --doc_stride 128 \ +# --output_dir ./roberta_res \ +# --overwrite_output_dir \ +# --logging_steps 20 \ +# --fp16 \ +# --dataloader_num_workers 1 \ +# --ort \ +# --skip_memory_metrics +# python ../../orttraining/tools/ci_test/compare_huggingface.py \ +# ci-pipeline-actual.json \ +# ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.roberta-large-rocm4.3.1.json +# displayName: 'Run Python Hugging-Face RoBERTa-L test' +# condition: succeededOrFailed() # ensure all tests are run - script: |- cd ./build/RelWithDebInfo export PYTHONPATH=$PWD python \ - /home/ciagent/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \ + /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \ --model_name_or_path distilbert-base-uncased \ --dataset_name wikitext \ --dataset_config_name wikitext-2-raw-v1 \ @@ -174,7 +183,7 @@ steps: --skip_memory_metrics python ../../orttraining/tools/ci_test/compare_huggingface.py \ ci-pipeline-actual.json \ - ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base.json + ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1.json displayName: 'Run Python Hugging-Face DistilBERT test' condition: succeededOrFailed() # ensure all tests are run @@ -182,7 +191,7 @@ steps: # cd ./build/RelWithDebInfo # export PYTHONPATH=$PWD # python \ -# /home/ciagent/huggingface-transformers/examples/pytorch/text-classification/run_glue.py \ +# /stage/huggingface-transformers/examples/pytorch/text-classification/run_glue.py \ # --model_name_or_path microsoft/deberta-v2-xxlarge \ # --task_name MRPC \ # --do_train \ @@ -204,7 +213,7 @@ steps: # cd ./build/RelWithDebInfo # export PYTHONPATH=$PWD # python \ -# /home/ciagent/huggingface-transformers/examples/pytorch/translation/run_translation.py \ +# /stage/huggingface-transformers/examples/pytorch/translation/run_translation.py \ # --source_prefix '"translate English to Romanian:"' \ # --dataset_name wmt16 \ # --dataset_config ro-en \ @@ -230,12 +239,12 @@ steps: # displayName: 'Run Python Hugging-Face T5-L test' # condition: succeededOrFailed() # ensure all tests are run -# update these if the E2E test data changes - script: |- python orttraining/tools/ci_test/download_azure_blob_archive.py \ --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ --target_dir training_e2e_test_data \ --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 + condition: succeededOrFailed() # ensure all tests are run displayName: 'Download onnxruntime_training_data.zip data' - script: |- diff --git a/tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile b/tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile new file mode 100644 index 0000000000..aac42add29 --- /dev/null +++ b/tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile @@ -0,0 +1,27 @@ +FROM rocm/pytorch:rocm4.3.1_ubuntu18.04_py3.6_pytorch_1.9.0 + +WORKDIR /stage + +# rocm-ci branch contains instrumentation needed for loss curves and perf +RUN git clone https://github.com/microsoft/huggingface-transformers.git &&\ + cd huggingface-transformers &&\ + git checkout rocm-ci &&\ + pip install -e . + +RUN pip install \ + numpy \ + onnx \ + cerberus \ + sympy \ + h5py \ + datasets==1.9.0 \ + requests \ + sacrebleu==1.5.1 \ + sacremoses \ + scipy \ + scikit-learn \ + sklearn \ + tokenizers \ + sentencepiece + +RUN pip install torch-ort --no-dependencies