From 8a148e44fb485bb84be935f0904cc1876b38a8ee Mon Sep 17 00:00:00 2001 From: Suffian Khan Date: Wed, 24 Feb 2021 20:18:03 -0800 Subject: [PATCH] make ci pipeline also run batch and convergence test (#6798) --- .../orttraining-pai-ci-pipeline.yml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml index 2699a2e9ad..fe891611b8 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml @@ -15,6 +15,14 @@ steps: echo "Selecting GPU based on HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES" displayName: 'Initialize environment' +# update these if the E2E test data changes +- script: |- + python orttraining/tools/ci_test/download_azure_blob_archive.py \ + --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ + --target_dir training_e2e_test_data \ + --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 + displayName: 'Download onnxruntime_training_data.zip data' + - script: |- python tools/ci_build/build.py \ --config RelWithDebInfo \ @@ -36,3 +44,19 @@ steps: ../../tools/ci_build/github/pai/pai_test_launcher.sh displayName: 'Run unit tests' +- script: |- + python orttraining/tools/ci_test/run_batch_size_test.py \ + --binary_dir build/RelWithDebInfo \ + --model_root training_e2e_test_data/models \ + --gpu_sku MI100_32G + displayName: 'Run batch size test' + condition: succeededOrFailed() # ensure all tests are run + +- script: |- + python orttraining/tools/ci_test/run_convergence_test.py \ + --binary_dir build/RelWithDebInfo \ + --model_root training_e2e_test_data/models \ + --training_data_root training_e2e_test_data/data \ + --gpu_sku MI100_32G + displayName: 'Run convergence test' + condition: succeededOrFailed() # ensure all tests are run