diff --git a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml index 2699a2e9ad..fe891611b8 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml @@ -15,6 +15,14 @@ steps: echo "Selecting GPU based on HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES" displayName: 'Initialize environment' +# update these if the E2E test data changes +- script: |- + python orttraining/tools/ci_test/download_azure_blob_archive.py \ + --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ + --target_dir training_e2e_test_data \ + --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 + displayName: 'Download onnxruntime_training_data.zip data' + - script: |- python tools/ci_build/build.py \ --config RelWithDebInfo \ @@ -36,3 +44,19 @@ steps: ../../tools/ci_build/github/pai/pai_test_launcher.sh displayName: 'Run unit tests' +- script: |- + python orttraining/tools/ci_test/run_batch_size_test.py \ + --binary_dir build/RelWithDebInfo \ + --model_root training_e2e_test_data/models \ + --gpu_sku MI100_32G + displayName: 'Run batch size test' + condition: succeededOrFailed() # ensure all tests are run + +- script: |- + python orttraining/tools/ci_test/run_convergence_test.py \ + --binary_dir build/RelWithDebInfo \ + --model_root training_e2e_test_data/models \ + --training_data_root training_e2e_test_data/data \ + --gpu_sku MI100_32G + displayName: 'Run convergence test' + condition: succeededOrFailed() # ensure all tests are run