Clean up training E2E test (#4078)

Update training E2E build to not go through CTest and call test scripts directly.

cmake/onnxruntime_training_e2e_tests.cmake

@@ -1,36 +0,0 @@
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License.
-
-# training end-to-end tests
-
-if (NOT IS_DIRECTORY ${onnxruntime_TRAINING_E2E_TEST_DATA_ROOT})
-  message(FATAL_ERROR "Training E2E test data directory is not valid: ${onnxruntime_TRAINING_E2E_TEST_DATA_ROOT}")
-endif()
-
-find_package(Python3 3.5 REQUIRED COMPONENTS Interpreter)
-
-# bert batch size test
-add_test(
-  NAME onnxruntime_training_bert_batch_size_test
-  COMMAND
-    ${Python3_EXECUTABLE} ${REPO_ROOT}/orttraining/tools/ci_test/run_batch_size_test.py
-      --binary_dir $<TARGET_FILE_DIR:onnxruntime_training_bert>
-      --model_root ${onnxruntime_TRAINING_E2E_TEST_DATA_ROOT}/models
-  CONFIGURATIONS RelWithDebInfo)
-
-# convergence test
-add_test(
-  NAME onnxruntime_training_bert_convergence_e2e_test
-  COMMAND
-    ${Python3_EXECUTABLE} ${REPO_ROOT}/orttraining/tools/ci_test/run_convergence_test.py
-      --binary_dir $<TARGET_FILE_DIR:onnxruntime_training_bert>
-      --training_data_root ${onnxruntime_TRAINING_E2E_TEST_DATA_ROOT}/data
-      --model_root ${onnxruntime_TRAINING_E2E_TEST_DATA_ROOT}/models
-  CONFIGURATIONS RelWithDebInfo)
-
-set_property(
-  TEST
-    onnxruntime_training_bert_batch_size_test
-    onnxruntime_training_bert_convergence_e2e_test
-  PROPERTY
-    LABELS training_e2e)

orttraining/tools/ci_test/run_batch_size_test.py

@@ -3,36 +3,42 @@
 # Licensed under the MIT License.
 
 import argparse
+import collections
 import subprocess
 import sys
 import os
 
+
 def parse_args():
     parser = argparse.ArgumentParser(description="Runs a BERT batch size test.")
-    parser.add_argument("--binary_dir", required=True,
-                        help="Path to the ORT binary directory.")
-    parser.add_argument("--model_root", required=True,
-                        help="Path to the model root directory.")
+    parser.add_argument("--binary_dir", required=True, help="Path to the ORT binary directory.")
+    parser.add_argument("--model_root", required=True, help="Path to the model root directory.")
     return parser.parse_args()
 
+
 def main():
     args = parse_args()
-    matrix = { # enable mixed-precision, sequence length, max batch size
-        "fp16-128": [True, 128, 66],
-        "fp16-512": [True, 512, 10],
-        "fp32-128": [False, 128, 33],
-        "fp32-512": [False, 512, 5]}
+
+    Config = collections.namedtuple("Config", ["enable_mixed_precision", "sequence_length", "max_batch_size"])
+    configs = [
+        Config(True, 128, 66),
+        Config(True, 512, 10),
+        Config(False, 128, 33),
+        Config(False, 512, 5),
+    ]
 
     # run BERT training
-    for m in matrix:
-        print("######## testing name - " + m + " ##############")
+    for config in configs:
+        print("##### testing name - {}-{} #####".format("fp16" if config.enable_mixed_precision else "fp32",
+                                                        config.sequence_length))
         cmds = [
             os.path.join(args.binary_dir, "onnxruntime_training_bert"),
             "--model_name", os.path.join(
-                args.model_root, "nv/bert-large/bert-large-uncased_L_24_H_1024_A_16_V_30528_S_512_Dp_0.1_optimized_layer_norm"),
-            "--train_batch_size", str(matrix[m][2]),
+                args.model_root,
+                "nv/bert-large/bert-large-uncased_L_24_H_1024_A_16_V_30528_S_512_Dp_0.1_optimized_layer_norm"),
+            "--train_batch_size", str(config.max_batch_size),
             "--mode", "perf",
-            "--max_seq_length", str(matrix[m][1]),
+            "--max_seq_length", str(config.sequence_length),
             "--num_train_steps", "10",
             "--display_loss_steps", "5",
             "--optimizer", "adam",
@@ -48,12 +54,13 @@ def main():
             "--enable_grad_norm_clip=false",
         ]
 
-        if matrix[m][0]:
+        if config.enable_mixed_precision:
             cmds.append("--use_mixed_precision"),
 
-        subprocess.run(cmds, timeout = 60).check_returncode()
+        subprocess.run(cmds, timeout=60).check_returncode()
 
     return 0
 
+
 if __name__ == "__main__":
     sys.exit(main())

tools/ci_build/build.py

@@ -95,21 +95,11 @@ def parse_arguments():
     parser.add_argument(
         "--skip_tests", action='store_true', help="Skip all tests.")
 
-    # Test options
-    parser.add_argument("--ctest_label_regex",
-                        help="Only run CTest tests with a label matching the pattern (passed to ctest --label-regex).")
-
     # Training options
     parser.add_argument(
         "--enable_nvtx_profile", action='store_true', help="Enable NVTX profile in ORT.")
     parser.add_argument(
         "--enable_training", action='store_true', help="Enable training in ORT.")
-    parser.add_argument(
-        "--enable_training_e2e_tests", action="store_true",
-        help="Enable the training end-to-end tests.")
-    parser.add_argument(
-        "--training_e2e_test_data_path",
-        help="Path to training end-to-end test data directory.")
     parser.add_argument(
         "--enable_training_python_frontend_e2e_tests", action="store_true",
         help="Enable the pytorch frontend training tests.")
@@ -627,8 +617,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home,
             "ON" if args.enable_nvtx_profile else "OFF"),
         "-Donnxruntime_ENABLE_TRAINING=" + (
             "ON" if args.enable_training else "OFF"),
-        "-Donnxruntime_ENABLE_TRAINING_E2E_TESTS=" + (
-            "ON" if args.enable_training_e2e_tests else "OFF"),
         "-Donnxruntime_USE_HOROVOD=" + (
             "ON" if args.use_horovod else "OFF"),
     ]
@@ -761,10 +749,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home,
     else:
         cmake_args += ["-Donnxruntime_PYBIND_EXPORT_OPSCHEMA=OFF"]
 
-    if args.training_e2e_test_data_path is not None:
-        cmake_args += ["-Donnxruntime_TRAINING_E2E_TEST_DATA_ROOT={}".format(
-            os.path.abspath(args.training_e2e_test_data_path))]
-
     cmake_args += ["-D{}".format(define) for define in cmake_extra_defines]
 
     if is_windows():
@@ -1132,9 +1116,6 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs,
                 cwd=cwd2, dll_path=dll_path)
         else:
             ctest_cmd = [ctest_path, "--build-config", config, "--verbose"]
-            if args.ctest_label_regex is not None:
-                ctest_cmd += ["--label-regex", args.ctest_label_regex]
-
             run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path)
 
         if args.enable_pybind:

tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-ci-pipeline.yml

@@ -10,19 +10,47 @@ jobs:
     clean: true
     submodules: recursive
 
-  - script: >
+  - script: |
       orttraining/tools/ci_test/download_e2e_test_data.py $(Build.BinariesDirectory)/training_e2e_test_data
     displayName: 'Download training end-to-end test data'
 
-  - script: >
-      tools/ci_build/github/linux/run_dockerbuild.sh
-      -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory)
-      -x "
-      --config RelWithDebInfo
-      --enable_training
-      --enable_training_e2e_tests --training_e2e_test_data_path /build/training_e2e_test_data
-      --update --build --test --ctest_label_regex training_e2e
-      "
-    displayName: 'Build and run end-to-end tests'
+  - script: |
+      tools/ci_build/github/linux/run_dockerbuild.sh \
+        -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) \
+        -x " \
+          --config RelWithDebInfo \
+          --enable_training \
+          --update --build \
+          "
+    displayName: 'Build'
+
+  - script: |
+      docker run \
+        --gpus all \
+        --rm \
+        --volume $(Build.SourcesDirectory):/onnxruntime_src \
+        --volume $(Build.BinariesDirectory):/build \
+        --volume $(Build.BinariesDirectory)/training_e2e_test_data:/training_e2e_test_data:ro \
+        onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6 \
+          /onnxruntime_src/orttraining/tools/ci_test/run_batch_size_test.py \
+            --binary_dir /build/RelWithDebInfo \
+            --model_root /training_e2e_test_data/models
+    displayName: 'Run batch size test'
+    condition: succeededOrFailed() # ensure all tests are run
+
+  - script: |
+      docker run \
+        --gpus all \
+        --rm \
+        --volume $(Build.SourcesDirectory):/onnxruntime_src \
+        --volume $(Build.BinariesDirectory):/build \
+        --volume $(Build.BinariesDirectory)/training_e2e_test_data:/training_e2e_test_data:ro \
+        onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6 \
+          /onnxruntime_src/orttraining/tools/ci_test/run_convergence_test.py \
+            --binary_dir /build/RelWithDebInfo \
+            --model_root /training_e2e_test_data/models \
+            --training_data_root /training_e2e_test_data/data
+    displayName: 'Run convergence test'
+    condition: succeededOrFailed() # ensure all tests are run
 
   - template: templates/clean-agent-build-directory-step.yml