From 38d76cc9046205b81f2b7af0698be7251f9f145d Mon Sep 17 00:00:00 2001 From: edgchen1 <18449977+edgchen1@users.noreply.github.com> Date: Fri, 29 May 2020 09:20:47 -0700 Subject: [PATCH] Clean up training E2E test (#4078) Update training E2E build to not go through CTest and call test scripts directly. --- cmake/onnxruntime_training_e2e_tests.cmake | 36 ------------- .../tools/ci_test/run_batch_size_test.py | 39 +++++++++------ tools/ci_build/build.py | 19 ------- ...raining-linux-gpu-e2e-test-ci-pipeline.yml | 50 +++++++++++++++---- 4 files changed, 62 insertions(+), 82 deletions(-) delete mode 100644 cmake/onnxruntime_training_e2e_tests.cmake diff --git a/cmake/onnxruntime_training_e2e_tests.cmake b/cmake/onnxruntime_training_e2e_tests.cmake deleted file mode 100644 index 5d39e7dd62..0000000000 --- a/cmake/onnxruntime_training_e2e_tests.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# training end-to-end tests - -if (NOT IS_DIRECTORY ${onnxruntime_TRAINING_E2E_TEST_DATA_ROOT}) - message(FATAL_ERROR "Training E2E test data directory is not valid: ${onnxruntime_TRAINING_E2E_TEST_DATA_ROOT}") -endif() - -find_package(Python3 3.5 REQUIRED COMPONENTS Interpreter) - -# bert batch size test -add_test( - NAME onnxruntime_training_bert_batch_size_test - COMMAND - ${Python3_EXECUTABLE} ${REPO_ROOT}/orttraining/tools/ci_test/run_batch_size_test.py - --binary_dir $ - --model_root ${onnxruntime_TRAINING_E2E_TEST_DATA_ROOT}/models - CONFIGURATIONS RelWithDebInfo) - -# convergence test -add_test( - NAME onnxruntime_training_bert_convergence_e2e_test - COMMAND - ${Python3_EXECUTABLE} ${REPO_ROOT}/orttraining/tools/ci_test/run_convergence_test.py - --binary_dir $ - --training_data_root ${onnxruntime_TRAINING_E2E_TEST_DATA_ROOT}/data - --model_root ${onnxruntime_TRAINING_E2E_TEST_DATA_ROOT}/models - CONFIGURATIONS RelWithDebInfo) - -set_property( - TEST - onnxruntime_training_bert_batch_size_test - onnxruntime_training_bert_convergence_e2e_test - PROPERTY - LABELS training_e2e) diff --git a/orttraining/tools/ci_test/run_batch_size_test.py b/orttraining/tools/ci_test/run_batch_size_test.py index cad1ad37b9..c9bdc6e9d7 100755 --- a/orttraining/tools/ci_test/run_batch_size_test.py +++ b/orttraining/tools/ci_test/run_batch_size_test.py @@ -3,36 +3,42 @@ # Licensed under the MIT License. import argparse +import collections import subprocess import sys import os + def parse_args(): parser = argparse.ArgumentParser(description="Runs a BERT batch size test.") - parser.add_argument("--binary_dir", required=True, - help="Path to the ORT binary directory.") - parser.add_argument("--model_root", required=True, - help="Path to the model root directory.") + parser.add_argument("--binary_dir", required=True, help="Path to the ORT binary directory.") + parser.add_argument("--model_root", required=True, help="Path to the model root directory.") return parser.parse_args() + def main(): args = parse_args() - matrix = { # enable mixed-precision, sequence length, max batch size - "fp16-128": [True, 128, 66], - "fp16-512": [True, 512, 10], - "fp32-128": [False, 128, 33], - "fp32-512": [False, 512, 5]} + + Config = collections.namedtuple("Config", ["enable_mixed_precision", "sequence_length", "max_batch_size"]) + configs = [ + Config(True, 128, 66), + Config(True, 512, 10), + Config(False, 128, 33), + Config(False, 512, 5), + ] # run BERT training - for m in matrix: - print("######## testing name - " + m + " ##############") + for config in configs: + print("##### testing name - {}-{} #####".format("fp16" if config.enable_mixed_precision else "fp32", + config.sequence_length)) cmds = [ os.path.join(args.binary_dir, "onnxruntime_training_bert"), "--model_name", os.path.join( - args.model_root, "nv/bert-large/bert-large-uncased_L_24_H_1024_A_16_V_30528_S_512_Dp_0.1_optimized_layer_norm"), - "--train_batch_size", str(matrix[m][2]), + args.model_root, + "nv/bert-large/bert-large-uncased_L_24_H_1024_A_16_V_30528_S_512_Dp_0.1_optimized_layer_norm"), + "--train_batch_size", str(config.max_batch_size), "--mode", "perf", - "--max_seq_length", str(matrix[m][1]), + "--max_seq_length", str(config.sequence_length), "--num_train_steps", "10", "--display_loss_steps", "5", "--optimizer", "adam", @@ -48,12 +54,13 @@ def main(): "--enable_grad_norm_clip=false", ] - if matrix[m][0]: + if config.enable_mixed_precision: cmds.append("--use_mixed_precision"), - subprocess.run(cmds, timeout = 60).check_returncode() + subprocess.run(cmds, timeout=60).check_returncode() return 0 + if __name__ == "__main__": sys.exit(main()) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index fe93b1a8fc..e8955db5fb 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -95,21 +95,11 @@ def parse_arguments(): parser.add_argument( "--skip_tests", action='store_true', help="Skip all tests.") - # Test options - parser.add_argument("--ctest_label_regex", - help="Only run CTest tests with a label matching the pattern (passed to ctest --label-regex).") - # Training options parser.add_argument( "--enable_nvtx_profile", action='store_true', help="Enable NVTX profile in ORT.") parser.add_argument( "--enable_training", action='store_true', help="Enable training in ORT.") - parser.add_argument( - "--enable_training_e2e_tests", action="store_true", - help="Enable the training end-to-end tests.") - parser.add_argument( - "--training_e2e_test_data_path", - help="Path to training end-to-end test data directory.") parser.add_argument( "--enable_training_python_frontend_e2e_tests", action="store_true", help="Enable the pytorch frontend training tests.") @@ -627,8 +617,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, "ON" if args.enable_nvtx_profile else "OFF"), "-Donnxruntime_ENABLE_TRAINING=" + ( "ON" if args.enable_training else "OFF"), - "-Donnxruntime_ENABLE_TRAINING_E2E_TESTS=" + ( - "ON" if args.enable_training_e2e_tests else "OFF"), "-Donnxruntime_USE_HOROVOD=" + ( "ON" if args.use_horovod else "OFF"), ] @@ -761,10 +749,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, else: cmake_args += ["-Donnxruntime_PYBIND_EXPORT_OPSCHEMA=OFF"] - if args.training_e2e_test_data_path is not None: - cmake_args += ["-Donnxruntime_TRAINING_E2E_TEST_DATA_ROOT={}".format( - os.path.abspath(args.training_e2e_test_data_path))] - cmake_args += ["-D{}".format(define) for define in cmake_extra_defines] if is_windows(): @@ -1132,9 +1116,6 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, cwd=cwd2, dll_path=dll_path) else: ctest_cmd = [ctest_path, "--build-config", config, "--verbose"] - if args.ctest_label_regex is not None: - ctest_cmd += ["--label-regex", args.ctest_label_regex] - run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path) if args.enable_pybind: diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-ci-pipeline.yml index 1536ba4804..d4a85abf47 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-ci-pipeline.yml @@ -10,19 +10,47 @@ jobs: clean: true submodules: recursive - - script: > + - script: | orttraining/tools/ci_test/download_e2e_test_data.py $(Build.BinariesDirectory)/training_e2e_test_data displayName: 'Download training end-to-end test data' - - script: > - tools/ci_build/github/linux/run_dockerbuild.sh - -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) - -x " - --config RelWithDebInfo - --enable_training - --enable_training_e2e_tests --training_e2e_test_data_path /build/training_e2e_test_data - --update --build --test --ctest_label_regex training_e2e - " - displayName: 'Build and run end-to-end tests' + - script: | + tools/ci_build/github/linux/run_dockerbuild.sh \ + -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) \ + -x " \ + --config RelWithDebInfo \ + --enable_training \ + --update --build \ + " + displayName: 'Build' + + - script: | + docker run \ + --gpus all \ + --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $(Build.BinariesDirectory)/training_e2e_test_data:/training_e2e_test_data:ro \ + onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6 \ + /onnxruntime_src/orttraining/tools/ci_test/run_batch_size_test.py \ + --binary_dir /build/RelWithDebInfo \ + --model_root /training_e2e_test_data/models + displayName: 'Run batch size test' + condition: succeededOrFailed() # ensure all tests are run + + - script: | + docker run \ + --gpus all \ + --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $(Build.BinariesDirectory)/training_e2e_test_data:/training_e2e_test_data:ro \ + onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6 \ + /onnxruntime_src/orttraining/tools/ci_test/run_convergence_test.py \ + --binary_dir /build/RelWithDebInfo \ + --model_root /training_e2e_test_data/models \ + --training_data_root /training_e2e_test_data/data + displayName: 'Run convergence test' + condition: succeededOrFailed() # ensure all tests are run - template: templates/clean-agent-build-directory-step.yml