diff --git a/onnxruntime/test/python/onnxruntime_test_ort_trainer.py b/onnxruntime/test/python/onnxruntime_test_ort_trainer.py index 49a11e5bc6..909cd6fce4 100644 --- a/onnxruntime/test/python/onnxruntime_test_ort_trainer.py +++ b/onnxruntime/test/python/onnxruntime_test_ort_trainer.py @@ -173,49 +173,5 @@ class TestOrtTrainer(unittest.TestCase): assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch") assert_allclose(expected_eval_loss, actual_eval_loss, rtol=rtol, err_msg="evaluation loss mismatch") - def testBertTrainingMixedPrecision(self): - # skip the test due to the lack of mixed precision capacity of ort CI. - return - - torch.manual_seed(1) - expected_losses = [11.078125, 11.0, 11.0390625, 11.0, 11.015625, 11.0, 10.9921875, 11.0703125] - expected_all_finites = [False, True, True, True, True, True, True, True] - expected_eval_loss = [11.046875] - actual_losses, actual_all_finites, actual_eval_loss = runBertTrainingTest( - gradient_accumulation_steps=1, use_mixed_precision=True, allreduce_post_accumulation=False, use_simple_model_desc=False) - - # to update expected outcomes, enable pdb and run the test with -s and copy paste outputs - # print('actual_losses ', actual_losses) - # print('actual_all_finite ', actual_all_finites) - # print('eval_loss', actual_eval_loss) - # import pdb; pdb.set_trace() - - rtol = 1e-01 - assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch") - assert_array_equal(expected_all_finites, actual_all_finites, "all_finite mismatch") - assert_allclose(expected_eval_loss, actual_eval_loss, rtol=rtol, err_msg="evaluation loss mismatch") - - def testBertTrainingGradientAccumulationMixedPrecision(self): - # skip the test due to the lack of mixed precision capacity of ort CI. - return - - torch.manual_seed(1) - expected_losses = [11.046875, 11.171875, 11.0234375, 11.046875, 10.8984375, 10.9921875, 11.078125, 10.96875] - expected_all_finites = [False, True] - expected_eval_loss = [11.0546875] - actual_losses, actual_all_finites, actual_eval_loss = runBertTrainingTest( - gradient_accumulation_steps=4, use_mixed_precision=True, allreduce_post_accumulation=False, use_simple_model_desc=False) - - # to update expected outcomes, enable pdb and run the test with -s and copy paste outputs - # print('actual_losses ', actual_losses) - # print('actual_all_finite ', actual_all_finites) - # print('eval_loss', actual_eval_loss) - # import pdb; pdb.set_trace() - - rtol = 1e-01 - assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch") - assert_array_equal(expected_all_finites, actual_all_finites, "all_finite mismatch") - assert_allclose(expected_eval_loss, actual_eval_loss, rtol=rtol, err_msg="evaluation loss mismatch") - if __name__ == '__main__': unittest.main(module=__name__, buffer=True) diff --git a/onnxruntime/test/python/onnxruntime_test_ort_trainer_with_mixed_precision.py b/onnxruntime/test/python/onnxruntime_test_ort_trainer_with_mixed_precision.py new file mode 100644 index 0000000000..6bd74e21bc --- /dev/null +++ b/onnxruntime/test/python/onnxruntime_test_ort_trainer_with_mixed_precision.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest +import pytest +from numpy.testing import assert_allclose, assert_array_equal + +import torch +from onnxruntime_test_ort_trainer import runBertTrainingTest + +class TestOrtTrainer(unittest.TestCase): + def testBertTrainingMixedPrecision(self): + torch.manual_seed(1) + expected_losses = [11.078125, 11.0, 11.0390625, 11.0, 11.015625, 11.0, 10.9921875, 11.0703125] + expected_all_finites = [False, True, True, True, True, True, True, True] + expected_eval_loss = [11.046875] + actual_losses, actual_all_finites, actual_eval_loss = runBertTrainingTest( + gradient_accumulation_steps=1, use_mixed_precision=True, allreduce_post_accumulation=False, use_simple_model_desc=False) + + rtol = 1e-01 + assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch") + assert_array_equal(expected_all_finites, actual_all_finites, "all_finite mismatch") + assert_allclose(expected_eval_loss, actual_eval_loss, rtol=rtol, err_msg="evaluation loss mismatch") + + def testBertTrainingGradientAccumulationMixedPrecision(self): + torch.manual_seed(1) + expected_losses = [11.046875, 11.171875, 11.0234375, 11.046875, 10.8984375, 10.9921875, 11.078125, 10.96875] + expected_all_finites = [False, True] + expected_eval_loss = [11.0546875] + actual_losses, actual_all_finites, actual_eval_loss = runBertTrainingTest( + gradient_accumulation_steps=4, use_mixed_precision=True, allreduce_post_accumulation=False, use_simple_model_desc=False) + + rtol = 1e-01 + assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch") + assert_array_equal(expected_all_finites, actual_all_finites, "all_finite mismatch") + assert_allclose(expected_eval_loss, actual_eval_loss, rtol=rtol, err_msg="evaluation loss mismatch") + +if __name__ == '__main__': + unittest.main(module=__name__, buffer=True) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 90e23f11a1..5a22561be3 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -80,6 +80,10 @@ Use the individual flags to only run the specified stages. parser.add_argument("--training_e2e_test_data_path", help="Path to training end-to-end test data directory.") + # Pytorch frontend test options + parser.add_argument("--enable_training_python_frontend_e2e_tests", action="store_true", + help="Enable the pytorch frontend training tests.") + # enable ONNX tests parser.add_argument("--enable_onnx_tests", action='store_true', help='''When running the Test phase, run onnx_test_running against available test data directories.''') @@ -620,6 +624,11 @@ def adb_push(source_dir, src, dest, **kwargs): def adb_shell(*args, **kwargs): return run_subprocess(['adb', 'shell', *args], **kwargs) +def run_training_python_frontend_e2e_tests(args, cwd, dll_path): + # frontend tests are to be added here: + log.info("Running python frontend e2e tests.") + run_subprocess([sys.executable, 'onnxruntime_test_ort_trainer_with_mixed_precision.py'], cwd=cwd, dll_path=dll_path) + def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enable_tvm = False, enable_tensorrt = False): for config in configs: log.info("Running tests for %s configuration", config) @@ -671,8 +680,13 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enab run_subprocess([sys.executable, 'onnxruntime_test_python.py'], cwd=cwd, dll_path=dll_path) if args.enable_training and args.use_cuda: + # run basic frontend tests run_subprocess([sys.executable, 'onnxruntime_test_ort_trainer.py'], cwd=cwd, dll_path=dll_path) + # run additional frontend tests for orttraining-linux-gpu-frontend_test_ci-pipeline + if args.enable_training_python_frontend_e2e_tests: + run_training_python_frontend_e2e_tests(args, cwd=cwd, dll_path=dll_path) + try: import onnx import scipy # gen_test_models.py used by onnx_test has a dependency on scipy diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml new file mode 100644 index 0000000000..37562df5d8 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml @@ -0,0 +1,32 @@ +# 'trigger: none' to use UI to schedule the pipeline runs. +# alternatively to schedule the pipeline run according to: +# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml +trigger: none + +jobs: +- job: Onnxruntime_Linux_GPU_Training_FrontEnd + + timeoutInMinutes: 120 + + steps: + - checkout: self + clean: true + submodules: recursive + + - template: templates/linux-set-variables-and-download.yml + + # insert a python frontend test data preparation step here + + - script: > + tools/ci_build/github/linux/run_dockerbuild.sh + -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) + -x " + --enable_training + --config RelWithDebInfo + --skip_onnx_tests + --build_wheel + --enable_training_python_frontend_e2e_tests + " + displayName: 'Build and run frontend tests' + + - template: templates/clean-agent-build-directory-step.yml \ No newline at end of file