saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

create pipeline for ci frontend tests (#3422)

Browse source 
create pipeline for nightly python front-end e2e tests
This commit is contained in:
liqunfu 2020-04-09 15:31:22 -07:00 committed by GitHub
parent a08f16471a
commit e7297e6c9d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 85 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
onnxruntime/test/python/onnxruntime_test_ort_trainer.py
View file

@ -173,49 +173,5 @@ class TestOrtTrainer(unittest.TestCase):
assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch")
assert_allclose(expected_eval_loss, actual_eval_loss, rtol=rtol, err_msg="evaluation loss mismatch")
def testBertTrainingMixedPrecision(self):
# skip the test due to the lack of mixed precision capacity of ort CI.
return
torch.manual_seed(1)
expected_losses = [11.078125, 11.0, 11.0390625, 11.0, 11.015625, 11.0, 10.9921875, 11.0703125]
expected_all_finites = [False, True, True, True, True, True, True, True]
expected_eval_loss = [11.046875]
actual_losses, actual_all_finites, actual_eval_loss = runBertTrainingTest(
gradient_accumulation_steps=1, use_mixed_precision=True, allreduce_post_accumulation=False, use_simple_model_desc=False)
# to update expected outcomes, enable pdb and run the test with -s and copy paste outputs
# print('actual_losses ', actual_losses)
# print('actual_all_finite ', actual_all_finites)
# print('eval_loss', actual_eval_loss)
# import pdb; pdb.set_trace()
rtol = 1e-01
assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch")
assert_array_equal(expected_all_finites, actual_all_finites, "all_finite mismatch")
assert_allclose(expected_eval_loss, actual_eval_loss, rtol=rtol, err_msg="evaluation loss mismatch")
def testBertTrainingGradientAccumulationMixedPrecision(self):
# skip the test due to the lack of mixed precision capacity of ort CI.
return
torch.manual_seed(1)
expected_losses = [11.046875, 11.171875, 11.0234375, 11.046875, 10.8984375, 10.9921875, 11.078125, 10.96875]
expected_all_finites = [False, True]
expected_eval_loss = [11.0546875]
actual_losses, actual_all_finites, actual_eval_loss = runBertTrainingTest(
gradient_accumulation_steps=4, use_mixed_precision=True, allreduce_post_accumulation=False, use_simple_model_desc=False)
# to update expected outcomes, enable pdb and run the test with -s and copy paste outputs
# print('actual_losses ', actual_losses)
# print('actual_all_finite ', actual_all_finites)
# print('eval_loss', actual_eval_loss)
# import pdb; pdb.set_trace()
rtol = 1e-01
assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch")
assert_array_equal(expected_all_finites, actual_all_finites, "all_finite mismatch")
assert_allclose(expected_eval_loss, actual_eval_loss, rtol=rtol, err_msg="evaluation loss mismatch")
if __name__ == '__main__':
unittest.main(module=__name__, buffer=True)

39
onnxruntime/test/python/onnxruntime_test_ort_trainer_with_mixed_precision.py Normal file
View file

@ -0,0 +1,39 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import unittest
import pytest
from numpy.testing import assert_allclose, assert_array_equal
import torch
from onnxruntime_test_ort_trainer import runBertTrainingTest
class TestOrtTrainer(unittest.TestCase):
def testBertTrainingMixedPrecision(self):
torch.manual_seed(1)
expected_losses = [11.078125, 11.0, 11.0390625, 11.0, 11.015625, 11.0, 10.9921875, 11.0703125]
expected_all_finites = [False, True, True, True, True, True, True, True]
expected_eval_loss = [11.046875]
actual_losses, actual_all_finites, actual_eval_loss = runBertTrainingTest(
gradient_accumulation_steps=1, use_mixed_precision=True, allreduce_post_accumulation=False, use_simple_model_desc=False)
rtol = 1e-01
assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch")
assert_array_equal(expected_all_finites, actual_all_finites, "all_finite mismatch")
assert_allclose(expected_eval_loss, actual_eval_loss, rtol=rtol, err_msg="evaluation loss mismatch")
def testBertTrainingGradientAccumulationMixedPrecision(self):
torch.manual_seed(1)
expected_losses = [11.046875, 11.171875, 11.0234375, 11.046875, 10.8984375, 10.9921875, 11.078125, 10.96875]
expected_all_finites = [False, True]
expected_eval_loss = [11.0546875]
actual_losses, actual_all_finites, actual_eval_loss = runBertTrainingTest(
gradient_accumulation_steps=4, use_mixed_precision=True, allreduce_post_accumulation=False, use_simple_model_desc=False)
rtol = 1e-01
assert_allclose(expected_losses, actual_losses, rtol=rtol, err_msg="loss mismatch")
assert_array_equal(expected_all_finites, actual_all_finites, "all_finite mismatch")
assert_allclose(expected_eval_loss, actual_eval_loss, rtol=rtol, err_msg="evaluation loss mismatch")
if __name__ == '__main__':
unittest.main(module=__name__, buffer=True)

14
tools/ci_build/build.py
View file

@ -80,6 +80,10 @@ Use the individual flags to only run the specified stages.
parser.add_argument("--training_e2e_test_data_path",
help="Path to training end-to-end test data directory.")
# Pytorch frontend test options
parser.add_argument("--enable_training_python_frontend_e2e_tests", action="store_true",
help="Enable the pytorch frontend training tests.")
# enable ONNX tests
parser.add_argument("--enable_onnx_tests", action='store_true',
help='''When running the Test phase, run onnx_test_running against available test data directories.''')
@ -620,6 +624,11 @@ def adb_push(source_dir, src, dest, **kwargs):
def adb_shell(*args, **kwargs):
return run_subprocess(['adb', 'shell', *args], **kwargs)
def run_training_python_frontend_e2e_tests(args, cwd, dll_path):
# frontend tests are to be added here:
log.info("Running python frontend e2e tests.")
run_subprocess([sys.executable, 'onnxruntime_test_ort_trainer_with_mixed_precision.py'], cwd=cwd, dll_path=dll_path)
def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enable_tvm = False, enable_tensorrt = False):
for config in configs:
log.info("Running tests for %s configuration", config)
@ -671,8 +680,13 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enab
run_subprocess([sys.executable, 'onnxruntime_test_python.py'], cwd=cwd, dll_path=dll_path)
if args.enable_training and args.use_cuda:
# run basic frontend tests
run_subprocess([sys.executable, 'onnxruntime_test_ort_trainer.py'], cwd=cwd, dll_path=dll_path)
# run additional frontend tests for orttraining-linux-gpu-frontend_test_ci-pipeline
if args.enable_training_python_frontend_e2e_tests:
run_training_python_frontend_e2e_tests(args, cwd=cwd, dll_path=dll_path)
try:
import onnx
import scipy # gen_test_models.py used by onnx_test has a dependency on scipy

32
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml Normal file
View file

@ -0,0 +1,32 @@
# 'trigger: none' to use UI to schedule the pipeline runs.
# alternatively to schedule the pipeline run according to:
# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml
trigger: none
jobs:
- job: Onnxruntime_Linux_GPU_Training_FrontEnd
timeoutInMinutes: 120
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/linux-set-variables-and-download.yml
# insert a python frontend test data preparation step here
- script: >
tools/ci_build/github/linux/run_dockerbuild.sh
-o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory)
-x "
--enable_training
--config RelWithDebInfo
--skip_onnx_tests
--build_wheel
--enable_training_python_frontend_e2e_tests
"
displayName: 'Build and run frontend tests'
- template: templates/clean-agent-build-directory-step.yml