From 7166586d7ebcb0169980b188615b8044f06de7f4 Mon Sep 17 00:00:00 2001 From: ytaous <4484531+ytaous@users.noreply.github.com> Date: Thu, 7 Oct 2021 15:47:14 -0700 Subject: [PATCH] Enable SkipCheck by default (#9215) * Enable SkipCheck by default * fix UTs * fix UT * fix UTs * fix UTs * address comments * fix UT * enable skipchecks * move _SkipCheck back * move _SkipCheck back * move _SkipCheck back * Update orttraining/orttraining/python/training/ortmodule/_inference_manager.py * Update orttraining/orttraining/python/training/ortmodule/_utils.py Co-authored-by: Ethan Tao Co-authored-by: Thiago Crepaldi --- .../ortmodule/_graph_execution_manager.py | 10 ++-- .../training/ortmodule/_inference_manager.py | 1 - .../python/training/ortmodule/_utils.py | 6 +-- .../python/orttraining_test_ortmodule_api.py | 51 ++++++++++++++++++- .../orttraining_test_ortmodule_fallback.py | 10 +++- 5 files changed, 66 insertions(+), 12 deletions(-) diff --git a/orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py b/orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py index 04d1f064e2..39836ca602 100644 --- a/orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py +++ b/orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py @@ -98,10 +98,12 @@ class GraphExecutionManager(GraphExecutionInterface): self._execution_agent = None # indicators of some logic have been executed previously thus could be skipped for faster training - self._skip_check = reduce(lambda x, y: x | y, - [_SkipCheck[name] for name in - _utils.parse_os_env_skip_check_flags('ORTMODULE_SKIPCHECK_POLICY', - _SkipCheck.SKIP_CHECK_DISABLED.name)]) + # default is enabled, if not define in os env + self._skip_check = _SkipCheck(_SkipCheck.SKIP_CHECK_DEVICE | _SkipCheck.SKIP_CHECK_BUILD_GRADIENT | _SkipCheck.SKIP_CHECK_EXECUTION_AGENT) + if os.getenv('ORTMODULE_SKIPCHECK_POLICY') is not None: + self._skip_check = reduce(lambda x, y: x | y, + [_SkipCheck[name] for name in + _utils.parse_os_env_skip_check_flags('ORTMODULE_SKIPCHECK_POLICY')]) self._first_skip_check_warning = True # Graph transformer config diff --git a/orttraining/orttraining/python/training/ortmodule/_inference_manager.py b/orttraining/orttraining/python/training/ortmodule/_inference_manager.py index b1f0184ea3..886aa5c2f1 100644 --- a/orttraining/orttraining/python/training/ortmodule/_inference_manager.py +++ b/orttraining/orttraining/python/training/ortmodule/_inference_manager.py @@ -142,7 +142,6 @@ class InferenceManager(GraphExecutionManager): self._fallback_manager.handle_exception(exception=e, log_level=self._debug_options.logging.log_level, override_policy=_FallbackPolicy.FALLBACK_FORCE_TORCH_FORWARD) - # Fallback to PyTorch due to failures *during* forward(), # (e.g. export, model/input post-processing, forward, output processing, etc) if self._fallback_manager.is_pending(): diff --git a/orttraining/orttraining/python/training/ortmodule/_utils.py b/orttraining/orttraining/python/training/ortmodule/_utils.py index 8edfae3419..87aca7ce86 100644 --- a/orttraining/orttraining/python/training/ortmodule/_utils.py +++ b/orttraining/orttraining/python/training/ortmodule/_utils.py @@ -197,10 +197,10 @@ def check_for_name_collisions_and_bind_methods_to_ortmodule(ortmodule: torch.nn. warnings.warn(f"User Module's attribute name {attribute_name} collides with ORTModule's attribute name. " "User Module's attribute may not be returned when trying to retrieve the attribute through ORTModule.") -def parse_os_env_skip_check_flags(env_name, default_skip_check_str): - """Returns a list of SkipChecks as defined by os env variable env_name or default provided""" +def parse_os_env_skip_check_flags(env_name): + """Returns a list of SkipChecks as defined by os env variable env_name""" - return os.getenv(env_name, default_skip_check_str).split('|') + return os.getenv(env_name).split('|') def get_exception_as_string(exception): assert isinstance(exception, Exception), 'exception must be a `Exception`' diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py index c21a4a9811..4f7b82b9de 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py @@ -2,7 +2,6 @@ # Licensed under the MIT License. # orttraining_test_ortmodule_api.py -from functools import reduce import itertools import math import random @@ -22,7 +21,6 @@ import os from distutils.version import LooseVersion from onnxruntime.training.ortmodule._custom_gradient_registry import register_gradient - from onnxruntime.training.ortmodule import (ORTModule, _utils, _io, @@ -580,6 +578,8 @@ def test_model_and_input_without_device(): out is not None def test_model_with_different_devices_same_session(): + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' + N, D_in, H, D_out = 64, 784, 500, 10 model = NeuralNetSinglePositionalArgument(D_in, H, D_out) model = ORTModule(model) @@ -594,6 +594,8 @@ def test_model_with_different_devices_same_session(): x = torch.randn(N, D_in, device=device) y = model(x) + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] + @pytest.mark.parametrize("device", ['cuda', 'cpu']) def test_input_requires_grad_saved(device): N, D_in, H, D_out = 32, 784, 500, 10 @@ -1377,6 +1379,8 @@ def test_bert_inputs_with_dynamic_shape(): @pytest.mark.parametrize("device", ['cuda', 'cpu']) def test_changes_input_requires_grad_reinitializes_module_gradient_graph_builder(device): + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' + N, D_in, H, D_out = 32, 784, 500, 10 model = NeuralNetSinglePositionalArgument(D_in, H, D_out).to(device) model = ORTModule(model) @@ -1394,6 +1398,8 @@ def test_changes_input_requires_grad_reinitializes_module_gradient_graph_builder assert module_gradient_graph_builder_training != \ model._torch_module._execution_manager(model._is_training())._graph_builder + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] + @pytest.mark.parametrize("device", ['cuda']) def test_input_requires_grad_backward_creates_input_grad_as_required0(device): N, D_in, H, D_out = 32, 784, 500, 10 @@ -1743,6 +1749,7 @@ def test_named_tuple_return_value_module(device): @pytest.mark.parametrize("device", ['cpu', 'cuda']) def test_exception_raised_for_custom_class_return_value_module(device): + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' N, D_in, H, D_out = 64, 784, 500, 10 pt_model = NeuralNetCustomClassOutput(D_in, H, D_out).to(device) @@ -1765,6 +1772,8 @@ def test_exception_raised_for_custom_class_return_value_module(device): with pytest.raises(_fallback.ORTModuleIOError) as runtime_error: ort_model(x, y, z) assert 'ORTModule does not support the following model output type' in str(runtime_error.value) + + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] def test_dynamic_axes_config(): device = 'cuda' @@ -2063,6 +2072,8 @@ def test_nested_return_value_module(device): ) def test_forward_data_and_model_on_different_devices(data_device, model_device): + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' + N, D_in, H, D_out = 64, 784, 500, 10 model = NeuralNetSinglePositionalArgument(D_in, H, D_out).to(model_device) ort_model = ORTModule(model) @@ -2084,6 +2095,8 @@ def test_forward_data_and_model_on_different_devices(data_device, model_device): ort_model(x) assert f"Input argument to forward found on device {torch.device(x.device)}, but expected it to be on module device {ort_model._torch_module._execution_manager(ort_model._is_training())._device}." in str(runtime_error.value) + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] + def test_forward_returns_none_type_as_output(): class NeuralNetNoneTypeOutput(torch.nn.Module): def __init__(self, input_size, num_classes): @@ -2180,6 +2193,9 @@ def test_model_wrapped_inside_torch_no_grad(): output = model(x) def test_model_initializer_requires_grad_changes_from_one_forward_to_next(): + + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' + device = 'cuda' N, D_in, H, D_out = 64, 784, 500, 10 model = NeuralNetPartialNoGradModel(D_in, H, D_out).to(device) @@ -2211,6 +2227,8 @@ def test_model_initializer_requires_grad_changes_from_one_forward_to_next(): assert torch.equal(weight_grad_2, weight_grad_3) assert torch.equal(bias_grad_2, bias_grad_3) + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] + def test_model_with_registered_buffers(): class NeuralNetWithRegisteredBuffer(torch.nn.Module): def __init__(self, input_size, hidden_size, num_classes): @@ -2553,6 +2571,9 @@ def test_train_eval_with_various_outputs(): _test_helpers.assert_values_are_close(pt_out, ort_out) def test_forward_dynamic_args(): + + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' + device = 'cuda' N, D_in, H, D_out = 64, 784, 500, 10 @@ -2591,9 +2612,14 @@ def test_forward_dynamic_args(): assert output is not None hash_args_size3 = hash(repr(model._torch_module._execution_manager(model._is_training())._input_info.schema)) assert hash_args_size3 != hash_args_size2 + + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] def test_forward_dynamic_kwargs(): + + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' + one = torch.FloatTensor([1]) model = NeuralNetSimplePositionalAndKeywordArguments() model = ORTModule(model) @@ -2643,6 +2669,8 @@ def test_forward_dynamic_kwargs(): assert hash_x2 != hash_x_y_z assert hash_x2 == hash_x + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] + @pytest.mark.parametrize("forward_statement", [# Only pos_X, pos_X as positionals @@ -2750,6 +2778,9 @@ def test_repro_iscontiguous(): def test_forward_call_default_input(): + + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' + class UnusedNet(torch.nn.Module): def __init__(self): super().__init__() @@ -2817,6 +2848,7 @@ def test_forward_call_default_input(): if model._is_training(): out.sum().backward() + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] def test_forward_call_kwargs_input_unexpected_order(): class OrderlyNet(torch.nn.Module): @@ -2871,6 +2903,9 @@ def test_forward_call_kwargs_input_unexpected_order(): def test_forward_call_lots_None(): + + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' + class NoneNet(torch.nn.Module): def __init__(self): super().__init__() @@ -2948,6 +2983,8 @@ def test_forward_call_lots_None(): run_step(a.item() + b.item() + c.item() + d.item() + e.item() + f.item() + y.item() + z.item(), **{'a': a, 'b': b, 'c': c, 'd': d, 'e': e, 'f': f, 'y': y, 'z': z}) + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] + @pytest.mark.parametrize("bool_argument", [True, False]) @pytest.mark.parametrize("int_argument", [100, 100000, 100000000, -100, -100000, -100000000]) @pytest.mark.parametrize("float_argument", [1.23, 11209123.12452, 12093702935.1249863, -1.23, -11209123.12452, -12093702935.1249863]) @@ -2988,6 +3025,9 @@ def test_primitive_inputs(bool_argument, int_argument, float_argument): @pytest.mark.parametrize("bool_arguments", [(True, False), (False, True)]) def test_changing_bool_input_re_exports_model(bool_arguments): + + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' + class PrimitiveTypesInputNet(torch.nn.Module): def __init__(self, input_size, hidden_size, num_classes): super(PrimitiveTypesInputNet, self).__init__() @@ -3024,6 +3064,8 @@ def test_changing_bool_input_re_exports_model(bool_arguments): assert exported_model1 != exported_model2 + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] + def test_model_with_registered_buffer_and_dropped_parameters(): class ModelWithBufferAndDroppedParameter(torch.nn.Module): def __init__(self, input_size, hidden_size, num_classes): @@ -3786,6 +3828,9 @@ def test_ortmodule_setattr_ortmodule_attribute(): assert ort_model._torch_module == True def test_ortmodule_setattr_signals_model_changed(): + + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' + class UserNet(torch.nn.Module): def __init__(self, input_flag): super(UserNet, self).__init__() @@ -3818,6 +3863,8 @@ def test_ortmodule_setattr_signals_model_changed(): assert exported_model1 != exported_model2 + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] + def test_ortmodule_attribute_name_collision_warning(): class UserNet(torch.nn.Module): def __init__(self): diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_fallback.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_fallback.py index e130794980..250014846d 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_fallback.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_fallback.py @@ -148,6 +148,7 @@ def test_ortmodule_fallback_device__mismatch(is_training, fallback_enabled, matc policy = 'FALLBACK_DISABLE' os.environ['ORTMODULE_FALLBACK_POLICY'] = policy os.environ['ORTMODULE_FALLBACK_RETRY'] = str(not persist_fallback) + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' data_device = 'cuda' N, D_in, H, D_out = 64, 784, 500, 10 @@ -169,8 +170,9 @@ def test_ortmodule_fallback_device__mismatch(is_training, fallback_enabled, matc if matching_policy: with pytest.raises(RuntimeError) as e: ort_model(inputs) - assert ("Expected all tensors to be on the same device, but found at least two devices, cpu and cuda:0!" - in str(e.value)) + assert \ + ("Tensor for argument #1 'self' is on CPU, but expected them to be on GPU (while checking arguments for addmm)" in str(e.value)) \ + or ("Expected all tensors to be on the same device, but found at least two devices, cpu and cuda:0!" in str(e.value)) else: with pytest.raises(_fallback.ORTModuleDeviceException) as e: ort_model(inputs) @@ -182,6 +184,7 @@ def test_ortmodule_fallback_device__mismatch(is_training, fallback_enabled, matc assert (f"Input argument to forward found on device {input_device}, " f"but expected it to be on module device {ort_model_device}." in str(e.value)) + del os.environ['ORTMODULE_SKIPCHECK_POLICY'] @pytest.mark.parametrize("is_training,fallback_enabled,matching_policy,persist_fallback", list(itertools.product([True, False], repeat=4))) @@ -539,6 +542,7 @@ def test_ortmodule_fallback_warn_message(is_training, persist_fallback): policy = 'FALLBACK_UNSUPPORTED_DEVICE' os.environ['ORTMODULE_FALLBACK_POLICY'] = policy os.environ['ORTMODULE_FALLBACK_RETRY'] = str(not persist_fallback) + os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED' data_device = 'cuda' N, D_in, H, D_out = 64, 784, 500, 10 @@ -558,3 +562,5 @@ def test_ortmodule_fallback_warn_message(is_training, persist_fallback): with pytest.warns(UserWarning) as warning_record: ort_model(inputs) assert "Fallback to PyTorch due to exception" in str(warning_record[0].message.args[0]) + + del os.environ['ORTMODULE_SKIPCHECK_POLICY']