saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enable SkipCheck by default (#9215)

Browse source 
* Enable SkipCheck by default

* fix UTs

* fix UT

* fix UTs

* fix UTs

* address comments

* fix UT

* enable skipchecks

* move _SkipCheck back

* move _SkipCheck back

* move _SkipCheck back

* Update orttraining/orttraining/python/training/ortmodule/_inference_manager.py

* Update orttraining/orttraining/python/training/ortmodule/_utils.py

Co-authored-by: Ethan Tao <ettao@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
Co-authored-by: Thiago Crepaldi <thiago.crepaldi@microsoft.com>
This commit is contained in:
ytaous 2021-10-07 15:47:14 -07:00 committed by GitHub
parent 88d5023885
commit 7166586d7e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 66 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py
View file

@ -98,10 +98,12 @@ class GraphExecutionManager(GraphExecutionInterface):
self._execution_agent = None
# indicators of some logic have been executed previously thus could be skipped for faster training
self._skip_check = reduce(lambda x, y: x | y,
[_SkipCheck[name] for name in
_utils.parse_os_env_skip_check_flags('ORTMODULE_SKIPCHECK_POLICY',
_SkipCheck.SKIP_CHECK_DISABLED.name)])
# default is enabled, if not define in os env
self._skip_check = _SkipCheck(_SkipCheck.SKIP_CHECK_DEVICE | _SkipCheck.SKIP_CHECK_BUILD_GRADIENT | _SkipCheck.SKIP_CHECK_EXECUTION_AGENT)
if os.getenv('ORTMODULE_SKIPCHECK_POLICY') is not None:
self._skip_check = reduce(lambda x, y: x | y,
[_SkipCheck[name] for name in
_utils.parse_os_env_skip_check_flags('ORTMODULE_SKIPCHECK_POLICY')])
self._first_skip_check_warning = True
# Graph transformer config

1
orttraining/orttraining/python/training/ortmodule/_inference_manager.py
View file

@ -142,7 +142,6 @@ class InferenceManager(GraphExecutionManager):
self._fallback_manager.handle_exception(exception=e,
log_level=self._debug_options.logging.log_level,
override_policy=_FallbackPolicy.FALLBACK_FORCE_TORCH_FORWARD)
# Fallback to PyTorch due to failures *during* forward(),
# (e.g. export, model/input post-processing, forward, output processing, etc)
if self._fallback_manager.is_pending():

6
orttraining/orttraining/python/training/ortmodule/_utils.py
View file

@ -197,10 +197,10 @@ def check_for_name_collisions_and_bind_methods_to_ortmodule(ortmodule: torch.nn.
warnings.warn(f"User Module's attribute name {attribute_name} collides with ORTModule's attribute name. "
"User Module's attribute may not be returned when trying to retrieve the attribute through ORTModule.")
def parse_os_env_skip_check_flags(env_name, default_skip_check_str):
"""Returns a list of SkipChecks as defined by os env variable env_name or default provided"""
def parse_os_env_skip_check_flags(env_name):
"""Returns a list of SkipChecks as defined by os env variable env_name"""
return os.getenv(env_name, default_skip_check_str).split('|')
return os.getenv(env_name).split('|')
def get_exception_as_string(exception):
assert isinstance(exception, Exception), 'exception must be a `Exception`'

51
orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py
View file

@ -2,7 +2,6 @@
# Licensed under the MIT License.
# orttraining_test_ortmodule_api.py
from functools import reduce
import itertools
import math
import random
@ -22,7 +21,6 @@ import os
from distutils.version import LooseVersion
from onnxruntime.training.ortmodule._custom_gradient_registry import register_gradient
from onnxruntime.training.ortmodule import (ORTModule,
_utils,
_io,
@ -580,6 +578,8 @@ def test_model_and_input_without_device():
out is not None
def test_model_with_different_devices_same_session():
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
N, D_in, H, D_out = 64, 784, 500, 10
model = NeuralNetSinglePositionalArgument(D_in, H, D_out)
model = ORTModule(model)
@ -594,6 +594,8 @@ def test_model_with_different_devices_same_session():
x = torch.randn(N, D_in, device=device)
y = model(x)
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
@pytest.mark.parametrize("device", ['cuda', 'cpu'])
def test_input_requires_grad_saved(device):
N, D_in, H, D_out = 32, 784, 500, 10
@ -1377,6 +1379,8 @@ def test_bert_inputs_with_dynamic_shape():
@pytest.mark.parametrize("device", ['cuda', 'cpu'])
def test_changes_input_requires_grad_reinitializes_module_gradient_graph_builder(device):
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
N, D_in, H, D_out = 32, 784, 500, 10
model = NeuralNetSinglePositionalArgument(D_in, H, D_out).to(device)
model = ORTModule(model)
@ -1394,6 +1398,8 @@ def test_changes_input_requires_grad_reinitializes_module_gradient_graph_builder
assert module_gradient_graph_builder_training != \
model._torch_module._execution_manager(model._is_training())._graph_builder
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
@pytest.mark.parametrize("device", ['cuda'])
def test_input_requires_grad_backward_creates_input_grad_as_required0(device):
N, D_in, H, D_out = 32, 784, 500, 10
@ -1743,6 +1749,7 @@ def test_named_tuple_return_value_module(device):
@pytest.mark.parametrize("device", ['cpu', 'cuda'])
def test_exception_raised_for_custom_class_return_value_module(device):
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
N, D_in, H, D_out = 64, 784, 500, 10
pt_model = NeuralNetCustomClassOutput(D_in, H, D_out).to(device)
@ -1765,6 +1772,8 @@ def test_exception_raised_for_custom_class_return_value_module(device):
with pytest.raises(_fallback.ORTModuleIOError) as runtime_error:
ort_model(x, y, z)
assert 'ORTModule does not support the following model output type' in str(runtime_error.value)
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
def test_dynamic_axes_config():
device = 'cuda'
@ -2063,6 +2072,8 @@ def test_nested_return_value_module(device):
)
def test_forward_data_and_model_on_different_devices(data_device, model_device):
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
N, D_in, H, D_out = 64, 784, 500, 10
model = NeuralNetSinglePositionalArgument(D_in, H, D_out).to(model_device)
ort_model = ORTModule(model)
@ -2084,6 +2095,8 @@ def test_forward_data_and_model_on_different_devices(data_device, model_device):
ort_model(x)
assert f"Input argument to forward found on device {torch.device(x.device)}, but expected it to be on module device {ort_model._torch_module._execution_manager(ort_model._is_training())._device}." in str(runtime_error.value)
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
def test_forward_returns_none_type_as_output():
class NeuralNetNoneTypeOutput(torch.nn.Module):
def __init__(self, input_size, num_classes):
@ -2180,6 +2193,9 @@ def test_model_wrapped_inside_torch_no_grad():
output = model(x)
def test_model_initializer_requires_grad_changes_from_one_forward_to_next():
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
device = 'cuda'
N, D_in, H, D_out = 64, 784, 500, 10
model = NeuralNetPartialNoGradModel(D_in, H, D_out).to(device)
@ -2211,6 +2227,8 @@ def test_model_initializer_requires_grad_changes_from_one_forward_to_next():
assert torch.equal(weight_grad_2, weight_grad_3)
assert torch.equal(bias_grad_2, bias_grad_3)
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
def test_model_with_registered_buffers():
class NeuralNetWithRegisteredBuffer(torch.nn.Module):
def __init__(self, input_size, hidden_size, num_classes):
@ -2553,6 +2571,9 @@ def test_train_eval_with_various_outputs():
_test_helpers.assert_values_are_close(pt_out, ort_out)
def test_forward_dynamic_args():
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
device = 'cuda'
N, D_in, H, D_out = 64, 784, 500, 10
@ -2591,9 +2612,14 @@ def test_forward_dynamic_args():
assert output is not None
hash_args_size3 = hash(repr(model._torch_module._execution_manager(model._is_training())._input_info.schema))
assert hash_args_size3 != hash_args_size2
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
def test_forward_dynamic_kwargs():
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
one = torch.FloatTensor([1])
model = NeuralNetSimplePositionalAndKeywordArguments()
model = ORTModule(model)
@ -2643,6 +2669,8 @@ def test_forward_dynamic_kwargs():
assert hash_x2 != hash_x_y_z
assert hash_x2 == hash_x
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
@pytest.mark.parametrize("forward_statement",
[# Only pos_X, pos_X as positionals
@ -2750,6 +2778,9 @@ def test_repro_iscontiguous():
def test_forward_call_default_input():
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
class UnusedNet(torch.nn.Module):
def __init__(self):
super().__init__()
@ -2817,6 +2848,7 @@ def test_forward_call_default_input():
if model._is_training():
out.sum().backward()
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
def test_forward_call_kwargs_input_unexpected_order():
class OrderlyNet(torch.nn.Module):
@ -2871,6 +2903,9 @@ def test_forward_call_kwargs_input_unexpected_order():
def test_forward_call_lots_None():
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
class NoneNet(torch.nn.Module):
def __init__(self):
super().__init__()
@ -2948,6 +2983,8 @@ def test_forward_call_lots_None():
run_step(a.item() + b.item() + c.item() + d.item() + e.item() + f.item() + y.item() + z.item(),
**{'a': a, 'b': b, 'c': c, 'd': d, 'e': e, 'f': f, 'y': y, 'z': z})
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
@pytest.mark.parametrize("bool_argument", [True, False])
@pytest.mark.parametrize("int_argument", [100, 100000, 100000000, -100, -100000, -100000000])
@pytest.mark.parametrize("float_argument", [1.23, 11209123.12452, 12093702935.1249863, -1.23, -11209123.12452, -12093702935.1249863])
@ -2988,6 +3025,9 @@ def test_primitive_inputs(bool_argument, int_argument, float_argument):
@pytest.mark.parametrize("bool_arguments", [(True, False), (False, True)])
def test_changing_bool_input_re_exports_model(bool_arguments):
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
class PrimitiveTypesInputNet(torch.nn.Module):
def __init__(self, input_size, hidden_size, num_classes):
super(PrimitiveTypesInputNet, self).__init__()
@ -3024,6 +3064,8 @@ def test_changing_bool_input_re_exports_model(bool_arguments):
assert exported_model1 != exported_model2
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
def test_model_with_registered_buffer_and_dropped_parameters():
class ModelWithBufferAndDroppedParameter(torch.nn.Module):
def __init__(self, input_size, hidden_size, num_classes):
@ -3786,6 +3828,9 @@ def test_ortmodule_setattr_ortmodule_attribute():
assert ort_model._torch_module == True
def test_ortmodule_setattr_signals_model_changed():
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
class UserNet(torch.nn.Module):
def __init__(self, input_flag):
super(UserNet, self).__init__()
@ -3818,6 +3863,8 @@ def test_ortmodule_setattr_signals_model_changed():
assert exported_model1 != exported_model2
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
def test_ortmodule_attribute_name_collision_warning():
class UserNet(torch.nn.Module):
def __init__(self):

10
orttraining/orttraining/test/python/orttraining_test_ortmodule_fallback.py
View file

@ -148,6 +148,7 @@ def test_ortmodule_fallback_device__mismatch(is_training, fallback_enabled, matc
policy = 'FALLBACK_DISABLE'
os.environ['ORTMODULE_FALLBACK_POLICY'] = policy
os.environ['ORTMODULE_FALLBACK_RETRY'] = str(not persist_fallback)
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
data_device = 'cuda'
N, D_in, H, D_out = 64, 784, 500, 10
@ -169,8 +170,9 @@ def test_ortmodule_fallback_device__mismatch(is_training, fallback_enabled, matc
if matching_policy:
with pytest.raises(RuntimeError) as e:
ort_model(inputs)
assert ("Expected all tensors to be on the same device, but found at least two devices, cpu and cuda:0!"
in str(e.value))
assert \
("Tensor for argument #1 'self' is on CPU, but expected them to be on GPU (while checking arguments for addmm)" in str(e.value)) \
or ("Expected all tensors to be on the same device, but found at least two devices, cpu and cuda:0!" in str(e.value))
else:
with pytest.raises(_fallback.ORTModuleDeviceException) as e:
ort_model(inputs)
@ -182,6 +184,7 @@ def test_ortmodule_fallback_device__mismatch(is_training, fallback_enabled, matc
assert (f"Input argument to forward found on device {input_device}, "
f"but expected it to be on module device {ort_model_device}." in str(e.value))
del os.environ['ORTMODULE_SKIPCHECK_POLICY']
@pytest.mark.parametrize("is_training,fallback_enabled,matching_policy,persist_fallback",
list(itertools.product([True, False], repeat=4)))
@ -539,6 +542,7 @@ def test_ortmodule_fallback_warn_message(is_training, persist_fallback):
policy = 'FALLBACK_UNSUPPORTED_DEVICE'
os.environ['ORTMODULE_FALLBACK_POLICY'] = policy
os.environ['ORTMODULE_FALLBACK_RETRY'] = str(not persist_fallback)
os.environ['ORTMODULE_SKIPCHECK_POLICY'] = 'SKIP_CHECK_DISABLED'
data_device = 'cuda'
N, D_in, H, D_out = 64, 784, 500, 10
@ -558,3 +562,5 @@ def test_ortmodule_fallback_warn_message(is_training, persist_fallback):
with pytest.warns(UserWarning) as warning_record:
ort_model(inputs)
assert "Fallback to PyTorch due to exception" in str(warning_record[0].message.args[0])
del os.environ['ORTMODULE_SKIPCHECK_POLICY']