saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix few small bugs (#17019)

Browse source 
### Fix few bugs

1. symbolic shape infer, there is no None check before get length. 
2. Rename PythonOp/PythonOpGrad's attribute `name` to `func_name`,
otherwise, when we use onnx.helper.make_node to create node, `name`
conflicts with node name.
3. Filter shape inference warnings for PythonOp for torch 2.0 or newer. 
4. Close file descriptor for log suppression. Without the fix, two extra
fd is left after the log suppression exit its context.
Before enter log suppression (left), Before exit log suppression (right)

![image](https://github.com/microsoft/onnxruntime/assets/10530022/3cd3057a-59f9-4c89-8359-d9b32c49a17e)
   With the fix, no fd added after context exit.

![image](https://github.com/microsoft/onnxruntime/assets/10530022/03454a8f-ab48-4552-bb9b-293a4f51be67)
This commit is contained in:
pengwa 2023-08-07 14:01:36 +08:00 committed by GitHub
parent a451318820
commit 3649376f09
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 47 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
onnxruntime/python/tools/symbolic_shape_infer.py
View file

@ -1859,8 +1859,11 @@ class SymbolicShapeInference:
if (
node.input[0] in self.sympy_data_
and [0] == axes
and starts is not None
and len(starts) == 1
and ends is not None
and len(ends) == 1
and steps is not None
and len(steps) == 1
):
input_sympy_data = self.sympy_data_[node.input[0]]
@ -2386,8 +2389,8 @@ class SymbolicShapeInference:
# The first output is autograd's context.
vi = self.known_vi_[node.output[0]]
vi.CopyFrom(helper.make_tensor_value_info(node.output[0], onnx.TensorProto.INT64, []))
if get_attribute(node, "name").decode() in ["_InspectActivation", "_IncrementStep"]:
# PythonOp with name being "_InspectActivation" or "_IncrementStep" will behave exactly same as a normal
if get_attribute(node, "func_name").decode() in ["_InspectActivation", "_IncrementStep"]:
# PythonOp with func_name being "_InspectActivation" or "_IncrementStep" will behave exactly same as a normal
# PythonOp when execution. The only difference is that
# 1). those ops having same number of tensor inputs and tensor outputs;
# 2). and the i-th output tensor's shape is same as i-th input tensor's shape.

5
orttraining/orttraining/core/graph/gradient_builder.cc
View file

@ -1760,8 +1760,9 @@ IMPLEMENT_GRADIENT_BUILDER(GetPythonOpGradient) {
std::vector<NodeDef> result;
auto src_attrs = SrcNodeAttributes();
std::vector<AttributeProto> attrs;
ORT_ENFORCE(utils::HasString(src_attrs.at("name")));
attrs.push_back(MakeAttribute("name", src_attrs.at("name").s()));
ORT_ENFORCE(src_attrs.count("func_name") > 0, "func_name attribute is missing.");
ORT_ENFORCE(utils::HasString(src_attrs.at("func_name")));
attrs.push_back(MakeAttribute("func_name", src_attrs.at("func_name").s()));
attrs.push_back(MakeAttribute("output_convention", src_attrs.at("input_convention").s()));
attrs.push_back(MakeAttribute("inplace", src_attrs.at("inplace").i()));

8
orttraining/orttraining/core/graph/training_op_defs.cc
View file

@ -3764,7 +3764,7 @@ Return true if all elements are true and false otherwise.
/*is_homogeneous*/ false,
/*min_arity*/ 1)
.Attr(
"name",
"func_name",
"Name of custom class.",
AttributeProto::STRING)
.Attr(
@ -3917,7 +3917,7 @@ Return true if all elements are true and false otherwise.
// This is a required field.
ORT_ENFORCE(output_tensor_types_proto, "PythonOp's must have \"output_tensor_types\" attribute.");
std::string func_name = getAttribute(ctx, "name", "");
std::string func_name = getAttribute(ctx, "func_name", "");
if (func_name == "_InspectActivation" || func_name == "_IncrementStep") {
// PythonOp with the name attribute being "_InspectActivation" or "_IncrementStep" will behave exactly the
// same as a normal PythonOp when execution. The only difference is that:
@ -3980,7 +3980,7 @@ Return true if all elements are true and false otherwise.
/*is_homogeneous*/ false,
/*min_arity*/ 1)
.Attr(
"name",
"func_name",
"Name of custom class.",
AttributeProto::STRING)
.Attr(
@ -4062,7 +4062,7 @@ Return true if all elements are true and false otherwise.
// This is a required field.
ORT_ENFORCE(output_tensor_types_proto, "PythonOpGrad's must have \"output_tensor_types\" attribute.");
std::string func_name = getAttribute(ctx, "name", "");
std::string func_name = getAttribute(ctx, "func_name", "");
if (func_name == "_InspectActivation" || func_name == "_IncrementStep") {
// PythonOpGrad with name attribute being "_InspectActivation" or "_IncrementStep" will behave exactly
// the same as a normal PythonOpGrad when execution. The only difference is that:

4
orttraining/orttraining/python/training/ortmodule/_custom_autograd_function_exporter.py
View file

@ -159,7 +159,7 @@ def _export_pt_1_10(g, n, *args, **kwargs):
# TODO: add fully-qualified name.
attrs = {
"name_s": name,
"func_name_s": name,
"inplace_i": inplace,
"input_convention_s": cconv,
"outputs": n.outputsSize(),
@ -247,7 +247,7 @@ def _post_process_enabling_autograd_fallback(exported_model):
node.output.append(output_names[0] + "_ctx")
node.output.extend(output_names)
for attr in node.attribute:
if attr.name == "name":
if attr.name == "func_name":
kclass_name = attr.s.decode("utf-8") if isinstance(attr.s, bytes) else attr.s
# If the duplicated function is used in ONNX graph, we will fail in case of a wrong function call.
# Todo: remove this trick once exporter can support fully qualified name for PythonOp.

13
orttraining/orttraining/python/training/ortmodule/_logger.py
View file

@ -171,6 +171,8 @@ def _suppress_os_stream_output(enable=True, on_exit: Optional[Callable] = None):
if enable:
# stdout and stderr is written to a tempfile instead
with tempfile.TemporaryFile() as fp:
old_stdout = None
old_stderr = None
try:
# Store original stdout and stderr file no.
old_stdout = os.dup(sys.stdout.fileno())
@ -185,11 +187,18 @@ def _suppress_os_stream_output(enable=True, on_exit: Optional[Callable] = None):
sys.stderr.flush()
# Restore stdout and stderr.
os.dup2(old_stdout, sys.stdout.fileno())
os.dup2(old_stderr, sys.stderr.fileno())
if old_stdout is not None:
os.dup2(old_stdout, sys.stdout.fileno())
if old_stderr is not None:
os.dup2(old_stderr, sys.stderr.fileno())
# Close file descriptors
os.close(old_stdout)
os.close(old_stderr)
if on_exit:
on_exit(fp)
else:
yield

34
orttraining/orttraining/python/training/ortmodule/options.py
View file

@ -136,22 +136,28 @@ class DebugOptions:
@property
def torch_exporter_filter(self):
"""Accessor for the filter export logs configuration."""
if self.log_level >= LogLevel.INFO and get_runtime_pytorch_version() < version.parse("2.0"):
torch_version = get_runtime_pytorch_version()
if self.log_level >= LogLevel.INFO:
if torch_version < version.parse("2.0"):
return [
# WARNING: The shape inference of com.microsoft::SoftmaxCrossEntropyLossInternal type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function.
# WARNING: The shape inference of com.microsoft::PythonOp type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function.
# WARNING: The shape inference of org.pytorch.aten::ATen type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function.
# WARNING: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function.
"type is missing, so it may result in wrong shape inference",
# Warning: Checker does not support models with experimental ops: ATen
"Checker does not support models with experimental ops:",
"Dropout is a training op and should not be exported in inference mode.",
# Warning: Shape inference does not support models with experimental operators: ATen
"Shape inference does not support models with experimental operators:",
# Warning: Unsupported operator Trilu. No schema registered for this operator.
# Warning: Unsupported operator ATen. No schema registered for this operator.
# Warning: Unsupported operator SoftmaxCrossEntropyLossInternal. No schema registered for this operator.
"No schema registered for this operator.",
]
return [
# WARNING: The shape inference of com.microsoft::SoftmaxCrossEntropyLossInternal type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function.
# WARNING: The shape inference of com.microsoft::PythonOp type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function.
# WARNING: The shape inference of org.pytorch.aten::ATen type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function.
# WARNING: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function.
# [W shape_type_inference.cpp:1974] Warning: The shape inference of com.microsoft::PythonOp type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (function UpdateReliable)
"type is missing, so it may result in wrong shape inference",
# Warning: Checker does not support models with experimental ops: ATen
"Checker does not support models with experimental ops:",
"Dropout is a training op and should not be exported in inference mode.",
# Warning: Shape inference does not support models with experimental operators: ATen
"Shape inference does not support models with experimental operators:",
# Warning: Unsupported operator Trilu. No schema registered for this operator.
# Warning: Unsupported operator ATen. No schema registered for this operator.
# Warning: Unsupported operator SoftmaxCrossEntropyLossInternal. No schema registered for this operator.
"No schema registered for this operator.",
]
return None

4
orttraining/orttraining/training_ops/cpu/torch/torch_custom_function_kernel_base.cc
View file

@ -27,7 +27,7 @@ std::vector<OrtValue> CreateOrtValueArgs(OpKernelContext* context,
}
void PythonOpBase::Init(const OpKernelInfo& info) {
ORT_THROW_IF_ERROR(info.GetAttr("name", &name_));
ORT_THROW_IF_ERROR(info.GetAttr("func_name", &name_));
ORT_THROW_IF_ERROR(info.GetAttr("inplace", &inplace_));
is_training_mode_ = static_cast<bool>(info.GetAttrOrDefault("training_mode", static_cast<int64_t>(0)));
@ -234,7 +234,7 @@ void PythonOpBase::SetOtherOutputs(OpKernelContext* context, std::vector<OrtValu
}
void PythonOpGradBase::Init(const OpKernelInfo& info) {
ORT_THROW_IF_ERROR(info.GetAttr("name", &name_));
ORT_THROW_IF_ERROR(info.GetAttr("func_name", &name_));
ORT_THROW_IF_ERROR(info.GetAttr("inplace", &inplace_));
ORT_THROW_IF_ERROR(info.GetAttrs("input_tensor_types", input_tensor_types_));
ORT_THROW_IF_ERROR(info.GetAttr("output_convention", &output_convention_));