From b3cb775cf9076321deaf7a6eb284fbc2da68adb8 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 23 Aug 2023 16:01:22 +1000 Subject: [PATCH] Two fixes involving minimal builds (#17000) ### Description - allocation planner was breaking if graph had no nodes - in this particular model a branch of an If node returned an outer scope value directly. - if model used non-tensor types and sparse tensors are disabled the call to IsSpareTensor causes an exception when prematurely terminates the code. - it's perfectly fine to check if a value is a sparse tensor when support for them is disabled. we just can't do anything with that OrtValue which is what the current ifdef's after the call to IsSparseTensor handle. ### Motivation and Context Fix model execution failure for partner with model that uses sequences in a minimal build with sparse tensors disabled. --- .../onnxruntime/core/framework/ort_value.h | 4 - .../core/framework/allocation_planner.cc | 44 ++++++---- .../test/framework/ort_model_only_test.cc | 36 ++++++++ .../test/testdata/ort_github_issue_17000.onnx | Bin 0 -> 509 bytes .../test/testdata/ort_github_issue_17000.ort | Bin 0 -> 3392 bytes .../test/testdata/ort_github_issue_17000.py | 77 ++++++++++++++++++ onnxruntime/test/testdata/required_ops.config | 4 +- .../testdata/required_ops_and_types.config | 7 +- 8 files changed, 146 insertions(+), 26 deletions(-) create mode 100644 onnxruntime/test/testdata/ort_github_issue_17000.onnx create mode 100644 onnxruntime/test/testdata/ort_github_issue_17000.ort create mode 100644 onnxruntime/test/testdata/ort_github_issue_17000.py diff --git a/include/onnxruntime/core/framework/ort_value.h b/include/onnxruntime/core/framework/ort_value.h index 48c4e4320d..a071f3182f 100644 --- a/include/onnxruntime/core/framework/ort_value.h +++ b/include/onnxruntime/core/framework/ort_value.h @@ -68,11 +68,7 @@ struct OrtValue { } bool IsSparseTensor() const { -#if !defined(DISABLE_SPARSE_TENSORS) return (type_ != nullptr && type_->IsSparseTensorType()); -#else - ORT_THROW("Sparse tensor is not supported in this build."); -#endif } onnxruntime::MLDataType Type() const { diff --git a/onnxruntime/core/framework/allocation_planner.cc b/onnxruntime/core/framework/allocation_planner.cc index 300db24a98..0bf27fdf5e 100644 --- a/onnxruntime/core/framework/allocation_planner.cc +++ b/onnxruntime/core/framework/allocation_planner.cc @@ -1715,31 +1715,39 @@ class PlannerImpl { void PartitionIntoStreams(const logging::Logger& /*logger*/, const ExecutionProviders& /*execution_providers*/, const PathString& /*partition_config_file*/) { - stream_nodes_.push_back({}); - node_stream_map_.resize(SafeInt(graph_viewer_.MaxNodeIndex()) + 1); - for (auto node_index : graph_viewer_.GetNodesInTopologicalOrder()) { - stream_nodes_[0].push_back(node_index); - node_stream_map_[node_index] = 0; + if (graph_viewer_.NumberOfNodes() > 0) { + stream_nodes_.push_back({}); + node_stream_map_.resize(SafeInt(graph_viewer_.MaxNodeIndex()) + 1); + for (auto node_index : graph_viewer_.GetNodesInTopologicalOrder()) { + stream_nodes_[0].push_back(node_index); + node_stream_map_[node_index] = 0; + } + num_logic_streams_ = 1; } - num_logic_streams_ = 1; } Status BuildExecutionPlan(const ExecutionProviders& execution_providers) { // 1. create logic stream instance auto& execution_plan = plan_.execution_plan; - ORT_ENFORCE(num_logic_streams_ == 1 && !stream_nodes_[0].empty()); - execution_plan.reserve(1); - auto first_node_index = stream_nodes_[0][0]; - auto* node = graph_viewer_.GetNode(first_node_index); - onnxruntime::ProviderType exec_provider_name = node->GetExecutionProviderType(); - const IExecutionProvider* ep = execution_providers.Get(exec_provider_name); - ORT_ENFORCE(ep); - auto node_device_mem_location = ep->GetOrtDeviceByMemType(OrtMemType::OrtMemTypeDefault); - execution_plan.emplace_back(std::make_unique(node_device_mem_location)); - // 2. add steps to the execution plan - for (auto node_index : stream_nodes_[0]) { - execution_plan[0]->steps_.emplace_back(std::make_unique(node_index)); + + if (graph_viewer_.NumberOfNodes() > 0) { + ORT_ENFORCE(num_logic_streams_ == 1 && !stream_nodes_[0].empty()); + execution_plan.reserve(1); + auto first_node_index = stream_nodes_[0][0]; + auto* node = graph_viewer_.GetNode(first_node_index); + onnxruntime::ProviderType exec_provider_name = node->GetExecutionProviderType(); + const IExecutionProvider* ep = execution_providers.Get(exec_provider_name); + ORT_ENFORCE(ep); + auto node_device_mem_location = ep->GetOrtDeviceByMemType(OrtMemType::OrtMemTypeDefault); + execution_plan.emplace_back(std::make_unique(node_device_mem_location)); + // 2. add steps to the execution plan + for (auto node_index : stream_nodes_[0]) { + execution_plan[0]->steps_.emplace_back(std::make_unique(node_index)); + } + } else { + // graph with no nodes. e.g. subgraph of If might return the input as-is or a constant value from an initializer } + return Status::OK(); } diff --git a/onnxruntime/test/framework/ort_model_only_test.cc b/onnxruntime/test/framework/ort_model_only_test.cc index f8da4e8959..e2cb82e47f 100644 --- a/onnxruntime/test/framework/ort_model_only_test.cc +++ b/onnxruntime/test/framework/ort_model_only_test.cc @@ -4,6 +4,7 @@ #include "core/flatbuffers/schema/ort.fbs.h" #include "core/framework/data_types.h" #include "core/framework/tensorprotoutils.h" +#include "core/framework/TensorSeq.h" #include "core/graph/model.h" #include "core/graph/onnx_protobuf.h" #include "core/session/onnxruntime_cxx_api.h" @@ -556,6 +557,41 @@ TEST(OrtModelOnlyTests, LoadOrtFormatModelFromBufferNoCopyInitializersUseBuffer) RunOrtModel(test_info); } +// regression test for 2 issues covered by PR #17000 (internally reported issue). +// 1) allocation planner broke in minimal build when subgraph had no nodes. +// 2) usage of a sequence data type caused an exception due to IsSparseTensor() throwing +// instead of allowing the calling code to have #ifdef'd code to handle when IsSparseTensor +// returned true and sparse tensors were disabled. +TEST(OrtModelOnlyTests, GithubIssue17000) { + // need to run the model to + auto model_uri = ORT_TSTR("testdata/ort_github_issue_17000.ort"); + + auto allocator = TestCPUExecutionProvider()->CreatePreferredAllocators()[0]; + + OrtValue item0, item1; + CreateMLValue(allocator, {1}, {1.f}, &item0); + CreateMLValue(allocator, {2}, {2.f, 3.f}, &item1); + + auto elem_type = DataTypeImpl::GetType(); + auto tensor_seq = std::make_unique(elem_type); + tensor_seq->SetElements({item0, item1}); + + auto mltype = DataTypeImpl::GetType(); + OrtValue value(tensor_seq.release(), mltype, mltype->GetDeleteFunc()); + + OrtModelTestInfo test_info; + test_info.model_filename = model_uri; + test_info.inputs.insert(std::make_pair("seq_in", value)); + test_info.output_names = {"still_has_elements"}; + test_info.output_verifier = [](const std::vector& fetches) { + const auto& output = fetches[0].Get(); + ASSERT_EQ(output.Shape().Size(), 1); + ASSERT_EQ(output.Data()[0], true); // removed one item from seq so should still have elements + }; + + RunOrtModel(test_info); +} + #if !defined(DISABLE_ML_OPS) // test that we can deserialize and run a previously saved ORT format model // for a model with sequence and map outputs diff --git a/onnxruntime/test/testdata/ort_github_issue_17000.onnx b/onnxruntime/test/testdata/ort_github_issue_17000.onnx new file mode 100644 index 0000000000000000000000000000000000000000..8320c19cb6de4fa6401f9516ddd420add910fb04 GIT binary patch literal 509 zcmZ`#!AiqG5N%>?(n(XoAPB)zZjDex(1XW-6pJ@+UY1RnCV_Qhvs?6Y#4qsDukt5s znye(Cdl_cl?0avX=iYsvzyijKwyc!sHdT;}{G2rFnt1j^+a1X@J(A3I0k>fGz%H+F zT5!ck&EabS$z*IW@?!n6;gFX`>yqvp_ukK{osT~&frJN8lu6hM}%-pBd_^uk%Xa}9`as!qwYPU d$pXFjpboZYWf$;As)jQ4!R6(mlJY&A{QxF1nKS?Z literal 0 HcmV?d00001 diff --git a/onnxruntime/test/testdata/ort_github_issue_17000.ort b/onnxruntime/test/testdata/ort_github_issue_17000.ort new file mode 100644 index 0000000000000000000000000000000000000000..08d9826dd5346d1187b6ad907ecc368d3c040579 GIT binary patch literal 3392 zcmai$O=whC6vyw_sU1HWQaebHAVGp;Aw#4sNXSC0QiYTTwIW3rrjwh@E0Z_z%`-_Q zP#5A#M2f58LP{wvO3A`S>7tY(C5Q_tBBi*HBE@AFHkE!&|G#(dbzUBY9_GwF_k5po z&wcL=8Dri#e)4TCX1f^_Rz#JxvNvXMlQGqojoJHKuUF0_#ys@0sgf~m(W{bGy-ex` zdR(|!IB{%tV*hK#+?NiX@hLNN$glXk{!g#>v>-o~li!h??a=zhI3r>n5bh98oUmul z*|=$^Y&@5?Cid?)=BV_C6^t0fWZl_wzGfs_^SmG%m29ODV^TWQvrRbhR$?2eO(^Iy z>4kcz`K00nKj(dnwVV#JlR|U{h2*jzJ^0tPJ}tyOE&j0Z72)&RjwFq)%}O()zxU(g z4;%pFMTKtC|rrlvPj8aLd__7)lxV=+~|4 z^A@8>4kmS0w^0>uSV-?)5Mnbfr5lp5pVDx&UhKTQjZv9%+vpUR6#+nu?Mi(W;QyJX7niuh!iHSL=-A*M+x) zoLj)dq63f@KCe7;XOfEk$v{;1$-brzR_c)Z^-jNJ+|Go_4hlS zcD-|sI3f0b$*^T^olVk+n=SdJx_(@;BG!nIUM%Zys%BJs*NuEH4oC)0XFC0JeWE0W z#M93l6n#@GKG{!ezowNl`uMN^KiBipd0o07Hr($?;TLM!V>t}xx?FM381w=lQ@H1N zO$>Nf#k@;#1=#bqyMIICb0*lLAAo%xc1k~=8#MV1%6LqOKK*{EIVbmh!T`tuLO@UQ zSAYkqKc7!g<&NazvH?ehA&1NA4Sgd18O1-W^{6mFu{YmJ0PlkOvZ~yfpYRCH9y8h( zA#0%$x+}poA^v4*F8%4|o|=E~w8+w$Xr2q`%!+0O$T7A{^{zczZ`;^>?``PU4%Jsd zN$Gv!ZT9wgr7y*!C(N+3?{@2**kUl?^_Z3S`Zk+>_I~0%ue9#{9P+%D?62hH7e$9V zyGL}$J@Gm#>i0J}{ad0#t;civhwVn|MAWpzXo*q{XRnc!wzk6ia zHqWI|C!S6^i_wft#5M!MaL?SQ3S0-oq0VqGxNl<$zw3p>N4Rf?^d9r~ zrF1}anAr!?K^LG;8+)g33U7yxOF17aqQe~T5*>WN3wbSi{ZZxiUC#C^(INh+oGoWH zkiF^3*;&ya<|3ToHdVzLa?XMhGt0SBSIFh_oImCz;*ew&zE`nFe^vO4bg9{F+ANK7 z_6_8GZ_VOt@$oX965mNqC!8g-XNMnhq6X0SUX+878@}ir-=^I(YPSp1 z8|3<2$@&__?%jV;0^XVXgwG_qExan65snM-c~fVLJ;0uvI>}sZ;ls3(G+Q-&qsp5x z_X>Ku#A|luYx7Yv>2y1@Y0WM#*d*dYHQL5pP~U=I?D5S#-e9BsSWXWe`qhYPoj6{u zEhKg^vP-ozX+&`p&q-uX7xc+D=;!q+S+_ZPV_wD312O1zmPfZ1#rg)18|_-$T56)g w-088H0lLgW@ISA*ek{bt_lBBk`ZAKMiaO~g&)riT;*az8{$OQ5pWYDu3;3K1)c^nh literal 0 HcmV?d00001 diff --git a/onnxruntime/test/testdata/ort_github_issue_17000.py b/onnxruntime/test/testdata/ort_github_issue_17000.py new file mode 100644 index 0000000000..43c10f5590 --- /dev/null +++ b/onnxruntime/test/testdata/ort_github_issue_17000.py @@ -0,0 +1,77 @@ +import numpy as np +import onnx +from onnx import TensorProto, helper, numpy_helper + + +def order_repeated_field(repeated_proto, key_name, order): + order = list(order) + repeated_proto.sort(key=lambda x: order.index(getattr(x, key_name))) + + +def make_node(op_type, inputs, outputs, name=None, doc_string=None, domain=None, **kwargs): + node = helper.make_node(op_type, inputs, outputs, name, doc_string, domain, **kwargs) + if doc_string == "": + node.doc_string = "" + order_repeated_field(node.attribute, "name", kwargs.keys()) + return node + + +def make_graph(*args, doc_string=None, **kwargs): + graph = helper.make_graph(*args, doc_string=doc_string, **kwargs) + if doc_string == "": + graph.doc_string = "" + return graph + + +test_graph = make_graph( + name="test_graph", + # model input of a sequence type to test IsSparseTensor issue + inputs=[ + helper.make_tensor_sequence_value_info("seq_in", TensorProto.FLOAT, shape=None), + ], + outputs=[ + helper.make_tensor_value_info("still_has_elements", TensorProto.BOOL, shape=[]), + ], + initializer=[ + numpy_helper.from_array(np.array(0, dtype="int64"), name="i0"), + ], + nodes=[ + make_node("SequenceLength", inputs=["seq_in"], outputs=["seq_len"], name="get_seq_len"), + make_node("Greater", inputs=["seq_len", "i0"], outputs=["has_elements"], name="get_has_elements"), + # If node with one branch that has no nodes to test the allocation planner issue + # if sequence has elements: + # remove one + # output bool of whether it still has elements + # else: + # output false (gives us branch with no nodes) + make_node( + "If", + name="test_if", + inputs=["has_elements"], + outputs=["still_has_elements"], + then_branch=make_graph( + name="then", + inputs=[], + outputs=[helper.make_tensor_value_info("then_bool_out", TensorProto.BOOL, shape=[])], + nodes=[ + make_node("SequenceErase", inputs=["seq_in", "i0"], outputs=["seq_less_one"]), + make_node("SequenceLength", inputs=["seq_less_one"], outputs=["new_seq_len"]), + make_node("Greater", inputs=["new_seq_len", "i0"], outputs=["then_bool_out"]), + ], + ), + else_branch=make_graph( + name="else", + initializer=[numpy_helper.from_array(np.array(False, dtype="bool"), name="else_bool_out")], + inputs=[], + outputs=[helper.make_tensor_value_info("else_bool_out", TensorProto.BOOL, shape=[])], + nodes=[], + ), + ), + ], +) + +# Graph with Sequence operations and an If node that has a subgraph with no nodes +model = helper.make_model(opset_imports=[helper.make_operatorsetid("ai.onnx", 14)], ir_version=7, graph=test_graph) + +onnx.shape_inference.infer_shapes(model, strict_mode=True) +onnx.save(model, "ort_github_issue_17000.onnx") diff --git a/onnxruntime/test/testdata/required_ops.config b/onnxruntime/test/testdata/required_ops.config index ac9d46666e..e70362bab4 100644 --- a/onnxruntime/test/testdata/required_ops.config +++ b/onnxruntime/test/testdata/required_ops.config @@ -3,9 +3,9 @@ ai.onnx;7;Abs,Add,And,BatchNormalization,Concat,Conv,Dropout,Flatten,Foo,Gather, ai.onnx;8;Add,Conv,Flatten,Gemm,MatMul,MaxPool,Mul,Relu,Reshape ai.onnx;9;Abs,Add,BatchNormalization,Cast,Clip,Concat,Constant,ConstantOfShape,Conv,Div,Equal,Gather,Gemm,Identity,If,LayerNormalization,LeakyRelu,Loop,MatMul,Mul,Pow,ReduceMean,Relu,Reshape,Scan,Shape,Sigmoid,Slice,Softmax,Softsign,Sqrt,Squeeze,Sub,Tanh,Transpose,Unsqueeze ai.onnx;10;Add,Cast,Concat,ConstantOfShape,Div,Dropout,Erf,Expand,Gather,Greater,Identity,If,LayerNormalization,Loop,MatMul,Mul,Neg,NonZero,Pow,ReduceMean,ReduceSum,Shape,Sqrt,Squeeze,Sub,Tanh,Transpose,Unsqueeze -ai.onnx;11;Abs,Add,ArgMax,BatchNormalization,Cast,Clip,Concat,Constant,ConstantOfShape,Conv,Div,Equal,Exp,Expand,Flatten,Gather,Gemm,Identity,If,LayerNormalization,Log,Loop,MatMul,MatMulInteger,Max,Min,Mul,Neg,Pow,RandomUniform,Range,ReduceMean,ReduceSum,ReduceSumSquare,Relu,Reshape,Scan,SequenceConstruct,SequenceInsert,SequenceLength,Shape,Sigmoid,Slice,Softmax,Split,Sqrt,Squeeze,Sub,Sum,Tanh,Transpose,Unsqueeze,Where +ai.onnx;11;Abs,Add,ArgMax,BatchNormalization,Cast,Clip,Concat,Constant,ConstantOfShape,Conv,Div,Equal,Exp,Expand,Flatten,Gather,Gemm,Identity,If,LayerNormalization,Log,Loop,MatMul,MatMulInteger,Max,Min,Mul,Neg,Pow,RandomUniform,Range,ReduceMean,ReduceSum,ReduceSumSquare,Relu,Reshape,Scan,SequenceConstruct,SequenceErase,SequenceInsert,SequenceLength,Shape,Sigmoid,Slice,Softmax,Split,Sqrt,Squeeze,Sub,Sum,Tanh,Transpose,Unsqueeze,Where ai.onnx;12;Add,And,Cast,Concat,Constant,ConstantOfShape,Conv,CumSum,Div,Dropout,DynamicQuantizeLinear,Equal,Erf,Expand,Flatten,Gather,GatherND,Gemm,GlobalAveragePool,Greater,Identity,If,IsInf,LayerNormalization,Less,Loop,MatMul,MatMulInteger,Min,Mul,Not,Pad,Pow,RandomNormalLike,RandomUniform,ReduceMean,ReduceSum,Relu,Reshape,Shape,Slice,Softmax,SoftmaxCrossEntropyLoss,SparseSoftmaxCrossEntropy,Split,Sqrt,Squeeze,Sub,Tanh,Transpose,Unsqueeze,Where -ai.onnx;13;Abs,Add,Cast,Concat,ConstantOfShape,Conv,DequantizeLinear,DynamicQuantizeLinear,Equal,Expand,FooBar,FooBar_Attr,Gather,Identity,LayerNormalization,MatMul,MatMulInteger,Mul,Pad,Pow,QuantizeLinear,Range,ReduceSum,Reshape,Shape,Tanh,Transpose,Unsqueeze,Where +ai.onnx;13;Abs,Add,Cast,Concat,ConstantOfShape,Conv,DequantizeLinear,DynamicQuantizeLinear,Equal,Expand,FooBar,FooBar_Attr,Gather,Greater,Identity,If,LayerNormalization,MatMul,MatMulInteger,Mul,Pad,Pow,QuantizeLinear,Range,ReduceSum,Reshape,Shape,Tanh,Transpose,Unsqueeze,Where ai.onnx;14;Add,ArgMax,Cast,Conv,Identity,Relu,Sigmoid,Sub ai.onnx;314159;Add ai.onnx.contrib;1;StringLower diff --git a/onnxruntime/test/testdata/required_ops_and_types.config b/onnxruntime/test/testdata/required_ops_and_types.config index 17687906d7..41f3742147 100644 --- a/onnxruntime/test/testdata/required_ops_and_types.config +++ b/onnxruntime/test/testdata/required_ops_and_types.config @@ -1,9 +1,12 @@ # required ops and types for ORT format models in testdata -ai.onnx;1;Conv{"inputs": {"0": ["float"]}},Foo,Identity +ai.onnx;1;Conv{"inputs": {"0": ["float"]}} ai.onnx;5;Reshape ai.onnx;6;Relu{"inputs": {"0": ["float"]}} ai.onnx;7;Add{"inputs": {"0": ["float"]}},Gemm{"inputs": {"0": ["float"]}},Mul{"inputs": {"0": ["float"]}} ai.onnx;8;MaxPool{"inputs": {"0": ["float"]}},Sum{"inputs": {"0": ["float"]}} ai.onnx;9;Cast{"inputs": {"0": ["float"]}, "outputs": {"0": ["bool"]}} -ai.onnx;11;ArgMax{"inputs": {"0": ["float"]}},If,Loop +ai.onnx;10;QLinearConv{"inputs": {"0": ["uint8_t"]}} +ai.onnx;11;ArgMax{"inputs": {"0": ["float"]}},Clip{"inputs": {"0": ["float"]}},Conv{"inputs": {"0": ["float"]}},If,Loop,SequenceErase,SequenceLength +ai.onnx;13;DequantizeLinear{"inputs": {"0": ["int32_t", "uint8_t"]}},Greater{"inputs": {"0": ["int64_t"]}},If,QuantizeLinear{"outputs": {"0": ["uint8_t"]}} ai.onnx.ml;1;ArrayFeatureExtractor,LinearClassifier,Normalizer,ZipMap +test;1;Foo