From b92fc66ea14a310bbc5f31f872a9138cfddb8b21 Mon Sep 17 00:00:00 2001 From: Hariharan Seshadri Date: Wed, 11 Nov 2020 23:44:14 -0800 Subject: [PATCH] Support opset-13 specs of controlflow ops (Loop, If) (#5665) --- cmake/external/onnx_minimal.cmake | 2 +- .../onnxruntime/core/framework/data_types.h | 5 +- onnxruntime/core/framework/TensorSeq.h | 5 +- onnxruntime/core/framework/data_types.cc | 61 +++++-- .../core/providers/cpu/controlflow/if.cc | 92 +++++++--- .../core/providers/cpu/controlflow/loop.cc | 76 ++++++-- .../providers/cpu/cpu_execution_provider.cc | 12 +- .../core/providers/cpu/tensor/unsqueeze.cc | 11 +- .../core/providers/cpu/tensor/unsqueeze.h | 4 +- .../core/providers/cuda/controlflow/if.cc | 15 +- .../core/providers/cuda/controlflow/loop.cc | 17 +- .../providers/cuda/cuda_execution_provider.cc | 12 +- onnxruntime/test/onnx/TestCase.cc | 168 ++++++++++++------ onnxruntime/test/onnx/TestCase.h | 2 + onnxruntime/test/onnx/main.cc | 6 +- onnxruntime/test/onnx/onnx_model_info.cc | 2 +- onnxruntime/test/onnx/onnx_model_info.h | 6 + onnxruntime/test/onnx/pb_helper.h | 4 +- onnxruntime/test/perftest/TFModelInfo.h | 1 + .../test/providers/cpu/controlflow/if_test.cc | 59 ++++++ .../providers/cpu/controlflow/loop_test.cc | 116 +++++++++++- .../onnx_backend_test_series_filters.jsonc | 4 +- onnxruntime/test/util/compare_ortvalue.cc | 76 ++++++-- 23 files changed, 603 insertions(+), 153 deletions(-) diff --git a/cmake/external/onnx_minimal.cmake b/cmake/external/onnx_minimal.cmake index b69885a280..c691965daf 100644 --- a/cmake/external/onnx_minimal.cmake +++ b/cmake/external/onnx_minimal.cmake @@ -14,7 +14,7 @@ endif() set(ONNX_SOURCE_ROOT ${PROJECT_SOURCE_DIR}/external/onnx) -add_library(onnx_proto ${ONNX_SOURCE_ROOT}/onnx/onnx-ml.proto ${ONNX_SOURCE_ROOT}/onnx/onnx-operators-ml.proto) +add_library(onnx_proto ${ONNX_SOURCE_ROOT}/onnx/onnx-ml.proto ${ONNX_SOURCE_ROOT}/onnx/onnx-operators-ml.proto ${ONNX_SOURCE_ROOT}/onnx/onnx-data.proto) target_include_directories(onnx_proto PUBLIC $ "${CMAKE_CURRENT_BINARY_DIR}") target_compile_definitions(onnx_proto PUBLIC $) diff --git a/include/onnxruntime/core/framework/data_types.h b/include/onnxruntime/core/framework/data_types.h index 5f539cfc43..4a6da345ed 100644 --- a/include/onnxruntime/core/framework/data_types.h +++ b/include/onnxruntime/core/framework/data_types.h @@ -284,12 +284,15 @@ class DataTypeImpl { static MLDataType GetDataType(const std::string&); static const std::vector& AllTensorTypes(); - static const std::vector& AllSequenceTensorTypes(); static const std::vector& AllFixedSizeTensorTypes(); + static const std::vector& AllSequenceTensorTypes(); + static const std::vector& AllFixedSizeSequenceTensorTypes(); static const std::vector& AllNumericTensorTypes(); static const std::vector& AllIEEEFloatTensorTypes(); static const std::vector& AllFixedSizeTensorExceptHalfTypes(); static const std::vector& AllIEEEFloatTensorExceptHalfTypes(); + static const std::vector& AllTensorAndSequenceTensorTypes(); + static const std::vector& AllFixedSizeTensorAndSequenceTensorTypes(); }; std::ostream& operator<<(std::ostream& out, MLDataType data_type); diff --git a/onnxruntime/core/framework/TensorSeq.h b/onnxruntime/core/framework/TensorSeq.h index 8c649abeec..a17a0866c8 100644 --- a/onnxruntime/core/framework/TensorSeq.h +++ b/onnxruntime/core/framework/TensorSeq.h @@ -28,6 +28,9 @@ class TensorSeq { } void SetElements(std::vector&& tensors) { + // The caller of this method ensures that : + // (1) `elem_type` is set before invoking this method + // (2) All tensors contain elements of the same primitive data type assert(tensors_.empty()); tensors_ = std::move(tensors); } @@ -38,7 +41,7 @@ class TensorSeq { return elem_type_ == o.elem_type_; } - bool IsSameDataType (const Tensor& o) const noexcept { + bool IsSameDataType(const Tensor& o) const noexcept { return elem_type_ == o.DataType()->AsPrimitiveDataType(); } diff --git a/onnxruntime/core/framework/data_types.cc b/onnxruntime/core/framework/data_types.cc index fe445c0de7..c39fc8ce59 100644 --- a/onnxruntime/core/framework/data_types.cc +++ b/onnxruntime/core/framework/data_types.cc @@ -915,26 +915,16 @@ const std::vector& DataTypeImpl::AllFixedSizeTensorTypes() { const std::vector& DataTypeImpl::AllTensorTypes() { static std::vector all_tensor_types = - {DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType()}; - + []() { + auto temp = AllFixedSizeTensorTypes(); + temp.push_back(DataTypeImpl::GetTensorType()); + return temp; + }(); return all_tensor_types; } -const std::vector& DataTypeImpl::AllSequenceTensorTypes() { - static std::vector all_sequence_tensor_types = +const std::vector& DataTypeImpl::AllFixedSizeSequenceTensorTypes() { + static std::vector all_fixed_size_sequence_tensor_types = {DataTypeImpl::GetSequenceTensorType(), DataTypeImpl::GetSequenceTensorType(), DataTypeImpl::GetSequenceTensorType(), @@ -947,9 +937,18 @@ const std::vector& DataTypeImpl::AllSequenceTensorTypes() { DataTypeImpl::GetSequenceTensorType(), DataTypeImpl::GetSequenceTensorType(), DataTypeImpl::GetSequenceTensorType(), - DataTypeImpl::GetSequenceTensorType(), - DataTypeImpl::GetSequenceTensorType()}; + DataTypeImpl::GetSequenceTensorType()}; + return all_fixed_size_sequence_tensor_types; +} + +const std::vector& DataTypeImpl::AllSequenceTensorTypes() { + static std::vector all_sequence_tensor_types = + []() { + auto temp = AllFixedSizeSequenceTensorTypes(); + temp.push_back(DataTypeImpl::GetSequenceTensorType()); + return temp; + }(); return all_sequence_tensor_types; } @@ -971,6 +970,30 @@ const std::vector& DataTypeImpl::AllNumericTensorTypes() { return all_numeric_size_tensor_types; } +const std::vector& DataTypeImpl::AllFixedSizeTensorAndSequenceTensorTypes() { + static std::vector all_fixed_size_tensor_and_sequence_tensor_types = + []() { + auto temp = AllFixedSizeTensorTypes(); + const auto& seq = AllFixedSizeSequenceTensorTypes(); + temp.insert(temp.end(), seq.begin(), seq.end()); + return temp; + }(); + + return all_fixed_size_tensor_and_sequence_tensor_types; +} + +const std::vector& DataTypeImpl::AllTensorAndSequenceTensorTypes() { + static std::vector all_tensor_and_sequence_types = + []() { + auto temp = AllTensorTypes(); + const auto& seq = AllSequenceTensorTypes(); + temp.insert(temp.end(), seq.begin(), seq.end()); + return temp; + }(); + + return all_tensor_and_sequence_types; +} + // helper to stream. expected to only be used for error output, so any typeid lookup // cost should be fine. alternative would be to add a static string field to DataTypeImpl // that we set in the register macro to the type name, and output that instead. diff --git a/onnxruntime/core/providers/cpu/controlflow/if.cc b/onnxruntime/core/providers/cpu/controlflow/if.cc index 51b9a40d13..f3d92c0192 100644 --- a/onnxruntime/core/providers/cpu/controlflow/if.cc +++ b/onnxruntime/core/providers/cpu/controlflow/if.cc @@ -19,7 +19,7 @@ namespace onnxruntime { /* ONNX_OPERATOR_SET_SCHEMA( If, - 1, + 13, OpSchema() .SetDoc("If conditional") .Input(0, "cond", "Condition for the if", "B") @@ -27,10 +27,24 @@ ONNX_OPERATOR_SET_SCHEMA( 0, "outputs", "Values that are live-out to the enclosing scope. The return values in " - "the `then_branch` and `else_branch` must be of the same shape and same " - "data type.", + "the `then_branch` and `else_branch` must be of the same data type. " + "The `then_branch` and `else_branch` may produce tensors with the same " + "element type and different shapes. " + "If corresponding outputs from the then-branch and the else-branch have " + "static shapes S1 and S2, then the shape of the corresponding output " + "variable of the if-node (if present) must be compatible with both S1 " + "and S2 as it represents the union of both possible shapes." + "For example, if in a model file, the the first " + "output of `then_branch` is typed float tensor with shape [2] and the " + "first output of `else_branch` is another float tensor with shape [3], " + "If's first output should have (a) no shape set, or (b) " + "a shape of rank 1 with neither `dim_value` nor `dim_param` set, or (c) " + "a shape of rank 1 with a unique `dim_param`. " + "In contrast, the first output cannot have the shape [2] since [2] and " + "[3] are not compatible.", "V", - OpSchema::Variadic) + OpSchema::Variadic, + false) .Attr( "then_branch", "Graph to run if condition is true. Has N outputs: values you wish to " @@ -43,8 +57,17 @@ ONNX_OPERATOR_SET_SCHEMA( " be live-out to the enclosing scope. The number of outputs must match" " the number of outputs in the then_branch.", AttributeProto::GRAPH) - .TypeConstraint("V", OpSchema::all_tensor_types(), "All Tensor types") - .TypeConstraint("B", {"tensor(bool)"}, "Only bool")); + .TypeConstraint( + "V", + [](){ + auto t = OpSchema::all_tensor_types(); + auto s = OpSchema::all_tensor_sequence_types(); + t.insert(t.end(), s.begin(), s.end()); + return t; + }(), + "All Tensor and Sequence types") + .TypeConstraint("B", {"tensor(bool)"}, "Only bool") + .TypeAndShapeInferenceFunction(IfInferenceFunction)); */ ONNX_CPU_OPERATOR_VERSIONED_KERNEL(If, @@ -56,11 +79,19 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL(If, // output shape rules requiring the output shapes of the 'THEN' and 'ELSE' // branches to be the same were relaxed in opset-11 +ONNX_CPU_OPERATOR_VERSIONED_KERNEL(If, + 11, 12, + KernelDefBuilder() + .TypeConstraint("B", DataTypeImpl::GetTensorType()) + .TypeConstraint("V", DataTypeImpl::AllTensorTypes()), + If); + +// sequence tensors were also supported in addition to existing support for tensors in opset-13 ONNX_CPU_OPERATOR_KERNEL(If, - 11, + 13, KernelDefBuilder() .TypeConstraint("B", DataTypeImpl::GetTensorType()) - .TypeConstraint("V", DataTypeImpl::AllTensorTypes()), + .TypeConstraint("V", DataTypeImpl::AllTensorAndSequenceTensorTypes()), If); struct If::Info { @@ -247,28 +278,40 @@ Status IfImpl::AllocateOutputTensors() { int index = 0; for (auto& graph_output : info_.subgraph.GetOutputs()) { - auto* graph_output_shape = graph_output->Shape(); - bool symbolic_dim_in_shape = false; + const auto* graph_output_type = graph_output->TypeAsProto(); - if (graph_output_shape) { - TensorShape output_shape = onnxruntime::utils::GetTensorShapeFromTensorShapeProto(*graph_output_shape); + if (graph_output_type->has_tensor_type()) { + auto* graph_output_shape = graph_output->Shape(); + bool symbolic_dim_in_shape = false; - // if size < 0 we have a symbolic dimension and need to use a temporary OrtValue in the subgraph execution - if (output_shape.Size() < 0) { - symbolic_dim_in_shape = true; - } else { - auto* tensor = context_.Output(index, output_shape); + if (graph_output_shape) { + TensorShape output_shape = onnxruntime::utils::GetTensorShapeFromTensorShapeProto(*graph_output_shape); - if (!tensor) - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for ", graph_output->Name()); + // if size < 0 we have a symbolic dimension and need to use a temporary OrtValue in the subgraph execution + if (output_shape.Size() < 0) { + symbolic_dim_in_shape = true; + } else { + auto* tensor = context_.Output(index, output_shape); - outputs_.push_back({AllocationType::IfOutput, *context_.GetOutputMLValue(index)}); + if (!tensor) + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for ", graph_output->Name()); + + outputs_.push_back({AllocationType::IfOutput, *context_.GetOutputMLValue(index)}); + } } - } - if (!graph_output_shape || symbolic_dim_in_shape) { - // we still need a value to put in the feeds we give to the execution frame, so just use an empty MLValue - outputs_.push_back({AllocationType::Delayed, {}}); + if (!graph_output_shape || symbolic_dim_in_shape) { + // we still need a value to put in the feeds we give to the execution frame, so just use an empty MLValue + outputs_.push_back({AllocationType::Delayed, {}}); + } + } else if (graph_output_type->has_sequence_type()) { + auto* seq_tensor = context_.Output(index); + if (!seq_tensor) + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for ", graph_output->Name()); + outputs_.push_back({AllocationType::IfOutput, *context_.GetOutputMLValue(index)}); + } else { + // Shouldn't hit this + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Only tensors or sequence of tensors are suppported"); } ++index; @@ -311,6 +354,7 @@ Status IfImpl::Execute(const FeedsFetchesManager& ffm) { // we don't update the provided OrtValue and return false for 'allocated'. // the execution frame will allocate a buffer on the required device, and the fetches copy // logic in utils::ExecuteSubgraph will handle moving it into the tensor we allocated here. + auto* tensor = context_.Output(i, shape); if (!tensor) return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for If output ", i); diff --git a/onnxruntime/core/providers/cpu/controlflow/loop.cc b/onnxruntime/core/providers/cpu/controlflow/loop.cc index 70259394c7..f352f28746 100644 --- a/onnxruntime/core/providers/cpu/controlflow/loop.cc +++ b/onnxruntime/core/providers/cpu/controlflow/loop.cc @@ -20,6 +20,7 @@ #include "core/framework/utils.h" #include "core/providers/cpu/tensor/utils.h" #include "core/framework/session_options.h" +#include "core/framework/TensorSeq.h" #include "gsl/gsl" @@ -34,9 +35,9 @@ namespace onnxruntime { /* ONNX_OPERATOR_SET_SCHEMA( Loop, - 1, + 13, OpSchema() - .SetDoc(Loop_ver1_doc) + .SetDoc(Loop_ver13_doc) .Input( 0, "M", @@ -56,13 +57,17 @@ ONNX_OPERATOR_SET_SCHEMA( "The initial values of any loop-carried dependencies (values that " "change across loop iterations)", "V", - OpSchema::Variadic) + OpSchema::Variadic, + false, + 0) .Output( 0, "v_final_and_scan_outputs", - "Final N loop carried dependency values then K scan_outputs", + "Final N loop carried dependency values then K scan_outputs. " + "Scan outputs must be Tensors.", "V", - OpSchema::Variadic) + OpSchema::Variadic, + false) .Attr( "body", "The graph run each iteration. It has 2+N inputs: (iteration_num, " @@ -73,9 +78,23 @@ ONNX_OPERATOR_SET_SCHEMA( " if the dimensions or data type of these scan_outputs change across loop" " iterations.", AttributeProto::GRAPH) - .TypeConstraint("V", OpSchema::all_tensor_types(), "All Tensor types") - .TypeConstraint("I", {"int64"}, "Only int64") - .TypeConstraint("B", {"bool"}, "Only bool") + .TypeConstraint( + "V", + []() { + auto t = OpSchema::all_tensor_types(); + auto s = OpSchema::all_tensor_sequence_types(); + t.insert(t.end(), s.begin(), s.end()); + return t; + }(), + "All Tensor and Sequence types") + .TypeConstraint( + "I", + {"tensor(int64)"}, + "tensor of int64, which should be a scalar.") + .TypeConstraint( + "B", + {"tensor(bool)"}, + "tensor of bool, which should be a scalar.") .TypeAndShapeInferenceFunction(LoopInferenceFunction)); */ @@ -87,12 +106,20 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL(Loop, .TypeConstraint("V", DataTypeImpl::AllTensorTypes()), Loop); +ONNX_CPU_OPERATOR_VERSIONED_KERNEL(Loop, + 11, 12, + KernelDefBuilder() + .TypeConstraint("I", DataTypeImpl::GetTensorType()) + .TypeConstraint("B", DataTypeImpl::GetTensorType()) + .TypeConstraint("V", DataTypeImpl::AllTensorTypes()), + Loop); + ONNX_CPU_OPERATOR_KERNEL(Loop, - 11, + 13, KernelDefBuilder() .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("B", DataTypeImpl::GetTensorType()) - .TypeConstraint("V", DataTypeImpl::AllTensorTypes()), + .TypeConstraint("V", DataTypeImpl::AllTensorAndSequenceTensorTypes()), Loop); struct Loop::Info { @@ -411,6 +438,7 @@ void LoopImpl::SaveOutputsAndUpdateFeeds(const std::vector& last_outpu // save loop outputs as we have to concatenate at the end for (int j = info_.num_loop_carried_vars; j < info_.num_outputs; ++j) { + ORT_ENFORCE(last_outputs[j + 1].IsTensor(), "All scan outputs MUST be tensors"); loop_output_tensors_[j - info_.num_loop_carried_vars].push_back(last_outputs[j + 1]); // skip 'cond' in output } } @@ -463,9 +491,31 @@ Status LoopImpl::Execute(const FeedsFetchesManager& ffm) { // As the loop carried variables may change shape across iterations there's no way to avoid a copy // as we need the final shape. auto copy_tensor_from_mlvalue_to_output = [this](const OrtValue& input, int output_idx) { - auto& data = input.Get(); - Tensor* output = context_.Output(output_idx, data.Shape()); - session_state_.GetDataTransferMgr().CopyTensor(input.Get(), *output); + auto type = input.Type(); + if (type == DataTypeImpl::GetType()) { + auto& data = input.Get(); + Tensor* output = context_.Output(output_idx, data.Shape()); + session_state_.GetDataTransferMgr().CopyTensor(input.Get(), *output); + } else if (type == DataTypeImpl::GetType()) { + std::vector tensors; + + auto& data = input.Get(); + TensorSeq* output = context_.Output(output_idx); + output->SetType(data.DataType()); + + AllocatorPtr alloc; + auto status = context_.GetTempSpaceAllocator(&alloc); + if (!status.IsOK()) { + ORT_THROW("Unable to get an allocator"); + } + for (auto it = data.begin(), end = data.end(); it != end; ++it) { + Tensor tmp(it->DataType(), onnxruntime::TensorShape(it->Shape()), alloc); + session_state_.GetDataTransferMgr().CopyTensor(*it, tmp); + tensors.push_back(std::move(tmp)); + } + + output->SetElements(std::move(tensors)); + } }; // copy to Loop output diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 3394a8f2e0..d5c0aaaa12 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -350,7 +350,7 @@ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOn class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, LogSoftmax); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float, Softmax); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, Softmax); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Loop); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, Loop); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, DepthToSpace); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Scan); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, Flatten); @@ -369,7 +369,7 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Ma class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, LpPool); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Conv); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ConvTranspose); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, If); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, If); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceLength); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceAt); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceEmpty); @@ -594,6 +594,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Resize); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, int32_t, Resize); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, uint8_t, Resize); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Loop); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, If); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Hardmax); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, LogSoftmax); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double, LogSoftmax); @@ -1115,6 +1117,7 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { float, ArgMin)>, BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, @@ -1124,7 +1127,6 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { Softmax)>, BuildKernelCreateInfo, - BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, @@ -1144,7 +1146,7 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, @@ -1558,6 +1560,8 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { int32_t, Resize)>, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc b/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc index 436f553eb8..930385406c 100644 --- a/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc +++ b/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc @@ -43,14 +43,15 @@ Status UnsqueezeBase::PrepareCompute(OpKernelContext* ctx, Prepare& p) const { std::vector axes; size_t num_inputs = ctx->InputCount(); - if (num_inputs == 2) { //axes is an input + if (num_inputs == 2) { //axes is an input const Tensor* axes_tensor = ctx->Input(1); ORT_ENFORCE(axes_tensor != nullptr, "Axes input is null"); - ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 1, - "An axes tensor must be a vector tensor."); - auto nDims = static_cast(axes_tensor->Shape()[0]); + ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 0 || + axes_tensor->Shape().NumDimensions() == 1, + "An axes tensor must be a scalar or a 1-D tensor."); + auto num_axes_elements = static_cast(axes_tensor->Shape().Size()); const auto* data = axes_tensor->template Data(); - axes.assign(data, data + nDims); + axes.assign(data, data + num_axes_elements); } else { axes.assign(axes_.begin(), axes_.end()); } diff --git a/onnxruntime/core/providers/cpu/tensor/unsqueeze.h b/onnxruntime/core/providers/cpu/tensor/unsqueeze.h index f5d18426c3..4e9cfad137 100644 --- a/onnxruntime/core/providers/cpu/tensor/unsqueeze.h +++ b/onnxruntime/core/providers/cpu/tensor/unsqueeze.h @@ -11,8 +11,8 @@ namespace onnxruntime { class UnsqueezeBase { protected: UnsqueezeBase(const OpKernelInfo& info) { - size_t numInputs = info.GetInputCount(); - if (numInputs == 1) { //axes must be a valid attribute + size_t num_inputs = info.GetInputCount(); + if (num_inputs == 1) { //axes must be a valid attribute ORT_ENFORCE(info.GetAttrs("axes", axes_).IsOK(), "Missing/Invalid 'axes' attribute value"); } } diff --git a/onnxruntime/core/providers/cuda/controlflow/if.cc b/onnxruntime/core/providers/cuda/controlflow/if.cc index 5898fe8d6a..5b074b49ac 100644 --- a/onnxruntime/core/providers/cuda/controlflow/if.cc +++ b/onnxruntime/core/providers/cuda/controlflow/if.cc @@ -23,9 +23,22 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(If, // output shape rules requiring the output shapes of the 'THEN' and 'ELSE' // branches to be the same were relaxed in opset-11 +ONNX_OPERATOR_VERSIONED_KERNEL_EX(If, + kOnnxDomain, + 11, 12, + kCudaExecutionProvider, + KernelDefBuilder() + .InputMemoryType(0) // 'cond' needs to be on CPU + .TypeConstraint("B", DataTypeImpl::GetTensorType()) + .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), + If); + +// sequence tensors were also supported in addition to existing support for tensors in opset-13, +// but we do not support sequence tensors in the cuda If kernel because there are no ops that handle +// sequence tensors on CUDA and supporting it for If doesn't add value while that is the case ONNX_OPERATOR_KERNEL_EX(If, kOnnxDomain, - 11, + 13, kCudaExecutionProvider, KernelDefBuilder() .InputMemoryType(0) // 'cond' needs to be on CPU diff --git a/onnxruntime/core/providers/cuda/controlflow/loop.cc b/onnxruntime/core/providers/cuda/controlflow/loop.cc index 4eece0562c..9b6e689956 100644 --- a/onnxruntime/core/providers/cuda/controlflow/loop.cc +++ b/onnxruntime/core/providers/cuda/controlflow/loop.cc @@ -23,9 +23,24 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(Loop, Loop); // zero variadic argument support was added in opset 11. using same implementation as for previous version +ONNX_OPERATOR_VERSIONED_KERNEL_EX(Loop, + kOnnxDomain, + 11, 12, + kCudaExecutionProvider, + KernelDefBuilder() + .InputMemoryType(0) // 'M' needs to be on CPU + .InputMemoryType(1) // 'cond' needs to be on CPU + .TypeConstraint("I", DataTypeImpl::GetTensorType()) + .TypeConstraint("B", DataTypeImpl::GetTensorType()) + .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), + Loop); + +// sequence tensors were also supported in addition to existing support for tensors in opset-13, +// but we do not support sequence tensors in the cuda Loop kernel because there are no ops that handle +// sequence tensors on CUDA and supporting it for Loop doesn't add value while that is the case ONNX_OPERATOR_KERNEL_EX(Loop, kOnnxDomain, - 11, + 13, kCudaExecutionProvider, KernelDefBuilder() .InputMemoryType(0) // 'M' needs to be on CPU diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc index 02ea9a8248..2a417d55d9 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc @@ -685,8 +685,8 @@ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDom class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, float, Gemm); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, double, Gemm); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, MLFloat16, Gemm); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, If); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, Loop); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, If); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, Loop); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, NonMaxSuppression); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, Range); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, float, ReduceL1); @@ -1000,6 +1000,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, MLFloat16, Resize); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, int32_t, Resize); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, uint8_t, Resize); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, If); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, Loop); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, Flatten); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, float, LRN); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, double, LRN); @@ -1381,8 +1383,8 @@ static Status RegisterCudaKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, @@ -1692,6 +1694,8 @@ static Status RegisterCudaKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index bb26a99363..be546422ad 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -19,6 +19,7 @@ #include "core/platform/path_lib.h" #include "core/session/onnxruntime_cxx_api.h" #include "core/framework/allocator.h" +#include "core/framework/TensorSeq.h" #include "re2/re2.h" #include @@ -131,7 +132,7 @@ static int ExtractFileNo(const std::basic_string& name) { } using PATH_STRING_TYPE = std::basic_string; -static void SortTensorFileNames(std::vector>& input_pb_files) { +static void SortFileNames(std::vector>& input_pb_files) { if (input_pb_files.size() <= 1) return; std::sort(input_pb_files.begin(), input_pb_files.end(), [](const std::basic_string& left, const std::basic_string& right) -> bool { @@ -299,10 +300,15 @@ class OnnxTestCase : public ITestCase { return std::string(); } - void ConvertTestData(const std::vector& test_data_pbs, - onnxruntime::test::HeapBuffer& b, bool is_input, + void ConvertTestData(const ONNX_NAMESPACE::TensorProto& test_data_pb, + onnxruntime::test::HeapBuffer& b, + bool is_input, size_t i, std::unordered_map& out) const; + void ConvertTestData(const ONNX_NAMESPACE::SequenceProto& test_data_pb, + onnxruntime::test::HeapBuffer& b, + bool is_input, size_t i, + std::unordered_map& out) const; std::once_flag model_parsed_; std::once_flag config_parsed_; double per_sample_tolerance_; @@ -318,6 +324,10 @@ class OnnxTestCase : public ITestCase { void GetRelativePerSampleTolerance(double* value) const override; void GetPostProcessing(bool* value) const override; + const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t i) const override { + return model_info_->GetInputInfoFromModel(i); + } + const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const override { return model_info_->GetOutputInfoFromModel(i); } @@ -403,21 +413,31 @@ static bool read_config_file(const std::basic_string& path, std: //load tensors from disk template -static void LoadTensors(const std::vector& pb_files, - std::vector* input_pbs) { - for (size_t i = 0; i != pb_files.size(); ++i) { - int tensor_fd; - auto st = Env::Default().FileOpenRd(pb_files.at(i), tensor_fd); - if (!st.IsOK()) { - ORT_THROW("open file '", ToMBString(pb_files.at(i)), "' failed:", st.ErrorMessage()); - } - google::protobuf::io::FileInputStream f(tensor_fd, protobuf_block_size_in_bytes); - f.SetCloseOnDelete(true); - ONNX_NAMESPACE::TensorProto tensor; - if (!tensor.ParseFromZeroCopyStream(&f)) { - ORT_THROW("parse file '", ToMBString(pb_files.at(i)), "' failed"); - } - input_pbs->emplace_back(tensor); +static void LoadTensor(const PATH_STRING_TYPE& pb_file, ONNX_NAMESPACE::TensorProto& input_pb) { + int tensor_fd; + auto st = Env::Default().FileOpenRd(pb_file, tensor_fd); + if (!st.IsOK()) { + ORT_THROW("open file '", ToMBString(pb_file), "' failed:", st.ErrorMessage()); + } + google::protobuf::io::FileInputStream f(tensor_fd, protobuf_block_size_in_bytes); + f.SetCloseOnDelete(true); + if (!input_pb.ParseFromZeroCopyStream(&f)) { + ORT_THROW("parse file '", ToMBString(pb_file), "' failed"); + } +} + +//load sequence tensors from disk +template +static void LoadSequenceTensor(const PATH_STRING_TYPE& pb_file, ONNX_NAMESPACE::SequenceProto& input_pb) { + int tensor_fd; + auto st = Env::Default().FileOpenRd(pb_file, tensor_fd); + if (!st.IsOK()) { + ORT_THROW("open file '", ToMBString(pb_file), "' failed:", st.ErrorMessage()); + } + google::protobuf::io::FileInputStream f(tensor_fd, protobuf_block_size_in_bytes); + f.SetCloseOnDelete(true); + if (!input_pb.ParseFromZeroCopyStream(&f)) { + ORT_THROW("parse file '", ToMBString(pb_file), "' failed"); } } @@ -457,6 +477,7 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, } std::vector test_data_pb_files; + const PATH_STRING_TYPE& dir_path = test_data_dirs_[id]; LoopDir(dir_path, [&test_data_pb_files, &dir_path, is_input](const PATH_CHAR_TYPE* filename, OrtFileType f_type) -> bool { @@ -473,41 +494,79 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, return true; }); - SortTensorFileNames(test_data_pb_files); + SortFileNames(test_data_pb_files); - std::vector test_data_pbs; - LoadTensors(test_data_pb_files, &test_data_pbs); - ConvertTestData(test_data_pbs, b, is_input, name_data_map); + for (size_t i = 0; i < test_data_pb_files.size(); ++i) { + const ONNX_NAMESPACE::ValueInfoProto* value_info_proto = is_input ? model_info_->GetInputInfoFromModel(i) : model_info_->GetOutputInfoFromModel(i); + if (!value_info_proto->has_type()) { + ORT_THROW("Model ", is_input ? "input " : "output ", i, " is missing type info"); + } + + if (value_info_proto->type().has_tensor_type()) { + ONNX_NAMESPACE::TensorProto test_pb; + LoadTensor(test_data_pb_files[i], test_pb); + ConvertTestData(test_pb, b, is_input, i, name_data_map); + } else if (value_info_proto->type().has_sequence_type()) { + ONNX_NAMESPACE::SequenceProto test_pb; + LoadSequenceTensor(test_data_pb_files[i], test_pb); + ConvertTestData(test_pb, b, is_input, i, name_data_map); + } else { + ORT_THROW("Unsupported type for the ", is_input ? "input " : "output ", i, " in the test runner"); + } + } } -void OnnxTestCase::ConvertTestData(const std::vector& test_data_pbs, +void OnnxTestCase::ConvertTestData(const ONNX_NAMESPACE::TensorProto& test_data_pb, onnxruntime::test::HeapBuffer& b, - bool is_input, std::unordered_map& out) const { - bool has_valid_names = true; - std::vector var_names(test_data_pbs.size()); - for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) { - std::string name = test_data_pbs[input_index].name(); - if (name.empty()) { - has_valid_names = false; - break; - } - var_names[input_index] = name; - } - if (!has_valid_names) { - size_t count = static_cast(is_input ? model_info_->GetInputCount() : model_info_->GetOutputCount()); - if (count != test_data_pbs.size()) { - ORT_THROW("data count mismatch, expect ", count, ", got ", test_data_pbs.size()); - } - for (size_t i = 0; i != count; ++i) { - var_names[i] = is_input ? model_info_->GetInputName(i) : model_info_->GetOutputName(i); - } - } - for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) { - std::string name = var_names[input_index]; - const ONNX_NAMESPACE::TensorProto& input = test_data_pbs[input_index]; - size_t len = 0; + bool is_input, size_t i, + std::unordered_map& out) const { + const std::string& name = test_data_pb.name(); + const std::string& name_finalized = !name.empty() + ? name + : (is_input ? model_info_->GetInputName(i) : model_info_->GetOutputName(i)); - auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(input, &len); + size_t len = 0; + + auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(test_data_pb, &len); + if (!status.IsOK()) { + ORT_THROW(status.ToString()); + } + void* p = len == 0 ? nullptr : b.AllocMemory(len); + Ort::Value v1{nullptr}; + onnxruntime::test::OrtCallback d; + OrtMemoryInfo cpu_memory_info(onnxruntime::CPU, OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeDefault); + status = onnxruntime::test::TensorProtoToMLValue(test_data_pb, onnxruntime::test::MemBuffer(p, len, cpu_memory_info), + v1, d); + if (!status.IsOK()) { + ORT_THROW(status.ToString()); + } + if (d.f) { + b.AddDeleter(d); + } + out.emplace(name_finalized, std::move(v1)); +} + +void OnnxTestCase::ConvertTestData(const ONNX_NAMESPACE::SequenceProto& test_data_pb, + onnxruntime::test::HeapBuffer& b, + bool is_input, size_t i, + std::unordered_map& out) const { + const std::string& name = test_data_pb.name(); + const std::string& name_finalized = !name.empty() + ? name + : (is_input ? model_info_->GetInputName(i) : model_info_->GetOutputName(i)); + + size_t len = 0; + + std::vector seq; + if (test_data_pb.elem_type() != ONNX_NAMESPACE::SequenceProto_DataType_TENSOR) { + ORT_THROW("Only parsing a sequence of tensors is currently supported"); + } + const auto& tensors = test_data_pb.tensor_values(); + const size_t val = tensors.size(); + seq.reserve(val); + + for (auto it = tensors.cbegin(); it != tensors.cend(); ++it) { + auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(*it, &len); if (!status.IsOK()) { ORT_THROW(status.ToString()); } @@ -515,7 +574,7 @@ void OnnxTestCase::ConvertTestData(const std::vector::max(), 0, - true - }; + true}; Ort::ThrowOnError(sf.OrtSessionOptionsAppendExecutionProvider_CUDA(sf, &cuda_options)); #else fprintf(stderr, "CUDA is not supported in this build"); @@ -555,8 +554,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) { {"cast_BFLOAT16_to_FLOAT", "onnx generate bfloat tensor as uint16 type", {}}, {"sequence_insert_at_back", "onnx currently not supporting loading segment", {}}, {"sequence_insert_at_front", "onnx currently not supporting loading segment", {}}, - {"if_seq", "NOT_IMPLEMENTED : Could not find an implementation for the node If(13)", {}}, - {"loop13_seq", "NOT_IMPLEMENTED : Could not find an implementation for the node Loop(13)", {}}, + {"loop13_seq", "ORT api does not currently support creating empty sequences (needed for this test)", {}}, }; #ifdef DISABLE_ML_OPS diff --git a/onnxruntime/test/onnx/onnx_model_info.cc b/onnxruntime/test/onnx/onnx_model_info.cc index 600978e0ca..0352783ec1 100644 --- a/onnxruntime/test/onnx/onnx_model_info.cc +++ b/onnxruntime/test/onnx/onnx_model_info.cc @@ -160,7 +160,7 @@ void OnnxModelInfo::InitOrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { } // Load input and output node args auto add_node_args = [&](const flatbuffers::Vector>* fbs_node_args, - std::vector node_args) -> Status { + std::vector& node_args) -> Status { if (fbs_node_args != nullptr) { node_args.reserve(fbs_node_args->size()); for (const auto* fbs_node_arg_name : *fbs_node_args) { diff --git a/onnxruntime/test/onnx/onnx_model_info.h b/onnxruntime/test/onnx/onnx_model_info.h index a8a048a7e7..aaf8ba8125 100644 --- a/onnxruntime/test/onnx/onnx_model_info.h +++ b/onnxruntime/test/onnx/onnx_model_info.h @@ -37,9 +37,15 @@ class OnnxModelInfo : public TestModelInfo { std::string GetModelVersion() const override { return onnx_commit_tag_; } const std::string& GetNodeName() const override { return node_name_; } + + const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t i) const override { + return &input_value_info_[i]; + } + const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const override { return &output_value_info_[i]; } + int GetInputCount() const override { return static_cast(input_value_info_.size()); } int GetOutputCount() const override { return static_cast(output_value_info_.size()); } const std::string& GetInputName(size_t i) const override { return input_value_info_[i].name(); } diff --git a/onnxruntime/test/onnx/pb_helper.h b/onnxruntime/test/onnx/pb_helper.h index 1259c50dd2..0adddde80c 100644 --- a/onnxruntime/test/onnx/pb_helper.h +++ b/onnxruntime/test/onnx/pb_helper.h @@ -42,7 +42,7 @@ #include #include #include -#include +#include #include "tml.pb.h" #ifdef __GNUC__ #pragma GCC diagnostic pop @@ -53,4 +53,4 @@ namespace onnxruntime { bool ParseDelimitedFromCodedStream(google::protobuf::MessageLite* message, google::protobuf::io::CodedInputStream* input, bool* clean_eof); -} \ No newline at end of file +} diff --git a/onnxruntime/test/perftest/TFModelInfo.h b/onnxruntime/test/perftest/TFModelInfo.h index 21dddca050..bf7f871a5a 100644 --- a/onnxruntime/test/perftest/TFModelInfo.h +++ b/onnxruntime/test/perftest/TFModelInfo.h @@ -12,6 +12,7 @@ class TFModelInfo : public TestModelInfo { const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); } const std::string& GetNodeName() const override { return node_name_; } + const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t) const override { return nullptr; } const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t) const override { return nullptr; } int GetInputCount() const override; diff --git a/onnxruntime/test/providers/cpu/controlflow/if_test.cc b/onnxruntime/test/providers/cpu/controlflow/if_test.cc index 83ced88d6a..866c5decf3 100644 --- a/onnxruntime/test/providers/cpu/controlflow/if_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/if_test.cc @@ -413,5 +413,64 @@ TEST(If, ConditionalBranchesOnlyContainConstantNodes_ElseBranchExecution) { test.Run(); } +// This is to test an "If" node with just a "SequenceEmpty" node in the "then" and "else" conditional branches +class IfOpTesterWithSequencesAsOutput : public OpTester { + public: + IfOpTesterWithSequencesAsOutput() : OpTester("If", 13) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + // Graph inputs are 0:Cond + ASSERT_EQ(graph_input_defs.size(), 1u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* if_cond_input = graph_input_defs[0]; + + std::vector inputs; + std::vector outputs; + + // add If node + { + inputs = {if_cond_input}; + outputs = {graph_output_defs[0]}; + + auto& if_node = graph.AddNode("if", "If", "If node", inputs, outputs); + + auto CreateSubgraphWithSequenceEmptyNode = [](bool then_branch, std::vector outputs) { + Model subgraph(then_branch ? "Then" : "Else", false, DefaultLoggingManager().DefaultLogger()); + auto& graph = subgraph.MainGraph(); + + // By default, the SequenceEmpty node will create an empty sequence of float tensors + ORT_IGNORE_RETURN_VALUE(graph.AddNode( + then_branch ? "SequenceEmpty_Then" : "SequenceEmpty_Else", + "SequenceEmpty", + then_branch ? "SequenceEmpty_Then" : "SequenceEmpty_Else", {}, outputs)); + + auto status = graph.Resolve(); + EXPECT_EQ(status, Status::OK()); + + auto& graphproto = graph.ToGraphProto(); + return graphproto; + }; + + if_node.AddAttribute("then_branch", CreateSubgraphWithSequenceEmptyNode(true, outputs)); + if_node.AddAttribute("else_branch", CreateSubgraphWithSequenceEmptyNode(false, outputs)); + } + } +}; + +// opset-13 allows sequences as outputs for 'If' nodes +TEST(If, TestIfWithSequencesAsOutput) { + IfOpTesterWithSequencesAsOutput test; + test.AddInput("If_input", {1}, {true}); + SeqTensors seq; // empty sequence of float tensors + test.AddSeqOutput("If_output", seq); + test.Run(); +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc index ff1ee3a0fa..4b06af4aa4 100644 --- a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc @@ -93,7 +93,7 @@ static const ONNX_NAMESPACE::GraphProto CreateSubgraph(const RunOptions& options bool is_iter_num_1d = options.subgraph_iter_num_1d_tensor; // Loop tests use unsqueeze operator in it's subgraph. Unsqueeze was updated in opset13 - // This test can continue to use opset12 or can be updated to use the latest opset once + // This test can continue to use opset12 or can be updated to use the latest opset once // unsqueeze op13 implementation is done. Model model("Loop subgraph", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), {{"", 12}}, {}, DefaultLoggingManager().DefaultLogger()); @@ -1048,5 +1048,119 @@ TEST(Loop, MixedExecutionProviders) { } #endif +// Test sequence as Loop carried dependency (opset-13 allows this) +TEST(Loop, SequenceAsLoopCarriedDependency) { + auto create_subgraph = []() { + Model model("sequence in Loop subgraph carried dependency", false, DefaultLoggingManager().DefaultLogger()); + auto& graph = model.MainGraph(); + + std::vector inputs; + std::vector outputs; + + /* Subgraph Adds inserted_tensor (float tensor) to a sequence across iterations + + Inputs: iter_num, cond_in, loop_var_0_in + + loop_var_0_in inserted_tensor cond_in iter_num + | | | (unused) + [SequenceInsert]-----/ [Identity] + | | + | cond_out + loop_var_0_out + */ + + // graph inputs types. + TypeProto int64_scalar; + int64_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT64); + int64_scalar.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + TypeProto bool_scalar; + bool_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_BOOL); + bool_scalar.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + + TypeProto float_tensor_sequence; + auto* tensor_type = float_tensor_sequence + .mutable_sequence_type() + ->mutable_elem_type() + ->mutable_tensor_type(); + + tensor_type->set_elem_type(TensorProto_DataType_FLOAT); + tensor_type->mutable_shape()->add_dim()->set_dim_value(1); + + TypeProto float_tensor; + float_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim(); + + // graph inputs + auto& iter_num_in = graph.GetOrCreateNodeArg("iter_num_in", &int64_scalar); + auto& cond_in = graph.GetOrCreateNodeArg("cond_in", &bool_scalar); + auto& loop_var_0_in = graph.GetOrCreateNodeArg("loop_var_0_in", &float_tensor_sequence); + + // graph outputs + auto& cond_out = graph.GetOrCreateNodeArg("cond_out", &bool_scalar); + auto& loop_var_0_out = graph.GetOrCreateNodeArg("loop_var_0_out", &float_tensor_sequence); + + // outer scope values. need type but not shape. + auto& inserted_tensor = graph.GetOrCreateNodeArg("inserted_tensor", &float_tensor); + + // add it to the outer scope so that we don't end up with it being considered a graph input + graph.AddOuterScopeNodeArg("inserted_tensor"); + + // cond_in -> cond_out + { + inputs = {&cond_in}; + outputs = {&cond_out}; + + graph.AddNode("cond_in_identity", "Identity", "Forward cond_in to cond_out", inputs, outputs); + } + + // iter_num_in -> loop_var_0_out + { + inputs = {&loop_var_0_in, &inserted_tensor}; + outputs = {&loop_var_0_out}; + + graph.AddNode("loop_var_out", "SequenceInsert", "append to sequence across iterations", inputs, outputs); + } + + graph.SetInputs({&iter_num_in, &cond_in, &loop_var_0_in}); + graph.SetOutputs({&cond_out, &loop_var_0_out}); + + // add an initializer for the tensor that will be inserted into the sequence on every iterations + { + TensorProto inserted_tensor_proto; + inserted_tensor_proto.set_name("inserted_tensor"); + inserted_tensor_proto.add_dims(1); + inserted_tensor_proto.add_float_data(1.f); + inserted_tensor_proto.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + + graph.AddInitializedTensor(inserted_tensor_proto); + } + + auto status = graph.Resolve(); + EXPECT_EQ(status, Status::OK()); + + return graph.ToGraphProto(); + }; + + OpTester test("Loop", 13); + auto body = create_subgraph(); + test.AddAttribute("body", body); + + test.AddInput("M", {1}, {3}); + test.AddInput("cond", {1}, {true}); + + SeqTensors seq_input; + test.AddSeqInput("loop_var_0_orig", seq_input); + + SeqTensors seq_output; + seq_output.AddTensor({1}, {1.f}); + seq_output.AddTensor({1}, {1.f}); + seq_output.AddTensor({1}, {1.f}); + test.AddSeqOutput("loop_var_0_final", seq_output); + + // Disable TensorRT on unsupported data type BOOL + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc index 5891a37ec0..7c0c0115b2 100644 --- a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc +++ b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc @@ -33,8 +33,8 @@ "^test_adam", // NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1) "^test_adam_multiple", // NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1) "^test_training_dropout.*", // NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel) - "^test_if_seq_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node If(13) - "^test_loop13_seq_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Loop(13) + "^test_if_seq_cpu", // ONNX backend test runner doesn't handle sequences: https://github.com/onnx/onnx/issues/3086 + "^test_loop13_seq_cpu", // ONNX backend test runner doesn't handle sequences: https://github.com/onnx/onnx/issues/3086 "^test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Resize(13) "^test_resize_downsample_scales_cubic_cpu", "^test_resize_downsample_scales_linear_cpu", diff --git a/onnxruntime/test/util/compare_ortvalue.cc b/onnxruntime/test/util/compare_ortvalue.cc index e5336afcef..b69f5159bc 100644 --- a/onnxruntime/test/util/compare_ortvalue.cc +++ b/onnxruntime/test/util/compare_ortvalue.cc @@ -40,6 +40,7 @@ #include "core/graph/onnx_protobuf.h" #include "core/framework/tensorprotoutils.h" #include "core/framework/utils.h" +#include "core/framework/TensorSeq.h" #include using namespace onnxruntime; @@ -302,10 +303,53 @@ namespace onnxruntime { std::pair CompareOrtValue(const OrtValue& o, const OrtValue& expected_mlvalue, double per_sample_tolerance, double relative_per_sample_tolerance, bool post_processing) { - if (o.IsTensor() != expected_mlvalue.IsTensor() || o.Type() != expected_mlvalue.Type()) { + if (o.Type() != expected_mlvalue.Type()) { return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); } - if (!o.IsTensor()) { + if (o.IsTensor()) { + const Tensor& outvalue = o.Get(); + const Tensor& expected_tensor = expected_mlvalue.Get(); + if (outvalue.DataType() != expected_tensor.DataType()) { + std::ostringstream oss; + oss << "expect " << ElementTypeToString(expected_tensor.DataType()) << " got " + << ElementTypeToString(outvalue.DataType()); + return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, oss.str()); + } + return CompareTwoTensors(outvalue, expected_tensor, per_sample_tolerance, relative_per_sample_tolerance, + post_processing); + } else if (o.IsTensorSequence()) { + auto& expected_tensor_seq = expected_mlvalue.Get(); + auto expected_tensor_count = expected_tensor_seq.Size(); + + auto& actual_tensor_seq = o.Get(); + auto actual_tensor_count = actual_tensor_seq.Size(); + + if (expected_tensor_count != actual_tensor_count) { + std::ostringstream oss; + oss << "expected tensor count in the sequence: " << expected_tensor_count << " got " + << actual_tensor_count; + return std::make_pair(COMPARE_RESULT::RESULT_DIFFERS, oss.str()); + } + + if (!expected_tensor_seq.IsSameDataType(actual_tensor_seq)) { + std::ostringstream oss; + oss << "expected tensor type in the sequence: " << expected_tensor_seq.DataType() << " got " + << actual_tensor_seq.DataType(); + return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, oss.str()); + } + + for (size_t i = 0; i < expected_tensor_count; ++i) { + auto res = CompareTwoTensors(actual_tensor_seq.Get(i), expected_tensor_seq.Get(i), per_sample_tolerance, relative_per_sample_tolerance, + post_processing); + if (res.first != COMPARE_RESULT::SUCCESS) { + return res; + } + } + + return std::make_pair(COMPARE_RESULT::SUCCESS, ""); + + } else { + // Maps #if !defined(DISABLE_ML_OPS) if (o.Type() == DataTypeImpl::GetType()) { return CompareSeqOfMapToFloat(o.Get(), expected_mlvalue.Get(), @@ -320,16 +364,6 @@ std::pair CompareOrtValue(const OrtValue& o, const return std::make_pair(COMPARE_RESULT::NOT_SUPPORT, "Map type is not supported in this build."); #endif } - const Tensor& outvalue = o.Get(); - const Tensor& expected_tensor = expected_mlvalue.Get(); - if (outvalue.DataType() != expected_tensor.DataType()) { - std::ostringstream oss; - oss << "expect " << ElementTypeToString(expected_tensor.DataType()) << " got " - << ElementTypeToString(outvalue.DataType()); - return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, oss.str()); - } - return CompareTwoTensors(outvalue, expected_tensor, per_sample_tolerance, relative_per_sample_tolerance, - post_processing); } std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const Ort::Value& o) { @@ -341,7 +375,7 @@ std::pair VerifyValueInfo(const ONNX_NAMESPACE::Val ::ONNX_NAMESPACE::TypeProto_Tensor t = v.type().tensor_type(); // below code doesn't work - // if (((TensorTypeBase*)o.Type())->GetElementType() != DataTypeImpl::ElementTypeFromProto(t.elem_type())) { + //if (((TensorTypeBase*)o.Type())->GetElementType() != DataTypeImpl::ElementTypeFromProto(t.elem_type())) { // return COMPARE_RESULT::TYPE_MISMATCH; //} auto info = o.GetTensorTypeAndShapeInfo(); @@ -368,11 +402,19 @@ std::pair VerifyValueInfo(const ONNX_NAMESPACE::Val VectorToString(shape, oss); return std::make_pair(COMPARE_RESULT::SHAPE_MISMATCH, oss.str()); } - } else { - // Cannot do this check for tensor type. + } else if (v.type().has_sequence_type()) { + // TODO: CXX API doesn't have IsTensorSequence() supported for Ort::Value + // TODO: Repeat whatever we did for Tensor above in a loop ? + return std::make_pair(COMPARE_RESULT::SUCCESS, ""); + } + + else { + // Cannot do this check for tensor/sequence of tensor type. // For tensor type, o.Type() is TensorTypeBase*, but p points to a subclass of TensorTypeBase - auto p = DataTypeImpl::TypeFromProto(v.type()); - if (((OrtValue*)(const OrtValue*)o)->Type() != p) { + // For sequences of tensor type, o.Type() is SequenceTensorTypeBase*, but p points to a subclass of SequenceTensorTypeBase + MLDataType p = DataTypeImpl::TypeFromProto(v.type()); + MLDataType q = ((OrtValue*)(const OrtValue*)o)->Type(); + if (q != p) { return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); } }