From 2d4d0abd366a000ac404231aa6ce5317af004528 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Thu, 10 Oct 2019 15:58:49 -0700 Subject: [PATCH] Add support for output seq(tensor) in python binding and test framework. Implement SequenceConstruct, SequenceEmpty, SequenceInsert and SequenceErase ops. (#2040) Add support for output seq(tensor) in python binding and test framework. Implement SequenceConstruct, SequenceEmpty, SequenceInsert and SequenceErase ops. (#2040) --- onnxruntime/core/framework/TensorSeq.h | 2 +- .../core/framework/onnxruntime_typeinfo.cc | 10 +- .../core/framework/onnxruntime_typeinfo.h | 2 + .../providers/cpu/cpu_execution_provider.cc | 8 + .../providers/cpu/sequence/sequence_ops.cc | 229 +++++++++++++++++- .../providers/cpu/sequence/sequence_ops.h | 29 +++ .../providers/nuphar/compiler/func_info.cc | 14 +- .../nuphar/nuphar_execution_provider.cc | 6 +- .../nuphar/runtime/sequential/basic.cc | 4 +- onnxruntime/core/session/onnxruntime_c_api.cc | 6 +- .../python/onnxruntime_pybind_mlvalue.cc | 48 +++- .../python/onnxruntime_pybind_mlvalue.h | 3 +- .../python/onnxruntime_pybind_state.cc | 80 +++--- .../cpu/sequence/sequence_ops_test.cc | 217 +++++++++++++++++ .../test/providers/provider_test_utils.cc | 37 ++- .../test/providers/provider_test_utils.h | 67 ++--- .../test/python/onnxruntime_test_python.py | 107 +++++--- .../test/testdata/sequence_construct.onnx | 18 ++ .../test/testdata/sequence_insert.onnx | 19 ++ 19 files changed, 771 insertions(+), 135 deletions(-) create mode 100644 onnxruntime/test/testdata/sequence_construct.onnx create mode 100644 onnxruntime/test/testdata/sequence_insert.onnx diff --git a/onnxruntime/core/framework/TensorSeq.h b/onnxruntime/core/framework/TensorSeq.h index 8a2d97a8e3..2bca29467e 100644 --- a/onnxruntime/core/framework/TensorSeq.h +++ b/onnxruntime/core/framework/TensorSeq.h @@ -17,7 +17,7 @@ struct TensorSeq { // The current design follows the Tensor methodology. // We also require this because the SequenceEmpty op expects the creation of a seq of a specific type // and the SequenceInsert op expects validation of tensors to be added to the seq against this type. - MLDataType dtype; + MLDataType dtype{}; // TODO: optimization opportunity - if all tensors in the seq are scalars, we can potentially represent them // as vector diff --git a/onnxruntime/core/framework/onnxruntime_typeinfo.cc b/onnxruntime/core/framework/onnxruntime_typeinfo.cc index 35a7253ce1..3e00d82f6c 100644 --- a/onnxruntime/core/framework/onnxruntime_typeinfo.cc +++ b/onnxruntime/core/framework/onnxruntime_typeinfo.cc @@ -14,8 +14,8 @@ using onnxruntime::BFloat16; using onnxruntime::DataTypeImpl; using onnxruntime::MLFloat16; -using onnxruntime::Tensor; using onnxruntime::SparseTensor; +using onnxruntime::Tensor; using onnxruntime::TensorShape; namespace on = ONNX_NAMESPACE; @@ -76,7 +76,7 @@ OrtStatus* OrtTypeInfo::FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, // Place Opaque first as tensors will be // mostly handled above and maps and sequences // are not common - switch (type_proto->value_case ()) { + switch (type_proto->value_case()) { case on::TypeProto::kOpaqueType: { *out = new OrtTypeInfo(ONNX_TYPE_OPAQUE, nullptr); return nullptr; @@ -110,7 +110,7 @@ OrtStatus* OrtTypeInfo::FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, return OrtApis::CreateStatus(ORT_NOT_IMPLEMENTED, "not implemented"); } -const DataTypeImpl* ElementTypeFromProto(int type) { +const DataTypeImpl* OrtTypeInfo::ElementTypeFromProto(int type) { switch (type) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: return DataTypeImpl::GetType(); @@ -176,7 +176,7 @@ OrtStatus* OrtTypeInfo::FromDataTypeImpl(const ONNX_NAMESPACE::TypeProto* input, TensorShape shape_data(std::move(dims)); for (int i = 0; i < s.dim_size(); ++i) { auto& t = s.dim(i); - switch (t.value_case ()) { + switch (t.value_case()) { case on::TensorShapeProto::Dimension::kDimValue: shape_data[i] = t.dim_value(); break; @@ -211,7 +211,7 @@ OrtStatus* OrtTypeInfo::FromDataTypeImpl(const ONNX_NAMESPACE::TypeProto* input, case on::TypeProto::VALUE_NOT_SET: break; default: - // Not implemented + // Not implemented break; } return OrtApis::CreateStatus(ORT_NOT_IMPLEMENTED, "not implemented"); diff --git a/onnxruntime/core/framework/onnxruntime_typeinfo.h b/onnxruntime/core/framework/onnxruntime_typeinfo.h index 63e783f052..ae05f366f4 100644 --- a/onnxruntime/core/framework/onnxruntime_typeinfo.h +++ b/onnxruntime/core/framework/onnxruntime_typeinfo.h @@ -33,6 +33,8 @@ struct OrtTypeInfo { const onnxruntime::DataTypeImpl* tensor_data_type, OrtTypeInfo** out); static OrtStatus* FromDataTypeImpl(const ONNX_NAMESPACE::TypeProto*, OrtTypeInfo** out); + static const onnxruntime::DataTypeImpl* ElementTypeFromProto(int type); + private: OrtTypeInfo(ONNXType type, OrtTensorTypeAndShapeInfo* data) noexcept; }; diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index d61b19b603..6fc2948e96 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -392,6 +392,10 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Co class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, If); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceLength); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceAt); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceEmpty); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceInsert); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceErase); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceConstruct); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ScatterND); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Gemm); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, GatherElements); @@ -1018,6 +1022,10 @@ void RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc b/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc index 7c7049838c..ba1a3db1cc 100644 --- a/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc +++ b/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc @@ -10,6 +10,9 @@ using namespace onnxruntime::common; namespace onnxruntime { +// TODO: The current implementation of sequence ops relies on tensor copies. Ideally we should try to avoid +// these copies. This has been postponed due to lack of time. + // SequenceLength ONNX_CPU_OPERATOR_KERNEL( SequenceLength, @@ -24,6 +27,7 @@ Status SequenceLength::Compute(OpKernelContext* context) const { ORT_ENFORCE(X != nullptr, "Got nullptr for sequence input."); auto* Y = context->Output(0, {}); + ORT_ENFORCE(Y != nullptr, "SequenceLength: Got nullptr for output tensor"); auto* Y_data = Y->template MutableData(); *Y_data = static_cast(X->tensors.size()); @@ -72,13 +76,6 @@ bool ValidateSeqIdx(int64_t input_seq_idx, int64_t seq_size) { return retval; } -template -static void CopyTensor(const Tensor& indexed_tensor, Tensor& output_tensor) { - const auto* input_data = indexed_tensor.template Data(); - auto* output_data = output_tensor.template MutableData(); - memcpy(output_data, input_data, indexed_tensor.SizeInBytes()); -} - Status SequenceAt::Compute(OpKernelContext* context) const { const auto* X = context->Input(0); ORT_ENFORCE(X != nullptr, "Got nullptr for sequence input."); @@ -96,9 +93,227 @@ Status SequenceAt::Compute(OpKernelContext* context) const { } const Tensor& indexed_tensor = X->tensors[input_seq_idx]; auto* Y = context->Output(0, indexed_tensor.Shape().GetDims()); + ORT_ENFORCE(Y != nullptr, "SequenceAt: Got nullptr for output tensor"); CopyCpuTensor(&indexed_tensor, Y); return Status::OK(); } +// SequenceEmpty +ONNX_CPU_OPERATOR_KERNEL( + SequenceEmpty, + 11, + KernelDefBuilder() + .TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes()), + SequenceEmpty); + +SequenceEmpty::SequenceEmpty(const OpKernelInfo& info) : OpKernel(info) { + if (!info.GetAttr("dtype", &dtype_).IsOK()) { + dtype_ = ONNX_NAMESPACE::TensorProto_DataType_FLOAT; + } +} + +Status SequenceEmpty::Compute(OpKernelContext* context) const { + auto* Y = context->Output(0); + ORT_ENFORCE(Y != nullptr, "SequenceEmpty: Got nullptr for output sequence"); + MLDataType seq_dtype{}; + switch (dtype_) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_BOOL: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_INT32: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_STRING: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_INT8: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_UINT8: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_UINT16: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_INT16: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_INT64: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_UINT32: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_UINT64: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: + seq_dtype = DataTypeImpl::GetType(); + break; + case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: + seq_dtype = DataTypeImpl::GetType(); + break; + default: + ORT_THROW("Unsupported 'dtype' value: ", dtype_); + } + + Y->dtype = seq_dtype; + return Status::OK(); +} + +// SequenceInsert +ONNX_CPU_OPERATOR_KERNEL( + SequenceInsert, + 11, + KernelDefBuilder() + .TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes()) + .TypeConstraint("I", std::vector{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + SequenceInsert); + +Status CreateCopyAndAppendCpuTensor(const Tensor& in_tensor, OpKernelContext* context, TensorSeq& tseq) { + AllocatorPtr alloc; + ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + Tensor tmp(in_tensor.DataType(), onnxruntime::TensorShape(in_tensor.Shape()), alloc); + CopyCpuTensor(&in_tensor, &tmp); + tseq.tensors.push_back(std::move(tmp)); + return Status::OK(); +} + +Status SequenceInsert::Compute(OpKernelContext* context) const { + const auto* S = context->Input(0); + ORT_ENFORCE(S != nullptr, "Got nullptr for sequence input."); + + const auto* X = context->Input(1); + ORT_ENFORCE(X != nullptr, "Got nullptr for input tensor."); + + // Data type of the input tensor MUST be same as that of the input sequence + if (S->dtype != X->DataType()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Data type of the input tensor MUST be same as that of the input sequence. Sequence data type (", + DataTypeImpl::ToString(S->dtype), "), input tensor data type (", DataTypeImpl::ToString(X->DataType()), ")"); + } + + const auto* I = context->Input(2); + int64_t num_tensors_input_seq = static_cast(S->tensors.size()); + int64_t input_seq_idx = num_tensors_input_seq + 1; // default is append + if (I) { // position is optional + input_seq_idx = GetSeqIdx(*I); + if (!ValidateSeqIdx(input_seq_idx, static_cast(num_tensors_input_seq))) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Invalid sequence index (", input_seq_idx, ") specified for sequence of size (", num_tensors_input_seq, ")"); + } + + if (input_seq_idx < 0) { + input_seq_idx = static_cast(num_tensors_input_seq) + input_seq_idx; + } + } + + auto* Y = context->Output(0); + ORT_ENFORCE(Y != nullptr, "SequenceInsert: Got nullptr for output sequence"); + Y->dtype = S->dtype; + Y->tensors.reserve(num_tensors_input_seq + 1); + for (int i = 0; i < num_tensors_input_seq; ++i) { + if (i == input_seq_idx) { + CreateCopyAndAppendCpuTensor(*X, context, *Y); + CreateCopyAndAppendCpuTensor(S->tensors[i], context, *Y); + } else { + CreateCopyAndAppendCpuTensor(S->tensors[i], context, *Y); + } + } + if (input_seq_idx == num_tensors_input_seq + 1) { + CreateCopyAndAppendCpuTensor(*X, context, *Y); + } + + return Status::OK(); +} + +// SequenceErase +ONNX_CPU_OPERATOR_KERNEL( + SequenceErase, + 11, + KernelDefBuilder() + .TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes()) + .TypeConstraint("I", std::vector{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + SequenceErase); + +Status SequenceErase::Compute(OpKernelContext* context) const { + const auto* S = context->Input(0); + ORT_ENFORCE(S != nullptr, "Got nullptr for sequence input."); + + const auto* I = context->Input(1); + int64_t num_tensors_input_seq = static_cast(S->tensors.size()); + int64_t input_seq_idx = num_tensors_input_seq - 1; // default is erase last one + if (I) { // position is optional + input_seq_idx = GetSeqIdx(*I); + if (!ValidateSeqIdx(input_seq_idx, static_cast(num_tensors_input_seq))) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Invalid sequence index (", input_seq_idx, ") specified for sequence of size (", num_tensors_input_seq, ")"); + } + + if (input_seq_idx < 0) { + input_seq_idx = static_cast(num_tensors_input_seq) + input_seq_idx; + } + } + + auto* Y = context->Output(0); + ORT_ENFORCE(Y != nullptr, "SequenceErase: Got nullptr for output sequence"); + Y->dtype = S->dtype; + Y->tensors.reserve(num_tensors_input_seq - 1); + for (int i = 0; i < num_tensors_input_seq; ++i) { + if (i == input_seq_idx) { + continue; + } + CreateCopyAndAppendCpuTensor(S->tensors[i], context, *Y); + } + + return Status::OK(); +} + +// SequenceConstruct +ONNX_CPU_OPERATOR_KERNEL( + SequenceConstruct, + 11, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::AllTensorTypes()) + .TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes()), + SequenceConstruct); + +Status SequenceConstruct::Compute(OpKernelContext* context) const { + auto num_inputs = Node().InputArgCount().front(); + ORT_ENFORCE(num_inputs >= 1, "Must have 1 or more inputs"); + + auto* Y = context->Output(0); + ORT_ENFORCE(Y != nullptr, "SequenceConstruct: Got nullptr for output sequence"); + + MLDataType first_dtype = context->Input(0)->DataType(); + // Before copying check if all tensors are of the same type. + for (int input_idx = 0; input_idx < num_inputs; ++input_idx) { + const auto* X = context->Input(input_idx); + if (input_idx > 0 && X->DataType() != first_dtype) { + ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Violation of the requirment that all input tensors must have the same data type."); + } + } + + // now copy the tensors to the output sequence + Y->dtype = first_dtype; + Y->tensors.reserve(num_inputs); + for (int input_idx = 0; input_idx < num_inputs; ++input_idx) { + const auto* X = context->Input(input_idx); + CreateCopyAndAppendCpuTensor(*X, context, *Y); + } + + return Status::OK(); +} } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/sequence/sequence_ops.h b/onnxruntime/core/providers/cpu/sequence/sequence_ops.h index 006d81bdfd..8b998a5f08 100644 --- a/onnxruntime/core/providers/cpu/sequence/sequence_ops.h +++ b/onnxruntime/core/providers/cpu/sequence/sequence_ops.h @@ -24,4 +24,33 @@ class SequenceAt final : public OpKernel { Status Compute(OpKernelContext* context) const override; }; +class SequenceEmpty final : public OpKernel { + public: + SequenceEmpty(const OpKernelInfo& info); + Status Compute(OpKernelContext* context) const override; + + private: + int64_t dtype_{}; +}; + +class SequenceInsert final : public OpKernel { + public: + SequenceInsert(const OpKernelInfo& info) : OpKernel(info) { + } + Status Compute(OpKernelContext* context) const override; +}; + +class SequenceErase final : public OpKernel { + public: + SequenceErase(const OpKernelInfo& info) : OpKernel(info) { + } + Status Compute(OpKernelContext* context) const override; +}; + +class SequenceConstruct final : public OpKernel { + public: + SequenceConstruct(const OpKernelInfo& info) : OpKernel(info) { + } + Status Compute(OpKernelContext* context) const override; +}; } //namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/func_info.cc b/onnxruntime/core/providers/nuphar/compiler/func_info.cc index 0eed552a25..fb24605a2d 100644 --- a/onnxruntime/core/providers/nuphar/compiler/func_info.cc +++ b/onnxruntime/core/providers/nuphar/compiler/func_info.cc @@ -6,13 +6,11 @@ #include "core/providers/nuphar/runtime/control_flow/scan_exec_ctx.h" #include "core/framework/op_kernel.h" #include "core/framework/tensorprotoutils.h" +#include "core/framework/onnxruntime_typeinfo.h" #include "core/codegen/common/common.h" #include "core/providers/nuphar/common/analysis/subgraph_codegen_stats.h" #include -// from onnxruntime_typeinf.cc, in global namespace -const onnxruntime::DataTypeImpl* ElementTypeFromProto(int type); - namespace onnxruntime { namespace nuphar { @@ -64,7 +62,7 @@ static void FillBasicFuncInfo(NupharFuncInfo* func_info, // fill in func args NupharFuncInfo::FuncArgMeta input_meta; - input_meta.dtype = ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); + input_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); input_meta.ort_arg_index = def_index; // fill in shape info and symobolic info @@ -160,7 +158,7 @@ static void FillBasicFuncInfo(NupharFuncInfo* func_info, } NupharFuncInfo::FuncArgMeta output_meta; - output_meta.dtype = ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); + output_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); output_meta.ort_arg_index = def_index; // fill in shape info and symobolic info @@ -345,7 +343,7 @@ static void FillScanExecInfo(NupharFuncInfo* func_info, } NupharFuncInfo::FuncArgMeta input_meta; - input_meta.dtype = ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); + input_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); input_meta.ort_arg_index = gsl::narrow_cast(ort_input_idx); // fill in shape info and symobolic info @@ -395,7 +393,7 @@ static void FillScanExecInfo(NupharFuncInfo* func_info, } NupharFuncInfo::FuncArgMeta input_meta; - input_meta.dtype = ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); + input_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); input_meta.ort_arg_index = gsl::narrow_cast(ort_input_idx); std::vector> symbols; @@ -514,7 +512,7 @@ static void FillScanExecInfo(NupharFuncInfo* func_info, } NupharFuncInfo::FuncArgMeta output_meta; - output_meta.dtype = ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); + output_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); output_meta.ort_arg_index = ort_arg_index; // shape and symbols diff --git a/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc b/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc index 00892fd2a6..0679a1ca72 100644 --- a/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc +++ b/onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc @@ -13,14 +13,12 @@ #include "core/providers/nuphar/compiler/x86/x86_target_info.h" #include "core/providers/nuphar/kernel.h" #include "core/providers/nuphar/partition/graph_partitioner.h" +#include "core/framework/onnxruntime_typeinfo.h" #include // TODO remove this after removing tvm::runtime using namespace onnxruntime::nuphar; -// from onnxruntime_typeinf.cc, in global namespace -const onnxruntime::DataTypeImpl* ElementTypeFromProto(int type); - namespace onnxruntime { // Initialization of thread local counter as subgraph id @@ -296,7 +294,7 @@ Status NupharExecutionProvider::SaveInitializer( shape_dims[i] = dims[i]; const TensorShape& shape = TensorShape::ReinterpretBaseType(shape_dims); - auto data_type = ElementTypeFromProto(proto->data_type()); + auto data_type = OrtTypeInfo::ElementTypeFromProto(proto->data_type()); auto t = onnxruntime::make_unique( data_type, shape, diff --git a/onnxruntime/core/providers/nuphar/runtime/sequential/basic.cc b/onnxruntime/core/providers/nuphar/runtime/sequential/basic.cc index fe69ce0970..062287d0cf 100644 --- a/onnxruntime/core/providers/nuphar/runtime/sequential/basic.cc +++ b/onnxruntime/core/providers/nuphar/runtime/sequential/basic.cc @@ -6,12 +6,10 @@ #include "core/codegen/common/common.h" #include "core/codegen/common/profile.h" #include "core/codegen/passes/utils/ort_tvm_utils.h" +#include "core/framework/onnxruntime_typeinfo.h" #include "gsl/gsl" #include -// from onnxruntime_typeinf.cc, in global namespace -const onnxruntime::DataTypeImpl* ElementTypeFromProto(int type); - namespace onnxruntime { namespace nuphar { diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index b2625b3e5d..56f05da4ff 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -836,7 +836,7 @@ OrtStatus* OrtGetValueImplSeqOfTensors(const OrtValue* p_ml_value, int index, Or } else if (tensor_elem_type == DataTypeImpl::GetType()) { st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); } else { - st = OrtApis::CreateStatus(ORT_FAIL, "Only sequences that contain float tensors are supported."); + st = OrtApis::CreateStatus(ORT_FAIL, "Invalid tensor element type in the input."); } return st; } @@ -993,10 +993,10 @@ static OrtStatus* OrtCreateValueImplSeqHelper(const OrtValue* const* in, size_t seq_ptr->tensors.resize(num_values); // use the data type of the first tensor as the data type of the seq - seq_ptr->dtype = reinterpret_cast(in[0])->Get().DataType(); + seq_ptr->dtype = static_cast(in[0])->Get().DataType(); for (size_t idx = 0; idx < num_values; ++idx) { - auto& one_tensor = reinterpret_cast(in[idx])->Get(); + auto& one_tensor = static_cast(in[idx])->Get(); auto tensor_elem_type = one_tensor.DataType(); // sequences must have tensors of the same data type diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc index a13e5f15d6..b5e9d7ad4e 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -13,6 +13,8 @@ #include "core/framework/tensor.h" #include "core/framework/allocator.h" #include "core/framework/TensorSeq.h" +#include "core/framework/data_types.h" +#include "core/framework/onnxruntime_typeinfo.h" using namespace std; namespace onnxruntime { @@ -105,7 +107,6 @@ std::unique_ptr CreateTensor(AllocatorPtr alloc, const std::string& name std::unique_ptr p_tensor; try { const int npy_type = PyArray_TYPE(darray); - // numpy requires long int as its dims. int ndim = PyArray_NDIM(darray); npy_intp* npy_dims = PyArray_DIMS(darray); @@ -196,18 +197,38 @@ std::unique_ptr CreateTensor(AllocatorPtr alloc, const std::string& name return p_tensor; } -void CreateSequenceOfTensors(AllocatorPtr alloc, const std::string& name_input, PyObject* pylist_obj, - OrtValue* p_mlvalue) { - auto list_size = PyList_Size(pylist_obj); - if (list_size <= 0) { - throw std::runtime_error("Got exception while creating seq(tensor) because input list size is " + std::to_string(list_size)); +void CreateSequenceOfTensors(AllocatorPtr alloc, const std::string& name_input, + const InputDefList* input_def_list, PyObject* pylist_obj, OrtValue* p_mlvalue) { + // get sequence type from the model + const auto& def_list = *input_def_list; + auto ret_it = std::find_if(std::begin(def_list), std::end(def_list), + [&name_input](const NodeArg* node_arg) { return name_input == node_arg->Name(); }); + if (ret_it == std::end(def_list)) { + throw std::runtime_error("Failed to find input with name: " + name_input + " in the model input def list"); } + const auto* type_proto = (*ret_it)->TypeAsProto(); + if (!type_proto || !(type_proto->has_sequence_type())) { + throw std::runtime_error("Either type_proto was null or it was not of sequence type"); + } + + // set the seq type auto p_seq_tensors = onnxruntime::make_unique(); - p_seq_tensors->tensors.resize(list_size); - for (Py_ssize_t i = 0; i < list_size; ++i) { - auto* py_obj = PyList_GetItem(pylist_obj, i); - auto p_tensor = CreateTensor(alloc, name_input, reinterpret_cast(py_obj)); - p_seq_tensors->tensors[i] = std::move(*p_tensor); + MLDataType seq_dtype = OrtTypeInfo::ElementTypeFromProto( + static_cast(type_proto->sequence_type().elem_type().tensor_type().elem_type())); + p_seq_tensors->dtype = seq_dtype; + + // populate the seq + auto list_size = PyList_Size(pylist_obj); + if (list_size > 0) { + p_seq_tensors->tensors.resize(list_size); + for (Py_ssize_t i = 0; i < list_size; ++i) { + auto* py_obj = PyList_GetItem(pylist_obj, i); + if (!PyObjectCheck_Array(py_obj)) { + throw std::runtime_error("CreateSequenceOfTensors: Input is not a tensor"); + } + auto p_tensor = CreateTensor(alloc, name_input, reinterpret_cast(py_obj)); + p_seq_tensors->tensors[i] = std::move(*(p_tensor.release())); + } } p_mlvalue->Init(p_seq_tensors.release(), @@ -428,14 +449,15 @@ void CreateGenericIterableMLValue(PyObject* iterator, AllocatorPtr alloc, const } } -void CreateGenericMLValue(AllocatorPtr alloc, const std::string& name_input, py::object& value, OrtValue* p_mlvalue) { +void CreateGenericMLValue(const onnxruntime::InputDefList* input_def_list, AllocatorPtr alloc, const std::string& name_input, + py::object& value, OrtValue* p_mlvalue) { if (PyObjectCheck_Array(value.ptr())) { // The most frequent case: input comes as an array. PyArrayObject* arr = reinterpret_cast(value.ptr()); CreateTensorMLValue(alloc, name_input, arr, p_mlvalue); } else if (PyList_Check(value.ptr())) { auto* seq_tensors = reinterpret_cast(value.ptr()); - CreateSequenceOfTensors(alloc, name_input, seq_tensors, p_mlvalue); + CreateSequenceOfTensors(alloc, name_input, input_def_list, seq_tensors, p_mlvalue); } else if (PyDict_Check(value.ptr())) { CreateMapMLValue_AgnosticVectorMap((PyObject*)NULL, value.ptr(), alloc, name_input, p_mlvalue); } else { diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.h b/onnxruntime/python/onnxruntime_pybind_mlvalue.h index 1c93906078..7914d198f3 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.h +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.h @@ -22,7 +22,8 @@ namespace py = pybind11; int OnnxRuntimeTensorToNumpyType(const DataTypeImpl* tensor_type); -void CreateGenericMLValue(AllocatorPtr alloc, const std::string& name_input, py::object& value, OrtValue* p_mlvalue); +void CreateGenericMLValue(const onnxruntime::InputDefList* input_def_list, AllocatorPtr alloc, const std::string& name_input, + py::object& value, OrtValue* p_mlvalue); } // namespace python } // namespace onnxruntime diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 66c5064856..11d3e1e24e 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -12,6 +12,7 @@ #include "core/graph/graph_viewer.h" #include "core/common/logging/logging.h" #include "core/common/logging/severity.h" +#include "core/framework/TensorSeq.h" #if USE_CUDA #define BACKEND_PROC "GPU" @@ -148,9 +149,53 @@ void AddNonTensor(OrtValue& val, std::vector& pyobjs) { pyobjs.push_back(py::cast(val.Get())); } +void GetPyObjFromTensor(const Tensor& rtensor, py::object& obj) { + std::vector npy_dims; + const TensorShape& shape = rtensor.Shape(); + + for (size_t n = 0; n < shape.NumDimensions(); ++n) { + npy_dims.push_back(shape[n]); + } + + MLDataType dtype = rtensor.DataType(); + const int numpy_type = OnnxRuntimeTensorToNumpyType(dtype); + obj = py::reinterpret_steal(PyArray_SimpleNew( + shape.NumDimensions(), npy_dims.data(), numpy_type)); + + void* outPtr = static_cast( + PyArray_DATA(reinterpret_cast(obj.ptr()))); + + if (numpy_type != NPY_OBJECT) { + memcpy(outPtr, rtensor.DataRaw(dtype), dtype->Size() * shape.Size()); + } else { + // Handle string type. + py::object* outObj = static_cast(outPtr); + const std::string* src = rtensor.template Data(); + for (int i = 0; i < rtensor.Shape().Size(); i++, src++) { + outObj[i] = py::cast(*src); + } + } +} + +template <> +void AddNonTensor(OrtValue& val, std::vector& pyobjs) { + const auto& seq_tensors = val.Get(); + size_t num_tensors = seq_tensors.tensors.size(); + py::list py_list; + for (size_t i = 0; i < num_tensors; ++i) { + const auto& rtensor = seq_tensors.tensors[i]; + py::object obj; + GetPyObjFromTensor(rtensor, obj); + py_list.append(obj); + } + pyobjs.push_back(py_list); +} + void AddNonTensorAsPyObj(OrtValue& val, std::vector& pyobjs) { // Should be in sync with core/framework/datatypes.h - if (val.Type() == DataTypeImpl::GetType()) { + if (val.Type() == DataTypeImpl::GetType()) { + AddNonTensor(val, pyobjs); + } else if (val.Type() == DataTypeImpl::GetType()) { AddNonTensor(val, pyobjs); } else if (val.Type() == DataTypeImpl::GetType()) { AddNonTensor(val, pyobjs); @@ -177,31 +222,8 @@ void AddNonTensorAsPyObj(OrtValue& val, std::vector& pyobjs) { void AddTensorAsPyObj(OrtValue& val, std::vector& pyobjs) { const Tensor& rtensor = val.Get(); - std::vector npy_dims; - const TensorShape& shape = rtensor.Shape(); - - for (size_t n = 0; n < shape.NumDimensions(); ++n) { - npy_dims.push_back(shape[n]); - } - - MLDataType dtype = rtensor.DataType(); - const int numpy_type = OnnxRuntimeTensorToNumpyType(dtype); - py::object obj = py::reinterpret_steal(PyArray_SimpleNew( - shape.NumDimensions(), npy_dims.data(), numpy_type)); - - void* outPtr = static_cast( - PyArray_DATA(reinterpret_cast(obj.ptr()))); - - if (numpy_type != NPY_OBJECT) { - memcpy(outPtr, rtensor.DataRaw(dtype), dtype->Size() * shape.Size()); - } else { - // Handle string type. - py::object* outObj = static_cast(outPtr); - const std::string* src = rtensor.template Data(); - for (int i = 0; i < rtensor.Shape().Size(); i++, src++) { - outObj[i] = py::cast(*src); - } - } + py::object obj; + GetPyObjFromTensor(rtensor, obj); pyobjs.push_back(obj); } @@ -692,7 +714,11 @@ including arg name, arg type (contains both type and shape).)pbdoc") NameMLValMap feeds; for (auto _ : pyfeeds) { OrtValue ml_value; - CreateGenericMLValue(GetAllocator(), _.first, _.second, &ml_value); + auto px = sess->GetModelInputs(); + if (!px.first.IsOK() || !px.second) { + throw std::runtime_error("Either failed to get model inputs from the session object or the input def list was null"); + } + CreateGenericMLValue(px.second, GetAllocator(), _.first, _.second, &ml_value); if (PyErr_Occurred()) { PyObject *ptype, *pvalue, *ptraceback; PyErr_Fetch(&ptype, &pvalue, &ptraceback); diff --git a/onnxruntime/test/providers/cpu/sequence/sequence_ops_test.cc b/onnxruntime/test/providers/cpu/sequence/sequence_ops_test.cc index f94ed9e823..a7cba0a347 100644 --- a/onnxruntime/test/providers/cpu/sequence/sequence_ops_test.cc +++ b/onnxruntime/test/providers/cpu/sequence/sequence_ops_test.cc @@ -7,6 +7,7 @@ namespace onnxruntime { namespace test { +// SequenceLength TEST(SequenceOpsTest, SequenceLengthPositiveFloat) { OpTester test("SequenceLength", 11); SeqTensors input; @@ -27,6 +28,7 @@ TEST(SequenceOpsTest, SequenceLengthPositiveInt64) { test.Run(); } +// SequenceAt TEST(SequenceOpsTest, SequenceAtPositiveIdx) { OpTester test("SequenceAt", 11); SeqTensors input; @@ -79,5 +81,220 @@ TEST(SequenceOpsTest, SequenceAtInvalidNegativeIdx) { test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index"); } +// SequenceEmpty +TEST(SequenceOpsTest, SequenceEmptyDefault) { + OpTester test("SequenceEmpty", 11); + test.AddSeqOutput("S", SeqTensors()); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceEmptyInt64) { + OpTester test("SequenceEmpty", 11); + test.AddAttribute("dtype", static_cast(7)); + test.AddSeqOutput("S", SeqTensors()); + test.Run(); +} + +// SequenceInsert +TEST(SequenceOpsTest, SequenceInsertPositiveDefaultFloat) { + OpTester test("SequenceInsert", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1., 2., 3., 4., 5., 6.}); + input.AddTensor({3, 3}, {1., 2., 3., 4., 5., 6., 7., 8., 9.}); + test.AddSeqInput("S", input); + test.AddInput("T", {3, 2}, {10., 20., 30., 40., 50., 60.}); + + SeqTensors output; + output.AddTensor({3, 2}, {1., 2., 3., 4., 5., 6.}); + output.AddTensor({3, 3}, {1., 2., 3., 4., 5., 6., 7., 8., 9.}); + output.AddTensor({3, 2}, {10., 20., 30., 40., 50., 60.}); + test.AddSeqOutput("S2", output); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceInsertPositiveDefault) { + OpTester test("SequenceInsert", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqInput("S", input); + test.AddInput("T", {3, 2}, {10, 20, 30, 40, 50, 60}); + + SeqTensors output; + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + test.AddSeqOutput("S2", output); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceInsertValidPositiveIdx) { + OpTester test("SequenceInsert", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqInput("S", input); + test.AddInput("T", {3, 2}, {10, 20, 30, 40, 50, 60}); + test.AddInput("I", {}, {1}); + + SeqTensors output; + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqOutput("S2", output); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceInsertValidNegativeIdx) { + OpTester test("SequenceInsert", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqInput("S", input); + test.AddInput("T", {3, 2}, {10, 20, 30, 40, 50, 60}); + test.AddInput("I", {}, {-2}); + + SeqTensors output; + output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqOutput("S2", output); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceInsertInvalidPositiveIdx) { + OpTester test("SequenceInsert", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqInput("S", input); + test.AddInput("T", {3, 2}, {10, 20, 30, 40, 50, 60}); + test.AddInput("I", {}, {99}); + + SeqTensors output; + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqOutput("S2", output); + test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index"); +} + +TEST(SequenceOpsTest, SequenceInsertInvalidNegativeIdx) { + OpTester test("SequenceInsert", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqInput("S", input); + test.AddInput("T", {3, 2}, {10, 20, 30, 40, 50, 60}); + test.AddInput("I", {}, {-99}); + + SeqTensors output; + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqOutput("S2", output); + test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index"); +} + +// SequenceErase +TEST(SequenceOpsTest, SequenceErasePositiveDefault) { + OpTester test("SequenceErase", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + input.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + test.AddSeqInput("S", input); + + SeqTensors output; + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqOutput("S2", output); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceEraseValidPositiveIdx) { + OpTester test("SequenceErase", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + input.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + test.AddSeqInput("S", input); + test.AddInput("I", {}, {1}); + + SeqTensors output; + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + test.AddSeqOutput("S2", output); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceEraseValidNegativeIdx) { + OpTester test("SequenceErase", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + input.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + input.AddTensor({2, 2}, {2, 4, 6, 8}); + test.AddSeqInput("S", input); + test.AddInput("I", {}, {-2}); + + SeqTensors output; + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + output.AddTensor({2, 2}, {2, 4, 6, 8}); + test.AddSeqOutput("S2", output); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceEraseInvalidPositiveIdx) { + OpTester test("SequenceErase", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + input.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + input.AddTensor({2, 2}, {2, 4, 6, 8}); + test.AddSeqInput("S", input); + test.AddInput("I", {}, {99}); + + SeqTensors output; + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + input.AddTensor({2, 2}, {2, 4, 6, 8}); + test.AddSeqOutput("S2", output); + test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index"); +} + +TEST(SequenceOpsTest, SequenceEraseInvalidNegativeIdx) { + OpTester test("SequenceErase", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + input.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + input.AddTensor({2, 2}, {2, 4, 6, 8}); + test.AddSeqInput("S", input); + test.AddInput("I", {}, {-99}); + + SeqTensors output; + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + input.AddTensor({2, 2}, {2, 4, 6, 8}); + test.AddSeqOutput("S2", output); + test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index"); +} + +// SequenceConstruct +TEST(SequenceOpsTest, SequenceConstructPositive) { + OpTester test("SequenceConstruct", 11); + test.AddInput("input_1", {3, 2}, {1, 2, 3, 4, 5, 6}); + test.AddInput("input_2", {3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddInput("input_3", {3, 2}, {10, 20, 30, 40, 50, 60}); + + SeqTensors output; + output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60}); + test.AddSeqOutput("S2", output); + test.Run(); +} } // namespace test } // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 7b5143cd4d..c16918eb66 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -240,6 +240,35 @@ void Check(const OpTester::Data& expected_data, const T& run_output, const std:: EXPECT_EQ(expected_data.data_.Get(), run_output) << "provider_type: " << provider_type; } +template <> +void Check(const OpTester::Data& expected_data, const TensorSeq& output_seq, + const std::string& provider_type) { + const auto& exp_seq = expected_data.data_.Get(); + + // first ensure data types match + EXPECT_EQ(exp_seq.dtype, output_seq.dtype) << "Data types don't match: Expected: " << DataTypeImpl::ToString(exp_seq.dtype) + << " Output: " << output_seq.dtype << " provider_type: " << provider_type; + + // check num of contained tensors + size_t expected_num_tensors = exp_seq.tensors.size(); + size_t output_num_tensors = output_seq.tensors.size(); + EXPECT_EQ(expected_num_tensors, output_num_tensors) << "Mismatch in number of tensors in the sequence" + << " Expected: " << expected_num_tensors << " Output: " + << output_num_tensors << " provider_type: " << provider_type; + + // now check the contents of the tensors + auto null_deleter = [](void*) {}; + + for (int i = 0; i < output_num_tensors; ++i) { + OrtValue temp_value; + // Reason for null_deleter: we don't want the tensor destructor to be called as part of this OrtValue destructor + // as we're creating this OrtValue only to reuse the Check functionality + temp_value.Init(const_cast(&exp_seq.tensors[i]), DataTypeImpl::GetType(), null_deleter); + OpTester::Data temp_data(NodeArg("dummy", nullptr), std::move(temp_value), optional(), optional()); + Check(temp_data, output_seq.tensors[i], provider_type); + } +} + template void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, OrtValue& ort_value, const std::string& provider_type) { @@ -260,11 +289,11 @@ void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, OrtValu void Check(const OpTester::Data& expected_data, OrtValue& ort_value, const std::string& provider_type) { #ifdef MICROSOFT_AUTOML - CheckDispatch(expected_data.data_.Type(), expected_data, ort_value, - provider_type); + CheckDispatch(expected_data.data_.Type(), expected_data, ort_value, + provider_type); #else - CheckDispatch(expected_data.data_.Type(), expected_data, ort_value, - provider_type); + CheckDispatch(expected_data.data_.Type(), expected_data, ort_value, + provider_type); #endif } diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h index 984eb3a08a..8f9ad9760a 100644 --- a/onnxruntime/test/providers/provider_test_utils.h +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -283,35 +283,12 @@ class OpTester { template void AddSeqInput(const char* name, const SeqTensors& seq_tensors) { - auto mltype = DataTypeImpl::GetType(); - ORT_ENFORCE(mltype != nullptr, "TensorSeq must be a registered cpp type"); - auto ptr = onnxruntime::make_unique(); - auto num_tensors = seq_tensors.tensors.size(); - ptr->tensors.resize(num_tensors); - for (int i = 0; i < num_tensors; ++i) { - TensorShape shape{seq_tensors.tensors[i].shape}; - auto values_count = static_cast(seq_tensors.tensors[i].data.size()); - ORT_ENFORCE(shape.Size() == values_count, values_count, - " input values doesn't match tensor size of ", shape.Size()); + AddSeqData(input_data_, name, seq_tensors); + } - auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU); - auto& tensor = ptr->tensors[i]; - - tensor = Tensor(DataTypeImpl::GetType(), - shape, - allocator); - - auto* data_ptr = tensor.template MutableData(); - for (int64_t x = 0; x < values_count; ++x) { - data_ptr[x] = seq_tensors.tensors[i].data[x]; - } - } - - OrtValue value; - value.Init(ptr.get(), mltype, mltype->GetDeleteFunc()); - ptr.release(); - input_data_.push_back(Data(NodeArg(name, &SequenceTensorType::s_sequence_tensor_type_proto), std::move(value), - optional(), optional())); + template + void AddSeqOutput(const char* name, const SeqTensors& seq_tensors) { + AddSeqData(output_data_, name, seq_tensors); } template @@ -495,6 +472,40 @@ class OpTester { } } + template + void AddSeqData(std::vector& data, const char* name, const SeqTensors& seq_tensors) { + auto mltype = DataTypeImpl::GetType(); + ORT_ENFORCE(mltype != nullptr, "TensorSeq must be a registered cpp type"); + auto ptr = onnxruntime::make_unique(); + ptr->dtype = DataTypeImpl::GetType(); + auto num_tensors = seq_tensors.tensors.size(); + ptr->tensors.resize(num_tensors); + for (int i = 0; i < num_tensors; ++i) { + TensorShape shape{seq_tensors.tensors[i].shape}; + auto values_count = static_cast(seq_tensors.tensors[i].data.size()); + ORT_ENFORCE(shape.Size() == values_count, values_count, + " input values doesn't match tensor size of ", shape.Size()); + + auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU); + auto& tensor = ptr->tensors[i]; + + tensor = Tensor(DataTypeImpl::GetType(), + shape, + allocator); + + auto* data_ptr = tensor.template MutableData(); + for (int64_t x = 0; x < values_count; ++x) { + data_ptr[x] = seq_tensors.tensors[i].data[x]; + } + } + + OrtValue value; + value.Init(ptr.get(), mltype, mltype->GetDeleteFunc()); + ptr.release(); + data.push_back(Data(NodeArg(name, &SequenceTensorType::s_sequence_tensor_type_proto), std::move(value), + optional(), optional())); + } + void ExecuteModel(Model& model, InferenceSession& session_object, ExpectResult expect_result, const std::string& expected_failure_string, const RunOptions* run_options, std::unordered_map feeds, std::vector output_names, diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py index 37ec1d1543..af0ae14dfe 100644 --- a/onnxruntime/test/python/onnxruntime_test_python.py +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -8,6 +8,7 @@ import numpy as np import onnxruntime as onnxrt import threading + class TestInferenceSession(unittest.TestCase): def get_name(self, name): @@ -42,26 +43,28 @@ class TestInferenceSession(unittest.TestCase): self.assertTrue(os.path.isfile(so.optimized_model_filepath)) def testGetProviders(self): - self.assertTrue('CPUExecutionProvider' in onnxrt.get_available_providers()) + self.assertTrue( + 'CPUExecutionProvider' in onnxrt.get_available_providers()) self.assertTrue('CPUExecutionProvider' in onnxrt.get_all_providers()) sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx")) self.assertTrue('CPUExecutionProvider' in sess.get_providers()) def testSetProviders(self): if 'CUDAExecutionProvider' in onnxrt.get_available_providers(): - sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx")) - # confirm that CUDA Provider is in list of registered providers. - self.assertTrue('CUDAExecutionProvider' in sess.get_providers()) - # reset the session and register only CPU Provider. - sess.set_providers(['CPUExecutionProvider']) - # confirm only CPU Provider is registered now. - self.assertEqual(['CPUExecutionProvider'], sess.get_providers()) + sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx")) + # confirm that CUDA Provider is in list of registered providers. + self.assertTrue('CUDAExecutionProvider' in sess.get_providers()) + # reset the session and register only CPU Provider. + sess.set_providers(['CPUExecutionProvider']) + # confirm only CPU Provider is registered now. + self.assertEqual(['CPUExecutionProvider'], sess.get_providers()) def testInvalidSetProviders(self): with self.assertRaises(ValueError) as context: - sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx")) - sess.set_providers(['InvalidProvider']) - self.assertTrue('[\'InvalidProvider\'] does not contain a subset of available providers' in str(context.exception)) + sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx")) + sess.set_providers(['InvalidProvider']) + self.assertTrue('[\'InvalidProvider\'] does not contain a subset of available providers' in str( + context.exception)) def testRunModel(self): sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx")) @@ -346,26 +349,6 @@ class TestInferenceSession(unittest.TestCase): res = sess.run([output_name], {x_name: x}) self.assertEqual(output_expected, res[0]) - def testSequenceLength(self): - sess = onnxrt.InferenceSession( - self.get_name("sequence_length.onnx")) - x = [np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3)), - np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3))] - - x_name = sess.get_inputs()[0].name - self.assertEqual(x_name, "X") - x_type = sess.get_inputs()[0].type - self.assertEqual(x_type, 'seq(tensor(float))') - - output_name = sess.get_outputs()[0].name - self.assertEqual(output_name, "Y") - output_type = sess.get_outputs()[0].type - self.assertEqual(output_type, 'tensor(int64)') - - output_expected = np.array(2, dtype=np.int64) - res = sess.run([output_name], {x_name: x}) - self.assertEqual(output_expected, res[0]) - def testZipMapInt64Float(self): sess = onnxrt.InferenceSession(self.get_name("zipmap_int64float.onnx")) x = np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], @@ -547,6 +530,68 @@ class TestInferenceSession(unittest.TestCase): res = sess.run([], {'input1:0': a, 'input:0':b}) + def testSequenceLength(self): + sess = onnxrt.InferenceSession(self.get_name("sequence_length.onnx")) + x = [np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3)), + np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3))] + + x_name = sess.get_inputs()[0].name + self.assertEqual(x_name, "X") + x_type = sess.get_inputs()[0].type + self.assertEqual(x_type, 'seq(tensor(float))') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "Y") + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'tensor(int64)') + + output_expected = np.array(2, dtype=np.int64) + res = sess.run([output_name], {x_name: x}) + self.assertEqual(output_expected, res[0]) + + def testSequenceConstruct(self): + sess = onnxrt.InferenceSession( + self.get_name("sequence_construct.onnx")) + + self.assertEqual(sess.get_inputs()[0].type, 'tensor(int64)') + self.assertEqual(sess.get_inputs()[1].type, 'tensor(int64)') + + self.assertEqual(sess.get_inputs()[0].name, "tensor1") + self.assertEqual(sess.get_inputs()[1].name, "tensor2") + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "output_sequence") + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'seq(tensor(int64))') + + output_expected = [np.array([1, 0, 3, 44, 23, 11], dtype=np.int64).reshape((2, 3)), + np.array([1, 2, 3, 4, 5, 6], dtype=np.int64).reshape((2, 3))] + + res = sess.run([output_name], {"tensor1": np.array([1, 0, 3, 44, 23, 11], dtype=np.int64).reshape((2, 3)), + "tensor2": np.array([1, 2, 3, 4, 5, 6], dtype=np.int64).reshape((2, 3))}) + + np.testing.assert_array_equal(output_expected, res[0]) + + def testSequenceInsert(self): + sess = onnxrt.InferenceSession(self.get_name("sequence_insert.onnx")) + + self.assertEqual(sess.get_inputs()[0].type, 'seq(tensor(int64))') + self.assertEqual(sess.get_inputs()[1].type, 'tensor(int64)') + + self.assertEqual(sess.get_inputs()[0].name, "input_seq") + self.assertEqual(sess.get_inputs()[1].name, "tensor") + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "output_sequence") + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'seq(tensor(int64))') + + output_expected = [ + np.array([1, 0, 3, 44, 23, 11], dtype=np.int64).reshape((2, 3))] + res = sess.run([output_name], {"tensor": np.array( + [1, 0, 3, 44, 23, 11], dtype=np.int64).reshape((2, 3)), "input_seq": []}) + np.testing.assert_array_equal(output_expected, res[0]) + if __name__ == '__main__': unittest.main() diff --git a/onnxruntime/test/testdata/sequence_construct.onnx b/onnxruntime/test/testdata/sequence_construct.onnx new file mode 100644 index 0000000000..5fdd0aaa78 --- /dev/null +++ b/onnxruntime/test/testdata/sequence_construct.onnx @@ -0,0 +1,18 @@ + onnx-example:¡ +6 +tensor1 +tensor2output_sequence"SequenceConstruct +test-modelZ +tensor1 +  + +Z +tensor2 +  + +b% +output_sequence" + +  + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/sequence_insert.onnx b/onnxruntime/test/testdata/sequence_insert.onnx new file mode 100644 index 0000000000..12dbcb0c65 --- /dev/null +++ b/onnxruntime/test/testdata/sequence_insert.onnx @@ -0,0 +1,19 @@ + onnx-example:¤ +4 + input_seq +tensoroutput_sequence"SequenceInsert +test-modelZ + input_seq" + +  + +Z +tensor +  + +b% +output_sequence" + +  + +B \ No newline at end of file