From ea60469af551f94b9b0452d0687f5be2c8b8a5ff Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Mon, 7 Oct 2019 15:35:09 -0700 Subject: [PATCH] Support seq(tensor), implement 2 sequence ops that use the new type. (#1983) * Mention OrtCreateSessionFromArray in C API doc * fix seq of tensors * changes on 9/30 * All tests passing * Add SequenceAt op * Fix shared_lib non_tensor_types test * Address some PR comments * Address PR comments * Add support in python bindings to accept seq(tensor) * Change data type from vector to TensorSeq * Change data type from vector to TensorSeq * Added some documentation * Added missing test model * Fix Linux build * Fix Mac build * Fix Mac build --- .../onnxruntime/core/framework/allocator.h | 2 + .../onnxruntime/core/framework/data_types.h | 22 +- include/onnxruntime/core/framework/tensor.h | 17 +- onnxruntime/core/framework/TensorSeq.h | 42 ++++ onnxruntime/core/framework/data_types.cc | 119 +++++++-- onnxruntime/core/framework/tensor.cc | 14 ++ .../providers/cpu/cpu_execution_provider.cc | 4 + .../providers/cpu/sequence/sequence_ops.cc | 104 ++++++++ .../providers/cpu/sequence/sequence_ops.h | 27 ++ onnxruntime/core/session/onnxruntime_c_api.cc | 231 +++++++++++++----- .../python/onnxruntime_pybind_mlvalue.cc | 46 +++- .../python/onnxruntime_pybind_state.cc | 114 ++++----- onnxruntime/test/framework/data_types_test.cc | 32 +-- .../cpu/sequence/sequence_ops_test.cc | 83 +++++++ .../test/providers/provider_test_utils.h | 61 +++++ .../test/python/onnxruntime_test_python.py | 20 ++ .../test/shared_lib/test_nontensor_types.cc | 27 ++ .../test/testdata/sequence_length.onnx | Bin 0 -> 96 bytes 18 files changed, 777 insertions(+), 188 deletions(-) create mode 100644 onnxruntime/core/framework/TensorSeq.h create mode 100644 onnxruntime/core/providers/cpu/sequence/sequence_ops.cc create mode 100644 onnxruntime/core/providers/cpu/sequence/sequence_ops.h create mode 100644 onnxruntime/test/providers/cpu/sequence/sequence_ops_test.cc create mode 100644 onnxruntime/test/testdata/sequence_length.onnx diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h index 50ec1b4b47..ab3741d51e 100644 --- a/include/onnxruntime/core/framework/allocator.h +++ b/include/onnxruntime/core/framework/allocator.h @@ -81,6 +81,8 @@ inline bool operator!=(const OrtDevice& left, const OrtDevice& other) { } struct OrtMemoryInfo { + OrtMemoryInfo() = default; // to allow default construction of Tensor + // use string for name, so we could have customized allocator in execution provider. const char* name; int id; diff --git a/include/onnxruntime/core/framework/data_types.h b/include/onnxruntime/core/framework/data_types.h index b11805378a..a496787b51 100644 --- a/include/onnxruntime/core/framework/data_types.h +++ b/include/onnxruntime/core/framework/data_types.h @@ -17,8 +17,10 @@ struct OrtValue; namespace ONNX_NAMESPACE { class TypeProto; } // namespace ONNX_NAMESPACE + namespace onnxruntime { /// Predefined registered types + //maps using MapStringToString = std::map; using MapStringToInt64 = std::map; @@ -30,10 +32,6 @@ using MapInt64ToFloat = std::map; using MapInt64ToDouble = std::map; //vectors/sequences -using VectorString = std::vector; -using VectorInt64 = std::vector; -using VectorFloat = std::vector; -using VectorDouble = std::vector; using VectorMapStringToFloat = std::vector; using VectorMapInt64ToFloat = std::vector; @@ -197,6 +195,9 @@ class DataTypeImpl { template static MLDataType GetTensorType(); + template + static MLDataType GetSequenceTensorType(); + // Return the MLDataType for a concrete sparse tensor type. template static MLDataType GetSparseTensorType(); @@ -212,6 +213,7 @@ class DataTypeImpl { static const TensorTypeBase* TensorTypeFromONNXEnum(int type); static const SparseTensorTypeBase* SparseTensorTypeFromONNXEnum(int type); + static const NonTensorTypeBase* SequenceTensorTypeFromONNXEnum(int type); static const char* ToString(MLDataType type); // Registers ONNX_NAMESPACE::DataType (internalized string) with @@ -221,6 +223,7 @@ class DataTypeImpl { static MLDataType GetDataType(const std::string&); static const std::vector& AllTensorTypes(); + static const std::vector& AllSequenceTensorTypes(); static const std::vector& AllFixedSizeTensorTypes(); static const std::vector& AllNumericTensorTypes(); static const std::vector& AllIEEEFloatTensorTypes(); @@ -752,6 +755,17 @@ class NonOnnxType : public DataTypeImpl { return SequenceType::Type(); \ } +#define ORT_REGISTER_SEQ_TENSOR_TYPE(ELEM_TYPE) \ + template <> \ + MLDataType SequenceTensorType::Type() { \ + static SequenceTensorType sequence_tensor_type; \ + return &sequence_tensor_type; \ + } \ + template <> \ + MLDataType DataTypeImpl::GetSequenceTensorType() { \ + return SequenceTensorType::Type(); \ + } + #define ORT_REGISTER_NON_ONNX_TYPE(TYPE) \ template <> \ MLDataType NonOnnxType::Type() { \ diff --git a/include/onnxruntime/core/framework/tensor.h b/include/onnxruntime/core/framework/tensor.h index f1d9bf52a9..ec8b8171f5 100644 --- a/include/onnxruntime/core/framework/tensor.h +++ b/include/onnxruntime/core/framework/tensor.h @@ -11,12 +11,11 @@ #include "gsl/gsl" #include "core/common/common.h" #include "core/framework/allocator.h" -#include "core/framework/data_types.h" #include "core/framework/tensor_shape.h" #include "onnxruntime_config.h" +#include "core/framework/data_types.h" namespace onnxruntime { - // TODO: Do we need this class or is IAllocator::MakeUniquePtr sufficient/better class BufferDeleter { public: @@ -58,6 +57,8 @@ using BufferNakedPtr = void*; */ class Tensor final { public: + Tensor() = default; // to allow creating vector to support seq(tensor) + /** * Create tensor with given type, shape, pre-allocate memory and allocator info. * This function won't check if the preallocated buffer(p_data) has enough room for the shape. @@ -171,17 +172,7 @@ class Tensor final { /** The number of bytes of data. */ - size_t SizeInBytes() const { - size_t ret; - int64_t l = shape_.Size(); - if (l >= static_cast(std::numeric_limits::max())) { - ORT_THROW("tensor size overflow"); - } - if (!IAllocator::CalcMemSizeForArray(static_cast(shape_.Size()), dtype_->Size(), &ret)) { - ORT_THROW("tensor size overflow"); - } - return ret; - } + size_t SizeInBytes() const; // More API methods. private: diff --git a/onnxruntime/core/framework/TensorSeq.h b/onnxruntime/core/framework/TensorSeq.h new file mode 100644 index 0000000000..8a2d97a8e3 --- /dev/null +++ b/onnxruntime/core/framework/TensorSeq.h @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/tensor.h" +#include + +namespace onnxruntime { +// Put this in a separate file to avoid circular dependency between tensor.h and data_types.h +// Data type to represent a sequence of tensors of the same type +struct TensorSeq { + using value_type = Tensor; // to satisfy SequenceType template + + // A sequence must be associated with only one data type and all tensors in the seq must be of that type + // One other alternative of storing the data type of a seq is to templatize the TensorSeq class. + // The current design follows the Tensor methodology. + // We also require this because the SequenceEmpty op expects the creation of a seq of a specific type + // and the SequenceInsert op expects validation of tensors to be added to the seq against this type. + MLDataType dtype; + + // TODO: optimization opportunity - if all tensors in the seq are scalars, we can potentially represent them + // as vector + std::vector tensors; +}; + +template +class SequenceTensorType : public NonTensorType { + public: + static MLDataType Type(); + + bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override { + return this->IsSequenceCompatible(type_proto); + } + + private: + SequenceTensorType() { + data_types_internal::SetSequenceType::Set(this->mutable_type_proto()); + } +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/data_types.cc b/onnxruntime/core/framework/data_types.cc index 78123a7b25..a07f36eb9c 100644 --- a/onnxruntime/core/framework/data_types.cc +++ b/onnxruntime/core/framework/data_types.cc @@ -3,6 +3,7 @@ #include "core/framework/data_types.h" #include "core/framework/tensor.h" +#include "core/framework/TensorSeq.h" #include "core/framework/sparse_tensor.h" #include "core/graph/onnx_protobuf.h" @@ -28,7 +29,8 @@ template <> MLDataType DataTypeImpl::GetType() { return TensorTypeBase::Type(); } -} // namespace onnxruntime + +} // namespace onnxruntime // This conflics with the above GetType<>() specialization #include "core/framework/tensorprotoutils.h" @@ -507,7 +509,7 @@ void NonTensorTypeBase::FromDataContainer(const void* /* data */, size_t /*data_ ORT_ENFORCE(false, "Not implemented"); } -void NonTensorTypeBase::ToDataContainer (const OrtValue& /* input */, size_t /*data_size */, void* /* data */) const { +void NonTensorTypeBase::ToDataContainer(const OrtValue& /* input */, size_t /*data_size */, void* /* data */) const { ORT_ENFORCE(false, "Not implemented"); } @@ -550,10 +552,22 @@ ORT_REGISTER_MAP(MapInt64ToInt64); ORT_REGISTER_MAP(MapInt64ToFloat); ORT_REGISTER_MAP(MapInt64ToDouble); -ORT_REGISTER_SEQ(VectorString); -ORT_REGISTER_SEQ(VectorFloat); -ORT_REGISTER_SEQ(VectorInt64); -ORT_REGISTER_SEQ(VectorDouble); +// Register sequence of tensor types +ORT_REGISTER_SEQ(TensorSeq) // required to ensure GetType works +ORT_REGISTER_SEQ_TENSOR_TYPE(int32_t); +ORT_REGISTER_SEQ_TENSOR_TYPE(float); +ORT_REGISTER_SEQ_TENSOR_TYPE(bool); +ORT_REGISTER_SEQ_TENSOR_TYPE(std::string); +ORT_REGISTER_SEQ_TENSOR_TYPE(int8_t); +ORT_REGISTER_SEQ_TENSOR_TYPE(uint8_t); +ORT_REGISTER_SEQ_TENSOR_TYPE(uint16_t); +ORT_REGISTER_SEQ_TENSOR_TYPE(int16_t); +ORT_REGISTER_SEQ_TENSOR_TYPE(int64_t); +ORT_REGISTER_SEQ_TENSOR_TYPE(double); +ORT_REGISTER_SEQ_TENSOR_TYPE(uint32_t); +ORT_REGISTER_SEQ_TENSOR_TYPE(uint64_t); +ORT_REGISTER_SEQ_TENSOR_TYPE(MLFloat16); +ORT_REGISTER_SEQ_TENSOR_TYPE(BFloat16); ORT_REGISTER_SEQ(VectorMapStringToFloat); ORT_REGISTER_SEQ(VectorMapInt64ToFloat); @@ -565,6 +579,12 @@ ORT_REGISTER_SEQ(VectorMapInt64ToFloat); reg_fn(mltype); \ } +#define REGISTER_SEQ_TENSOR_PROTO(TYPE, reg_fn) \ + { \ + MLDataType mltype = DataTypeImpl::GetSequenceTensorType(); \ + reg_fn(mltype); \ + } + #define REGISTER_SPARSE_TENSOR_PROTO(TYPE, reg_fn) \ { \ MLDataType mltype = DataTypeImpl::GetSparseTensorType(); \ @@ -619,10 +639,20 @@ void RegisterAllProtos(const std::function& reg_fn) { REGISTER_ONNX_PROTO(MapInt64ToFloat, reg_fn); REGISTER_ONNX_PROTO(MapInt64ToDouble, reg_fn); - REGISTER_ONNX_PROTO(VectorString, reg_fn); - REGISTER_ONNX_PROTO(VectorFloat, reg_fn); - REGISTER_ONNX_PROTO(VectorInt64, reg_fn); - REGISTER_ONNX_PROTO(VectorDouble, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(int32_t, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(float, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(bool, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(std::string, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(int8_t, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(uint8_t, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(uint16_t, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(int16_t, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(int64_t, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(double, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(uint32_t, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(uint64_t, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(MLFloat16, reg_fn); + REGISTER_SEQ_TENSOR_PROTO(BFloat16, reg_fn); REGISTER_ONNX_PROTO(VectorMapStringToFloat, reg_fn); REGISTER_ONNX_PROTO(VectorMapInt64ToFloat, reg_fn); @@ -724,6 +754,41 @@ const TensorTypeBase* DataTypeImpl::TensorTypeFromONNXEnum(int type) { } } +const NonTensorTypeBase* DataTypeImpl::SequenceTensorTypeFromONNXEnum(int type) { + switch (type) { + case TensorProto_DataType_FLOAT: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_BOOL: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_INT32: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_DOUBLE: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_STRING: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_UINT8: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_UINT16: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_INT8: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_INT16: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_INT64: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_UINT32: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_UINT64: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_FLOAT16: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + case TensorProto_DataType_BFLOAT16: + return DataTypeImpl::GetSequenceTensorType()->AsNonTensorTypeBase(); + default: + ORT_NOT_IMPLEMENTED("tensor type ", type, " is not supported"); + } +} + const SparseTensorTypeBase* DataTypeImpl::SparseTensorTypeFromONNXEnum(int type) { switch (type) { case TensorProto_DataType_FLOAT: @@ -868,19 +933,7 @@ MLDataType DataTypeImpl::TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto) { } // MapType break; case TypeProto::ValueCase::kTensorType: { - auto val_elem_type = val_type.tensor_type().elem_type(); - switch (val_elem_type) { - case TensorProto_DataType_STRING: - return DataTypeImpl::GetType(); - case TensorProto_DataType_INT64: - return DataTypeImpl::GetType(); - case TensorProto_DataType_FLOAT: - return DataTypeImpl::GetType(); - case TensorProto_DataType_DOUBLE: - return DataTypeImpl::GetType(); - default: - break; - } + return DataTypeImpl::GetType(); } // kTensorType break; default: @@ -989,6 +1042,26 @@ const std::vector& DataTypeImpl::AllTensorTypes() { return all_tensor_types; } +const std::vector& DataTypeImpl::AllSequenceTensorTypes() { + static std::vector all_sequence_tensor_types = + {DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType(), + DataTypeImpl::GetSequenceTensorType()}; + + return all_sequence_tensor_types; +} + const std::vector& DataTypeImpl::AllNumericTensorTypes() { static std::vector all_numeric_size_tensor_types = {DataTypeImpl::GetTensorType(), diff --git a/onnxruntime/core/framework/tensor.cc b/onnxruntime/core/framework/tensor.cc index 3ef4bcfe51..00595c30e2 100644 --- a/onnxruntime/core/framework/tensor.cc +++ b/onnxruntime/core/framework/tensor.cc @@ -5,6 +5,8 @@ #include #include "core/framework/allocatormgr.h" +#include "core/framework/data_types.h" + using namespace std; namespace onnxruntime { @@ -27,6 +29,18 @@ Tensor::Tensor(MLDataType p_type, const TensorShape& shape, std::shared_ptr= static_cast(std::numeric_limits::max())) { + ORT_THROW("tensor size overflow"); + } + if (!IAllocator::CalcMemSizeForArray(static_cast(shape_.Size()), dtype_->Size(), &ret)) { + ORT_THROW("tensor size overflow"); + } + return ret; +} + void Tensor::Init(MLDataType p_type, const TensorShape& shape, void* p_raw_data, AllocatorPtr deleter, int64_t offset) { int64_t shape_size = shape.Size(); if (shape_size < 0) ORT_THROW("shape.Size() must >=0"); diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 3283330328..8aa6ab91e3 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -389,6 +389,8 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Lp class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Conv); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ConvTranspose); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, If); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceLength); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceAt); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ScatterND); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Gemm); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, GatherElements); @@ -1009,6 +1011,8 @@ void RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc b/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc new file mode 100644 index 0000000000..7c7049838c --- /dev/null +++ b/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cpu/sequence/sequence_ops.h" +#include "core/framework/tensorprotoutils.h" +#include "core/providers/cpu/tensor/utils.h" +#include "core/framework/TensorSeq.h" + +using namespace onnxruntime::common; + +namespace onnxruntime { + +// SequenceLength +ONNX_CPU_OPERATOR_KERNEL( + SequenceLength, + 11, + KernelDefBuilder() + .TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes()) + .TypeConstraint("I", DataTypeImpl::GetTensorType()), + SequenceLength); + +Status SequenceLength::Compute(OpKernelContext* context) const { + const auto* X = context->Input(0); + ORT_ENFORCE(X != nullptr, "Got nullptr for sequence input."); + + auto* Y = context->Output(0, {}); + auto* Y_data = Y->template MutableData(); + *Y_data = static_cast(X->tensors.size()); + + return Status::OK(); +} + +// SequenceAt +ONNX_CPU_OPERATOR_KERNEL( + SequenceAt, + 11, + KernelDefBuilder() + .TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes()) + .TypeConstraint("T", DataTypeImpl::AllTensorTypes()) + .TypeConstraint("I", std::vector{ + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + SequenceAt); + +static int64_t GetSeqIdx(const Tensor& idx_tensor) { + int64_t seq_idx = INT_MAX; + auto idx_tensor_dtype = utils::GetTensorProtoType(idx_tensor); + switch (idx_tensor_dtype) { + case ONNX_NAMESPACE::TensorProto_DataType_INT32: { + const auto* idx_data = idx_tensor.Data(); + seq_idx = static_cast(*idx_data); + break; + } + case ONNX_NAMESPACE::TensorProto_DataType_INT64: { + const auto* idx_data = idx_tensor.Data(); + seq_idx = *idx_data; + break; + } + default: + ORT_THROW("Unsupported data type: ", idx_tensor_dtype); + } + return seq_idx; +} + +bool ValidateSeqIdx(int64_t input_seq_idx, int64_t seq_size) { + bool retval = false; + if (input_seq_idx < 0) { + retval = input_seq_idx <= -1 && input_seq_idx >= -seq_size; + } else { + retval = input_seq_idx < seq_size; + } + return retval; +} + +template +static void CopyTensor(const Tensor& indexed_tensor, Tensor& output_tensor) { + const auto* input_data = indexed_tensor.template Data(); + auto* output_data = output_tensor.template MutableData(); + memcpy(output_data, input_data, indexed_tensor.SizeInBytes()); +} + +Status SequenceAt::Compute(OpKernelContext* context) const { + const auto* X = context->Input(0); + ORT_ENFORCE(X != nullptr, "Got nullptr for sequence input."); + + const auto* I = context->Input(1); + ORT_ENFORCE(I != nullptr, "Got nullptr input for index tensor"); + int64_t input_seq_idx = GetSeqIdx(*I); + if (!ValidateSeqIdx(input_seq_idx, static_cast(X->tensors.size()))) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Invalid sequence index (", input_seq_idx, ") specified for sequence of size (", X->tensors.size(), ")"); + } + + if (input_seq_idx < 0) { + input_seq_idx = static_cast(X->tensors.size()) + input_seq_idx; + } + const Tensor& indexed_tensor = X->tensors[input_seq_idx]; + auto* Y = context->Output(0, indexed_tensor.Shape().GetDims()); + CopyCpuTensor(&indexed_tensor, Y); + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/sequence/sequence_ops.h b/onnxruntime/core/providers/cpu/sequence/sequence_ops.h new file mode 100644 index 0000000000..006d81bdfd --- /dev/null +++ b/onnxruntime/core/providers/cpu/sequence/sequence_ops.h @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { + +class SequenceLength final : public OpKernel { + public: + SequenceLength(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +class SequenceAt final : public OpKernel { + public: + SequenceAt(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override; +}; + +} //namespace onnxruntime diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 000e27638c..5ef0478067 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -24,6 +24,7 @@ #include "core/session/ort_apis.h" #include "core/framework/data_types.h" #include "abi_session_options_impl.h" +#include "core/framework/TensorSeq.h" using namespace onnxruntime::logging; using onnxruntime::BFloat16; @@ -101,9 +102,9 @@ ORT_API_STATUS_IMPL(OrtApis::CreateEnvWithCustomLogger, OrtLoggingFunction loggi std::string name = logid; std::unique_ptr logger = onnxruntime::make_unique(logging_function, logger_param); auto default_logging_manager = onnxruntime::make_unique(std::move(logger), - static_cast(default_warning_level), false, - LoggingManager::InstanceType::Default, - &name); + static_cast(default_warning_level), false, + LoggingManager::InstanceType::Default, + &name); std::unique_ptr env; Status status = Environment::Create(env); if (status.IsOK()) @@ -117,9 +118,9 @@ ORT_API_STATUS_IMPL(OrtApis::CreateEnv, OrtLoggingLevel default_warning_level, API_IMPL_BEGIN std::string name = logid; auto default_logging_manager = onnxruntime::make_unique(std::unique_ptr{new CLogSink{}}, - static_cast(default_warning_level), false, - LoggingManager::InstanceType::Default, - &name); + static_cast(default_warning_level), false, + LoggingManager::InstanceType::Default, + &name); std::unique_ptr env; Status status = Environment::Create(env); if (status.IsOK()) { @@ -143,6 +144,25 @@ OrtStatus* CreateTensorImpl(const int64_t* shape, size_t shape_len, OrtAllocator return nullptr; } +template +OrtStatus* CreateTensorImplForSeq(const int64_t* shape, size_t shape_len, + Tensor& out) { + std::vector shapes(shape_len); + for (size_t i = 0; i != shape_len; ++i) { + shapes[i] = shape[i]; + } + OrtAllocator* allocator; + // TODO(pranav): what allocator should be used to create the tensor here? + // for the sake of simplicity of the API using the default one here + auto st = OrtApis::GetAllocatorWithDefaultOptions(&allocator); + if (st) { + return st; + } + std::shared_ptr alloc_ptr = std::make_shared(allocator); + out = Tensor(DataTypeImpl::GetType(), onnxruntime::TensorShape(shapes), alloc_ptr); + return nullptr; +} + /** * * this function will create a copy of the allocator info @@ -664,6 +684,13 @@ OrtStatus* OrtGetNumSequenceElements(const OrtValue* p_ml_value, size_t* out) { return nullptr; } +template <> +OrtStatus* OrtGetNumSequenceElements(const OrtValue* p_ml_value, size_t* out) { + auto& data = p_ml_value->Get(); + *out = data.tensors.size(); + return nullptr; +} + static OrtStatus* OrtGetValueCountImpl(const OrtValue* value, size_t* out) { ONNXType value_type; if (auto status = OrtApis::GetValueType(value, &value_type)) @@ -676,15 +703,8 @@ static OrtStatus* OrtGetValueCountImpl(const OrtValue* value, size_t* out) { auto v = reinterpret_cast(value); auto type = v->Type(); // Note: keep these in sync with the registered types in data_types.h - if (type == DataTypeImpl::GetType()) { - return OrtGetNumSequenceElements(v, out); - } - if (type == DataTypeImpl::GetType()) { - return OrtGetNumSequenceElements(v, out); - } else if (type == DataTypeImpl::GetType()) { - return OrtGetNumSequenceElements(v, out); - } else if (type == DataTypeImpl::GetType()) { - return OrtGetNumSequenceElements(v, out); + if (type == DataTypeImpl::GetType()) { + return OrtGetNumSequenceElements(v, out); } else if (type == DataTypeImpl::GetType()) { return OrtGetNumSequenceElements(v, out); } else if (type == DataTypeImpl::GetType()) { @@ -773,16 +793,52 @@ OrtStatus* PopulateTensorWithData(OrtValue* oval, const std::string return nullptr; } +template +OrtStatus* OrtGetValueImplSeqOfTensorsHelper(OrtAllocator* allocator, const Tensor& tensor, + OrtValue** out) { + const auto& shape = tensor.Shape(); + const auto* tensor_data = tensor.Data(); + OrtStatus* st = OrtApis::CreateTensorAsOrtValue(allocator, shape.GetDims().data(), shape.NumDimensions(), + GetONNXTensorElementDataType(), out); + return st ? st : PopulateTensorWithData(*out, tensor_data, shape.Size()); +} + template -OrtStatus* OrtGetValueImplSeqOfPrimitives(const OrtValue* p_ml_value, int index, OrtAllocator* allocator, - OrtValue** out) { - using ElemType = typename T::value_type; +OrtStatus* OrtGetValueImplSeqOfTensors(const OrtValue* p_ml_value, int index, OrtAllocator* allocator, + OrtValue** out) { auto& data = p_ml_value->Get(); - auto& data_elem = data.at(index); - std::vector dims = {1}; - OrtStatus* st = OrtApis::CreateTensorAsOrtValue(allocator, dims.data(), dims.size(), - GetONNXTensorElementDataType(), out); - return st ? st : PopulateTensorWithData(*out, &data_elem, 1); + auto& one_tensor = data.tensors.at(index); + + auto tensor_elem_type = one_tensor.DataType(); + OrtStatus* st{}; + if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtGetValueImplSeqOfTensorsHelper(allocator, one_tensor, out); + } else { + st = OrtApis::CreateStatus(ORT_FAIL, "Only sequences that contain float tensors are supported."); + } + return st; } static OrtStatus* OrtGetValueImplSeq(const OrtValue* value, int index, OrtAllocator* allocator, @@ -790,15 +846,8 @@ static OrtStatus* OrtGetValueImplSeq(const OrtValue* value, int index, OrtAlloca auto p_ml_value = reinterpret_cast(value); auto type = p_ml_value->Type(); // Note: keep these in sync with the registered types in data_types.h - if (type == DataTypeImpl::GetType()) { - return OrtGetValueImplSeqOfPrimitives(p_ml_value, index, allocator, out); - } - if (type == DataTypeImpl::GetType()) { - return OrtGetValueImplSeqOfPrimitives(p_ml_value, index, allocator, out); - } else if (type == DataTypeImpl::GetType()) { - return OrtGetValueImplSeqOfPrimitives(p_ml_value, index, allocator, out); - } else if (type == DataTypeImpl::GetType()) { - return OrtGetValueImplSeqOfPrimitives(p_ml_value, index, allocator, out); + if (type == DataTypeImpl::GetType()) { + return OrtGetValueImplSeqOfTensors(p_ml_value, index, allocator, out); } else if (type == DataTypeImpl::GetType()) { return OrtGetValueImplSeqOfMap(p_ml_value, index, out); } else if (type == DataTypeImpl::GetType()) { @@ -897,44 +946,106 @@ ORT_API_STATUS_IMPL(OrtApis::GetValue, const OrtValue* value, int index, OrtAllo template static OrtStatus* OrtCreateValueImplSeqHelperMap(const OrtValue* const* in, size_t num_values, OrtValue** out) { using SeqType = std::vector; - auto vec_ptr = onnxruntime::make_unique(); - vec_ptr->reserve(num_values); + auto seq_ptr = onnxruntime::make_unique(); + seq_ptr->reserve(num_values); for (size_t idx = 0; idx < num_values; ++idx) { auto& m = reinterpret_cast(in[idx])->Get(); - vec_ptr->push_back(m); + seq_ptr->push_back(m); } // create OrtValue with this vector auto value = onnxruntime::make_unique(); - value->Init(vec_ptr.release(), + value->Init(seq_ptr.release(), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); *out = value.release(); return nullptr; } -template -static OrtStatus* OrtCreateValueImplSeqHelper(const OrtValue* const* in, size_t num_values, OrtValue** out) { - using SeqType = std::vector; - auto vec_ptr = onnxruntime::make_unique(); - vec_ptr->reserve(num_values); +template +static OrtStatus* OrtCreateValueImplSeqHelperTensor(const Tensor& tensor, + Tensor& out) { + auto data = tensor.Data(); + if (!data) { + return OrtApis::CreateStatus(ORT_FAIL, "Encountered nullptr."); + } + OrtAllocator* allocator; + OrtStatus* st = OrtApis::GetAllocatorWithDefaultOptions(&allocator); + if (st) { + return st; + } + + st = CreateTensorImplForSeq(tensor.Shape().GetDims().data(), tensor.Shape().NumDimensions(), out); + if (st) { + return st; + } + + size_t num_elems = tensor.Shape().Size(); + auto* out_data = out.MutableData(); + for (size_t i = 0; i < num_elems; ++i) { + *out_data++ = *data++; + } + return nullptr; +} + +static OrtStatus* OrtCreateValueImplSeqHelper(const OrtValue* const* in, size_t num_values, + OrtValue** out) { + auto seq_ptr = std::make_unique(); + seq_ptr->tensors.resize(num_values); + + // use the data type of the first tensor as the data type of the seq + seq_ptr->dtype = reinterpret_cast(in[0])->Get().DataType(); + for (size_t idx = 0; idx < num_values; ++idx) { - auto& tensor = reinterpret_cast(in[idx])->Get(); - auto data = tensor.Data(); - if (!data) { - return OrtApis::CreateStatus(ORT_FAIL, "Encountered nullptr."); + auto& one_tensor = reinterpret_cast(in[idx])->Get(); + auto tensor_elem_type = one_tensor.DataType(); + + // sequences must have tensors of the same data type + if (idx > 0 && (tensor_elem_type != seq_ptr->dtype)) { + return OrtApis::CreateStatus(ORT_FAIL, + "Sequences must have tensors of the same data type. There was at least one tensor in the input that was different."); + } + + OrtStatus* st{}; + if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtCreateValueImplSeqHelperTensor(one_tensor, seq_ptr->tensors[idx]); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtCreateValueImplSeqHelperTensor(one_tensor, seq_ptr->tensors[idx]); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtCreateValueImplSeqHelperTensor(one_tensor, seq_ptr->tensors[idx]); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtCreateValueImplSeqHelperTensor(one_tensor, seq_ptr->tensors[idx]); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtCreateValueImplSeqHelperTensor(one_tensor, seq_ptr->tensors[idx]); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtCreateValueImplSeqHelperTensor(one_tensor, seq_ptr->tensors[idx]); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtCreateValueImplSeqHelperTensor(one_tensor, seq_ptr->tensors[idx]); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtCreateValueImplSeqHelperTensor(one_tensor, seq_ptr->tensors[idx]); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtCreateValueImplSeqHelperTensor(one_tensor, seq_ptr->tensors[idx]); + } else if (tensor_elem_type == DataTypeImpl::GetType()) { + st = OrtCreateValueImplSeqHelperTensor(one_tensor, seq_ptr->tensors[idx]); + } else { + std::string err_msg = std::string("Unsupported data type: ") + DataTypeImpl::ToString(tensor_elem_type); + st = OrtApis::CreateStatus(ORT_FAIL, err_msg.c_str()); + } + + if (st) { + return st; } - vec_ptr->push_back(*data); } // create OrtValue with this vector auto value = onnxruntime::make_unique(); - value->Init(vec_ptr.release(), - DataTypeImpl::GetType(), - DataTypeImpl::GetType()->GetDeleteFunc()); + value->Init(seq_ptr.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); *out = value.release(); return nullptr; } -static OrtStatus* OrtCreateValueImplSeq(const OrtValue* const* in, size_t num_values, OrtValue** out) { +static OrtStatus* OrtCreateValueImplSeq(const OrtValue* const* in, size_t num_values, + OrtValue** out) { // We only support limited sequence types. For the sake of simplicity the type of the first // OrtValue* in OrtValue** will determine the type of the vector used to create the output OrtValue // this type should be either a tensor of limited types or map of limited types @@ -964,19 +1075,7 @@ static OrtStatus* OrtCreateValueImplSeq(const OrtValue* const* in, size_t num_va // finally create the output vector/MLValue auto first_mlvalue = reinterpret_cast(ovfirst); if (first_value_type == ONNX_TYPE_TENSOR) { - auto vec_type = first_mlvalue->Get().DataType(); - if (vec_type == DataTypeImpl::GetType()) { - return OrtCreateValueImplSeqHelper(in, num_values, out); - } - if (vec_type == DataTypeImpl::GetType()) { - return OrtCreateValueImplSeqHelper(in, num_values, out); - } else if (vec_type == DataTypeImpl::GetType()) { - return OrtCreateValueImplSeqHelper(in, num_values, out); - } else if (vec_type == DataTypeImpl::GetType()) { - return OrtCreateValueImplSeqHelper(in, num_values, out); - } else { - return OrtApis::CreateStatus(ORT_FAIL, "Type not supported."); - } + return OrtCreateValueImplSeqHelper(in, num_values, out); } else if (first_value_type == ONNX_TYPE_MAP) { auto map_type = first_mlvalue->Type(); if (map_type == DataTypeImpl::GetType()) { @@ -1064,7 +1163,8 @@ static OrtStatus* OrtCreateValueImplMap(const OrtValue* const* in, size_t num_va return OrtApis::CreateStatus(ORT_FAIL, "Key type is not supported yet."); } -static OrtStatus* OrtCreateValueImpl(const OrtValue* const* in, size_t num_values, enum ONNXType value_type, OrtValue** out) { +static OrtStatus* OrtCreateValueImpl(const OrtValue* const* in, size_t num_values, enum ONNXType value_type, + OrtValue** out) { if (num_values <= 0) { return OrtApis::CreateStatus(ORT_FAIL, "Number of values should be at least 1."); } @@ -1077,7 +1177,8 @@ static OrtStatus* OrtCreateValueImpl(const OrtValue* const* in, size_t num_value return OrtApis::CreateStatus(ORT_FAIL, "Input is not of type sequence or map."); } -ORT_API_STATUS_IMPL(OrtApis::CreateValue, const OrtValue* const* in, size_t num_values, enum ONNXType value_type, OrtValue** out) { +ORT_API_STATUS_IMPL(OrtApis::CreateValue, const OrtValue* const* in, size_t num_values, enum ONNXType value_type, + OrtValue** out) { API_IMPL_BEGIN return OrtCreateValueImpl(in, num_values, value_type, out); API_IMPL_END diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc index 29ed538af3..a13e5f15d6 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -12,6 +12,7 @@ #include "core/framework/tensor_shape.h" #include "core/framework/tensor.h" #include "core/framework/allocator.h" +#include "core/framework/TensorSeq.h" using namespace std; namespace onnxruntime { @@ -95,13 +96,13 @@ bool PyObjectCheck_Array(PyObject* o) { return PyObject_HasAttrString(o, "__array_finalize__"); } -void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyArrayObject* pyObject, - OrtValue* p_mlvalue) { +std::unique_ptr CreateTensor(AllocatorPtr alloc, const std::string& name_input, PyArrayObject* pyObject) { PyArrayObject* darray = PyArray_GETCONTIGUOUS(pyObject); if (darray == NULL) { throw std::runtime_error(std::string("The object must be a contiguous array for input '") + name_input + std::string("'.")); } bool dref = false; + std::unique_ptr p_tensor; try { const int npy_type = PyArray_TYPE(darray); @@ -115,7 +116,7 @@ void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyAr TensorShape shape(dims); auto element_type = NumpyToOnnxRuntimeTensorType(npy_type); - std::unique_ptr p_tensor = onnxruntime::make_unique(element_type, shape, alloc); + p_tensor = onnxruntime::make_unique(element_type, shape, alloc); if (npy_type == NPY_UNICODE) { // Copy string data which needs to be done after Tensor is allocated. // Strings are Python strings or numpy.unicode string. @@ -176,10 +177,6 @@ void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyAr } memcpy(buffer, static_cast(PyArray_DATA(darray)), len); } - - p_mlvalue->Init(p_tensor.release(), - DataTypeImpl::GetType(), - DataTypeImpl::GetType()->GetDeleteFunc()); } catch (...) { if (!dref) { Py_XDECREF(darray); @@ -195,6 +192,38 @@ void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyAr if (!dref) { Py_XDECREF(darray); } + + return p_tensor; +} + +void CreateSequenceOfTensors(AllocatorPtr alloc, const std::string& name_input, PyObject* pylist_obj, + OrtValue* p_mlvalue) { + auto list_size = PyList_Size(pylist_obj); + if (list_size <= 0) { + throw std::runtime_error("Got exception while creating seq(tensor) because input list size is " + std::to_string(list_size)); + } + auto p_seq_tensors = onnxruntime::make_unique(); + p_seq_tensors->tensors.resize(list_size); + for (Py_ssize_t i = 0; i < list_size; ++i) { + auto* py_obj = PyList_GetItem(pylist_obj, i); + auto p_tensor = CreateTensor(alloc, name_input, reinterpret_cast(py_obj)); + p_seq_tensors->tensors[i] = std::move(*p_tensor); + } + + p_mlvalue->Init(p_seq_tensors.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); +} + +void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyArrayObject* pyObject, + OrtValue* p_mlvalue) { + auto p_tensor = CreateTensor(alloc, name_input, pyObject); + if (!p_tensor) { + throw std::runtime_error("Got exception while creating tensor for input: " + name_input); + } + p_mlvalue->Init(p_tensor.release(), + DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); } std::string _get_type_name(int64_t&) { @@ -404,6 +433,9 @@ void CreateGenericMLValue(AllocatorPtr alloc, const std::string& name_input, py: // The most frequent case: input comes as an array. PyArrayObject* arr = reinterpret_cast(value.ptr()); CreateTensorMLValue(alloc, name_input, arr, p_mlvalue); + } else if (PyList_Check(value.ptr())) { + auto* seq_tensors = reinterpret_cast(value.ptr()); + CreateSequenceOfTensors(alloc, name_input, seq_tensors, p_mlvalue); } else if (PyDict_Check(value.ptr())) { CreateMapMLValue_AgnosticVectorMap((PyObject*)NULL, value.ptr(), alloc, name_input, p_mlvalue); } else { diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index e5a95a6152..66c5064856 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -147,6 +147,7 @@ template void AddNonTensor(OrtValue& val, std::vector& pyobjs) { pyobjs.push_back(py::cast(val.Get())); } + void AddNonTensorAsPyObj(OrtValue& val, std::vector& pyobjs) { // Should be in sync with core/framework/datatypes.h if (val.Type() == DataTypeImpl::GetType()) { @@ -165,14 +166,6 @@ void AddNonTensorAsPyObj(OrtValue& val, std::vector& pyobjs) { AddNonTensor(val, pyobjs); } else if (val.Type() == DataTypeImpl::GetType()) { AddNonTensor(val, pyobjs); - } else if (val.Type() == DataTypeImpl::GetType()) { - AddNonTensor(val, pyobjs); - } else if (val.Type() == DataTypeImpl::GetType()) { - AddNonTensor(val, pyobjs); - } else if (val.Type() == DataTypeImpl::GetType()) { - AddNonTensor(val, pyobjs); - } else if (val.Type() == DataTypeImpl::GetType()) { - AddNonTensor(val, pyobjs); } else if (val.Type() == DataTypeImpl::GetType()) { AddNonTensor(val, pyobjs); } else if (val.Type() == DataTypeImpl::GetType()) { @@ -629,57 +622,55 @@ including arg name, arg type (contains both type and shape).)pbdoc") return *(na.Type()); }, "node type") - .def( - "__str__", [](const onnxruntime::NodeArg& na) -> std::string { - std::ostringstream res; - res << "NodeArg(name='" << na.Name() << "', type='" << *(na.Type()) << "', shape="; - auto shape = na.Shape(); - std::vector arr; - if (shape == nullptr || shape->dim_size() == 0) { - res << "[]"; + .def("__str__", [](const onnxruntime::NodeArg& na) -> std::string { + std::ostringstream res; + res << "NodeArg(name='" << na.Name() << "', type='" << *(na.Type()) << "', shape="; + auto shape = na.Shape(); + std::vector arr; + if (shape == nullptr || shape->dim_size() == 0) { + res << "[]"; + } else { + res << "["; + for (int i = 0; i < shape->dim_size(); ++i) { + if (utils::HasDimValue(shape->dim(i))) { + res << shape->dim(i).dim_value(); + } else if (utils::HasDimParam(shape->dim(i))) { + res << "'" << shape->dim(i).dim_param() << "'"; } else { - res << "["; - for (int i = 0; i < shape->dim_size(); ++i) { - if (utils::HasDimValue(shape->dim(i))) { - res << shape->dim(i).dim_value(); - } else if (utils::HasDimParam(shape->dim(i))) { - res << "'" << shape->dim(i).dim_param() << "'"; - } else { - res << "None"; - } - - if (i < shape->dim_size() - 1) { - res << ", "; - } - } - res << "]"; - } - res << ")"; - - return std::string(res.str()); - }, - "converts the node into a readable string") - .def_property_readonly( - "shape", [](const onnxruntime::NodeArg& na) -> std::vector { - auto shape = na.Shape(); - std::vector arr; - if (shape == nullptr || shape->dim_size() == 0) { - return arr; + res << "None"; } - arr.resize(shape->dim_size()); - for (int i = 0; i < shape->dim_size(); ++i) { - if (utils::HasDimValue(shape->dim(i))) { - arr[i] = py::cast(shape->dim(i).dim_value()); - } else if (utils::HasDimParam(shape->dim(i))) { - arr[i] = py::cast(shape->dim(i).dim_param()); - } else { - arr[i] = py::none(); - } + if (i < shape->dim_size() - 1) { + res << ", "; } - return arr; - }, - "node shape (assuming the node holds a tensor)"); + } + res << "]"; + } + res << ")"; + + return std::string(res.str()); + }, + "converts the node into a readable string") + .def_property_readonly("shape", [](const onnxruntime::NodeArg& na) -> std::vector { + auto shape = na.Shape(); + std::vector arr; + if (shape == nullptr || shape->dim_size() == 0) { + return arr; + } + + arr.resize(shape->dim_size()); + for (int i = 0; i < shape->dim_size(); ++i) { + if (utils::HasDimValue(shape->dim(i))) { + arr[i] = py::cast(shape->dim(i).dim_value()); + } else if (utils::HasDimParam(shape->dim(i))) { + arr[i] = py::cast(shape->dim(i).dim_param()); + } else { + arr[i] = py::none(); + } + } + return arr; + }, + "node shape (assuming the node holds a tensor)"); py::class_(m, "SessionObjectInitializer"); py::class_(m, "InferenceSession", R"pbdoc(This is the main class used to run a model.)pbdoc") @@ -691,13 +682,12 @@ including arg name, arg type (contains both type and shape).)pbdoc") InitializeSession(sess, provider_types); }, R"pbdoc(Load a model saved in ONNX format.)pbdoc") - .def( - "read_bytes", [](InferenceSession* sess, const py::bytes& serializedModel, std::vector& provider_types) { - std::istringstream buffer(serializedModel); - OrtPybindThrowIfError(sess->Load(buffer)); - InitializeSession(sess, provider_types); - }, - R"pbdoc(Load a model serialized in ONNX format.)pbdoc") + .def("read_bytes", [](InferenceSession* sess, const py::bytes& serializedModel, std::vector& provider_types) { + std::istringstream buffer(serializedModel); + OrtPybindThrowIfError(sess->Load(buffer)); + InitializeSession(sess, provider_types); + }, + R"pbdoc(Load a model serialized in ONNX format.)pbdoc") .def("run", [](InferenceSession* sess, std::vector output_names, std::map pyfeeds, RunOptions* run_options = nullptr) -> std::vector { NameMLValMap feeds; for (auto _ : pyfeeds) { diff --git a/onnxruntime/test/framework/data_types_test.cc b/onnxruntime/test/framework/data_types_test.cc index ec3522e467..afcf6a2f60 100644 --- a/onnxruntime/test/framework/data_types_test.cc +++ b/onnxruntime/test/framework/data_types_test.cc @@ -28,6 +28,7 @@ struct TestMap { // Try recursive type registration and compatibility tests using TestMapToMapInt64ToFloat = TestMap; +using VectorInt64 = std::vector; using TestMapStringToVectorInt64 = TestMap; // Trial to see if we resolve the setter properly @@ -39,6 +40,7 @@ struct TestSequence { using value_type = T; }; +using VectorString = std::vector; using TestSequenceOfSequence = TestSequence; /// Adding an Opaque type with type parameters @@ -80,6 +82,8 @@ ORT_REGISTER_MAP(TestMapMLFloat16ToFloat); ORT_REGISTER_SEQ(MyOpaqueSeqCpp_1); ORT_REGISTER_SEQ(MyOpaqueSeqCpp_2); ORT_REGISTER_SEQ(TestSequenceOfSequence); +ORT_REGISTER_SEQ(VectorString); +ORT_REGISTER_SEQ(VectorInt64); ORT_REGISTER_OPAQUE_TYPE(TestOpaqueType_1, TestOpaqueDomain_1, TestOpaqueName_1); ORT_REGISTER_OPAQUE_TYPE(TestOpaqueType_2, TestOpaqueDomain_2, TestOpaqueName_2); @@ -390,6 +394,20 @@ TEST_F(DataTypeTest, BFloat16Test) { TEST_F(DataTypeTest, DataUtilsTest) { using namespace ONNX_NAMESPACE::Utils; + // Test simple seq + { + const std::string seq_float("seq(tensor(float))"); + const auto* seq_proto = DataTypeImpl::GetSequenceTensorType()->GetTypeProto(); + EXPECT_NE(seq_proto, nullptr); + DataType seq_dt = DataTypeUtils::ToType(*seq_proto); + EXPECT_NE(seq_dt, nullptr); + EXPECT_EQ(seq_float, *seq_dt); + DataType seq_from_str = DataTypeUtils::ToType(*seq_dt); + // Expect internalized strings + EXPECT_EQ(seq_dt, seq_from_str); + const auto& from_dt_proto = DataTypeUtils::ToTypeProto(seq_dt); + EXPECT_TRUE(DataTypeImpl::GetSequenceTensorType()->IsCompatible(from_dt_proto)); + } // Test Tensor { const std::string tensor_uint64("tensor(uint64)"); @@ -494,20 +512,6 @@ TEST_F(DataTypeTest, DataUtilsTest) { const auto& from_dt_proto = DataTypeUtils::ToTypeProto(map_dt); EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); } - // Test simple seq - { - const std::string seq_float("seq(tensor(float))"); - const auto* seq_proto = DataTypeImpl::GetType()->GetTypeProto(); - EXPECT_NE(seq_proto, nullptr); - DataType seq_dt = DataTypeUtils::ToType(*seq_proto); - EXPECT_NE(seq_dt, nullptr); - EXPECT_EQ(seq_float, *seq_dt); - DataType seq_from_str = DataTypeUtils::ToType(*seq_dt); - // Expect internalized strings - EXPECT_EQ(seq_dt, seq_from_str); - const auto& from_dt_proto = DataTypeUtils::ToTypeProto(seq_dt); - EXPECT_TRUE(DataTypeImpl::GetType()->IsCompatible(from_dt_proto)); - } // Test Sequence with recursion { const std::string seq_map_str_float("seq(map(string,tensor(float)))"); diff --git a/onnxruntime/test/providers/cpu/sequence/sequence_ops_test.cc b/onnxruntime/test/providers/cpu/sequence/sequence_ops_test.cc new file mode 100644 index 0000000000..f94ed9e823 --- /dev/null +++ b/onnxruntime/test/providers/cpu/sequence/sequence_ops_test.cc @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(SequenceOpsTest, SequenceLengthPositiveFloat) { + OpTester test("SequenceLength", 11); + SeqTensors input; + input.AddTensor({3, 2}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}); + input.AddTensor({3, 3}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}); + test.AddSeqInput("S", input); + test.AddOutput("I", {}, {2}); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceLengthPositiveInt64) { + OpTester test("SequenceLength", 11); + SeqTensors input; + input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6}); + input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9}); + test.AddSeqInput("S", input); + test.AddOutput("I", {}, {2}); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceAtPositiveIdx) { + OpTester test("SequenceAt", 11); + SeqTensors input; + std::vector output_vec{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f}; + std::vector output_shape{3, 3}; + input.AddTensor({3, 2}, {10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f}); + input.AddTensor(output_shape, output_vec); + test.AddSeqInput("S", input); + test.AddInput("I", {}, {1}); + test.AddOutput("T", output_shape, output_vec); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceAtNegativeIdx) { + OpTester test("SequenceAt", 11); + SeqTensors input; + std::vector output_vec{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f}; + std::vector output_shape{3, 3}; + input.AddTensor({3, 2}, {10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f}); + input.AddTensor(output_shape, output_vec); + test.AddSeqInput("S", input); + test.AddInput("I", {}, {-1}); + test.AddOutput("T", output_shape, output_vec); + test.Run(); +} + +TEST(SequenceOpsTest, SequenceAtInvalidPositiveIdx) { + OpTester test("SequenceAt", 11); + SeqTensors input; + std::vector output_vec{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f}; + std::vector output_shape{3, 3}; + input.AddTensor({3, 2}, {10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f}); + input.AddTensor(output_shape, output_vec); + test.AddSeqInput("S", input); + test.AddInput("I", {}, {10}); + test.AddOutput("T", output_shape, output_vec); + test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index"); +} + +TEST(SequenceOpsTest, SequenceAtInvalidNegativeIdx) { + OpTester test("SequenceAt", 11); + SeqTensors input; + std::vector output_vec{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f}; + std::vector output_shape{3, 3}; + input.AddTensor({3, 2}, {10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f}); + input.AddTensor(output_shape, output_vec); + test.AddSeqInput("S", input); + test.AddInput("I", {}, {-10}); + test.AddOutput("T", output_shape, output_vec); + test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index"); +} + +} // namespace test +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h index 8f965c86e6..6803806730 100644 --- a/onnxruntime/test/providers/provider_test_utils.h +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -16,6 +16,7 @@ #include "core/framework/data_types.h" #include "test/test_environment.h" #include "test/framework/TestAllocatorManager.h" +#include "core/framework/TensorSeq.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -27,6 +28,20 @@ class InferenceSession; struct SessionOptions; namespace test { +template +struct SeqTensors { + void AddTensor(const std::vector& shape0, const std::vector& data0) { + tensors.push_back(Tensor{shape0, data0}); + } + + template + struct Tensor { + std::vector shape; + std::vector data; + }; + std::vector> tensors; +}; + // unfortunately std::optional is in C++17 so use a miniversion of it template class optional { @@ -154,6 +169,20 @@ struct VectorOfMapType { template const VectorOfMapTypeProto VectorOfMapType::s_vec_map_type_proto; +template +struct SequenceTensorTypeProto : ONNX_NAMESPACE::TypeProto { + SequenceTensorTypeProto() { + MLDataType dt = DataTypeImpl::GetTensorType(); + const auto* elem_proto = dt->GetTypeProto(); + mutable_sequence_type()->mutable_elem_type()->CopyFrom(*elem_proto); + auto* tensor_type = mutable_sequence_type()->mutable_elem_type()->mutable_tensor_type(); + tensor_type->set_elem_type(TypeToDataType()); + } +}; + +template +const SequenceTensorTypeProto s_sequence_tensor_type_proto; + // To use OpTester: // 1. Create one with the op name // 2. Call AddAttribute with any attributes @@ -215,6 +244,38 @@ class OpTester { input_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional(), optional())); } + template + void AddSeqInput(const char* name, const SeqTensors& seq_tensors) { + auto mltype = DataTypeImpl::GetType(); + ORT_ENFORCE(mltype != nullptr, "TensorSeq must be a registered cpp type"); + auto ptr = std::make_unique(); + auto num_tensors = seq_tensors.tensors.size(); + ptr->tensors.resize(num_tensors); + for (int i = 0; i < num_tensors; ++i) { + TensorShape shape{seq_tensors.tensors[i].shape}; + auto values_count = static_cast(seq_tensors.tensors[i].data.size()); + ORT_ENFORCE(shape.Size() == values_count, values_count, + " input values doesn't match tensor size of ", shape.Size()); + + auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU); + auto& tensor = ptr->tensors[i]; + + tensor = Tensor(DataTypeImpl::GetType(), + shape, + allocator); + + auto* data_ptr = tensor.template MutableData(); + for (int64_t x = 0; x < values_count; ++x) { + data_ptr[x] = seq_tensors.tensors[i].data[x]; + } + } + + OrtValue value; + value.Init(ptr.get(), mltype, mltype->GetDeleteFunc()); + ptr.release(); + input_data_.push_back(Data(NodeArg(name, &s_sequence_tensor_type_proto), std::move(value), optional(), optional())); + } + template void AddInput(const char* name, const std::map& val) { std::unique_ptr> ptr = onnxruntime::make_unique>(val); diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py index 0d7e7ac026..37ec1d1543 100644 --- a/onnxruntime/test/python/onnxruntime_test_python.py +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -346,6 +346,26 @@ class TestInferenceSession(unittest.TestCase): res = sess.run([output_name], {x_name: x}) self.assertEqual(output_expected, res[0]) + def testSequenceLength(self): + sess = onnxrt.InferenceSession( + self.get_name("sequence_length.onnx")) + x = [np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3)), + np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3))] + + x_name = sess.get_inputs()[0].name + self.assertEqual(x_name, "X") + x_type = sess.get_inputs()[0].type + self.assertEqual(x_type, 'seq(tensor(float))') + + output_name = sess.get_outputs()[0].name + self.assertEqual(output_name, "Y") + output_type = sess.get_outputs()[0].type + self.assertEqual(output_type, 'tensor(int64)') + + output_expected = np.array(2, dtype=np.int64) + res = sess.run([output_name], {x_name: x}) + self.assertEqual(output_expected, res[0]) + def testZipMapInt64Float(self): sess = onnxrt.InferenceSession(self.get_name("zipmap_int64float.onnx")) x = np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], diff --git a/onnxruntime/test/shared_lib/test_nontensor_types.cc b/onnxruntime/test/shared_lib/test_nontensor_types.cc index cdbae006d5..02e7d17b9a 100644 --- a/onnxruntime/test/shared_lib/test_nontensor_types.cc +++ b/onnxruntime/test/shared_lib/test_nontensor_types.cc @@ -149,3 +149,30 @@ TEST_F(CApiTest, CreateGetVectorOfMapsStringFloat) { // support zipmap output t std::set(std::begin(values), std::end(values))); } } + +TEST_F(CApiTest, CreateGetSeqTensors) { + // Creation + auto default_allocator = onnxruntime::make_unique(); + Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault); + + std::vector in; + std::vector vals{3, 1, 2, 0}; + std::vector dims{1, 4}; + const int N = 2; + for (int i = 0; i < N; ++i) { + // create tensor + Ort::Value tensor = Ort::Value::CreateTensor(info, vals.data(), vals.size() * sizeof(int64_t), + dims.data(), dims.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64); + in.push_back(std::move(tensor)); + } + + Ort::Value seq_ort = Ort::Value::CreateSequence(in); + + // Fetch + for (int idx = 0; idx < N; ++idx) { + Ort::Value out = seq_ort.GetValue(idx, default_allocator.get()); + int64_t* ret = out.GetTensorMutableData(); + ASSERT_EQ(std::set(ret, ret + vals.size()), + std::set(std::begin(vals), std::end(vals))); + } +} diff --git a/onnxruntime/test/testdata/sequence_length.onnx b/onnxruntime/test/testdata/sequence_length.onnx new file mode 100644 index 0000000000000000000000000000000000000000..84a62d826be525e6ab4d993fc5deec7c33dfa957 GIT binary patch literal 96 zcmd;J6XMCw%d5~$tw_u*$Vs*G;u7Oxj1XdsRN@OxEi6sVOHTDk%}X!I5aKFHEiTc` v%}+_qi4q5?5mFN1;^X4sU=-ruV&Y%|V&)`nF2+b9HZB$pb|D5QCIM~$mK76# literal 0 HcmV?d00001