saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-12 17:57:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support seq(tensor), implement 2 sequence ops that use the new type. (#1983)

Browse source 
* Mention OrtCreateSessionFromArray in C API doc

* fix seq of tensors

* changes on 9/30

* All tests passing

* Add SequenceAt op

* Fix shared_lib non_tensor_types test

* Address some PR comments

* Address PR comments

* Add support in python bindings to accept seq(tensor)

* Change data type from vector<Tensor> to TensorSeq

* Change data type from vector<Tensor> to TensorSeq

* Added some documentation

* Added missing test model

* Fix Linux build

* Fix Mac build

* Fix Mac build
This commit is contained in:
Pranav Sharma 2019-10-07 15:35:09 -07:00 committed by GitHub
parent 00e24ae4fe
commit ea60469af5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 777 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
include/onnxruntime/core/framework/allocator.h
View file

@ -81,6 +81,8 @@ inline bool operator!=(const OrtDevice& left, const OrtDevice& other) {
}
struct OrtMemoryInfo {
OrtMemoryInfo() = default; // to allow default construction of Tensor
// use string for name, so we could have customized allocator in execution provider.
const char* name;
int id;

22
include/onnxruntime/core/framework/data_types.h
View file

@ -17,8 +17,10 @@ struct OrtValue;
namespace ONNX_NAMESPACE {
class TypeProto;
} // namespace ONNX_NAMESPACE
namespace onnxruntime {
/// Predefined registered types
//maps
using MapStringToString = std::map<std::string, std::string>;
using MapStringToInt64 = std::map<std::string, int64_t>;
@ -30,10 +32,6 @@ using MapInt64ToFloat = std::map<int64_t, float>;
using MapInt64ToDouble = std::map<int64_t, double>;
//vectors/sequences
using VectorString = std::vector<std::string>;
using VectorInt64 = std::vector<int64_t>;
using VectorFloat = std::vector<float>;
using VectorDouble = std::vector<double>;
using VectorMapStringToFloat = std::vector<MapStringToFloat>;
using VectorMapInt64ToFloat = std::vector<MapInt64ToFloat>;
@ -197,6 +195,9 @@ class DataTypeImpl {
template <typename elemT>
static MLDataType GetTensorType();
template <typename elemT>
static MLDataType GetSequenceTensorType();
// Return the MLDataType for a concrete sparse tensor type.
template <typename elemT>
static MLDataType GetSparseTensorType();
@ -212,6 +213,7 @@ class DataTypeImpl {
static const TensorTypeBase* TensorTypeFromONNXEnum(int type);
static const SparseTensorTypeBase* SparseTensorTypeFromONNXEnum(int type);
static const NonTensorTypeBase* SequenceTensorTypeFromONNXEnum(int type);
static const char* ToString(MLDataType type);
// Registers ONNX_NAMESPACE::DataType (internalized string) with
@ -221,6 +223,7 @@ class DataTypeImpl {
static MLDataType GetDataType(const std::string&);
static const std::vector<MLDataType>& AllTensorTypes();
static const std::vector<MLDataType>& AllSequenceTensorTypes();
static const std::vector<MLDataType>& AllFixedSizeTensorTypes();
static const std::vector<MLDataType>& AllNumericTensorTypes();
static const std::vector<MLDataType>& AllIEEEFloatTensorTypes();
@ -752,6 +755,17 @@ class NonOnnxType : public DataTypeImpl {
return SequenceType<TYPE>::Type(); \
}
#define ORT_REGISTER_SEQ_TENSOR_TYPE(ELEM_TYPE) \
template <> \
MLDataType SequenceTensorType<ELEM_TYPE>::Type() { \
static SequenceTensorType<ELEM_TYPE> sequence_tensor_type; \
return &sequence_tensor_type; \
} \
template <> \
MLDataType DataTypeImpl::GetSequenceTensorType<ELEM_TYPE>() { \
return SequenceTensorType<ELEM_TYPE>::Type(); \
}
#define ORT_REGISTER_NON_ONNX_TYPE(TYPE) \
template <> \
MLDataType NonOnnxType<TYPE>::Type() { \

17
include/onnxruntime/core/framework/tensor.h
View file

@ -11,12 +11,11 @@
#include "gsl/gsl"
#include "core/common/common.h"
#include "core/framework/allocator.h"
#include "core/framework/data_types.h"
#include "core/framework/tensor_shape.h"
#include "onnxruntime_config.h"
#include "core/framework/data_types.h"
namespace onnxruntime {
// TODO: Do we need this class or is IAllocator::MakeUniquePtr sufficient/better
class BufferDeleter {
public:
@ -58,6 +57,8 @@ using BufferNakedPtr = void*;
*/
class Tensor final {
public:
Tensor() = default; // to allow creating vector<Tensor> to support seq(tensor)
/**
* Create tensor with given type, shape, pre-allocate memory and allocator info.
* This function won't check if the preallocated buffer(p_data) has enough room for the shape.
@ -171,17 +172,7 @@ class Tensor final {
/**
The number of bytes of data.
*/
size_t SizeInBytes() const {
size_t ret;
int64_t l = shape_.Size();
if (l >= static_cast<int64_t>(std::numeric_limits<ptrdiff_t>::max())) {
ORT_THROW("tensor size overflow");
}
if (!IAllocator::CalcMemSizeForArray(static_cast<size_t>(shape_.Size()), dtype_->Size(), &ret)) {
ORT_THROW("tensor size overflow");
}
return ret;
}
size_t SizeInBytes() const;
// More API methods.
private:

42
onnxruntime/core/framework/TensorSeq.h Normal file
View file

@ -0,0 +1,42 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/framework/tensor.h"
#include <vector>
namespace onnxruntime {
// Put this in a separate file to avoid circular dependency between tensor.h and data_types.h
// Data type to represent a sequence of tensors of the same type
struct TensorSeq {
using value_type = Tensor; // to satisfy SequenceType template
// A sequence must be associated with only one data type and all tensors in the seq must be of that type
// One other alternative of storing the data type of a seq is to templatize the TensorSeq class.
// The current design follows the Tensor methodology.
// We also require this because the SequenceEmpty op expects the creation of a seq of a specific type
// and the SequenceInsert op expects validation of tensors to be added to the seq against this type.
MLDataType dtype;
// TODO: optimization opportunity - if all tensors in the seq are scalars, we can potentially represent them
// as vector<primitive type>
std::vector<Tensor> tensors;
};
template <typename TensorElemType>
class SequenceTensorType : public NonTensorType<TensorSeq> {
public:
static MLDataType Type();
bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override {
return this->IsSequenceCompatible(type_proto);
}
private:
SequenceTensorType() {
data_types_internal::SetSequenceType<TensorElemType>::Set(this->mutable_type_proto());
}
};
} // namespace onnxruntime

119
onnxruntime/core/framework/data_types.cc
View file

@ -3,6 +3,7 @@
#include "core/framework/data_types.h"
#include "core/framework/tensor.h"
#include "core/framework/TensorSeq.h"
#include "core/framework/sparse_tensor.h"
#include "core/graph/onnx_protobuf.h"
@ -28,7 +29,8 @@ template <>
MLDataType DataTypeImpl::GetType<Tensor>() {
return TensorTypeBase::Type();
}
} // namespace onnxruntime
} // namespace onnxruntime
// This conflics with the above GetType<>() specialization
#include "core/framework/tensorprotoutils.h"
@ -507,7 +509,7 @@ void NonTensorTypeBase::FromDataContainer(const void* /* data */, size_t /*data_
ORT_ENFORCE(false, "Not implemented");
}
void NonTensorTypeBase::ToDataContainer (const OrtValue& /* input */, size_t /*data_size */, void* /* data */) const {
void NonTensorTypeBase::ToDataContainer(const OrtValue& /* input */, size_t /*data_size */, void* /* data */) const {
ORT_ENFORCE(false, "Not implemented");
}
@ -550,10 +552,22 @@ ORT_REGISTER_MAP(MapInt64ToInt64);
ORT_REGISTER_MAP(MapInt64ToFloat);
ORT_REGISTER_MAP(MapInt64ToDouble);
ORT_REGISTER_SEQ(VectorString);
ORT_REGISTER_SEQ(VectorFloat);
ORT_REGISTER_SEQ(VectorInt64);
ORT_REGISTER_SEQ(VectorDouble);
// Register sequence of tensor types
ORT_REGISTER_SEQ(TensorSeq) // required to ensure GetType<TensorSeq> works
ORT_REGISTER_SEQ_TENSOR_TYPE(int32_t);
ORT_REGISTER_SEQ_TENSOR_TYPE(float);
ORT_REGISTER_SEQ_TENSOR_TYPE(bool);
ORT_REGISTER_SEQ_TENSOR_TYPE(std::string);
ORT_REGISTER_SEQ_TENSOR_TYPE(int8_t);
ORT_REGISTER_SEQ_TENSOR_TYPE(uint8_t);
ORT_REGISTER_SEQ_TENSOR_TYPE(uint16_t);
ORT_REGISTER_SEQ_TENSOR_TYPE(int16_t);
ORT_REGISTER_SEQ_TENSOR_TYPE(int64_t);
ORT_REGISTER_SEQ_TENSOR_TYPE(double);
ORT_REGISTER_SEQ_TENSOR_TYPE(uint32_t);
ORT_REGISTER_SEQ_TENSOR_TYPE(uint64_t);
ORT_REGISTER_SEQ_TENSOR_TYPE(MLFloat16);
ORT_REGISTER_SEQ_TENSOR_TYPE(BFloat16);
ORT_REGISTER_SEQ(VectorMapStringToFloat);
ORT_REGISTER_SEQ(VectorMapInt64ToFloat);
@ -565,6 +579,12 @@ ORT_REGISTER_SEQ(VectorMapInt64ToFloat);
reg_fn(mltype); \
}
#define REGISTER_SEQ_TENSOR_PROTO(TYPE, reg_fn) \
{ \
MLDataType mltype = DataTypeImpl::GetSequenceTensorType<TYPE>(); \
reg_fn(mltype); \
}
#define REGISTER_SPARSE_TENSOR_PROTO(TYPE, reg_fn) \
{ \
MLDataType mltype = DataTypeImpl::GetSparseTensorType<TYPE>(); \
@ -619,10 +639,20 @@ void RegisterAllProtos(const std::function<void(MLDataType)>& reg_fn) {
REGISTER_ONNX_PROTO(MapInt64ToFloat, reg_fn);
REGISTER_ONNX_PROTO(MapInt64ToDouble, reg_fn);
REGISTER_ONNX_PROTO(VectorString, reg_fn);
REGISTER_ONNX_PROTO(VectorFloat, reg_fn);
REGISTER_ONNX_PROTO(VectorInt64, reg_fn);
REGISTER_ONNX_PROTO(VectorDouble, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(int32_t, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(float, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(bool, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(std::string, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(int8_t, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(uint8_t, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(uint16_t, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(int16_t, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(int64_t, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(double, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(uint32_t, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(uint64_t, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(MLFloat16, reg_fn);
REGISTER_SEQ_TENSOR_PROTO(BFloat16, reg_fn);
REGISTER_ONNX_PROTO(VectorMapStringToFloat, reg_fn);
REGISTER_ONNX_PROTO(VectorMapInt64ToFloat, reg_fn);
@ -724,6 +754,41 @@ const TensorTypeBase* DataTypeImpl::TensorTypeFromONNXEnum(int type) {
}
}
const NonTensorTypeBase* DataTypeImpl::SequenceTensorTypeFromONNXEnum(int type) {
switch (type) {
case TensorProto_DataType_FLOAT:
return DataTypeImpl::GetSequenceTensorType<float>()->AsNonTensorTypeBase();
case TensorProto_DataType_BOOL:
return DataTypeImpl::GetSequenceTensorType<bool>()->AsNonTensorTypeBase();
case TensorProto_DataType_INT32:
return DataTypeImpl::GetSequenceTensorType<int32_t>()->AsNonTensorTypeBase();
case TensorProto_DataType_DOUBLE:
return DataTypeImpl::GetSequenceTensorType<double>()->AsNonTensorTypeBase();
case TensorProto_DataType_STRING:
return DataTypeImpl::GetSequenceTensorType<std::string>()->AsNonTensorTypeBase();
case TensorProto_DataType_UINT8:
return DataTypeImpl::GetSequenceTensorType<uint8_t>()->AsNonTensorTypeBase();
case TensorProto_DataType_UINT16:
return DataTypeImpl::GetSequenceTensorType<uint16_t>()->AsNonTensorTypeBase();
case TensorProto_DataType_INT8:
return DataTypeImpl::GetSequenceTensorType<int8_t>()->AsNonTensorTypeBase();
case TensorProto_DataType_INT16:
return DataTypeImpl::GetSequenceTensorType<int16_t>()->AsNonTensorTypeBase();
case TensorProto_DataType_INT64:
return DataTypeImpl::GetSequenceTensorType<int64_t>()->AsNonTensorTypeBase();
case TensorProto_DataType_UINT32:
return DataTypeImpl::GetSequenceTensorType<uint32_t>()->AsNonTensorTypeBase();
case TensorProto_DataType_UINT64:
return DataTypeImpl::GetSequenceTensorType<uint64_t>()->AsNonTensorTypeBase();
case TensorProto_DataType_FLOAT16:
return DataTypeImpl::GetSequenceTensorType<MLFloat16>()->AsNonTensorTypeBase();
case TensorProto_DataType_BFLOAT16:
return DataTypeImpl::GetSequenceTensorType<BFloat16>()->AsNonTensorTypeBase();
default:
ORT_NOT_IMPLEMENTED("tensor type ", type, " is not supported");
}
}
const SparseTensorTypeBase* DataTypeImpl::SparseTensorTypeFromONNXEnum(int type) {
switch (type) {
case TensorProto_DataType_FLOAT:
@ -868,19 +933,7 @@ MLDataType DataTypeImpl::TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto) {
} // MapType
break;
case TypeProto::ValueCase::kTensorType: {
auto val_elem_type = val_type.tensor_type().elem_type();
switch (val_elem_type) {
case TensorProto_DataType_STRING:
return DataTypeImpl::GetType<VectorString>();
case TensorProto_DataType_INT64:
return DataTypeImpl::GetType<VectorInt64>();
case TensorProto_DataType_FLOAT:
return DataTypeImpl::GetType<VectorFloat>();
case TensorProto_DataType_DOUBLE:
return DataTypeImpl::GetType<VectorDouble>();
default:
break;
}
return DataTypeImpl::GetType<TensorSeq>();
} // kTensorType
break;
default:
@ -989,6 +1042,26 @@ const std::vector<MLDataType>& DataTypeImpl::AllTensorTypes() {
return all_tensor_types;
}
const std::vector<MLDataType>& DataTypeImpl::AllSequenceTensorTypes() {
static std::vector<MLDataType> all_sequence_tensor_types =
{DataTypeImpl::GetSequenceTensorType<float>(),
DataTypeImpl::GetSequenceTensorType<double>(),
DataTypeImpl::GetSequenceTensorType<int64_t>(),
DataTypeImpl::GetSequenceTensorType<uint64_t>(),
DataTypeImpl::GetSequenceTensorType<int32_t>(),
DataTypeImpl::GetSequenceTensorType<uint32_t>(),
DataTypeImpl::GetSequenceTensorType<int16_t>(),
DataTypeImpl::GetSequenceTensorType<uint16_t>(),
DataTypeImpl::GetSequenceTensorType<int8_t>(),
DataTypeImpl::GetSequenceTensorType<uint8_t>(),
DataTypeImpl::GetSequenceTensorType<MLFloat16>(),
DataTypeImpl::GetSequenceTensorType<BFloat16>(),
DataTypeImpl::GetSequenceTensorType<bool>(),
DataTypeImpl::GetSequenceTensorType<std::string>()};
return all_sequence_tensor_types;
}
const std::vector<MLDataType>& DataTypeImpl::AllNumericTensorTypes() {
static std::vector<MLDataType> all_numeric_size_tensor_types =
{DataTypeImpl::GetTensorType<float>(),

14
onnxruntime/core/framework/tensor.cc
View file

@ -5,6 +5,8 @@
#include <utility>
#include "core/framework/allocatormgr.h"
#include "core/framework/data_types.h"
using namespace std;
namespace onnxruntime {
@ -27,6 +29,18 @@ Tensor::Tensor(MLDataType p_type, const TensorShape& shape, std::shared_ptr<IAll
Init(p_type, shape, p_data, allocator, offset);
}
size_t Tensor::SizeInBytes() const {
size_t ret;
int64_t l = shape_.Size();
if (l >= static_cast<int64_t>(std::numeric_limits<ptrdiff_t>::max())) {
ORT_THROW("tensor size overflow");
}
if (!IAllocator::CalcMemSizeForArray(static_cast<size_t>(shape_.Size()), dtype_->Size(), &ret)) {
ORT_THROW("tensor size overflow");
}
return ret;
}
void Tensor::Init(MLDataType p_type, const TensorShape& shape, void* p_raw_data, AllocatorPtr deleter, int64_t offset) {
int64_t shape_size = shape.Size();
if (shape_size < 0) ORT_THROW("shape.Size() must >=0");

4
onnxruntime/core/providers/cpu/cpu_execution_provider.cc
View file

@ -389,6 +389,8 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Lp
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Conv);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ConvTranspose);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, If);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceLength);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceAt);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ScatterND);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Gemm);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, GatherElements);
@ -1009,6 +1011,8 @@ void RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Conv)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ConvTranspose)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, If)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceLength)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceAt)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ScatterND)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Gemm)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, GatherElements)>,

104
onnxruntime/core/providers/cpu/sequence/sequence_ops.cc Normal file
View file

@ -0,0 +1,104 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/cpu/sequence/sequence_ops.h"
#include "core/framework/tensorprotoutils.h"
#include "core/providers/cpu/tensor/utils.h"
#include "core/framework/TensorSeq.h"
using namespace onnxruntime::common;
namespace onnxruntime {
// SequenceLength
ONNX_CPU_OPERATOR_KERNEL(
SequenceLength,
11,
KernelDefBuilder()
.TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes())
.TypeConstraint("I", DataTypeImpl::GetTensorType<int64_t>()),
SequenceLength);
Status SequenceLength::Compute(OpKernelContext* context) const {
const auto* X = context->Input<TensorSeq>(0);
ORT_ENFORCE(X != nullptr, "Got nullptr for sequence input.");
auto* Y = context->Output(0, {});
auto* Y_data = Y->template MutableData<int64_t>();
*Y_data = static_cast<int64_t>(X->tensors.size());
return Status::OK();
}
// SequenceAt
ONNX_CPU_OPERATOR_KERNEL(
SequenceAt,
11,
KernelDefBuilder()
.TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes())
.TypeConstraint("T", DataTypeImpl::AllTensorTypes())
.TypeConstraint("I", std::vector<MLDataType>{
DataTypeImpl::GetTensorType<int32_t>(),
DataTypeImpl::GetTensorType<int64_t>()}),
SequenceAt);
static int64_t GetSeqIdx(const Tensor& idx_tensor) {
int64_t seq_idx = INT_MAX;
auto idx_tensor_dtype = utils::GetTensorProtoType(idx_tensor);
switch (idx_tensor_dtype) {
case ONNX_NAMESPACE::TensorProto_DataType_INT32: {
const auto* idx_data = idx_tensor.Data<int32_t>();
seq_idx = static_cast<int64_t>(*idx_data);
break;
}
case ONNX_NAMESPACE::TensorProto_DataType_INT64: {
const auto* idx_data = idx_tensor.Data<int64_t>();
seq_idx = *idx_data;
break;
}
default:
ORT_THROW("Unsupported data type: ", idx_tensor_dtype);
}
return seq_idx;
}
bool ValidateSeqIdx(int64_t input_seq_idx, int64_t seq_size) {
bool retval = false;
if (input_seq_idx < 0) {
retval = input_seq_idx <= -1 && input_seq_idx >= -seq_size;
} else {
retval = input_seq_idx < seq_size;
}
return retval;
}
template <typename T>
static void CopyTensor(const Tensor& indexed_tensor, Tensor& output_tensor) {
const auto* input_data = indexed_tensor.template Data<T>();
auto* output_data = output_tensor.template MutableData<T>();
memcpy(output_data, input_data, indexed_tensor.SizeInBytes());
}
Status SequenceAt::Compute(OpKernelContext* context) const {
const auto* X = context->Input<TensorSeq>(0);
ORT_ENFORCE(X != nullptr, "Got nullptr for sequence input.");
const auto* I = context->Input<Tensor>(1);
ORT_ENFORCE(I != nullptr, "Got nullptr input for index tensor");
int64_t input_seq_idx = GetSeqIdx(*I);
if (!ValidateSeqIdx(input_seq_idx, static_cast<int64_t>(X->tensors.size()))) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"Invalid sequence index (", input_seq_idx, ") specified for sequence of size (", X->tensors.size(), ")");
}
if (input_seq_idx < 0) {
input_seq_idx = static_cast<int64_t>(X->tensors.size()) + input_seq_idx;
}
const Tensor& indexed_tensor = X->tensors[input_seq_idx];
auto* Y = context->Output(0, indexed_tensor.Shape().GetDims());
CopyCpuTensor(&indexed_tensor, Y);
return Status::OK();
}
} // namespace onnxruntime

27
onnxruntime/core/providers/cpu/sequence/sequence_ops.h Normal file
View file

@ -0,0 +1,27 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/common/common.h"
#include "core/framework/op_kernel.h"
namespace onnxruntime {
class SequenceLength final : public OpKernel {
public:
SequenceLength(const OpKernelInfo& info) : OpKernel(info) {
}
Status Compute(OpKernelContext* context) const override;
};
class SequenceAt final : public OpKernel {
public:
SequenceAt(const OpKernelInfo& info) : OpKernel(info) {
}
Status Compute(OpKernelContext* context) const override;
};
} //namespace onnxruntime

231
onnxruntime/core/session/onnxruntime_c_api.cc
View file

@ -24,6 +24,7 @@
#include "core/session/ort_apis.h"
#include "core/framework/data_types.h"
#include "abi_session_options_impl.h"
#include "core/framework/TensorSeq.h"
using namespace onnxruntime::logging;
using onnxruntime::BFloat16;
@ -101,9 +102,9 @@ ORT_API_STATUS_IMPL(OrtApis::CreateEnvWithCustomLogger, OrtLoggingFunction loggi
std::string name = logid;
std::unique_ptr<ISink> logger = onnxruntime::make_unique<LoggingWrapper>(logging_function, logger_param);
auto default_logging_manager = onnxruntime::make_unique<LoggingManager>(std::move(logger),
static_cast<Severity>(default_warning_level), false,
LoggingManager::InstanceType::Default,
&name);
static_cast<Severity>(default_warning_level), false,
LoggingManager::InstanceType::Default,
&name);
std::unique_ptr<Environment> env;
Status status = Environment::Create(env);
if (status.IsOK())
@ -117,9 +118,9 @@ ORT_API_STATUS_IMPL(OrtApis::CreateEnv, OrtLoggingLevel default_warning_level,
API_IMPL_BEGIN
std::string name = logid;
auto default_logging_manager = onnxruntime::make_unique<LoggingManager>(std::unique_ptr<ISink>{new CLogSink{}},
static_cast<Severity>(default_warning_level), false,
LoggingManager::InstanceType::Default,
&name);
static_cast<Severity>(default_warning_level), false,
LoggingManager::InstanceType::Default,
&name);
std::unique_ptr<Environment> env;
Status status = Environment::Create(env);
if (status.IsOK()) {
@ -143,6 +144,25 @@ OrtStatus* CreateTensorImpl(const int64_t* shape, size_t shape_len, OrtAllocator
return nullptr;
}
template <typename T>
OrtStatus* CreateTensorImplForSeq(const int64_t* shape, size_t shape_len,
Tensor& out) {
std::vector<int64_t> shapes(shape_len);
for (size_t i = 0; i != shape_len; ++i) {
shapes[i] = shape[i];
}
OrtAllocator* allocator;
// TODO(pranav): what allocator should be used to create the tensor here?
// for the sake of simplicity of the API using the default one here
auto st = OrtApis::GetAllocatorWithDefaultOptions(&allocator);
if (st) {
return st;
}
std::shared_ptr<IAllocator> alloc_ptr = std::make_shared<onnxruntime::AllocatorWrapper>(allocator);
out = Tensor(DataTypeImpl::GetType<T>(), onnxruntime::TensorShape(shapes), alloc_ptr);
return nullptr;
}
/**
*
* this function will create a copy of the allocator info
@ -664,6 +684,13 @@ OrtStatus* OrtGetNumSequenceElements(const OrtValue* p_ml_value, size_t* out) {
return nullptr;
}
template <>
OrtStatus* OrtGetNumSequenceElements<TensorSeq>(const OrtValue* p_ml_value, size_t* out) {
auto& data = p_ml_value->Get<TensorSeq>();
*out = data.tensors.size();
return nullptr;
}
static OrtStatus* OrtGetValueCountImpl(const OrtValue* value, size_t* out) {
ONNXType value_type;
if (auto status = OrtApis::GetValueType(value, &value_type))
@ -676,15 +703,8 @@ static OrtStatus* OrtGetValueCountImpl(const OrtValue* value, size_t* out) {
auto v = reinterpret_cast<const OrtValue*>(value);
auto type = v->Type();
// Note: keep these in sync with the registered types in data_types.h
if (type == DataTypeImpl::GetType<VectorString>()) {
return OrtGetNumSequenceElements<VectorString>(v, out);
}
if (type == DataTypeImpl::GetType<VectorInt64>()) {
return OrtGetNumSequenceElements<VectorInt64>(v, out);
} else if (type == DataTypeImpl::GetType<VectorFloat>()) {
return OrtGetNumSequenceElements<VectorFloat>(v, out);
} else if (type == DataTypeImpl::GetType<VectorDouble>()) {
return OrtGetNumSequenceElements<VectorDouble>(v, out);
if (type == DataTypeImpl::GetType<TensorSeq>()) {
return OrtGetNumSequenceElements<TensorSeq>(v, out);
} else if (type == DataTypeImpl::GetType<VectorMapStringToFloat>()) {
return OrtGetNumSequenceElements<VectorMapStringToFloat>(v, out);
} else if (type == DataTypeImpl::GetType<VectorMapInt64ToFloat>()) {
@ -773,16 +793,52 @@ OrtStatus* PopulateTensorWithData<std::string>(OrtValue* oval, const std::string
return nullptr;
}
template <typename TensorElemType>
OrtStatus* OrtGetValueImplSeqOfTensorsHelper(OrtAllocator* allocator, const Tensor& tensor,
OrtValue** out) {
const auto& shape = tensor.Shape();
const auto* tensor_data = tensor.Data<TensorElemType>();
OrtStatus* st = OrtApis::CreateTensorAsOrtValue(allocator, shape.GetDims().data(), shape.NumDimensions(),
GetONNXTensorElementDataType<TensorElemType>(), out);
return st ? st : PopulateTensorWithData<TensorElemType>(*out, tensor_data, shape.Size());
}
template <typename T>
OrtStatus* OrtGetValueImplSeqOfPrimitives(const OrtValue* p_ml_value, int index, OrtAllocator* allocator,
OrtValue** out) {
using ElemType = typename T::value_type;
OrtStatus* OrtGetValueImplSeqOfTensors(const OrtValue* p_ml_value, int index, OrtAllocator* allocator,
OrtValue** out) {
auto& data = p_ml_value->Get<T>();
auto& data_elem = data.at(index);
std::vector<int64_t> dims = {1};
OrtStatus* st = OrtApis::CreateTensorAsOrtValue(allocator, dims.data(), dims.size(),
GetONNXTensorElementDataType<ElemType>(), out);
return st ? st : PopulateTensorWithData<ElemType>(*out, &data_elem, 1);
auto& one_tensor = data.tensors.at(index);
auto tensor_elem_type = one_tensor.DataType();
OrtStatus* st{};
if (tensor_elem_type == DataTypeImpl::GetType<bool>()) {
st = OrtGetValueImplSeqOfTensorsHelper<bool>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<float>()) {
st = OrtGetValueImplSeqOfTensorsHelper<float>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<MLFloat16>()) {
st = OrtGetValueImplSeqOfTensorsHelper<MLFloat16>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<double>()) {
st = OrtGetValueImplSeqOfTensorsHelper<double>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<int8_t>()) {
st = OrtGetValueImplSeqOfTensorsHelper<int8_t>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<uint8_t>()) {
st = OrtGetValueImplSeqOfTensorsHelper<uint8_t>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<int16_t>()) {
st = OrtGetValueImplSeqOfTensorsHelper<int16_t>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<uint16_t>()) {
st = OrtGetValueImplSeqOfTensorsHelper<uint16_t>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<int32_t>()) {
st = OrtGetValueImplSeqOfTensorsHelper<int32_t>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<uint32_t>()) {
st = OrtGetValueImplSeqOfTensorsHelper<uint32_t>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<int64_t>()) {
st = OrtGetValueImplSeqOfTensorsHelper<int64_t>(allocator, one_tensor, out);
} else if (tensor_elem_type == DataTypeImpl::GetType<std::string>()) {
st = OrtGetValueImplSeqOfTensorsHelper<std::string>(allocator, one_tensor, out);
} else {
st = OrtApis::CreateStatus(ORT_FAIL, "Only sequences that contain float tensors are supported.");
}
return st;
}
static OrtStatus* OrtGetValueImplSeq(const OrtValue* value, int index, OrtAllocator* allocator,
@ -790,15 +846,8 @@ static OrtStatus* OrtGetValueImplSeq(const OrtValue* value, int index, OrtAlloca
auto p_ml_value = reinterpret_cast<const OrtValue*>(value);
auto type = p_ml_value->Type();
// Note: keep these in sync with the registered types in data_types.h
if (type == DataTypeImpl::GetType<VectorString>()) {
return OrtGetValueImplSeqOfPrimitives<VectorString>(p_ml_value, index, allocator, out);
}
if (type == DataTypeImpl::GetType<VectorInt64>()) {
return OrtGetValueImplSeqOfPrimitives<VectorInt64>(p_ml_value, index, allocator, out);
} else if (type == DataTypeImpl::GetType<VectorFloat>()) {
return OrtGetValueImplSeqOfPrimitives<VectorFloat>(p_ml_value, index, allocator, out);
} else if (type == DataTypeImpl::GetType<VectorDouble>()) {
return OrtGetValueImplSeqOfPrimitives<VectorDouble>(p_ml_value, index, allocator, out);
if (type == DataTypeImpl::GetType<TensorSeq>()) {
return OrtGetValueImplSeqOfTensors<TensorSeq>(p_ml_value, index, allocator, out);
} else if (type == DataTypeImpl::GetType<VectorMapStringToFloat>()) {
return OrtGetValueImplSeqOfMap<VectorMapStringToFloat>(p_ml_value, index, out);
} else if (type == DataTypeImpl::GetType<VectorMapInt64ToFloat>()) {
@ -897,44 +946,106 @@ ORT_API_STATUS_IMPL(OrtApis::GetValue, const OrtValue* value, int index, OrtAllo
template <typename T>
static OrtStatus* OrtCreateValueImplSeqHelperMap(const OrtValue* const* in, size_t num_values, OrtValue** out) {
using SeqType = std::vector<T>;
auto vec_ptr = onnxruntime::make_unique<SeqType>();
vec_ptr->reserve(num_values);
auto seq_ptr = onnxruntime::make_unique<SeqType>();
seq_ptr->reserve(num_values);
for (size_t idx = 0; idx < num_values; ++idx) {
auto& m = reinterpret_cast<const OrtValue*>(in[idx])->Get<T>();
vec_ptr->push_back(m);
seq_ptr->push_back(m);
}
// create OrtValue with this vector
auto value = onnxruntime::make_unique<OrtValue>();
value->Init(vec_ptr.release(),
value->Init(seq_ptr.release(),
DataTypeImpl::GetType<SeqType>(),
DataTypeImpl::GetType<SeqType>()->GetDeleteFunc());
*out = value.release();
return nullptr;
}
template <typename T>
static OrtStatus* OrtCreateValueImplSeqHelper(const OrtValue* const* in, size_t num_values, OrtValue** out) {
using SeqType = std::vector<T>;
auto vec_ptr = onnxruntime::make_unique<SeqType>();
vec_ptr->reserve(num_values);
template <typename TensorElemType>
static OrtStatus* OrtCreateValueImplSeqHelperTensor(const Tensor& tensor,
Tensor& out) {
auto data = tensor.Data<TensorElemType>();
if (!data) {
return OrtApis::CreateStatus(ORT_FAIL, "Encountered nullptr.");
}
OrtAllocator* allocator;
OrtStatus* st = OrtApis::GetAllocatorWithDefaultOptions(&allocator);
if (st) {
return st;
}
st = CreateTensorImplForSeq<TensorElemType>(tensor.Shape().GetDims().data(), tensor.Shape().NumDimensions(), out);
if (st) {
return st;
}
size_t num_elems = tensor.Shape().Size();
auto* out_data = out.MutableData<TensorElemType>();
for (size_t i = 0; i < num_elems; ++i) {
*out_data++ = *data++;
}
return nullptr;
}
static OrtStatus* OrtCreateValueImplSeqHelper(const OrtValue* const* in, size_t num_values,
OrtValue** out) {
auto seq_ptr = std::make_unique<TensorSeq>();
seq_ptr->tensors.resize(num_values);
// use the data type of the first tensor as the data type of the seq
seq_ptr->dtype = reinterpret_cast<const OrtValue*>(in[0])->Get<Tensor>().DataType();
for (size_t idx = 0; idx < num_values; ++idx) {
auto& tensor = reinterpret_cast<const OrtValue*>(in[idx])->Get<Tensor>();
auto data = tensor.Data<T>();
if (!data) {
return OrtApis::CreateStatus(ORT_FAIL, "Encountered nullptr.");
auto& one_tensor = reinterpret_cast<const OrtValue*>(in[idx])->Get<Tensor>();
auto tensor_elem_type = one_tensor.DataType();
// sequences must have tensors of the same data type
if (idx > 0 && (tensor_elem_type != seq_ptr->dtype)) {
return OrtApis::CreateStatus(ORT_FAIL,
"Sequences must have tensors of the same data type. There was at least one tensor in the input that was different.");
}
OrtStatus* st{};
if (tensor_elem_type == DataTypeImpl::GetType<bool>()) {
st = OrtCreateValueImplSeqHelperTensor<bool>(one_tensor, seq_ptr->tensors[idx]);
} else if (tensor_elem_type == DataTypeImpl::GetType<float>()) {
st = OrtCreateValueImplSeqHelperTensor<float>(one_tensor, seq_ptr->tensors[idx]);
} else if (tensor_elem_type == DataTypeImpl::GetType<double>()) {
st = OrtCreateValueImplSeqHelperTensor<double>(one_tensor, seq_ptr->tensors[idx]);
} else if (tensor_elem_type == DataTypeImpl::GetType<int8_t>()) {
st = OrtCreateValueImplSeqHelperTensor<int8_t>(one_tensor, seq_ptr->tensors[idx]);
} else if (tensor_elem_type == DataTypeImpl::GetType<uint8_t>()) {
st = OrtCreateValueImplSeqHelperTensor<uint8_t>(one_tensor, seq_ptr->tensors[idx]);
} else if (tensor_elem_type == DataTypeImpl::GetType<int16_t>()) {
st = OrtCreateValueImplSeqHelperTensor<int16_t>(one_tensor, seq_ptr->tensors[idx]);
} else if (tensor_elem_type == DataTypeImpl::GetType<uint16_t>()) {
st = OrtCreateValueImplSeqHelperTensor<uint16_t>(one_tensor, seq_ptr->tensors[idx]);
} else if (tensor_elem_type == DataTypeImpl::GetType<int32_t>()) {
st = OrtCreateValueImplSeqHelperTensor<int32_t>(one_tensor, seq_ptr->tensors[idx]);
} else if (tensor_elem_type == DataTypeImpl::GetType<uint32_t>()) {
st = OrtCreateValueImplSeqHelperTensor<uint32_t>(one_tensor, seq_ptr->tensors[idx]);
} else if (tensor_elem_type == DataTypeImpl::GetType<int64_t>()) {
st = OrtCreateValueImplSeqHelperTensor<int64_t>(one_tensor, seq_ptr->tensors[idx]);
} else {
std::string err_msg = std::string("Unsupported data type: ") + DataTypeImpl::ToString(tensor_elem_type);
st = OrtApis::CreateStatus(ORT_FAIL, err_msg.c_str());
}
if (st) {
return st;
}
vec_ptr->push_back(*data);
}
// create OrtValue with this vector
auto value = onnxruntime::make_unique<OrtValue>();
value->Init(vec_ptr.release(),
DataTypeImpl::GetType<SeqType>(),
DataTypeImpl::GetType<SeqType>()->GetDeleteFunc());
value->Init(seq_ptr.release(),
DataTypeImpl::GetType<TensorSeq>(),
DataTypeImpl::GetType<TensorSeq>()->GetDeleteFunc());
*out = value.release();
return nullptr;
}
static OrtStatus* OrtCreateValueImplSeq(const OrtValue* const* in, size_t num_values, OrtValue** out) {
static OrtStatus* OrtCreateValueImplSeq(const OrtValue* const* in, size_t num_values,
OrtValue** out) {
// We only support limited sequence types. For the sake of simplicity the type of the first
// OrtValue* in OrtValue** will determine the type of the vector used to create the output OrtValue
// this type should be either a tensor of limited types or map of limited types
@ -964,19 +1075,7 @@ static OrtStatus* OrtCreateValueImplSeq(const OrtValue* const* in, size_t num_va
// finally create the output vector/MLValue
auto first_mlvalue = reinterpret_cast<const OrtValue*>(ovfirst);
if (first_value_type == ONNX_TYPE_TENSOR) {
auto vec_type = first_mlvalue->Get<Tensor>().DataType();
if (vec_type == DataTypeImpl::GetType<std::string>()) {
return OrtCreateValueImplSeqHelper<std::string>(in, num_values, out);
}
if (vec_type == DataTypeImpl::GetType<int64_t>()) {
return OrtCreateValueImplSeqHelper<int64_t>(in, num_values, out);
} else if (vec_type == DataTypeImpl::GetType<float>()) {
return OrtCreateValueImplSeqHelper<float>(in, num_values, out);
} else if (vec_type == DataTypeImpl::GetType<double>()) {
return OrtCreateValueImplSeqHelper<double>(in, num_values, out);
} else {
return OrtApis::CreateStatus(ORT_FAIL, "Type not supported.");
}
return OrtCreateValueImplSeqHelper(in, num_values, out);
} else if (first_value_type == ONNX_TYPE_MAP) {
auto map_type = first_mlvalue->Type();
if (map_type == DataTypeImpl::GetType<MapStringToFloat>()) {
@ -1064,7 +1163,8 @@ static OrtStatus* OrtCreateValueImplMap(const OrtValue* const* in, size_t num_va
return OrtApis::CreateStatus(ORT_FAIL, "Key type is not supported yet.");
}
static OrtStatus* OrtCreateValueImpl(const OrtValue* const* in, size_t num_values, enum ONNXType value_type, OrtValue** out) {
static OrtStatus* OrtCreateValueImpl(const OrtValue* const* in, size_t num_values, enum ONNXType value_type,
OrtValue** out) {
if (num_values <= 0) {
return OrtApis::CreateStatus(ORT_FAIL, "Number of values should be at least 1.");
}
@ -1077,7 +1177,8 @@ static OrtStatus* OrtCreateValueImpl(const OrtValue* const* in, size_t num_value
return OrtApis::CreateStatus(ORT_FAIL, "Input is not of type sequence or map.");
}
ORT_API_STATUS_IMPL(OrtApis::CreateValue, const OrtValue* const* in, size_t num_values, enum ONNXType value_type, OrtValue** out) {
ORT_API_STATUS_IMPL(OrtApis::CreateValue, const OrtValue* const* in, size_t num_values, enum ONNXType value_type,
OrtValue** out) {
API_IMPL_BEGIN
return OrtCreateValueImpl(in, num_values, value_type, out);
API_IMPL_END

46
onnxruntime/python/onnxruntime_pybind_mlvalue.cc
View file

@ -12,6 +12,7 @@
#include "core/framework/tensor_shape.h"
#include "core/framework/tensor.h"
#include "core/framework/allocator.h"
#include "core/framework/TensorSeq.h"
using namespace std;
namespace onnxruntime {
@ -95,13 +96,13 @@ bool PyObjectCheck_Array(PyObject* o) {
return PyObject_HasAttrString(o, "__array_finalize__");
}
void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyArrayObject* pyObject,
OrtValue* p_mlvalue) {
std::unique_ptr<Tensor> CreateTensor(AllocatorPtr alloc, const std::string& name_input, PyArrayObject* pyObject) {
PyArrayObject* darray = PyArray_GETCONTIGUOUS(pyObject);
if (darray == NULL) {
throw std::runtime_error(std::string("The object must be a contiguous array for input '") + name_input + std::string("'."));
}
bool dref = false;
std::unique_ptr<Tensor> p_tensor;
try {
const int npy_type = PyArray_TYPE(darray);
@ -115,7 +116,7 @@ void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyAr
TensorShape shape(dims);
auto element_type = NumpyToOnnxRuntimeTensorType(npy_type);
std::unique_ptr<Tensor> p_tensor = onnxruntime::make_unique<Tensor>(element_type, shape, alloc);
p_tensor = onnxruntime::make_unique<Tensor>(element_type, shape, alloc);
if (npy_type == NPY_UNICODE) {
// Copy string data which needs to be done after Tensor is allocated.
// Strings are Python strings or numpy.unicode string.
@ -176,10 +177,6 @@ void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyAr
}
memcpy(buffer, static_cast<void*>(PyArray_DATA(darray)), len);
}
p_mlvalue->Init(p_tensor.release(),
DataTypeImpl::GetType<Tensor>(),
DataTypeImpl::GetType<Tensor>()->GetDeleteFunc());
} catch (...) {
if (!dref) {
Py_XDECREF(darray);
@ -195,6 +192,38 @@ void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyAr
if (!dref) {
Py_XDECREF(darray);
}
return p_tensor;
}
void CreateSequenceOfTensors(AllocatorPtr alloc, const std::string& name_input, PyObject* pylist_obj,
OrtValue* p_mlvalue) {
auto list_size = PyList_Size(pylist_obj);
if (list_size <= 0) {
throw std::runtime_error("Got exception while creating seq(tensor) because input list size is " + std::to_string(list_size));
}
auto p_seq_tensors = onnxruntime::make_unique<TensorSeq>();
p_seq_tensors->tensors.resize(list_size);
for (Py_ssize_t i = 0; i < list_size; ++i) {
auto* py_obj = PyList_GetItem(pylist_obj, i);
auto p_tensor = CreateTensor(alloc, name_input, reinterpret_cast<PyArrayObject*>(py_obj));
p_seq_tensors->tensors[i] = std::move(*p_tensor);
}
p_mlvalue->Init(p_seq_tensors.release(),
DataTypeImpl::GetType<TensorSeq>(),
DataTypeImpl::GetType<TensorSeq>()->GetDeleteFunc());
}
void CreateTensorMLValue(AllocatorPtr alloc, const std::string& name_input, PyArrayObject* pyObject,
OrtValue* p_mlvalue) {
auto p_tensor = CreateTensor(alloc, name_input, pyObject);
if (!p_tensor) {
throw std::runtime_error("Got exception while creating tensor for input: " + name_input);
}
p_mlvalue->Init(p_tensor.release(),
DataTypeImpl::GetType<Tensor>(),
DataTypeImpl::GetType<Tensor>()->GetDeleteFunc());
}
std::string _get_type_name(int64_t&) {
@ -404,6 +433,9 @@ void CreateGenericMLValue(AllocatorPtr alloc, const std::string& name_input, py:
// The most frequent case: input comes as an array.
PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(value.ptr());
CreateTensorMLValue(alloc, name_input, arr, p_mlvalue);
} else if (PyList_Check(value.ptr())) {
auto* seq_tensors = reinterpret_cast<PyObject*>(value.ptr());
CreateSequenceOfTensors(alloc, name_input, seq_tensors, p_mlvalue);
} else if (PyDict_Check(value.ptr())) {
CreateMapMLValue_AgnosticVectorMap((PyObject*)NULL, value.ptr(), alloc, name_input, p_mlvalue);
} else {

114
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -147,6 +147,7 @@ template <typename T>
void AddNonTensor(OrtValue& val, std::vector<py::object>& pyobjs) {
pyobjs.push_back(py::cast(val.Get<T>()));
}
void AddNonTensorAsPyObj(OrtValue& val, std::vector<py::object>& pyobjs) {
// Should be in sync with core/framework/datatypes.h
if (val.Type() == DataTypeImpl::GetType<MapStringToString>()) {
@ -165,14 +166,6 @@ void AddNonTensorAsPyObj(OrtValue& val, std::vector<py::object>& pyobjs) {
AddNonTensor<MapInt64ToFloat>(val, pyobjs);
} else if (val.Type() == DataTypeImpl::GetType<MapInt64ToDouble>()) {
AddNonTensor<MapInt64ToDouble>(val, pyobjs);
} else if (val.Type() == DataTypeImpl::GetType<VectorString>()) {
AddNonTensor<VectorString>(val, pyobjs);
} else if (val.Type() == DataTypeImpl::GetType<VectorInt64>()) {
AddNonTensor<VectorInt64>(val, pyobjs);
} else if (val.Type() == DataTypeImpl::GetType<VectorFloat>()) {
AddNonTensor<VectorFloat>(val, pyobjs);
} else if (val.Type() == DataTypeImpl::GetType<VectorDouble>()) {
AddNonTensor<VectorDouble>(val, pyobjs);
} else if (val.Type() == DataTypeImpl::GetType<VectorMapStringToFloat>()) {
AddNonTensor<VectorMapStringToFloat>(val, pyobjs);
} else if (val.Type() == DataTypeImpl::GetType<VectorMapInt64ToFloat>()) {
@ -629,57 +622,55 @@ including arg name, arg type (contains both type and shape).)pbdoc")
return *(na.Type());
},
"node type")
.def(
"__str__", [](const onnxruntime::NodeArg& na) -> std::string {
std::ostringstream res;
res << "NodeArg(name='" << na.Name() << "', type='" << *(na.Type()) << "', shape=";
auto shape = na.Shape();
std::vector<py::object> arr;
if (shape == nullptr || shape->dim_size() == 0) {
res << "[]";
.def("__str__", [](const onnxruntime::NodeArg& na) -> std::string {
std::ostringstream res;
res << "NodeArg(name='" << na.Name() << "', type='" << *(na.Type()) << "', shape=";
auto shape = na.Shape();
std::vector<py::object> arr;
if (shape == nullptr || shape->dim_size() == 0) {
res << "[]";
} else {
res << "[";
for (int i = 0; i < shape->dim_size(); ++i) {
if (utils::HasDimValue(shape->dim(i))) {
res << shape->dim(i).dim_value();
} else if (utils::HasDimParam(shape->dim(i))) {
res << "'" << shape->dim(i).dim_param() << "'";
} else {
res << "[";
for (int i = 0; i < shape->dim_size(); ++i) {
if (utils::HasDimValue(shape->dim(i))) {
res << shape->dim(i).dim_value();
} else if (utils::HasDimParam(shape->dim(i))) {
res << "'" << shape->dim(i).dim_param() << "'";
} else {
res << "None";
}
if (i < shape->dim_size() - 1) {
res << ", ";
}
}
res << "]";
}
res << ")";
return std::string(res.str());
},
"converts the node into a readable string")
.def_property_readonly(
"shape", [](const onnxruntime::NodeArg& na) -> std::vector<py::object> {
auto shape = na.Shape();
std::vector<py::object> arr;
if (shape == nullptr || shape->dim_size() == 0) {
return arr;
res << "None";
}
arr.resize(shape->dim_size());
for (int i = 0; i < shape->dim_size(); ++i) {
if (utils::HasDimValue(shape->dim(i))) {
arr[i] = py::cast(shape->dim(i).dim_value());
} else if (utils::HasDimParam(shape->dim(i))) {
arr[i] = py::cast(shape->dim(i).dim_param());
} else {
arr[i] = py::none();
}
if (i < shape->dim_size() - 1) {
res << ", ";
}
return arr;
},
"node shape (assuming the node holds a tensor)");
}
res << "]";
}
res << ")";
return std::string(res.str());
},
"converts the node into a readable string")
.def_property_readonly("shape", [](const onnxruntime::NodeArg& na) -> std::vector<py::object> {
auto shape = na.Shape();
std::vector<py::object> arr;
if (shape == nullptr || shape->dim_size() == 0) {
return arr;
}
arr.resize(shape->dim_size());
for (int i = 0; i < shape->dim_size(); ++i) {
if (utils::HasDimValue(shape->dim(i))) {
arr[i] = py::cast(shape->dim(i).dim_value());
} else if (utils::HasDimParam(shape->dim(i))) {
arr[i] = py::cast(shape->dim(i).dim_param());
} else {
arr[i] = py::none();
}
}
return arr;
},
"node shape (assuming the node holds a tensor)");
py::class_<SessionObjectInitializer>(m, "SessionObjectInitializer");
py::class_<InferenceSession>(m, "InferenceSession", R"pbdoc(This is the main class used to run a model.)pbdoc")
@ -691,13 +682,12 @@ including arg name, arg type (contains both type and shape).)pbdoc")
InitializeSession(sess, provider_types);
},
R"pbdoc(Load a model saved in ONNX format.)pbdoc")
.def(
"read_bytes", [](InferenceSession* sess, const py::bytes& serializedModel, std::vector<std::string>& provider_types) {
std::istringstream buffer(serializedModel);
OrtPybindThrowIfError(sess->Load(buffer));
InitializeSession(sess, provider_types);
},
R"pbdoc(Load a model serialized in ONNX format.)pbdoc")
.def("read_bytes", [](InferenceSession* sess, const py::bytes& serializedModel, std::vector<std::string>& provider_types) {
std::istringstream buffer(serializedModel);
OrtPybindThrowIfError(sess->Load(buffer));
InitializeSession(sess, provider_types);
},
R"pbdoc(Load a model serialized in ONNX format.)pbdoc")
.def("run", [](InferenceSession* sess, std::vector<std::string> output_names, std::map<std::string, py::object> pyfeeds, RunOptions* run_options = nullptr) -> std::vector<py::object> {
NameMLValMap feeds;
for (auto _ : pyfeeds) {

32
onnxruntime/test/framework/data_types_test.cc
View file

@ -28,6 +28,7 @@ struct TestMap {
// Try recursive type registration and compatibility tests
using TestMapToMapInt64ToFloat = TestMap<int64_t, MapInt64ToFloat>;
using VectorInt64 = std::vector<int64_t>;
using TestMapStringToVectorInt64 = TestMap<std::string, VectorInt64>;
// Trial to see if we resolve the setter properly
@ -39,6 +40,7 @@ struct TestSequence {
using value_type = T;
};
using VectorString = std::vector<std::string>;
using TestSequenceOfSequence = TestSequence<VectorString>;
/// Adding an Opaque type with type parameters
@ -80,6 +82,8 @@ ORT_REGISTER_MAP(TestMapMLFloat16ToFloat);
ORT_REGISTER_SEQ(MyOpaqueSeqCpp_1);
ORT_REGISTER_SEQ(MyOpaqueSeqCpp_2);
ORT_REGISTER_SEQ(TestSequenceOfSequence);
ORT_REGISTER_SEQ(VectorString);
ORT_REGISTER_SEQ(VectorInt64);
ORT_REGISTER_OPAQUE_TYPE(TestOpaqueType_1, TestOpaqueDomain_1, TestOpaqueName_1);
ORT_REGISTER_OPAQUE_TYPE(TestOpaqueType_2, TestOpaqueDomain_2, TestOpaqueName_2);
@ -390,6 +394,20 @@ TEST_F(DataTypeTest, BFloat16Test) {
TEST_F(DataTypeTest, DataUtilsTest) {
using namespace ONNX_NAMESPACE::Utils;
// Test simple seq
{
const std::string seq_float("seq(tensor(float))");
const auto* seq_proto = DataTypeImpl::GetSequenceTensorType<float>()->GetTypeProto();
EXPECT_NE(seq_proto, nullptr);
DataType seq_dt = DataTypeUtils::ToType(*seq_proto);
EXPECT_NE(seq_dt, nullptr);
EXPECT_EQ(seq_float, *seq_dt);
DataType seq_from_str = DataTypeUtils::ToType(*seq_dt);
// Expect internalized strings
EXPECT_EQ(seq_dt, seq_from_str);
const auto& from_dt_proto = DataTypeUtils::ToTypeProto(seq_dt);
EXPECT_TRUE(DataTypeImpl::GetSequenceTensorType<float>()->IsCompatible(from_dt_proto));
}
// Test Tensor
{
const std::string tensor_uint64("tensor(uint64)");
@ -494,20 +512,6 @@ TEST_F(DataTypeTest, DataUtilsTest) {
const auto& from_dt_proto = DataTypeUtils::ToTypeProto(map_dt);
EXPECT_TRUE(DataTypeImpl::GetType<MyOpaqueMapCpp_2>()->IsCompatible(from_dt_proto));
}
// Test simple seq
{
const std::string seq_float("seq(tensor(float))");
const auto* seq_proto = DataTypeImpl::GetType<VectorFloat>()->GetTypeProto();
EXPECT_NE(seq_proto, nullptr);
DataType seq_dt = DataTypeUtils::ToType(*seq_proto);
EXPECT_NE(seq_dt, nullptr);
EXPECT_EQ(seq_float, *seq_dt);
DataType seq_from_str = DataTypeUtils::ToType(*seq_dt);
// Expect internalized strings
EXPECT_EQ(seq_dt, seq_from_str);
const auto& from_dt_proto = DataTypeUtils::ToTypeProto(seq_dt);
EXPECT_TRUE(DataTypeImpl::GetType<VectorFloat>()->IsCompatible(from_dt_proto));
}
// Test Sequence with recursion
{
const std::string seq_map_str_float("seq(map(string,tensor(float)))");

83
onnxruntime/test/providers/cpu/sequence/sequence_ops_test.cc Normal file
View file

@ -0,0 +1,83 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
namespace onnxruntime {
namespace test {
TEST(SequenceOpsTest, SequenceLengthPositiveFloat) {
OpTester test("SequenceLength", 11);
SeqTensors<float> input;
input.AddTensor({3, 2}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
input.AddTensor({3, 3}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
test.AddSeqInput("S", input);
test.AddOutput<int64_t>("I", {}, {2});
test.Run();
}
TEST(SequenceOpsTest, SequenceLengthPositiveInt64) {
OpTester test("SequenceLength", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqInput("S", input);
test.AddOutput<int64_t>("I", {}, {2});
test.Run();
}
TEST(SequenceOpsTest, SequenceAtPositiveIdx) {
OpTester test("SequenceAt", 11);
SeqTensors<float> input;
std::vector<float> output_vec{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f};
std::vector<int64_t> output_shape{3, 3};
input.AddTensor({3, 2}, {10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f});
input.AddTensor(output_shape, output_vec);
test.AddSeqInput("S", input);
test.AddInput("I", {}, {1});
test.AddOutput<float>("T", output_shape, output_vec);
test.Run();
}
TEST(SequenceOpsTest, SequenceAtNegativeIdx) {
OpTester test("SequenceAt", 11);
SeqTensors<float> input;
std::vector<float> output_vec{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f};
std::vector<int64_t> output_shape{3, 3};
input.AddTensor({3, 2}, {10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f});
input.AddTensor(output_shape, output_vec);
test.AddSeqInput("S", input);
test.AddInput("I", {}, {-1});
test.AddOutput<float>("T", output_shape, output_vec);
test.Run();
}
TEST(SequenceOpsTest, SequenceAtInvalidPositiveIdx) {
OpTester test("SequenceAt", 11);
SeqTensors<float> input;
std::vector<float> output_vec{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f};
std::vector<int64_t> output_shape{3, 3};
input.AddTensor({3, 2}, {10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f});
input.AddTensor(output_shape, output_vec);
test.AddSeqInput("S", input);
test.AddInput("I", {}, {10});
test.AddOutput<float>("T", output_shape, output_vec);
test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index");
}
TEST(SequenceOpsTest, SequenceAtInvalidNegativeIdx) {
OpTester test("SequenceAt", 11);
SeqTensors<float> input;
std::vector<float> output_vec{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f};
std::vector<int64_t> output_shape{3, 3};
input.AddTensor({3, 2}, {10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f});
input.AddTensor(output_shape, output_vec);
test.AddSeqInput("S", input);
test.AddInput("I", {}, {-10});
test.AddOutput<float>("T", output_shape, output_vec);
test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index");
}
} // namespace test
} // namespace onnxruntime

61
onnxruntime/test/providers/provider_test_utils.h
View file

@ -16,6 +16,7 @@
#include "core/framework/data_types.h"
#include "test/test_environment.h"
#include "test/framework/TestAllocatorManager.h"
#include "core/framework/TensorSeq.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
@ -27,6 +28,20 @@ class InferenceSession;
struct SessionOptions;
namespace test {
template <typename T>
struct SeqTensors {
void AddTensor(const std::vector<int64_t>& shape0, const std::vector<T>& data0) {
tensors.push_back(Tensor<T>{shape0, data0});
}
template <typename U>
struct Tensor {
std::vector<int64_t> shape;
std::vector<U> data;
};
std::vector<Tensor<T>> tensors;
};
// unfortunately std::optional is in C++17 so use a miniversion of it
template <typename T>
class optional {
@ -154,6 +169,20 @@ struct VectorOfMapType {
template <typename TKey, typename TVal>
const VectorOfMapTypeProto<TKey, TVal> VectorOfMapType<TKey, TVal>::s_vec_map_type_proto;
template <typename ElemType>
struct SequenceTensorTypeProto : ONNX_NAMESPACE::TypeProto {
SequenceTensorTypeProto() {
MLDataType dt = DataTypeImpl::GetTensorType<ElemType>();
const auto* elem_proto = dt->GetTypeProto();
mutable_sequence_type()->mutable_elem_type()->CopyFrom(*elem_proto);
auto* tensor_type = mutable_sequence_type()->mutable_elem_type()->mutable_tensor_type();
tensor_type->set_elem_type(TypeToDataType<ElemType>());
}
};
template <typename ElemType>
const SequenceTensorTypeProto<ElemType> s_sequence_tensor_type_proto;
// To use OpTester:
// 1. Create one with the op name
// 2. Call AddAttribute with any attributes
@ -215,6 +244,38 @@ class OpTester {
input_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional<float>(), optional<float>()));
}
template <typename T>
void AddSeqInput(const char* name, const SeqTensors<T>& seq_tensors) {
auto mltype = DataTypeImpl::GetType<TensorSeq>();
ORT_ENFORCE(mltype != nullptr, "TensorSeq must be a registered cpp type");
auto ptr = std::make_unique<TensorSeq>();
auto num_tensors = seq_tensors.tensors.size();
ptr->tensors.resize(num_tensors);
for (int i = 0; i < num_tensors; ++i) {
TensorShape shape{seq_tensors.tensors[i].shape};
auto values_count = static_cast<int64_t>(seq_tensors.tensors[i].data.size());
ORT_ENFORCE(shape.Size() == values_count, values_count,
" input values doesn't match tensor size of ", shape.Size());
auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU);
auto& tensor = ptr->tensors[i];
tensor = Tensor(DataTypeImpl::GetType<T>(),
shape,
allocator);
auto* data_ptr = tensor.template MutableData<T>();
for (int64_t x = 0; x < values_count; ++x) {
data_ptr[x] = seq_tensors.tensors[i].data[x];
}
}
OrtValue value;
value.Init(ptr.get(), mltype, mltype->GetDeleteFunc());
ptr.release();
input_data_.push_back(Data(NodeArg(name, &s_sequence_tensor_type_proto<T>), std::move(value), optional<float>(), optional<float>()));
}
template <typename TKey, typename TVal>
void AddInput(const char* name, const std::map<TKey, TVal>& val) {
std::unique_ptr<std::map<TKey, TVal>> ptr = onnxruntime::make_unique<std::map<TKey, TVal>>(val);

20
onnxruntime/test/python/onnxruntime_test_python.py
View file

@ -346,6 +346,26 @@ class TestInferenceSession(unittest.TestCase):
res = sess.run([output_name], {x_name: x})
self.assertEqual(output_expected, res[0])
def testSequenceLength(self):
sess = onnxrt.InferenceSession(
self.get_name("sequence_length.onnx"))
x = [np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3)),
np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3))]
x_name = sess.get_inputs()[0].name
self.assertEqual(x_name, "X")
x_type = sess.get_inputs()[0].type
self.assertEqual(x_type, 'seq(tensor(float))')
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "Y")
output_type = sess.get_outputs()[0].type
self.assertEqual(output_type, 'tensor(int64)')
output_expected = np.array(2, dtype=np.int64)
res = sess.run([output_name], {x_name: x})
self.assertEqual(output_expected, res[0])
def testZipMapInt64Float(self):
sess = onnxrt.InferenceSession(self.get_name("zipmap_int64float.onnx"))
x = np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0],

27
onnxruntime/test/shared_lib/test_nontensor_types.cc
View file

@ -149,3 +149,30 @@ TEST_F(CApiTest, CreateGetVectorOfMapsStringFloat) { // support zipmap output t
std::set<float>(std::begin(values), std::end(values)));
}
}
TEST_F(CApiTest, CreateGetSeqTensors) {
// Creation
auto default_allocator = onnxruntime::make_unique<MockedOrtAllocator>();
Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault);
std::vector<Ort::Value> in;
std::vector<int64_t> vals{3, 1, 2, 0};
std::vector<int64_t> dims{1, 4};
const int N = 2;
for (int i = 0; i < N; ++i) {
// create tensor
Ort::Value tensor = Ort::Value::CreateTensor(info, vals.data(), vals.size() * sizeof(int64_t),
dims.data(), dims.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64);
in.push_back(std::move(tensor));
}
Ort::Value seq_ort = Ort::Value::CreateSequence(in);
// Fetch
for (int idx = 0; idx < N; ++idx) {
Ort::Value out = seq_ort.GetValue(idx, default_allocator.get());
int64_t* ret = out.GetTensorMutableData<int64_t>();
ASSERT_EQ(std::set<int64_t>(ret, ret + vals.size()),
std::set<int64_t>(std::begin(vals), std::end(vals)));
}
}

BIN
onnxruntime/test/testdata/sequence_length.onnx vendored Normal file
View file

Binary file not shown.