saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add support for output seq(tensor) in python binding and test framework. Implement SequenceConstruct, SequenceEmpty, SequenceInsert and SequenceErase ops. (#2040)

Browse source 
Add support for output seq(tensor) in python binding and test framework. Implement SequenceConstruct, SequenceEmpty, SequenceInsert and SequenceErase ops. (#2040)
This commit is contained in:
Pranav Sharma 2019-10-10 15:58:49 -07:00 committed by GitHub
parent ddbc2086e4
commit 2d4d0abd36
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 771 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
onnxruntime/core/framework/TensorSeq.h
View file

@ -17,7 +17,7 @@ struct TensorSeq {
// The current design follows the Tensor methodology.
// We also require this because the SequenceEmpty op expects the creation of a seq of a specific type
// and the SequenceInsert op expects validation of tensors to be added to the seq against this type.
MLDataType dtype;
MLDataType dtype{};
// TODO: optimization opportunity - if all tensors in the seq are scalars, we can potentially represent them
// as vector<primitive type>

10
onnxruntime/core/framework/onnxruntime_typeinfo.cc
View file

@ -14,8 +14,8 @@
using onnxruntime::BFloat16;
using onnxruntime::DataTypeImpl;
using onnxruntime::MLFloat16;
using onnxruntime::Tensor;
using onnxruntime::SparseTensor;
using onnxruntime::Tensor;
using onnxruntime::TensorShape;
namespace on = ONNX_NAMESPACE;
@ -76,7 +76,7 @@ OrtStatus* OrtTypeInfo::FromDataTypeImpl(const onnxruntime::DataTypeImpl* input,
// Place Opaque first as tensors will be
// mostly handled above and maps and sequences
// are not common
switch (type_proto->value_case ()) {
switch (type_proto->value_case()) {
case on::TypeProto::kOpaqueType: {
*out = new OrtTypeInfo(ONNX_TYPE_OPAQUE, nullptr);
return nullptr;
@ -110,7 +110,7 @@ OrtStatus* OrtTypeInfo::FromDataTypeImpl(const onnxruntime::DataTypeImpl* input,
return OrtApis::CreateStatus(ORT_NOT_IMPLEMENTED, "not implemented");
}
const DataTypeImpl* ElementTypeFromProto(int type) {
const DataTypeImpl* OrtTypeInfo::ElementTypeFromProto(int type) {
switch (type) {
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
return DataTypeImpl::GetType<float>();
@ -176,7 +176,7 @@ OrtStatus* OrtTypeInfo::FromDataTypeImpl(const ONNX_NAMESPACE::TypeProto* input,
TensorShape shape_data(std::move(dims));
for (int i = 0; i < s.dim_size(); ++i) {
auto& t = s.dim(i);
switch (t.value_case ()) {
switch (t.value_case()) {
case on::TensorShapeProto::Dimension::kDimValue:
shape_data[i] = t.dim_value();
break;
@ -211,7 +211,7 @@ OrtStatus* OrtTypeInfo::FromDataTypeImpl(const ONNX_NAMESPACE::TypeProto* input,
case on::TypeProto::VALUE_NOT_SET:
break;
default:
// Not implemented
// Not implemented
break;
}
return OrtApis::CreateStatus(ORT_NOT_IMPLEMENTED, "not implemented");

2
onnxruntime/core/framework/onnxruntime_typeinfo.h
View file

@ -33,6 +33,8 @@ struct OrtTypeInfo {
const onnxruntime::DataTypeImpl* tensor_data_type, OrtTypeInfo** out);
static OrtStatus* FromDataTypeImpl(const ONNX_NAMESPACE::TypeProto*, OrtTypeInfo** out);
static const onnxruntime::DataTypeImpl* ElementTypeFromProto(int type);
private:
OrtTypeInfo(ONNXType type, OrtTensorTypeAndShapeInfo* data) noexcept;
};

8
onnxruntime/core/providers/cpu/cpu_execution_provider.cc
View file

@ -392,6 +392,10 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Co
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, If);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceLength);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceAt);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceEmpty);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceInsert);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceErase);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceConstruct);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ScatterND);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Gemm);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, GatherElements);
@ -1018,6 +1022,10 @@ void RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, If)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceLength)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceAt)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceEmpty)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceInsert)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceErase)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceConstruct)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ScatterND)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Gemm)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, GatherElements)>,

229
onnxruntime/core/providers/cpu/sequence/sequence_ops.cc
View file

@ -10,6 +10,9 @@ using namespace onnxruntime::common;
namespace onnxruntime {
// TODO: The current implementation of sequence ops relies on tensor copies. Ideally we should try to avoid
// these copies. This has been postponed due to lack of time.
// SequenceLength
ONNX_CPU_OPERATOR_KERNEL(
SequenceLength,
@ -24,6 +27,7 @@ Status SequenceLength::Compute(OpKernelContext* context) const {
ORT_ENFORCE(X != nullptr, "Got nullptr for sequence input.");
auto* Y = context->Output(0, {});
ORT_ENFORCE(Y != nullptr, "SequenceLength: Got nullptr for output tensor");
auto* Y_data = Y->template MutableData<int64_t>();
*Y_data = static_cast<int64_t>(X->tensors.size());
@ -72,13 +76,6 @@ bool ValidateSeqIdx(int64_t input_seq_idx, int64_t seq_size) {
return retval;
}
template <typename T>
static void CopyTensor(const Tensor& indexed_tensor, Tensor& output_tensor) {
const auto* input_data = indexed_tensor.template Data<T>();
auto* output_data = output_tensor.template MutableData<T>();
memcpy(output_data, input_data, indexed_tensor.SizeInBytes());
}
Status SequenceAt::Compute(OpKernelContext* context) const {
const auto* X = context->Input<TensorSeq>(0);
ORT_ENFORCE(X != nullptr, "Got nullptr for sequence input.");
@ -96,9 +93,227 @@ Status SequenceAt::Compute(OpKernelContext* context) const {
}
const Tensor& indexed_tensor = X->tensors[input_seq_idx];
auto* Y = context->Output(0, indexed_tensor.Shape().GetDims());
ORT_ENFORCE(Y != nullptr, "SequenceAt: Got nullptr for output tensor");
CopyCpuTensor(&indexed_tensor, Y);
return Status::OK();
}
// SequenceEmpty
ONNX_CPU_OPERATOR_KERNEL(
SequenceEmpty,
11,
KernelDefBuilder()
.TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes()),
SequenceEmpty);
SequenceEmpty::SequenceEmpty(const OpKernelInfo& info) : OpKernel(info) {
if (!info.GetAttr("dtype", &dtype_).IsOK()) {
dtype_ = ONNX_NAMESPACE::TensorProto_DataType_FLOAT;
}
}
Status SequenceEmpty::Compute(OpKernelContext* context) const {
auto* Y = context->Output<TensorSeq>(0);
ORT_ENFORCE(Y != nullptr, "SequenceEmpty: Got nullptr for output sequence");
MLDataType seq_dtype{};
switch (dtype_) {
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
seq_dtype = DataTypeImpl::GetType<float>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_BOOL:
seq_dtype = DataTypeImpl::GetType<bool>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_INT32:
seq_dtype = DataTypeImpl::GetType<int>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE:
seq_dtype = DataTypeImpl::GetType<double>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_STRING:
seq_dtype = DataTypeImpl::GetType<std::string>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_INT8:
seq_dtype = DataTypeImpl::GetType<int8_t>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
seq_dtype = DataTypeImpl::GetType<uint8_t>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_UINT16:
seq_dtype = DataTypeImpl::GetType<uint16_t>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_INT16:
seq_dtype = DataTypeImpl::GetType<int16_t>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_INT64:
seq_dtype = DataTypeImpl::GetType<int64_t>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_UINT32:
seq_dtype = DataTypeImpl::GetType<uint32_t>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_UINT64:
seq_dtype = DataTypeImpl::GetType<uint64_t>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16:
seq_dtype = DataTypeImpl::GetType<MLFloat16>();
break;
case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16:
seq_dtype = DataTypeImpl::GetType<BFloat16>();
break;
default:
ORT_THROW("Unsupported 'dtype' value: ", dtype_);
}
Y->dtype = seq_dtype;
return Status::OK();
}
// SequenceInsert
ONNX_CPU_OPERATOR_KERNEL(
SequenceInsert,
11,
KernelDefBuilder()
.TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes())
.TypeConstraint("I", std::vector<MLDataType>{
DataTypeImpl::GetTensorType<int32_t>(),
DataTypeImpl::GetTensorType<int64_t>()}),
SequenceInsert);
Status CreateCopyAndAppendCpuTensor(const Tensor& in_tensor, OpKernelContext* context, TensorSeq& tseq) {
AllocatorPtr alloc;
ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc));
Tensor tmp(in_tensor.DataType(), onnxruntime::TensorShape(in_tensor.Shape()), alloc);
CopyCpuTensor(&in_tensor, &tmp);
tseq.tensors.push_back(std::move(tmp));
return Status::OK();
}
Status SequenceInsert::Compute(OpKernelContext* context) const {
const auto* S = context->Input<TensorSeq>(0);
ORT_ENFORCE(S != nullptr, "Got nullptr for sequence input.");
const auto* X = context->Input<Tensor>(1);
ORT_ENFORCE(X != nullptr, "Got nullptr for input tensor.");
// Data type of the input tensor MUST be same as that of the input sequence
if (S->dtype != X->DataType()) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"Data type of the input tensor MUST be same as that of the input sequence. Sequence data type (",
DataTypeImpl::ToString(S->dtype), "), input tensor data type (", DataTypeImpl::ToString(X->DataType()), ")");
}
const auto* I = context->Input<Tensor>(2);
int64_t num_tensors_input_seq = static_cast<int64_t>(S->tensors.size());
int64_t input_seq_idx = num_tensors_input_seq + 1; // default is append
if (I) { // position is optional
input_seq_idx = GetSeqIdx(*I);
if (!ValidateSeqIdx(input_seq_idx, static_cast<int64_t>(num_tensors_input_seq))) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"Invalid sequence index (", input_seq_idx, ") specified for sequence of size (", num_tensors_input_seq, ")");
}
if (input_seq_idx < 0) {
input_seq_idx = static_cast<int64_t>(num_tensors_input_seq) + input_seq_idx;
}
}
auto* Y = context->Output<TensorSeq>(0);
ORT_ENFORCE(Y != nullptr, "SequenceInsert: Got nullptr for output sequence");
Y->dtype = S->dtype;
Y->tensors.reserve(num_tensors_input_seq + 1);
for (int i = 0; i < num_tensors_input_seq; ++i) {
if (i == input_seq_idx) {
CreateCopyAndAppendCpuTensor(*X, context, *Y);
CreateCopyAndAppendCpuTensor(S->tensors[i], context, *Y);
} else {
CreateCopyAndAppendCpuTensor(S->tensors[i], context, *Y);
}
}
if (input_seq_idx == num_tensors_input_seq + 1) {
CreateCopyAndAppendCpuTensor(*X, context, *Y);
}
return Status::OK();
}
// SequenceErase
ONNX_CPU_OPERATOR_KERNEL(
SequenceErase,
11,
KernelDefBuilder()
.TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes())
.TypeConstraint("I", std::vector<MLDataType>{
DataTypeImpl::GetTensorType<int32_t>(),
DataTypeImpl::GetTensorType<int64_t>()}),
SequenceErase);
Status SequenceErase::Compute(OpKernelContext* context) const {
const auto* S = context->Input<TensorSeq>(0);
ORT_ENFORCE(S != nullptr, "Got nullptr for sequence input.");
const auto* I = context->Input<Tensor>(1);
int64_t num_tensors_input_seq = static_cast<int64_t>(S->tensors.size());
int64_t input_seq_idx = num_tensors_input_seq - 1; // default is erase last one
if (I) { // position is optional
input_seq_idx = GetSeqIdx(*I);
if (!ValidateSeqIdx(input_seq_idx, static_cast<int64_t>(num_tensors_input_seq))) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"Invalid sequence index (", input_seq_idx, ") specified for sequence of size (", num_tensors_input_seq, ")");
}
if (input_seq_idx < 0) {
input_seq_idx = static_cast<int64_t>(num_tensors_input_seq) + input_seq_idx;
}
}
auto* Y = context->Output<TensorSeq>(0);
ORT_ENFORCE(Y != nullptr, "SequenceErase: Got nullptr for output sequence");
Y->dtype = S->dtype;
Y->tensors.reserve(num_tensors_input_seq - 1);
for (int i = 0; i < num_tensors_input_seq; ++i) {
if (i == input_seq_idx) {
continue;
}
CreateCopyAndAppendCpuTensor(S->tensors[i], context, *Y);
}
return Status::OK();
}
// SequenceConstruct
ONNX_CPU_OPERATOR_KERNEL(
SequenceConstruct,
11,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::AllTensorTypes())
.TypeConstraint("S", DataTypeImpl::AllSequenceTensorTypes()),
SequenceConstruct);
Status SequenceConstruct::Compute(OpKernelContext* context) const {
auto num_inputs = Node().InputArgCount().front();
ORT_ENFORCE(num_inputs >= 1, "Must have 1 or more inputs");
auto* Y = context->Output<TensorSeq>(0);
ORT_ENFORCE(Y != nullptr, "SequenceConstruct: Got nullptr for output sequence");
MLDataType first_dtype = context->Input<Tensor>(0)->DataType();
// Before copying check if all tensors are of the same type.
for (int input_idx = 0; input_idx < num_inputs; ++input_idx) {
const auto* X = context->Input<Tensor>(input_idx);
if (input_idx > 0 && X->DataType() != first_dtype) {
ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"Violation of the requirment that all input tensors must have the same data type.");
}
}
// now copy the tensors to the output sequence
Y->dtype = first_dtype;
Y->tensors.reserve(num_inputs);
for (int input_idx = 0; input_idx < num_inputs; ++input_idx) {
const auto* X = context->Input<Tensor>(input_idx);
CreateCopyAndAppendCpuTensor(*X, context, *Y);
}
return Status::OK();
}
} // namespace onnxruntime

29
onnxruntime/core/providers/cpu/sequence/sequence_ops.h
View file

@ -24,4 +24,33 @@ class SequenceAt final : public OpKernel {
Status Compute(OpKernelContext* context) const override;
};
class SequenceEmpty final : public OpKernel {
public:
SequenceEmpty(const OpKernelInfo& info);
Status Compute(OpKernelContext* context) const override;
private:
int64_t dtype_{};
};
class SequenceInsert final : public OpKernel {
public:
SequenceInsert(const OpKernelInfo& info) : OpKernel(info) {
}
Status Compute(OpKernelContext* context) const override;
};
class SequenceErase final : public OpKernel {
public:
SequenceErase(const OpKernelInfo& info) : OpKernel(info) {
}
Status Compute(OpKernelContext* context) const override;
};
class SequenceConstruct final : public OpKernel {
public:
SequenceConstruct(const OpKernelInfo& info) : OpKernel(info) {
}
Status Compute(OpKernelContext* context) const override;
};
} //namespace onnxruntime

14
onnxruntime/core/providers/nuphar/compiler/func_info.cc
View file

@ -6,13 +6,11 @@
#include "core/providers/nuphar/runtime/control_flow/scan_exec_ctx.h"
#include "core/framework/op_kernel.h"
#include "core/framework/tensorprotoutils.h"
#include "core/framework/onnxruntime_typeinfo.h"
#include "core/codegen/common/common.h"
#include "core/providers/nuphar/common/analysis/subgraph_codegen_stats.h"
#include <unordered_map>
// from onnxruntime_typeinf.cc, in global namespace
const onnxruntime::DataTypeImpl* ElementTypeFromProto(int type);
namespace onnxruntime {
namespace nuphar {
@ -64,7 +62,7 @@ static void FillBasicFuncInfo(NupharFuncInfo* func_info,
// fill in func args
NupharFuncInfo::FuncArgMeta input_meta;
input_meta.dtype = ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type());
input_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type());
input_meta.ort_arg_index = def_index;
// fill in shape info and symobolic info
@ -160,7 +158,7 @@ static void FillBasicFuncInfo(NupharFuncInfo* func_info,
}
NupharFuncInfo::FuncArgMeta output_meta;
output_meta.dtype = ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type());
output_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type());
output_meta.ort_arg_index = def_index;
// fill in shape info and symobolic info
@ -345,7 +343,7 @@ static void FillScanExecInfo(NupharFuncInfo* func_info,
}
NupharFuncInfo::FuncArgMeta input_meta;
input_meta.dtype = ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type());
input_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type());
input_meta.ort_arg_index = gsl::narrow_cast<int>(ort_input_idx);
// fill in shape info and symobolic info
@ -395,7 +393,7 @@ static void FillScanExecInfo(NupharFuncInfo* func_info,
}
NupharFuncInfo::FuncArgMeta input_meta;
input_meta.dtype = ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type());
input_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type());
input_meta.ort_arg_index = gsl::narrow_cast<int>(ort_input_idx);
std::vector<std::pair<size_t, std::string>> symbols;
@ -514,7 +512,7 @@ static void FillScanExecInfo(NupharFuncInfo* func_info,
}
NupharFuncInfo::FuncArgMeta output_meta;
output_meta.dtype = ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type());
output_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type());
output_meta.ort_arg_index = ort_arg_index;
// shape and symbols

6
onnxruntime/core/providers/nuphar/nuphar_execution_provider.cc
View file

@ -13,14 +13,12 @@
#include "core/providers/nuphar/compiler/x86/x86_target_info.h"
#include "core/providers/nuphar/kernel.h"
#include "core/providers/nuphar/partition/graph_partitioner.h"
#include "core/framework/onnxruntime_typeinfo.h"
#include <tvm/runtime/device_api.h> // TODO remove this after removing tvm::runtime
using namespace onnxruntime::nuphar;
// from onnxruntime_typeinf.cc, in global namespace
const onnxruntime::DataTypeImpl* ElementTypeFromProto(int type);
namespace onnxruntime {
// Initialization of thread local counter as subgraph id
@ -296,7 +294,7 @@ Status NupharExecutionProvider::SaveInitializer(
shape_dims[i] = dims[i];
const TensorShape& shape = TensorShape::ReinterpretBaseType(shape_dims);
auto data_type = ElementTypeFromProto(proto->data_type());
auto data_type = OrtTypeInfo::ElementTypeFromProto(proto->data_type());
auto t = onnxruntime::make_unique<Tensor>(
data_type,
shape,

4
onnxruntime/core/providers/nuphar/runtime/sequential/basic.cc
View file

@ -6,12 +6,10 @@
#include "core/codegen/common/common.h"
#include "core/codegen/common/profile.h"
#include "core/codegen/passes/utils/ort_tvm_utils.h"
#include "core/framework/onnxruntime_typeinfo.h"
#include "gsl/gsl"
#include <tvm/tvm.h>
// from onnxruntime_typeinf.cc, in global namespace
const onnxruntime::DataTypeImpl* ElementTypeFromProto(int type);
namespace onnxruntime {
namespace nuphar {

6
onnxruntime/core/session/onnxruntime_c_api.cc
View file

@ -836,7 +836,7 @@ OrtStatus* OrtGetValueImplSeqOfTensors(const OrtValue* p_ml_value, int index, Or
} else if (tensor_elem_type == DataTypeImpl::GetType<std::string>()) {
st = OrtGetValueImplSeqOfTensorsHelper<std::string>(allocator, one_tensor, out);
} else {
st = OrtApis::CreateStatus(ORT_FAIL, "Only sequences that contain float tensors are supported.");
st = OrtApis::CreateStatus(ORT_FAIL, "Invalid tensor element type in the input.");
}
return st;
}
@ -993,10 +993,10 @@ static OrtStatus* OrtCreateValueImplSeqHelper(const OrtValue* const* in, size_t
seq_ptr->tensors.resize(num_values);
// use the data type of the first tensor as the data type of the seq
seq_ptr->dtype = reinterpret_cast<const OrtValue*>(in[0])->Get<Tensor>().DataType();
seq_ptr->dtype = static_cast<const OrtValue*>(in[0])->Get<Tensor>().DataType();
for (size_t idx = 0; idx < num_values; ++idx) {
auto& one_tensor = reinterpret_cast<const OrtValue*>(in[idx])->Get<Tensor>();
auto& one_tensor = static_cast<const OrtValue*>(in[idx])->Get<Tensor>();
auto tensor_elem_type = one_tensor.DataType();
// sequences must have tensors of the same data type

48
onnxruntime/python/onnxruntime_pybind_mlvalue.cc
View file

@ -13,6 +13,8 @@
#include "core/framework/tensor.h"
#include "core/framework/allocator.h"
#include "core/framework/TensorSeq.h"
#include "core/framework/data_types.h"
#include "core/framework/onnxruntime_typeinfo.h"
using namespace std;
namespace onnxruntime {
@ -105,7 +107,6 @@ std::unique_ptr<Tensor> CreateTensor(AllocatorPtr alloc, const std::string& name
std::unique_ptr<Tensor> p_tensor;
try {
const int npy_type = PyArray_TYPE(darray);
// numpy requires long int as its dims.
int ndim = PyArray_NDIM(darray);
npy_intp* npy_dims = PyArray_DIMS(darray);
@ -196,18 +197,38 @@ std::unique_ptr<Tensor> CreateTensor(AllocatorPtr alloc, const std::string& name
return p_tensor;
}
void CreateSequenceOfTensors(AllocatorPtr alloc, const std::string& name_input, PyObject* pylist_obj,
OrtValue* p_mlvalue) {
auto list_size = PyList_Size(pylist_obj);
if (list_size <= 0) {
throw std::runtime_error("Got exception while creating seq(tensor) because input list size is " + std::to_string(list_size));
void CreateSequenceOfTensors(AllocatorPtr alloc, const std::string& name_input,
const InputDefList* input_def_list, PyObject* pylist_obj, OrtValue* p_mlvalue) {
// get sequence type from the model
const auto& def_list = *input_def_list;
auto ret_it = std::find_if(std::begin(def_list), std::end(def_list),
[&name_input](const NodeArg* node_arg) { return name_input == node_arg->Name(); });
if (ret_it == std::end(def_list)) {
throw std::runtime_error("Failed to find input with name: " + name_input + " in the model input def list");
}
const auto* type_proto = (*ret_it)->TypeAsProto();
if (!type_proto || !(type_proto->has_sequence_type())) {
throw std::runtime_error("Either type_proto was null or it was not of sequence type");
}
// set the seq type
auto p_seq_tensors = onnxruntime::make_unique<TensorSeq>();
p_seq_tensors->tensors.resize(list_size);
for (Py_ssize_t i = 0; i < list_size; ++i) {
auto* py_obj = PyList_GetItem(pylist_obj, i);
auto p_tensor = CreateTensor(alloc, name_input, reinterpret_cast<PyArrayObject*>(py_obj));
p_seq_tensors->tensors[i] = std::move(*p_tensor);
MLDataType seq_dtype = OrtTypeInfo::ElementTypeFromProto(
static_cast<ONNX_NAMESPACE::TensorProto_DataType>(type_proto->sequence_type().elem_type().tensor_type().elem_type()));
p_seq_tensors->dtype = seq_dtype;
// populate the seq
auto list_size = PyList_Size(pylist_obj);
if (list_size > 0) {
p_seq_tensors->tensors.resize(list_size);
for (Py_ssize_t i = 0; i < list_size; ++i) {
auto* py_obj = PyList_GetItem(pylist_obj, i);
if (!PyObjectCheck_Array(py_obj)) {
throw std::runtime_error("CreateSequenceOfTensors: Input is not a tensor");
}
auto p_tensor = CreateTensor(alloc, name_input, reinterpret_cast<PyArrayObject*>(py_obj));
p_seq_tensors->tensors[i] = std::move(*(p_tensor.release()));
}
}
p_mlvalue->Init(p_seq_tensors.release(),
@ -428,14 +449,15 @@ void CreateGenericIterableMLValue(PyObject* iterator, AllocatorPtr alloc, const
}
}
void CreateGenericMLValue(AllocatorPtr alloc, const std::string& name_input, py::object& value, OrtValue* p_mlvalue) {
void CreateGenericMLValue(const onnxruntime::InputDefList* input_def_list, AllocatorPtr alloc, const std::string& name_input,
py::object& value, OrtValue* p_mlvalue) {
if (PyObjectCheck_Array(value.ptr())) {
// The most frequent case: input comes as an array.
PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(value.ptr());
CreateTensorMLValue(alloc, name_input, arr, p_mlvalue);
} else if (PyList_Check(value.ptr())) {
auto* seq_tensors = reinterpret_cast<PyObject*>(value.ptr());
CreateSequenceOfTensors(alloc, name_input, seq_tensors, p_mlvalue);
CreateSequenceOfTensors(alloc, name_input, input_def_list, seq_tensors, p_mlvalue);
} else if (PyDict_Check(value.ptr())) {
CreateMapMLValue_AgnosticVectorMap((PyObject*)NULL, value.ptr(), alloc, name_input, p_mlvalue);
} else {

3
onnxruntime/python/onnxruntime_pybind_mlvalue.h
View file

@ -22,7 +22,8 @@ namespace py = pybind11;
int OnnxRuntimeTensorToNumpyType(const DataTypeImpl* tensor_type);
void CreateGenericMLValue(AllocatorPtr alloc, const std::string& name_input, py::object& value, OrtValue* p_mlvalue);
void CreateGenericMLValue(const onnxruntime::InputDefList* input_def_list, AllocatorPtr alloc, const std::string& name_input,
py::object& value, OrtValue* p_mlvalue);
} // namespace python
} // namespace onnxruntime

80
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -12,6 +12,7 @@
#include "core/graph/graph_viewer.h"
#include "core/common/logging/logging.h"
#include "core/common/logging/severity.h"
#include "core/framework/TensorSeq.h"
#if USE_CUDA
#define BACKEND_PROC "GPU"
@ -148,9 +149,53 @@ void AddNonTensor(OrtValue& val, std::vector<py::object>& pyobjs) {
pyobjs.push_back(py::cast(val.Get<T>()));
}
void GetPyObjFromTensor(const Tensor& rtensor, py::object& obj) {
std::vector<npy_intp> npy_dims;
const TensorShape& shape = rtensor.Shape();
for (size_t n = 0; n < shape.NumDimensions(); ++n) {
npy_dims.push_back(shape[n]);
}
MLDataType dtype = rtensor.DataType();
const int numpy_type = OnnxRuntimeTensorToNumpyType(dtype);
obj = py::reinterpret_steal<py::object>(PyArray_SimpleNew(
shape.NumDimensions(), npy_dims.data(), numpy_type));
void* outPtr = static_cast<void*>(
PyArray_DATA(reinterpret_cast<PyArrayObject*>(obj.ptr())));
if (numpy_type != NPY_OBJECT) {
memcpy(outPtr, rtensor.DataRaw(dtype), dtype->Size() * shape.Size());
} else {
// Handle string type.
py::object* outObj = static_cast<py::object*>(outPtr);
const std::string* src = rtensor.template Data<std::string>();
for (int i = 0; i < rtensor.Shape().Size(); i++, src++) {
outObj[i] = py::cast(*src);
}
}
}
template <>
void AddNonTensor<TensorSeq>(OrtValue& val, std::vector<py::object>& pyobjs) {
const auto& seq_tensors = val.Get<TensorSeq>();
size_t num_tensors = seq_tensors.tensors.size();
py::list py_list;
for (size_t i = 0; i < num_tensors; ++i) {
const auto& rtensor = seq_tensors.tensors[i];
py::object obj;
GetPyObjFromTensor(rtensor, obj);
py_list.append(obj);
}
pyobjs.push_back(py_list);
}
void AddNonTensorAsPyObj(OrtValue& val, std::vector<py::object>& pyobjs) {
// Should be in sync with core/framework/datatypes.h
if (val.Type() == DataTypeImpl::GetType<MapStringToString>()) {
if (val.Type() == DataTypeImpl::GetType<TensorSeq>()) {
AddNonTensor<TensorSeq>(val, pyobjs);
} else if (val.Type() == DataTypeImpl::GetType<MapStringToString>()) {
AddNonTensor<MapStringToString>(val, pyobjs);
} else if (val.Type() == DataTypeImpl::GetType<MapStringToInt64>()) {
AddNonTensor<MapStringToInt64>(val, pyobjs);
@ -177,31 +222,8 @@ void AddNonTensorAsPyObj(OrtValue& val, std::vector<py::object>& pyobjs) {
void AddTensorAsPyObj(OrtValue& val, std::vector<py::object>& pyobjs) {
const Tensor& rtensor = val.Get<Tensor>();
std::vector<npy_intp> npy_dims;
const TensorShape& shape = rtensor.Shape();
for (size_t n = 0; n < shape.NumDimensions(); ++n) {
npy_dims.push_back(shape[n]);
}
MLDataType dtype = rtensor.DataType();
const int numpy_type = OnnxRuntimeTensorToNumpyType(dtype);
py::object obj = py::reinterpret_steal<py::object>(PyArray_SimpleNew(
shape.NumDimensions(), npy_dims.data(), numpy_type));
void* outPtr = static_cast<void*>(
PyArray_DATA(reinterpret_cast<PyArrayObject*>(obj.ptr())));
if (numpy_type != NPY_OBJECT) {
memcpy(outPtr, rtensor.DataRaw(dtype), dtype->Size() * shape.Size());
} else {
// Handle string type.
py::object* outObj = static_cast<py::object*>(outPtr);
const std::string* src = rtensor.template Data<std::string>();
for (int i = 0; i < rtensor.Shape().Size(); i++, src++) {
outObj[i] = py::cast(*src);
}
}
py::object obj;
GetPyObjFromTensor(rtensor, obj);
pyobjs.push_back(obj);
}
@ -692,7 +714,11 @@ including arg name, arg type (contains both type and shape).)pbdoc")
NameMLValMap feeds;
for (auto _ : pyfeeds) {
OrtValue ml_value;
CreateGenericMLValue(GetAllocator(), _.first, _.second, &ml_value);
auto px = sess->GetModelInputs();
if (!px.first.IsOK() || !px.second) {
throw std::runtime_error("Either failed to get model inputs from the session object or the input def list was null");
}
CreateGenericMLValue(px.second, GetAllocator(), _.first, _.second, &ml_value);
if (PyErr_Occurred()) {
PyObject *ptype, *pvalue, *ptraceback;
PyErr_Fetch(&ptype, &pvalue, &ptraceback);

217
onnxruntime/test/providers/cpu/sequence/sequence_ops_test.cc
View file

@ -7,6 +7,7 @@
namespace onnxruntime {
namespace test {
// SequenceLength
TEST(SequenceOpsTest, SequenceLengthPositiveFloat) {
OpTester test("SequenceLength", 11);
SeqTensors<float> input;
@ -27,6 +28,7 @@ TEST(SequenceOpsTest, SequenceLengthPositiveInt64) {
test.Run();
}
// SequenceAt
TEST(SequenceOpsTest, SequenceAtPositiveIdx) {
OpTester test("SequenceAt", 11);
SeqTensors<float> input;
@ -79,5 +81,220 @@ TEST(SequenceOpsTest, SequenceAtInvalidNegativeIdx) {
test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index");
}
// SequenceEmpty
TEST(SequenceOpsTest, SequenceEmptyDefault) {
OpTester test("SequenceEmpty", 11);
test.AddSeqOutput("S", SeqTensors<float>());
test.Run();
}
TEST(SequenceOpsTest, SequenceEmptyInt64) {
OpTester test("SequenceEmpty", 11);
test.AddAttribute("dtype", static_cast<int64_t>(7));
test.AddSeqOutput("S", SeqTensors<int64_t>());
test.Run();
}
// SequenceInsert
TEST(SequenceOpsTest, SequenceInsertPositiveDefaultFloat) {
OpTester test("SequenceInsert", 11);
SeqTensors<float> input;
input.AddTensor({3, 2}, {1., 2., 3., 4., 5., 6.});
input.AddTensor({3, 3}, {1., 2., 3., 4., 5., 6., 7., 8., 9.});
test.AddSeqInput("S", input);
test.AddInput<float>("T", {3, 2}, {10., 20., 30., 40., 50., 60.});
SeqTensors<float> output;
output.AddTensor({3, 2}, {1., 2., 3., 4., 5., 6.});
output.AddTensor({3, 3}, {1., 2., 3., 4., 5., 6., 7., 8., 9.});
output.AddTensor({3, 2}, {10., 20., 30., 40., 50., 60.});
test.AddSeqOutput("S2", output);
test.Run();
}
TEST(SequenceOpsTest, SequenceInsertPositiveDefault) {
OpTester test("SequenceInsert", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqInput("S", input);
test.AddInput<int64_t>("T", {3, 2}, {10, 20, 30, 40, 50, 60});
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
test.AddSeqOutput("S2", output);
test.Run();
}
TEST(SequenceOpsTest, SequenceInsertValidPositiveIdx) {
OpTester test("SequenceInsert", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqInput("S", input);
test.AddInput<int64_t>("T", {3, 2}, {10, 20, 30, 40, 50, 60});
test.AddInput<int64_t>("I", {}, {1});
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqOutput("S2", output);
test.Run();
}
TEST(SequenceOpsTest, SequenceInsertValidNegativeIdx) {
OpTester test("SequenceInsert", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqInput("S", input);
test.AddInput<int64_t>("T", {3, 2}, {10, 20, 30, 40, 50, 60});
test.AddInput<int64_t>("I", {}, {-2});
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqOutput("S2", output);
test.Run();
}
TEST(SequenceOpsTest, SequenceInsertInvalidPositiveIdx) {
OpTester test("SequenceInsert", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqInput("S", input);
test.AddInput<int64_t>("T", {3, 2}, {10, 20, 30, 40, 50, 60});
test.AddInput<int64_t>("I", {}, {99});
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqOutput("S2", output);
test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index");
}
TEST(SequenceOpsTest, SequenceInsertInvalidNegativeIdx) {
OpTester test("SequenceInsert", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqInput("S", input);
test.AddInput<int64_t>("T", {3, 2}, {10, 20, 30, 40, 50, 60});
test.AddInput<int64_t>("I", {}, {-99});
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqOutput("S2", output);
test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index");
}
// SequenceErase
TEST(SequenceOpsTest, SequenceErasePositiveDefault) {
OpTester test("SequenceErase", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
input.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
test.AddSeqInput("S", input);
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddSeqOutput("S2", output);
test.Run();
}
TEST(SequenceOpsTest, SequenceEraseValidPositiveIdx) {
OpTester test("SequenceErase", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
input.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
test.AddSeqInput("S", input);
test.AddInput<int64_t>("I", {}, {1});
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
test.AddSeqOutput("S2", output);
test.Run();
}
TEST(SequenceOpsTest, SequenceEraseValidNegativeIdx) {
OpTester test("SequenceErase", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
input.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
input.AddTensor({2, 2}, {2, 4, 6, 8});
test.AddSeqInput("S", input);
test.AddInput<int64_t>("I", {}, {-2});
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
output.AddTensor({2, 2}, {2, 4, 6, 8});
test.AddSeqOutput("S2", output);
test.Run();
}
TEST(SequenceOpsTest, SequenceEraseInvalidPositiveIdx) {
OpTester test("SequenceErase", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
input.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
input.AddTensor({2, 2}, {2, 4, 6, 8});
test.AddSeqInput("S", input);
test.AddInput<int64_t>("I", {}, {99});
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
input.AddTensor({2, 2}, {2, 4, 6, 8});
test.AddSeqOutput("S2", output);
test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index");
}
TEST(SequenceOpsTest, SequenceEraseInvalidNegativeIdx) {
OpTester test("SequenceErase", 11);
SeqTensors<int64_t> input;
input.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
input.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
input.AddTensor({2, 2}, {2, 4, 6, 8});
test.AddSeqInput("S", input);
test.AddInput<int64_t>("I", {}, {-99});
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
input.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
input.AddTensor({2, 2}, {2, 4, 6, 8});
test.AddSeqOutput("S2", output);
test.Run(OpTester::ExpectResult::kExpectFailure, "Invalid sequence index");
}
// SequenceConstruct
TEST(SequenceOpsTest, SequenceConstructPositive) {
OpTester test("SequenceConstruct", 11);
test.AddInput<int64_t>("input_1", {3, 2}, {1, 2, 3, 4, 5, 6});
test.AddInput<int64_t>("input_2", {3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
test.AddInput<int64_t>("input_3", {3, 2}, {10, 20, 30, 40, 50, 60});
SeqTensors<int64_t> output;
output.AddTensor({3, 2}, {1, 2, 3, 4, 5, 6});
output.AddTensor({3, 3}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
output.AddTensor({3, 2}, {10, 20, 30, 40, 50, 60});
test.AddSeqOutput("S2", output);
test.Run();
}
} // namespace test
} // namespace onnxruntime

37
onnxruntime/test/providers/provider_test_utils.cc
View file

@ -240,6 +240,35 @@ void Check(const OpTester::Data& expected_data, const T& run_output, const std::
EXPECT_EQ(expected_data.data_.Get<T>(), run_output) << "provider_type: " << provider_type;
}
template <>
void Check<TensorSeq>(const OpTester::Data& expected_data, const TensorSeq& output_seq,
const std::string& provider_type) {
const auto& exp_seq = expected_data.data_.Get<TensorSeq>();
// first ensure data types match
EXPECT_EQ(exp_seq.dtype, output_seq.dtype) << "Data types don't match: Expected: " << DataTypeImpl::ToString(exp_seq.dtype)
<< " Output: " << output_seq.dtype << " provider_type: " << provider_type;
// check num of contained tensors
size_t expected_num_tensors = exp_seq.tensors.size();
size_t output_num_tensors = output_seq.tensors.size();
EXPECT_EQ(expected_num_tensors, output_num_tensors) << "Mismatch in number of tensors in the sequence"
<< " Expected: " << expected_num_tensors << " Output: "
<< output_num_tensors << " provider_type: " << provider_type;
// now check the contents of the tensors
auto null_deleter = [](void*) {};
for (int i = 0; i < output_num_tensors; ++i) {
OrtValue temp_value;
// Reason for null_deleter: we don't want the tensor destructor to be called as part of this OrtValue destructor
// as we're creating this OrtValue only to reuse the Check functionality
temp_value.Init(const_cast<Tensor*>(&exp_seq.tensors[i]), DataTypeImpl::GetType<Tensor>(), null_deleter);
OpTester::Data temp_data(NodeArg("dummy", nullptr), std::move(temp_value), optional<float>(), optional<float>());
Check(temp_data, output_seq.tensors[i], provider_type);
}
}
template <typename Type>
void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, OrtValue& ort_value,
const std::string& provider_type) {
@ -260,11 +289,11 @@ void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, OrtValu
void Check(const OpTester::Data& expected_data, OrtValue& ort_value, const std::string& provider_type) {
#ifdef MICROSOFT_AUTOML
CheckDispatch<dtf::TimePoint, VectorMapStringToFloat, VectorMapInt64ToFloat>(expected_data.data_.Type(), expected_data, ort_value,
provider_type);
CheckDispatch<dtf::TimePoint, VectorMapStringToFloat, VectorMapInt64ToFloat, TensorSeq>(expected_data.data_.Type(), expected_data, ort_value,
provider_type);
#else
CheckDispatch<VectorMapStringToFloat, VectorMapInt64ToFloat>(expected_data.data_.Type(), expected_data, ort_value,
provider_type);
CheckDispatch<VectorMapStringToFloat, VectorMapInt64ToFloat, TensorSeq>(expected_data.data_.Type(), expected_data, ort_value,
provider_type);
#endif
}

67
onnxruntime/test/providers/provider_test_utils.h
View file

@ -283,35 +283,12 @@ class OpTester {
template <typename T>
void AddSeqInput(const char* name, const SeqTensors<T>& seq_tensors) {
auto mltype = DataTypeImpl::GetType<TensorSeq>();
ORT_ENFORCE(mltype != nullptr, "TensorSeq must be a registered cpp type");
auto ptr = onnxruntime::make_unique<TensorSeq>();
auto num_tensors = seq_tensors.tensors.size();
ptr->tensors.resize(num_tensors);
for (int i = 0; i < num_tensors; ++i) {
TensorShape shape{seq_tensors.tensors[i].shape};
auto values_count = static_cast<int64_t>(seq_tensors.tensors[i].data.size());
ORT_ENFORCE(shape.Size() == values_count, values_count,
" input values doesn't match tensor size of ", shape.Size());
AddSeqData<T>(input_data_, name, seq_tensors);
}
auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU);
auto& tensor = ptr->tensors[i];
tensor = Tensor(DataTypeImpl::GetType<T>(),
shape,
allocator);
auto* data_ptr = tensor.template MutableData<T>();
for (int64_t x = 0; x < values_count; ++x) {
data_ptr[x] = seq_tensors.tensors[i].data[x];
}
}
OrtValue value;
value.Init(ptr.get(), mltype, mltype->GetDeleteFunc());
ptr.release();
input_data_.push_back(Data(NodeArg(name, &SequenceTensorType<T>::s_sequence_tensor_type_proto), std::move(value),
optional<float>(), optional<float>()));
template <typename T>
void AddSeqOutput(const char* name, const SeqTensors<T>& seq_tensors) {
AddSeqData<T>(output_data_, name, seq_tensors);
}
template <typename TKey, typename TVal>
@ -495,6 +472,40 @@ class OpTester {
}
}
template <typename T>
void AddSeqData(std::vector<Data>& data, const char* name, const SeqTensors<T>& seq_tensors) {
auto mltype = DataTypeImpl::GetType<TensorSeq>();
ORT_ENFORCE(mltype != nullptr, "TensorSeq must be a registered cpp type");
auto ptr = onnxruntime::make_unique<TensorSeq>();
ptr->dtype = DataTypeImpl::GetType<T>();
auto num_tensors = seq_tensors.tensors.size();
ptr->tensors.resize(num_tensors);
for (int i = 0; i < num_tensors; ++i) {
TensorShape shape{seq_tensors.tensors[i].shape};
auto values_count = static_cast<int64_t>(seq_tensors.tensors[i].data.size());
ORT_ENFORCE(shape.Size() == values_count, values_count,
" input values doesn't match tensor size of ", shape.Size());
auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU);
auto& tensor = ptr->tensors[i];
tensor = Tensor(DataTypeImpl::GetType<T>(),
shape,
allocator);
auto* data_ptr = tensor.template MutableData<T>();
for (int64_t x = 0; x < values_count; ++x) {
data_ptr[x] = seq_tensors.tensors[i].data[x];
}
}
OrtValue value;
value.Init(ptr.get(), mltype, mltype->GetDeleteFunc());
ptr.release();
data.push_back(Data(NodeArg(name, &SequenceTensorType<T>::s_sequence_tensor_type_proto), std::move(value),
optional<float>(), optional<float>()));
}
void ExecuteModel(Model& model, InferenceSession& session_object, ExpectResult expect_result,
const std::string& expected_failure_string, const RunOptions* run_options,
std::unordered_map<std::string, OrtValue> feeds, std::vector<std::string> output_names,

107
onnxruntime/test/python/onnxruntime_test_python.py
View file

@ -8,6 +8,7 @@ import numpy as np
import onnxruntime as onnxrt
import threading
class TestInferenceSession(unittest.TestCase):
def get_name(self, name):
@ -42,26 +43,28 @@ class TestInferenceSession(unittest.TestCase):
self.assertTrue(os.path.isfile(so.optimized_model_filepath))
def testGetProviders(self):
self.assertTrue('CPUExecutionProvider' in onnxrt.get_available_providers())
self.assertTrue(
'CPUExecutionProvider' in onnxrt.get_available_providers())
self.assertTrue('CPUExecutionProvider' in onnxrt.get_all_providers())
sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx"))
self.assertTrue('CPUExecutionProvider' in sess.get_providers())
def testSetProviders(self):
if 'CUDAExecutionProvider' in onnxrt.get_available_providers():
sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx"))
# confirm that CUDA Provider is in list of registered providers.
self.assertTrue('CUDAExecutionProvider' in sess.get_providers())
# reset the session and register only CPU Provider.
sess.set_providers(['CPUExecutionProvider'])
# confirm only CPU Provider is registered now.
self.assertEqual(['CPUExecutionProvider'], sess.get_providers())
sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx"))
# confirm that CUDA Provider is in list of registered providers.
self.assertTrue('CUDAExecutionProvider' in sess.get_providers())
# reset the session and register only CPU Provider.
sess.set_providers(['CPUExecutionProvider'])
# confirm only CPU Provider is registered now.
self.assertEqual(['CPUExecutionProvider'], sess.get_providers())
def testInvalidSetProviders(self):
with self.assertRaises(ValueError) as context:
sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx"))
sess.set_providers(['InvalidProvider'])
self.assertTrue('[\'InvalidProvider\'] does not contain a subset of available providers' in str(context.exception))
sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx"))
sess.set_providers(['InvalidProvider'])
self.assertTrue('[\'InvalidProvider\'] does not contain a subset of available providers' in str(
context.exception))
def testRunModel(self):
sess = onnxrt.InferenceSession(self.get_name("mul_1.onnx"))
@ -346,26 +349,6 @@ class TestInferenceSession(unittest.TestCase):
res = sess.run([output_name], {x_name: x})
self.assertEqual(output_expected, res[0])
def testSequenceLength(self):
sess = onnxrt.InferenceSession(
self.get_name("sequence_length.onnx"))
x = [np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3)),
np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3))]
x_name = sess.get_inputs()[0].name
self.assertEqual(x_name, "X")
x_type = sess.get_inputs()[0].type
self.assertEqual(x_type, 'seq(tensor(float))')
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "Y")
output_type = sess.get_outputs()[0].type
self.assertEqual(output_type, 'tensor(int64)')
output_expected = np.array(2, dtype=np.int64)
res = sess.run([output_name], {x_name: x})
self.assertEqual(output_expected, res[0])
def testZipMapInt64Float(self):
sess = onnxrt.InferenceSession(self.get_name("zipmap_int64float.onnx"))
x = np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0],
@ -547,6 +530,68 @@ class TestInferenceSession(unittest.TestCase):
res = sess.run([], {'input1:0': a, 'input:0':b})
def testSequenceLength(self):
sess = onnxrt.InferenceSession(self.get_name("sequence_length.onnx"))
x = [np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3)),
np.array([1.0, 0.0, 3.0, 44.0, 23.0, 11.0], dtype=np.float32).reshape((2, 3))]
x_name = sess.get_inputs()[0].name
self.assertEqual(x_name, "X")
x_type = sess.get_inputs()[0].type
self.assertEqual(x_type, 'seq(tensor(float))')
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "Y")
output_type = sess.get_outputs()[0].type
self.assertEqual(output_type, 'tensor(int64)')
output_expected = np.array(2, dtype=np.int64)
res = sess.run([output_name], {x_name: x})
self.assertEqual(output_expected, res[0])
def testSequenceConstruct(self):
sess = onnxrt.InferenceSession(
self.get_name("sequence_construct.onnx"))
self.assertEqual(sess.get_inputs()[0].type, 'tensor(int64)')
self.assertEqual(sess.get_inputs()[1].type, 'tensor(int64)')
self.assertEqual(sess.get_inputs()[0].name, "tensor1")
self.assertEqual(sess.get_inputs()[1].name, "tensor2")
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "output_sequence")
output_type = sess.get_outputs()[0].type
self.assertEqual(output_type, 'seq(tensor(int64))')
output_expected = [np.array([1, 0, 3, 44, 23, 11], dtype=np.int64).reshape((2, 3)),
np.array([1, 2, 3, 4, 5, 6], dtype=np.int64).reshape((2, 3))]
res = sess.run([output_name], {"tensor1": np.array([1, 0, 3, 44, 23, 11], dtype=np.int64).reshape((2, 3)),
"tensor2": np.array([1, 2, 3, 4, 5, 6], dtype=np.int64).reshape((2, 3))})
np.testing.assert_array_equal(output_expected, res[0])
def testSequenceInsert(self):
sess = onnxrt.InferenceSession(self.get_name("sequence_insert.onnx"))
self.assertEqual(sess.get_inputs()[0].type, 'seq(tensor(int64))')
self.assertEqual(sess.get_inputs()[1].type, 'tensor(int64)')
self.assertEqual(sess.get_inputs()[0].name, "input_seq")
self.assertEqual(sess.get_inputs()[1].name, "tensor")
output_name = sess.get_outputs()[0].name
self.assertEqual(output_name, "output_sequence")
output_type = sess.get_outputs()[0].type
self.assertEqual(output_type, 'seq(tensor(int64))')
output_expected = [
np.array([1, 0, 3, 44, 23, 11], dtype=np.int64).reshape((2, 3))]
res = sess.run([output_name], {"tensor": np.array(
[1, 0, 3, 44, 23, 11], dtype=np.int64).reshape((2, 3)), "input_seq": []})
np.testing.assert_array_equal(output_expected, res[0])
if __name__ == '__main__':
unittest.main()

18
onnxruntime/test/testdata/sequence_construct.onnx vendored Normal file
View file

@ -0,0 +1,18 @@
 onnx-example:¡
6
tensor1
tensor2output_sequence"SequenceConstruct
test-modelZ
tensor1


Z
tensor2


b%
output_sequence"



B

19
onnxruntime/test/testdata/sequence_insert.onnx vendored Normal file
View file

@ -0,0 +1,19 @@
 onnx-example:¤
4
input_seq
tensoroutput_sequence"SequenceInsert
test-modelZ
input_seq"



Z
tensor


b%
output_sequence"



B