saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-26 19:52:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support opset-13 specs of controlflow ops (Loop, If) (#5665)

Browse source
This commit is contained in:
Hariharan Seshadri 2020-11-11 23:44:14 -08:00 committed by GitHub
parent 07dc25e939
commit b92fc66ea1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 603 additions and 153 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmake/external/onnx_minimal.cmake vendored
View file

@ -14,7 +14,7 @@ endif()
set(ONNX_SOURCE_ROOT ${PROJECT_SOURCE_DIR}/external/onnx)
add_library(onnx_proto ${ONNX_SOURCE_ROOT}/onnx/onnx-ml.proto ${ONNX_SOURCE_ROOT}/onnx/onnx-operators-ml.proto)
add_library(onnx_proto ${ONNX_SOURCE_ROOT}/onnx/onnx-ml.proto ${ONNX_SOURCE_ROOT}/onnx/onnx-operators-ml.proto ${ONNX_SOURCE_ROOT}/onnx/onnx-data.proto)
target_include_directories(onnx_proto PUBLIC $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_INCLUDE_DIRECTORIES> "${CMAKE_CURRENT_BINARY_DIR}")
target_compile_definitions(onnx_proto PUBLIC $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_COMPILE_DEFINITIONS>)

5
include/onnxruntime/core/framework/data_types.h
View file

@ -284,12 +284,15 @@ class DataTypeImpl {
static MLDataType GetDataType(const std::string&);
static const std::vector<MLDataType>& AllTensorTypes();
static const std::vector<MLDataType>& AllSequenceTensorTypes();
static const std::vector<MLDataType>& AllFixedSizeTensorTypes();
static const std::vector<MLDataType>& AllSequenceTensorTypes();
static const std::vector<MLDataType>& AllFixedSizeSequenceTensorTypes();
static const std::vector<MLDataType>& AllNumericTensorTypes();
static const std::vector<MLDataType>& AllIEEEFloatTensorTypes();
static const std::vector<MLDataType>& AllFixedSizeTensorExceptHalfTypes();
static const std::vector<MLDataType>& AllIEEEFloatTensorExceptHalfTypes();
static const std::vector<MLDataType>& AllTensorAndSequenceTensorTypes();
static const std::vector<MLDataType>& AllFixedSizeTensorAndSequenceTensorTypes();
};
std::ostream& operator<<(std::ostream& out, MLDataType data_type);

5
onnxruntime/core/framework/TensorSeq.h
View file

@ -28,6 +28,9 @@ class TensorSeq {
}
void SetElements(std::vector<Tensor>&& tensors) {
// The caller of this method ensures that :
// (1) `elem_type` is set before invoking this method
// (2) All tensors contain elements of the same primitive data type
assert(tensors_.empty());
tensors_ = std::move(tensors);
}
@ -38,7 +41,7 @@ class TensorSeq {
return elem_type_ == o.elem_type_;
}
bool IsSameDataType (const Tensor& o) const noexcept {
bool IsSameDataType(const Tensor& o) const noexcept {
return elem_type_ == o.DataType()->AsPrimitiveDataType();
}

61
onnxruntime/core/framework/data_types.cc
View file

@ -915,26 +915,16 @@ const std::vector<MLDataType>& DataTypeImpl::AllFixedSizeTensorTypes() {
const std::vector<MLDataType>& DataTypeImpl::AllTensorTypes() {
static std::vector<MLDataType> all_tensor_types =
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<int64_t>(),
DataTypeImpl::GetTensorType<uint64_t>(),
DataTypeImpl::GetTensorType<int32_t>(),
DataTypeImpl::GetTensorType<uint32_t>(),
DataTypeImpl::GetTensorType<int16_t>(),
DataTypeImpl::GetTensorType<uint16_t>(),
DataTypeImpl::GetTensorType<int8_t>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
DataTypeImpl::GetTensorType<BFloat16>(),
DataTypeImpl::GetTensorType<bool>(),
DataTypeImpl::GetTensorType<std::string>()};
[]() {
auto temp = AllFixedSizeTensorTypes();
temp.push_back(DataTypeImpl::GetTensorType<std::string>());
return temp;
}();
return all_tensor_types;
}
const std::vector<MLDataType>& DataTypeImpl::AllSequenceTensorTypes() {
static std::vector<MLDataType> all_sequence_tensor_types =
const std::vector<MLDataType>& DataTypeImpl::AllFixedSizeSequenceTensorTypes() {
static std::vector<MLDataType> all_fixed_size_sequence_tensor_types =
{DataTypeImpl::GetSequenceTensorType<float>(),
DataTypeImpl::GetSequenceTensorType<double>(),
DataTypeImpl::GetSequenceTensorType<int64_t>(),
@ -947,9 +937,18 @@ const std::vector<MLDataType>& DataTypeImpl::AllSequenceTensorTypes() {
DataTypeImpl::GetSequenceTensorType<uint8_t>(),
DataTypeImpl::GetSequenceTensorType<MLFloat16>(),
DataTypeImpl::GetSequenceTensorType<BFloat16>(),
DataTypeImpl::GetSequenceTensorType<bool>(),
DataTypeImpl::GetSequenceTensorType<std::string>()};
DataTypeImpl::GetSequenceTensorType<bool>()};
return all_fixed_size_sequence_tensor_types;
}
const std::vector<MLDataType>& DataTypeImpl::AllSequenceTensorTypes() {
static std::vector<MLDataType> all_sequence_tensor_types =
[]() {
auto temp = AllFixedSizeSequenceTensorTypes();
temp.push_back(DataTypeImpl::GetSequenceTensorType<std::string>());
return temp;
}();
return all_sequence_tensor_types;
}
@ -971,6 +970,30 @@ const std::vector<MLDataType>& DataTypeImpl::AllNumericTensorTypes() {
return all_numeric_size_tensor_types;
}
const std::vector<MLDataType>& DataTypeImpl::AllFixedSizeTensorAndSequenceTensorTypes() {
static std::vector<MLDataType> all_fixed_size_tensor_and_sequence_tensor_types =
[]() {
auto temp = AllFixedSizeTensorTypes();
const auto& seq = AllFixedSizeSequenceTensorTypes();
temp.insert(temp.end(), seq.begin(), seq.end());
return temp;
}();
return all_fixed_size_tensor_and_sequence_tensor_types;
}
const std::vector<MLDataType>& DataTypeImpl::AllTensorAndSequenceTensorTypes() {
static std::vector<MLDataType> all_tensor_and_sequence_types =
[]() {
auto temp = AllTensorTypes();
const auto& seq = AllSequenceTensorTypes();
temp.insert(temp.end(), seq.begin(), seq.end());
return temp;
}();
return all_tensor_and_sequence_types;
}
// helper to stream. expected to only be used for error output, so any typeid lookup
// cost should be fine. alternative would be to add a static string field to DataTypeImpl
// that we set in the register macro to the type name, and output that instead.

92
onnxruntime/core/providers/cpu/controlflow/if.cc
View file

@ -19,7 +19,7 @@ namespace onnxruntime {
/*
ONNX_OPERATOR_SET_SCHEMA(
If,
1,
13,
OpSchema()
.SetDoc("If conditional")
.Input(0, "cond", "Condition for the if", "B")
@ -27,10 +27,24 @@ ONNX_OPERATOR_SET_SCHEMA(
0,
"outputs",
"Values that are live-out to the enclosing scope. The return values in "
"the `then_branch` and `else_branch` must be of the same shape and same "
"data type.",
"the `then_branch` and `else_branch` must be of the same data type. "
"The `then_branch` and `else_branch` may produce tensors with the same "
"element type and different shapes. "
"If corresponding outputs from the then-branch and the else-branch have "
"static shapes S1 and S2, then the shape of the corresponding output "
"variable of the if-node (if present) must be compatible with both S1 "
"and S2 as it represents the union of both possible shapes."
"For example, if in a model file, the the first "
"output of `then_branch` is typed float tensor with shape [2] and the "
"first output of `else_branch` is another float tensor with shape [3], "
"If's first output should have (a) no shape set, or (b) "
"a shape of rank 1 with neither `dim_value` nor `dim_param` set, or (c) "
"a shape of rank 1 with a unique `dim_param`. "
"In contrast, the first output cannot have the shape [2] since [2] and "
"[3] are not compatible.",
"V",
OpSchema::Variadic)
OpSchema::Variadic,
false)
.Attr(
"then_branch",
"Graph to run if condition is true. Has N outputs: values you wish to "
@ -43,8 +57,17 @@ ONNX_OPERATOR_SET_SCHEMA(
" be live-out to the enclosing scope. The number of outputs must match"
" the number of outputs in the then_branch.",
AttributeProto::GRAPH)
.TypeConstraint("V", OpSchema::all_tensor_types(), "All Tensor types")
.TypeConstraint("B", {"tensor(bool)"}, "Only bool"));
.TypeConstraint(
"V",
[](){
auto t = OpSchema::all_tensor_types();
auto s = OpSchema::all_tensor_sequence_types();
t.insert(t.end(), s.begin(), s.end());
return t;
}(),
"All Tensor and Sequence types")
.TypeConstraint("B", {"tensor(bool)"}, "Only bool")
.TypeAndShapeInferenceFunction(IfInferenceFunction));
*/
ONNX_CPU_OPERATOR_VERSIONED_KERNEL(If,
@ -56,11 +79,19 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL(If,
// output shape rules requiring the output shapes of the 'THEN' and 'ELSE'
// branches to be the same were relaxed in opset-11
ONNX_CPU_OPERATOR_VERSIONED_KERNEL(If,
11, 12,
KernelDefBuilder()
.TypeConstraint("B", DataTypeImpl::GetTensorType<bool>())
.TypeConstraint("V", DataTypeImpl::AllTensorTypes()),
If);
// sequence tensors were also supported in addition to existing support for tensors in opset-13
ONNX_CPU_OPERATOR_KERNEL(If,
11,
13,
KernelDefBuilder()
.TypeConstraint("B", DataTypeImpl::GetTensorType<bool>())
.TypeConstraint("V", DataTypeImpl::AllTensorTypes()),
.TypeConstraint("V", DataTypeImpl::AllTensorAndSequenceTensorTypes()),
If);
struct If::Info {
@ -247,28 +278,40 @@ Status IfImpl::AllocateOutputTensors() {
int index = 0;
for (auto& graph_output : info_.subgraph.GetOutputs()) {
auto* graph_output_shape = graph_output->Shape();
bool symbolic_dim_in_shape = false;
const auto* graph_output_type = graph_output->TypeAsProto();
if (graph_output_shape) {
TensorShape output_shape = onnxruntime::utils::GetTensorShapeFromTensorShapeProto(*graph_output_shape);
if (graph_output_type->has_tensor_type()) {
auto* graph_output_shape = graph_output->Shape();
bool symbolic_dim_in_shape = false;
// if size < 0 we have a symbolic dimension and need to use a temporary OrtValue in the subgraph execution
if (output_shape.Size() < 0) {
symbolic_dim_in_shape = true;
} else {
auto* tensor = context_.Output(index, output_shape);
if (graph_output_shape) {
TensorShape output_shape = onnxruntime::utils::GetTensorShapeFromTensorShapeProto(*graph_output_shape);
if (!tensor)
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for ", graph_output->Name());
// if size < 0 we have a symbolic dimension and need to use a temporary OrtValue in the subgraph execution
if (output_shape.Size() < 0) {
symbolic_dim_in_shape = true;
} else {
auto* tensor = context_.Output(index, output_shape);
outputs_.push_back({AllocationType::IfOutput, *context_.GetOutputMLValue(index)});
if (!tensor)
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for ", graph_output->Name());
outputs_.push_back({AllocationType::IfOutput, *context_.GetOutputMLValue(index)});
}
}
}
if (!graph_output_shape || symbolic_dim_in_shape) {
// we still need a value to put in the feeds we give to the execution frame, so just use an empty MLValue
outputs_.push_back({AllocationType::Delayed, {}});
if (!graph_output_shape || symbolic_dim_in_shape) {
// we still need a value to put in the feeds we give to the execution frame, so just use an empty MLValue
outputs_.push_back({AllocationType::Delayed, {}});
}
} else if (graph_output_type->has_sequence_type()) {
auto* seq_tensor = context_.Output<TensorSeq>(index);
if (!seq_tensor)
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for ", graph_output->Name());
outputs_.push_back({AllocationType::IfOutput, *context_.GetOutputMLValue(index)});
} else {
// Shouldn't hit this
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Only tensors or sequence of tensors are suppported");
}
++index;
@ -311,6 +354,7 @@ Status IfImpl::Execute(const FeedsFetchesManager& ffm) {
// we don't update the provided OrtValue and return false for 'allocated'.
// the execution frame will allocate a buffer on the required device, and the fetches copy
// logic in utils::ExecuteSubgraph will handle moving it into the tensor we allocated here.
auto* tensor = context_.Output(i, shape);
if (!tensor)
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create output tensor for If output ", i);

76
onnxruntime/core/providers/cpu/controlflow/loop.cc
View file

@ -20,6 +20,7 @@
#include "core/framework/utils.h"
#include "core/providers/cpu/tensor/utils.h"
#include "core/framework/session_options.h"
#include "core/framework/TensorSeq.h"
#include "gsl/gsl"
@ -34,9 +35,9 @@ namespace onnxruntime {
/*
ONNX_OPERATOR_SET_SCHEMA(
Loop,
1,
13,
OpSchema()
.SetDoc(Loop_ver1_doc)
.SetDoc(Loop_ver13_doc)
.Input(
0,
"M",
@ -56,13 +57,17 @@ ONNX_OPERATOR_SET_SCHEMA(
"The initial values of any loop-carried dependencies (values that "
"change across loop iterations)",
"V",
OpSchema::Variadic)
OpSchema::Variadic,
false,
0)
.Output(
0,
"v_final_and_scan_outputs",
"Final N loop carried dependency values then K scan_outputs",
"Final N loop carried dependency values then K scan_outputs. "
"Scan outputs must be Tensors.",
"V",
OpSchema::Variadic)
OpSchema::Variadic,
false)
.Attr(
"body",
"The graph run each iteration. It has 2+N inputs: (iteration_num, "
@ -73,9 +78,23 @@ ONNX_OPERATOR_SET_SCHEMA(
" if the dimensions or data type of these scan_outputs change across loop"
" iterations.",
AttributeProto::GRAPH)
.TypeConstraint("V", OpSchema::all_tensor_types(), "All Tensor types")
.TypeConstraint("I", {"int64"}, "Only int64")
.TypeConstraint("B", {"bool"}, "Only bool")
.TypeConstraint(
"V",
[]() {
auto t = OpSchema::all_tensor_types();
auto s = OpSchema::all_tensor_sequence_types();
t.insert(t.end(), s.begin(), s.end());
return t;
}(),
"All Tensor and Sequence types")
.TypeConstraint(
"I",
{"tensor(int64)"},
"tensor of int64, which should be a scalar.")
.TypeConstraint(
"B",
{"tensor(bool)"},
"tensor of bool, which should be a scalar.")
.TypeAndShapeInferenceFunction(LoopInferenceFunction));
*/
@ -87,12 +106,20 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL(Loop,
.TypeConstraint("V", DataTypeImpl::AllTensorTypes()),
Loop);
ONNX_CPU_OPERATOR_VERSIONED_KERNEL(Loop,
11, 12,
KernelDefBuilder()
.TypeConstraint("I", DataTypeImpl::GetTensorType<int64_t>())
.TypeConstraint("B", DataTypeImpl::GetTensorType<bool>())
.TypeConstraint("V", DataTypeImpl::AllTensorTypes()),
Loop);
ONNX_CPU_OPERATOR_KERNEL(Loop,
11,
13,
KernelDefBuilder()
.TypeConstraint("I", DataTypeImpl::GetTensorType<int64_t>())
.TypeConstraint("B", DataTypeImpl::GetTensorType<bool>())
.TypeConstraint("V", DataTypeImpl::AllTensorTypes()),
.TypeConstraint("V", DataTypeImpl::AllTensorAndSequenceTensorTypes()),
Loop);
struct Loop::Info {
@ -411,6 +438,7 @@ void LoopImpl::SaveOutputsAndUpdateFeeds(const std::vector<OrtValue>& last_outpu
// save loop outputs as we have to concatenate at the end
for (int j = info_.num_loop_carried_vars; j < info_.num_outputs; ++j) {
ORT_ENFORCE(last_outputs[j + 1].IsTensor(), "All scan outputs MUST be tensors");
loop_output_tensors_[j - info_.num_loop_carried_vars].push_back(last_outputs[j + 1]); // skip 'cond' in output
}
}
@ -463,9 +491,31 @@ Status LoopImpl::Execute(const FeedsFetchesManager& ffm) {
// As the loop carried variables may change shape across iterations there's no way to avoid a copy
// as we need the final shape.
auto copy_tensor_from_mlvalue_to_output = [this](const OrtValue& input, int output_idx) {
auto& data = input.Get<Tensor>();
Tensor* output = context_.Output(output_idx, data.Shape());
session_state_.GetDataTransferMgr().CopyTensor(input.Get<Tensor>(), *output);
auto type = input.Type();
if (type == DataTypeImpl::GetType<Tensor>()) {
auto& data = input.Get<Tensor>();
Tensor* output = context_.Output(output_idx, data.Shape());
session_state_.GetDataTransferMgr().CopyTensor(input.Get<Tensor>(), *output);
} else if (type == DataTypeImpl::GetType<TensorSeq>()) {
std::vector<Tensor> tensors;
auto& data = input.Get<TensorSeq>();
TensorSeq* output = context_.Output<TensorSeq>(output_idx);
output->SetType(data.DataType());
AllocatorPtr alloc;
auto status = context_.GetTempSpaceAllocator(&alloc);
if (!status.IsOK()) {
ORT_THROW("Unable to get an allocator");
}
for (auto it = data.begin(), end = data.end(); it != end; ++it) {
Tensor tmp(it->DataType(), onnxruntime::TensorShape(it->Shape()), alloc);
session_state_.GetDataTransferMgr().CopyTensor(*it, tmp);
tensors.push_back(std::move(tmp));
}
output->SetElements(std::move(tensors));
}
};
// copy to Loop output

12
onnxruntime/core/providers/cpu/cpu_execution_provider.cc
View file

@ -350,7 +350,7 @@ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOn
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, LogSoftmax);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float, Softmax);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, Softmax);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Loop);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, Loop);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, DepthToSpace);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Scan);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, Flatten);
@ -369,7 +369,7 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Ma
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, LpPool);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Conv);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ConvTranspose);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, If);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, If);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceLength);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceAt);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceEmpty);
@ -594,6 +594,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain,
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Resize);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, int32_t, Resize);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, uint8_t, Resize);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Loop);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, If);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Hardmax);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, LogSoftmax);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double, LogSoftmax);
@ -1115,6 +1117,7 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
float, ArgMin)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12,
int32_t, ArgMin)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, Loop)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, Hardmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float,
LogSoftmax)>,
@ -1124,7 +1127,6 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
Softmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float,
Softmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Loop)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, DepthToSpace)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Scan)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, Flatten)>,
@ -1144,7 +1146,7 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, LpPool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Conv)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ConvTranspose)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, If)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, If)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceLength)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceAt)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, SequenceEmpty)>,
@ -1558,6 +1560,8 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
int32_t, Resize)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13,
uint8_t, Resize)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Loop)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, If)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Hardmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float,
LogSoftmax)>,

11
onnxruntime/core/providers/cpu/tensor/unsqueeze.cc
View file

@ -43,14 +43,15 @@ Status UnsqueezeBase::PrepareCompute(OpKernelContext* ctx, Prepare& p) const {
std::vector<int64_t> axes;
size_t num_inputs = ctx->InputCount();
if (num_inputs == 2) { //axes is an input
if (num_inputs == 2) { //axes is an input
const Tensor* axes_tensor = ctx->Input<Tensor>(1);
ORT_ENFORCE(axes_tensor != nullptr, "Axes input is null");
ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 1,
"An axes tensor must be a vector tensor.");
auto nDims = static_cast<size_t>(axes_tensor->Shape()[0]);
ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 0 ||
axes_tensor->Shape().NumDimensions() == 1,
"An axes tensor must be a scalar or a 1-D tensor.");
auto num_axes_elements = static_cast<size_t>(axes_tensor->Shape().Size());
const auto* data = axes_tensor->template Data<int64_t>();
axes.assign(data, data + nDims);
axes.assign(data, data + num_axes_elements);
} else {
axes.assign(axes_.begin(), axes_.end());
}

4
onnxruntime/core/providers/cpu/tensor/unsqueeze.h
View file

@ -11,8 +11,8 @@ namespace onnxruntime {
class UnsqueezeBase {
protected:
UnsqueezeBase(const OpKernelInfo& info) {
size_t numInputs = info.GetInputCount();
if (numInputs == 1) { //axes must be a valid attribute
size_t num_inputs = info.GetInputCount();
if (num_inputs == 1) { //axes must be a valid attribute
ORT_ENFORCE(info.GetAttrs("axes", axes_).IsOK(), "Missing/Invalid 'axes' attribute value");
}
}

15
onnxruntime/core/providers/cuda/controlflow/if.cc
View file

@ -23,9 +23,22 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(If,
// output shape rules requiring the output shapes of the 'THEN' and 'ELSE'
// branches to be the same were relaxed in opset-11
ONNX_OPERATOR_VERSIONED_KERNEL_EX(If,
kOnnxDomain,
11, 12,
kCudaExecutionProvider,
KernelDefBuilder()
.InputMemoryType<OrtMemTypeCPUInput>(0) // 'cond' needs to be on CPU
.TypeConstraint("B", DataTypeImpl::GetTensorType<bool>())
.TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()),
If);
// sequence tensors were also supported in addition to existing support for tensors in opset-13,
// but we do not support sequence tensors in the cuda If kernel because there are no ops that handle
// sequence tensors on CUDA and supporting it for If doesn't add value while that is the case
ONNX_OPERATOR_KERNEL_EX(If,
kOnnxDomain,
11,
13,
kCudaExecutionProvider,
KernelDefBuilder()
.InputMemoryType<OrtMemTypeCPUInput>(0) // 'cond' needs to be on CPU

17
onnxruntime/core/providers/cuda/controlflow/loop.cc
View file

@ -23,9 +23,24 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(Loop,
Loop);
// zero variadic argument support was added in opset 11. using same implementation as for previous version
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Loop,
kOnnxDomain,
11, 12,
kCudaExecutionProvider,
KernelDefBuilder()
.InputMemoryType<OrtMemTypeCPUInput>(0) // 'M' needs to be on CPU
.InputMemoryType<OrtMemTypeCPUInput>(1) // 'cond' needs to be on CPU
.TypeConstraint("I", DataTypeImpl::GetTensorType<int64_t>())
.TypeConstraint("B", DataTypeImpl::GetTensorType<bool>())
.TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()),
Loop);
// sequence tensors were also supported in addition to existing support for tensors in opset-13,
// but we do not support sequence tensors in the cuda Loop kernel because there are no ops that handle
// sequence tensors on CUDA and supporting it for Loop doesn't add value while that is the case
ONNX_OPERATOR_KERNEL_EX(Loop,
kOnnxDomain,
11,
13,
kCudaExecutionProvider,
KernelDefBuilder()
.InputMemoryType<OrtMemTypeCPUInput>(0) // 'M' needs to be on CPU

12
onnxruntime/core/providers/cuda/cuda_execution_provider.cc
View file

@ -685,8 +685,8 @@ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDom
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, float, Gemm);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, double, Gemm);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, MLFloat16, Gemm);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, If);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, Loop);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, If);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, Loop);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, NonMaxSuppression);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, Range);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, float, ReduceL1);
@ -1000,6 +1000,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain,
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, MLFloat16, Resize);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, int32_t, Resize);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, uint8_t, Resize);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, If);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, Loop);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, Flatten);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, float, LRN);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, double, LRN);
@ -1381,8 +1383,8 @@ static Status RegisterCudaKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, MLFloat16, Gemm)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, float, Gemm)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, double, Gemm)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, If)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, Loop)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, If)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, Loop)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, NonMaxSuppression)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, Range)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 11, 12, float, ReduceL1)>,
@ -1692,6 +1694,8 @@ static Status RegisterCudaKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, MLFloat16, Resize)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, int32_t, Resize)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, uint8_t, Resize)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, If)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, Loop)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, Flatten)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, float, LRN)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 13, double, LRN)>,

168
onnxruntime/test/onnx/TestCase.cc
View file

@ -19,6 +19,7 @@
#include "core/platform/path_lib.h"
#include "core/session/onnxruntime_cxx_api.h"
#include "core/framework/allocator.h"
#include "core/framework/TensorSeq.h"
#include "re2/re2.h"
#include <cctype>
@ -131,7 +132,7 @@ static int ExtractFileNo(const std::basic_string<CHAR_T>& name) {
}
using PATH_STRING_TYPE = std::basic_string<PATH_CHAR_TYPE>;
static void SortTensorFileNames(std::vector<std::basic_string<PATH_CHAR_TYPE>>& input_pb_files) {
static void SortFileNames(std::vector<std::basic_string<PATH_CHAR_TYPE>>& input_pb_files) {
if (input_pb_files.size() <= 1) return;
std::sort(input_pb_files.begin(), input_pb_files.end(),
[](const std::basic_string<PATH_CHAR_TYPE>& left, const std::basic_string<PATH_CHAR_TYPE>& right) -> bool {
@ -299,10 +300,15 @@ class OnnxTestCase : public ITestCase {
return std::string();
}
void ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto>& test_data_pbs,
onnxruntime::test::HeapBuffer& b, bool is_input,
void ConvertTestData(const ONNX_NAMESPACE::TensorProto& test_data_pb,
onnxruntime::test::HeapBuffer& b,
bool is_input, size_t i,
std::unordered_map<std::string, Ort::Value>& out) const;
void ConvertTestData(const ONNX_NAMESPACE::SequenceProto& test_data_pb,
onnxruntime::test::HeapBuffer& b,
bool is_input, size_t i,
std::unordered_map<std::string, Ort::Value>& out) const;
std::once_flag model_parsed_;
std::once_flag config_parsed_;
double per_sample_tolerance_;
@ -318,6 +324,10 @@ class OnnxTestCase : public ITestCase {
void GetRelativePerSampleTolerance(double* value) const override;
void GetPostProcessing(bool* value) const override;
const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t i) const override {
return model_info_->GetInputInfoFromModel(i);
}
const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const override {
return model_info_->GetOutputInfoFromModel(i);
}
@ -403,21 +413,31 @@ static bool read_config_file(const std::basic_string<PATH_CHAR_TYPE>& path, std:
//load tensors from disk
template <typename PATH_STRING_TYPE>
static void LoadTensors(const std::vector<PATH_STRING_TYPE>& pb_files,
std::vector<ONNX_NAMESPACE::TensorProto>* input_pbs) {
for (size_t i = 0; i != pb_files.size(); ++i) {
int tensor_fd;
auto st = Env::Default().FileOpenRd(pb_files.at(i), tensor_fd);
if (!st.IsOK()) {
ORT_THROW("open file '", ToMBString(pb_files.at(i)), "' failed:", st.ErrorMessage());
}
google::protobuf::io::FileInputStream f(tensor_fd, protobuf_block_size_in_bytes);
f.SetCloseOnDelete(true);
ONNX_NAMESPACE::TensorProto tensor;
if (!tensor.ParseFromZeroCopyStream(&f)) {
ORT_THROW("parse file '", ToMBString(pb_files.at(i)), "' failed");
}
input_pbs->emplace_back(tensor);
static void LoadTensor(const PATH_STRING_TYPE& pb_file, ONNX_NAMESPACE::TensorProto& input_pb) {
int tensor_fd;
auto st = Env::Default().FileOpenRd(pb_file, tensor_fd);
if (!st.IsOK()) {
ORT_THROW("open file '", ToMBString(pb_file), "' failed:", st.ErrorMessage());
}
google::protobuf::io::FileInputStream f(tensor_fd, protobuf_block_size_in_bytes);
f.SetCloseOnDelete(true);
if (!input_pb.ParseFromZeroCopyStream(&f)) {
ORT_THROW("parse file '", ToMBString(pb_file), "' failed");
}
}
//load sequence tensors from disk
template <typename PATH_STRING_TYPE>
static void LoadSequenceTensor(const PATH_STRING_TYPE& pb_file, ONNX_NAMESPACE::SequenceProto& input_pb) {
int tensor_fd;
auto st = Env::Default().FileOpenRd(pb_file, tensor_fd);
if (!st.IsOK()) {
ORT_THROW("open file '", ToMBString(pb_file), "' failed:", st.ErrorMessage());
}
google::protobuf::io::FileInputStream f(tensor_fd, protobuf_block_size_in_bytes);
f.SetCloseOnDelete(true);
if (!input_pb.ParseFromZeroCopyStream(&f)) {
ORT_THROW("parse file '", ToMBString(pb_file), "' failed");
}
}
@ -457,6 +477,7 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b,
}
std::vector<PATH_STRING_TYPE> test_data_pb_files;
const PATH_STRING_TYPE& dir_path = test_data_dirs_[id];
LoopDir(dir_path,
[&test_data_pb_files, &dir_path, is_input](const PATH_CHAR_TYPE* filename, OrtFileType f_type) -> bool {
@ -473,41 +494,79 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b,
return true;
});
SortTensorFileNames(test_data_pb_files);
SortFileNames(test_data_pb_files);
std::vector<ONNX_NAMESPACE::TensorProto> test_data_pbs;
LoadTensors(test_data_pb_files, &test_data_pbs);
ConvertTestData(test_data_pbs, b, is_input, name_data_map);
for (size_t i = 0; i < test_data_pb_files.size(); ++i) {
const ONNX_NAMESPACE::ValueInfoProto* value_info_proto = is_input ? model_info_->GetInputInfoFromModel(i) : model_info_->GetOutputInfoFromModel(i);
if (!value_info_proto->has_type()) {
ORT_THROW("Model ", is_input ? "input " : "output ", i, " is missing type info");
}
if (value_info_proto->type().has_tensor_type()) {
ONNX_NAMESPACE::TensorProto test_pb;
LoadTensor(test_data_pb_files[i], test_pb);
ConvertTestData(test_pb, b, is_input, i, name_data_map);
} else if (value_info_proto->type().has_sequence_type()) {
ONNX_NAMESPACE::SequenceProto test_pb;
LoadSequenceTensor(test_data_pb_files[i], test_pb);
ConvertTestData(test_pb, b, is_input, i, name_data_map);
} else {
ORT_THROW("Unsupported type for the ", is_input ? "input " : "output ", i, " in the test runner");
}
}
}
void OnnxTestCase::ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto>& test_data_pbs,
void OnnxTestCase::ConvertTestData(const ONNX_NAMESPACE::TensorProto& test_data_pb,
onnxruntime::test::HeapBuffer& b,
bool is_input, std::unordered_map<std::string, Ort::Value>& out) const {
bool has_valid_names = true;
std::vector<std::string> var_names(test_data_pbs.size());
for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) {
std::string name = test_data_pbs[input_index].name();
if (name.empty()) {
has_valid_names = false;
break;
}
var_names[input_index] = name;
}
if (!has_valid_names) {
size_t count = static_cast<size_t>(is_input ? model_info_->GetInputCount() : model_info_->GetOutputCount());
if (count != test_data_pbs.size()) {
ORT_THROW("data count mismatch, expect ", count, ", got ", test_data_pbs.size());
}
for (size_t i = 0; i != count; ++i) {
var_names[i] = is_input ? model_info_->GetInputName(i) : model_info_->GetOutputName(i);
}
}
for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) {
std::string name = var_names[input_index];
const ONNX_NAMESPACE::TensorProto& input = test_data_pbs[input_index];
size_t len = 0;
bool is_input, size_t i,
std::unordered_map<std::string, Ort::Value>& out) const {
const std::string& name = test_data_pb.name();
const std::string& name_finalized = !name.empty()
? name
: (is_input ? model_info_->GetInputName(i) : model_info_->GetOutputName(i));
auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(input, &len);
size_t len = 0;
auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(test_data_pb, &len);
if (!status.IsOK()) {
ORT_THROW(status.ToString());
}
void* p = len == 0 ? nullptr : b.AllocMemory(len);
Ort::Value v1{nullptr};
onnxruntime::test::OrtCallback d;
OrtMemoryInfo cpu_memory_info(onnxruntime::CPU, OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeDefault);
status = onnxruntime::test::TensorProtoToMLValue(test_data_pb, onnxruntime::test::MemBuffer(p, len, cpu_memory_info),
v1, d);
if (!status.IsOK()) {
ORT_THROW(status.ToString());
}
if (d.f) {
b.AddDeleter(d);
}
out.emplace(name_finalized, std::move(v1));
}
void OnnxTestCase::ConvertTestData(const ONNX_NAMESPACE::SequenceProto& test_data_pb,
onnxruntime::test::HeapBuffer& b,
bool is_input, size_t i,
std::unordered_map<std::string, Ort::Value>& out) const {
const std::string& name = test_data_pb.name();
const std::string& name_finalized = !name.empty()
? name
: (is_input ? model_info_->GetInputName(i) : model_info_->GetOutputName(i));
size_t len = 0;
std::vector<Ort::Value> seq;
if (test_data_pb.elem_type() != ONNX_NAMESPACE::SequenceProto_DataType_TENSOR) {
ORT_THROW("Only parsing a sequence of tensors is currently supported");
}
const auto& tensors = test_data_pb.tensor_values();
const size_t val = tensors.size();
seq.reserve(val);
for (auto it = tensors.cbegin(); it != tensors.cend(); ++it) {
auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(*it, &len);
if (!status.IsOK()) {
ORT_THROW(status.ToString());
}
@ -515,7 +574,7 @@ void OnnxTestCase::ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto
Ort::Value v1{nullptr};
onnxruntime::test::OrtCallback d;
OrtMemoryInfo cpu_memory_info(onnxruntime::CPU, OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeDefault);
status = onnxruntime::test::TensorProtoToMLValue(input, onnxruntime::test::MemBuffer(p, len, cpu_memory_info),
status = onnxruntime::test::TensorProtoToMLValue(*it, onnxruntime::test::MemBuffer(p, len, cpu_memory_info),
v1, d);
if (!status.IsOK()) {
ORT_THROW(status.ToString());
@ -523,7 +582,16 @@ void OnnxTestCase::ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto
if (d.f) {
b.AddDeleter(d);
}
out.emplace(name, std::move(v1));
seq.push_back(std::move(v1));
}
if (seq.size() == 0) {
// TODO: implement support for creating empty sequences. Not urgent yet since we don't have real world models.
// For now, only the single node ONNX test - `test_loop13_seq` requires it (will keep it disabled for now).
ORT_THROW("Creation of empty sequences is currently not supported in the test runner");
} else {
out.emplace(name_finalized, Ort::Value::CreateSequence(seq));
}
}

2
onnxruntime/test/onnx/TestCase.h
View file

@ -33,6 +33,7 @@ class ITestCase {
bool is_input) const = 0;
virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0;
virtual const std::string& GetNodeName() const = 0;
virtual const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t i) const = 0;
virtual const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const = 0;
virtual const std::string& GetTestCaseName() const = 0;
@ -59,6 +60,7 @@ class TestModelInfo {
return test_case_dir;
}
virtual const std::string& GetNodeName() const = 0;
virtual const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t i) const = 0;
virtual const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const = 0;
virtual int GetInputCount() const = 0;
virtual int GetOutputCount() const = 0;

6
onnxruntime/test/onnx/main.cc
View file

@ -330,8 +330,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
OrtCudnnConvAlgoSearch::EXHAUSTIVE,
std::numeric_limits<size_t>::max(),
0,
true
};
true};
Ort::ThrowOnError(sf.OrtSessionOptionsAppendExecutionProvider_CUDA(sf, &cuda_options));
#else
fprintf(stderr, "CUDA is not supported in this build");
@ -555,8 +554,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
{"cast_BFLOAT16_to_FLOAT", "onnx generate bfloat tensor as uint16 type", {}},
{"sequence_insert_at_back", "onnx currently not supporting loading segment", {}},
{"sequence_insert_at_front", "onnx currently not supporting loading segment", {}},
{"if_seq", "NOT_IMPLEMENTED : Could not find an implementation for the node If(13)", {}},
{"loop13_seq", "NOT_IMPLEMENTED : Could not find an implementation for the node Loop(13)", {}},
{"loop13_seq", "ORT api does not currently support creating empty sequences (needed for this test)", {}},
};
#ifdef DISABLE_ML_OPS

2
onnxruntime/test/onnx/onnx_model_info.cc
View file

@ -160,7 +160,7 @@ void OnnxModelInfo::InitOrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url) {
}
// Load input and output node args
auto add_node_args = [&](const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>* fbs_node_args,
std::vector<ONNX_NAMESPACE::ValueInfoProto> node_args) -> Status {
std::vector<ONNX_NAMESPACE::ValueInfoProto>& node_args) -> Status {
if (fbs_node_args != nullptr) {
node_args.reserve(fbs_node_args->size());
for (const auto* fbs_node_arg_name : *fbs_node_args) {

6
onnxruntime/test/onnx/onnx_model_info.h
View file

@ -37,9 +37,15 @@ class OnnxModelInfo : public TestModelInfo {
std::string GetModelVersion() const override { return onnx_commit_tag_; }
const std::string& GetNodeName() const override { return node_name_; }
const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t i) const override {
return &input_value_info_[i];
}
const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const override {
return &output_value_info_[i];
}
int GetInputCount() const override { return static_cast<int>(input_value_info_.size()); }
int GetOutputCount() const override { return static_cast<int>(output_value_info_.size()); }
const std::string& GetInputName(size_t i) const override { return input_value_info_[i].name(); }

4
onnxruntime/test/onnx/pb_helper.h
View file

@ -42,7 +42,7 @@
#include <google/protobuf/message_lite.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <onnx/onnx_pb.h>
#include <onnx/onnx-data_pb.h>
#include "tml.pb.h"
#ifdef __GNUC__
#pragma GCC diagnostic pop
@ -53,4 +53,4 @@ namespace onnxruntime {
bool ParseDelimitedFromCodedStream(google::protobuf::MessageLite* message,
google::protobuf::io::CodedInputStream* input,
bool* clean_eof);
}
}

1
onnxruntime/test/perftest/TFModelInfo.h
View file

@ -12,6 +12,7 @@ class TFModelInfo : public TestModelInfo {
const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); }
const std::string& GetNodeName() const override { return node_name_; }
const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t) const override { return nullptr; }
const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t) const override { return nullptr; }
int GetInputCount() const override;

59
onnxruntime/test/providers/cpu/controlflow/if_test.cc
View file

@ -413,5 +413,64 @@ TEST(If, ConditionalBranchesOnlyContainConstantNodes_ElseBranchExecution) {
test.Run();
}
// This is to test an "If" node with just a "SequenceEmpty" node in the "then" and "else" conditional branches
class IfOpTesterWithSequencesAsOutput : public OpTester {
public:
IfOpTesterWithSequencesAsOutput() : OpTester("If", 13) {
}
protected:
void AddNodes(onnxruntime::Graph& graph,
std::vector<onnxruntime::NodeArg*>& graph_input_defs,
std::vector<onnxruntime::NodeArg*>& graph_output_defs,
std::vector<std::function<void(onnxruntime::Node& node)>>& /*add_attribute_funcs*/) override {
// Graph inputs are 0:Cond
ASSERT_EQ(graph_input_defs.size(), 1u);
ASSERT_EQ(graph_output_defs.size(), 1u);
NodeArg* if_cond_input = graph_input_defs[0];
std::vector<NodeArg*> inputs;
std::vector<NodeArg*> outputs;
// add If node
{
inputs = {if_cond_input};
outputs = {graph_output_defs[0]};
auto& if_node = graph.AddNode("if", "If", "If node", inputs, outputs);
auto CreateSubgraphWithSequenceEmptyNode = [](bool then_branch, std::vector<NodeArg*> outputs) {
Model subgraph(then_branch ? "Then" : "Else", false, DefaultLoggingManager().DefaultLogger());
auto& graph = subgraph.MainGraph();
// By default, the SequenceEmpty node will create an empty sequence of float tensors
ORT_IGNORE_RETURN_VALUE(graph.AddNode(
then_branch ? "SequenceEmpty_Then" : "SequenceEmpty_Else",
"SequenceEmpty",
then_branch ? "SequenceEmpty_Then" : "SequenceEmpty_Else", {}, outputs));
auto status = graph.Resolve();
EXPECT_EQ(status, Status::OK());
auto& graphproto = graph.ToGraphProto();
return graphproto;
};
if_node.AddAttribute("then_branch", CreateSubgraphWithSequenceEmptyNode(true, outputs));
if_node.AddAttribute("else_branch", CreateSubgraphWithSequenceEmptyNode(false, outputs));
}
}
};
// opset-13 allows sequences as outputs for 'If' nodes
TEST(If, TestIfWithSequencesAsOutput) {
IfOpTesterWithSequencesAsOutput test;
test.AddInput<bool>("If_input", {1}, {true});
SeqTensors<float> seq; // empty sequence of float tensors
test.AddSeqOutput("If_output", seq);
test.Run();
}
} // namespace test
} // namespace onnxruntime

116
onnxruntime/test/providers/cpu/controlflow/loop_test.cc
View file

@ -93,7 +93,7 @@ static const ONNX_NAMESPACE::GraphProto CreateSubgraph(const RunOptions& options
bool is_iter_num_1d = options.subgraph_iter_num_1d_tensor;
// Loop tests use unsqueeze operator in it's subgraph. Unsqueeze was updated in opset13
// This test can continue to use opset12 or can be updated to use the latest opset once
// This test can continue to use opset12 or can be updated to use the latest opset once
// unsqueeze op13 implementation is done.
Model model("Loop subgraph", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), {{"", 12}},
{}, DefaultLoggingManager().DefaultLogger());
@ -1048,5 +1048,119 @@ TEST(Loop, MixedExecutionProviders) {
}
#endif
// Test sequence as Loop carried dependency (opset-13 allows this)
TEST(Loop, SequenceAsLoopCarriedDependency) {
auto create_subgraph = []() {
Model model("sequence in Loop subgraph carried dependency", false, DefaultLoggingManager().DefaultLogger());
auto& graph = model.MainGraph();
std::vector<NodeArg*> inputs;
std::vector<NodeArg*> outputs;
/* Subgraph Adds inserted_tensor (float tensor) to a sequence across iterations
Inputs: iter_num, cond_in, loop_var_0_in
loop_var_0_in inserted_tensor cond_in iter_num
| | | (unused)
[SequenceInsert]-----/ [Identity]
| |
| cond_out
loop_var_0_out
*/
// graph inputs types.
TypeProto int64_scalar;
int64_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_INT64);
int64_scalar.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
TypeProto bool_scalar;
bool_scalar.mutable_tensor_type()->set_elem_type(TensorProto_DataType_BOOL);
bool_scalar.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
TypeProto float_tensor_sequence;
auto* tensor_type = float_tensor_sequence
.mutable_sequence_type()
->mutable_elem_type()
->mutable_tensor_type();
tensor_type->set_elem_type(TensorProto_DataType_FLOAT);
tensor_type->mutable_shape()->add_dim()->set_dim_value(1);
TypeProto float_tensor;
float_tensor.mutable_tensor_type()->set_elem_type(TensorProto_DataType_FLOAT);
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim();
// graph inputs
auto& iter_num_in = graph.GetOrCreateNodeArg("iter_num_in", &int64_scalar);
auto& cond_in = graph.GetOrCreateNodeArg("cond_in", &bool_scalar);
auto& loop_var_0_in = graph.GetOrCreateNodeArg("loop_var_0_in", &float_tensor_sequence);
// graph outputs
auto& cond_out = graph.GetOrCreateNodeArg("cond_out", &bool_scalar);
auto& loop_var_0_out = graph.GetOrCreateNodeArg("loop_var_0_out", &float_tensor_sequence);
// outer scope values. need type but not shape.
auto& inserted_tensor = graph.GetOrCreateNodeArg("inserted_tensor", &float_tensor);
// add it to the outer scope so that we don't end up with it being considered a graph input
graph.AddOuterScopeNodeArg("inserted_tensor");
// cond_in -> cond_out
{
inputs = {&cond_in};
outputs = {&cond_out};
graph.AddNode("cond_in_identity", "Identity", "Forward cond_in to cond_out", inputs, outputs);
}
// iter_num_in -> loop_var_0_out
{
inputs = {&loop_var_0_in, &inserted_tensor};
outputs = {&loop_var_0_out};
graph.AddNode("loop_var_out", "SequenceInsert", "append to sequence across iterations", inputs, outputs);
}
graph.SetInputs({&iter_num_in, &cond_in, &loop_var_0_in});
graph.SetOutputs({&cond_out, &loop_var_0_out});
// add an initializer for the tensor that will be inserted into the sequence on every iterations
{
TensorProto inserted_tensor_proto;
inserted_tensor_proto.set_name("inserted_tensor");
inserted_tensor_proto.add_dims(1);
inserted_tensor_proto.add_float_data(1.f);
inserted_tensor_proto.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT);
graph.AddInitializedTensor(inserted_tensor_proto);
}
auto status = graph.Resolve();
EXPECT_EQ(status, Status::OK());
return graph.ToGraphProto();
};
OpTester test("Loop", 13);
auto body = create_subgraph();
test.AddAttribute<GraphProto>("body", body);
test.AddInput<int64_t>("M", {1}, {3});
test.AddInput<bool>("cond", {1}, {true});
SeqTensors<float> seq_input;
test.AddSeqInput("loop_var_0_orig", seq_input);
SeqTensors<float> seq_output;
seq_output.AddTensor({1}, {1.f});
seq_output.AddTensor({1}, {1.f});
seq_output.AddTensor({1}, {1.f});
test.AddSeqOutput("loop_var_0_final", seq_output);
// Disable TensorRT on unsupported data type BOOL
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
}
} // namespace test
} // namespace onnxruntime

4
onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc vendored
View file

@ -33,8 +33,8 @@
"^test_adam", // NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1)
"^test_adam_multiple", // NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1)
"^test_training_dropout.*", // NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel)
"^test_if_seq_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node If(13)
"^test_loop13_seq_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Loop(13)
"^test_if_seq_cpu", // ONNX backend test runner doesn't handle sequences: https://github.com/onnx/onnx/issues/3086
"^test_loop13_seq_cpu", // ONNX backend test runner doesn't handle sequences: https://github.com/onnx/onnx/issues/3086
"^test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Resize(13)
"^test_resize_downsample_scales_cubic_cpu",
"^test_resize_downsample_scales_linear_cpu",

76
onnxruntime/test/util/compare_ortvalue.cc
View file

@ -40,6 +40,7 @@
#include "core/graph/onnx_protobuf.h"
#include "core/framework/tensorprotoutils.h"
#include "core/framework/utils.h"
#include "core/framework/TensorSeq.h"
#include <core/session/onnxruntime_cxx_api.h>
using namespace onnxruntime;
@ -302,10 +303,53 @@ namespace onnxruntime {
std::pair<COMPARE_RESULT, std::string> CompareOrtValue(const OrtValue& o, const OrtValue& expected_mlvalue,
double per_sample_tolerance,
double relative_per_sample_tolerance, bool post_processing) {
if (o.IsTensor() != expected_mlvalue.IsTensor() || o.Type() != expected_mlvalue.Type()) {
if (o.Type() != expected_mlvalue.Type()) {
return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, "");
}
if (!o.IsTensor()) {
if (o.IsTensor()) {
const Tensor& outvalue = o.Get<Tensor>();
const Tensor& expected_tensor = expected_mlvalue.Get<Tensor>();
if (outvalue.DataType() != expected_tensor.DataType()) {
std::ostringstream oss;
oss << "expect " << ElementTypeToString(expected_tensor.DataType()) << " got "
<< ElementTypeToString(outvalue.DataType());
return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, oss.str());
}
return CompareTwoTensors(outvalue, expected_tensor, per_sample_tolerance, relative_per_sample_tolerance,
post_processing);
} else if (o.IsTensorSequence()) {
auto& expected_tensor_seq = expected_mlvalue.Get<TensorSeq>();
auto expected_tensor_count = expected_tensor_seq.Size();
auto& actual_tensor_seq = o.Get<TensorSeq>();
auto actual_tensor_count = actual_tensor_seq.Size();
if (expected_tensor_count != actual_tensor_count) {
std::ostringstream oss;
oss << "expected tensor count in the sequence: " << expected_tensor_count << " got "
<< actual_tensor_count;
return std::make_pair(COMPARE_RESULT::RESULT_DIFFERS, oss.str());
}
if (!expected_tensor_seq.IsSameDataType(actual_tensor_seq)) {
std::ostringstream oss;
oss << "expected tensor type in the sequence: " << expected_tensor_seq.DataType() << " got "
<< actual_tensor_seq.DataType();
return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, oss.str());
}
for (size_t i = 0; i < expected_tensor_count; ++i) {
auto res = CompareTwoTensors(actual_tensor_seq.Get(i), expected_tensor_seq.Get(i), per_sample_tolerance, relative_per_sample_tolerance,
post_processing);
if (res.first != COMPARE_RESULT::SUCCESS) {
return res;
}
}
return std::make_pair(COMPARE_RESULT::SUCCESS, "");
} else {
// Maps
#if !defined(DISABLE_ML_OPS)
if (o.Type() == DataTypeImpl::GetType<VectorMapInt64ToFloat>()) {
return CompareSeqOfMapToFloat(o.Get<VectorMapInt64ToFloat>(), expected_mlvalue.Get<VectorMapInt64ToFloat>(),
@ -320,16 +364,6 @@ std::pair<COMPARE_RESULT, std::string> CompareOrtValue(const OrtValue& o, const
return std::make_pair(COMPARE_RESULT::NOT_SUPPORT, "Map type is not supported in this build.");
#endif
}
const Tensor& outvalue = o.Get<Tensor>();
const Tensor& expected_tensor = expected_mlvalue.Get<Tensor>();
if (outvalue.DataType() != expected_tensor.DataType()) {
std::ostringstream oss;
oss << "expect " << ElementTypeToString(expected_tensor.DataType()) << " got "
<< ElementTypeToString(outvalue.DataType());
return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, oss.str());
}
return CompareTwoTensors(outvalue, expected_tensor, per_sample_tolerance, relative_per_sample_tolerance,
post_processing);
}
std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const Ort::Value& o) {
@ -341,7 +375,7 @@ std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::Val
::ONNX_NAMESPACE::TypeProto_Tensor t = v.type().tensor_type();
// below code doesn't work
// if (((TensorTypeBase*)o.Type())->GetElementType() != DataTypeImpl::ElementTypeFromProto(t.elem_type())) {
//if (((TensorTypeBase*)o.Type())->GetElementType() != DataTypeImpl::ElementTypeFromProto(t.elem_type())) {
// return COMPARE_RESULT::TYPE_MISMATCH;
//}
auto info = o.GetTensorTypeAndShapeInfo();
@ -368,11 +402,19 @@ std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::Val
VectorToString(shape, oss);
return std::make_pair(COMPARE_RESULT::SHAPE_MISMATCH, oss.str());
}
} else {
// Cannot do this check for tensor type.
} else if (v.type().has_sequence_type()) {
// TODO: CXX API doesn't have IsTensorSequence() supported for Ort::Value
// TODO: Repeat whatever we did for Tensor above in a loop ?
return std::make_pair(COMPARE_RESULT::SUCCESS, "");
}
else {
// Cannot do this check for tensor/sequence of tensor type.
// For tensor type, o.Type() is TensorTypeBase*, but p points to a subclass of TensorTypeBase
auto p = DataTypeImpl::TypeFromProto(v.type());
if (((OrtValue*)(const OrtValue*)o)->Type() != p) {
// For sequences of tensor type, o.Type() is SequenceTensorTypeBase*, but p points to a subclass of SequenceTensorTypeBase
MLDataType p = DataTypeImpl::TypeFromProto(v.type());
MLDataType q = ((OrtValue*)(const OrtValue*)o)->Type();
if (q != p) {
return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, "");
}
}