saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add opset-11 TopK CPU kernel (#1912)

Browse source 
* initial commit

* Update

* Update top_k.cc

* PR comments

* Add more tests

* Update

* Add another test case

* Update

* Resolve conflicts

* Update

* Nits

* Nits

* Nits

* Pick sorted content using 2 different approaches

* Update to logic

* PR comments

* PR feedback

* Update

* Fix build

* Fix build

* Update
This commit is contained in:
Hariharan Seshadri 2019-10-09 19:09:30 -07:00 committed by GitHub
parent 8fda6593fe
commit d186c19c45
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 686 additions and 269 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
onnxruntime/core/providers/cpu/cpu_execution_provider.cc
View file

@ -278,7 +278,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain,
// Opset 10
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, StringNormalizer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, TopK);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, TopK);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, MaxPool);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, AveragePool);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, Mod);
@ -401,6 +401,7 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Pa
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, GatherND);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Range);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Unique);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, TopK);
void RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
static const BuildKernelCreateInfoFn function_table[] = {
@ -874,7 +875,8 @@ void RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
// Opset 10
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, StringNormalizer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, TopK)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10,
TopK)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10,
MaxPool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10,
@ -1024,12 +1026,14 @@ void RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Pad)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, GatherND)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Range)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Unique)>};
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Unique)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, TopK)>,
};
for (auto& function_table_entry : function_table) {
kernel_registry.Register(function_table_entry());
}
}
} // namespace onnxruntime
// Forward declarations of ml op kernels
namespace ml {

373
onnxruntime/core/providers/cpu/math/top_k.cc
View file

@ -1,18 +1,18 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "core/providers/cpu/math/top_k.h"
#include "core/providers/common.h"
@ -22,115 +22,199 @@
#include "core/framework/tensor.h"
#include "core/util/math_cpuonly.h"
#include <queue>
#include <algorithm>
#include <cmath>
using namespace std;
namespace onnxruntime {
// Helper methods
static int64_t SizeToDim(size_t k, const vector<int64_t>& dims) {
ORT_ENFORCE(k <= dims.size());
int64_t r = 1;
for (size_t i = 0; i < k; ++i) {
r *= dims[i];
}
return r;
}
static int64_t SizeFromDim(size_t k, const vector<int64_t>& dims) {
ORT_ENFORCE(k <= dims.size());
int64_t r = 1;
for (size_t i = k; i < dims.size(); ++i) {
r *= dims[i];
}
return r;
}
template <typename T>
struct ValueCmp {
bool operator()(
const pair<T, int64_t>& lhs,
const pair<T, int64_t>& rhs) {
return (
lhs.first > rhs.first ||
(lhs.first == rhs.first && lhs.second < rhs.second));
struct GreaterValueCmp {
bool operator()(const pair<T, int64_t>& lhs, const pair<T, int64_t>& rhs) {
return (lhs.first > rhs.first ||
// when values are equal, we want lhs to get higher "priority"
// if its corresponding index comes first (i.e.) is lower
(lhs.first == rhs.first && lhs.second < rhs.second));
}
};
// Core TopK implementation
Status TopKImpl(OpKernelContext* p_op_kernel_context, const Tensor* X, const int axis, const unsigned k) {
template <typename T>
struct LesserValueCmp {
bool operator()(const pair<T, int64_t>& lhs, const pair<T, int64_t>& rhs) {
return (lhs.first < rhs.first ||
// when values are equal, we want lhs to get higher "priority"
// if its corresponding index comes first (i.e.) is lower
(lhs.first == rhs.first && lhs.second < rhs.second));
}
};
const vector<int64_t>& in_dims = X->Shape().GetDims();
// Will return axis_ as is if positive or fixes it in case it is negative
auto axis_parsed = HandleNegativeAxis(axis, in_dims.size());
// Check to ensure k is within the bounds of what is available in that specific axis
if (in_dims.at(axis_parsed) < k) {
ostringstream err_msg;
err_msg << "k argment [" << k << "] should not be greater than specified axis dim value [" << in_dims.at(axis_parsed) << "]";
return Status(common::ONNXRUNTIME, common::FAIL, err_msg.str());
// Static helpers that implement the core logic for each of the 'TopK' operator flavor
// Selects the top k elements (largest or smallest based on template parameter)
template <class Comparator>
static vector<pair<float, int64_t>> select_top_k(const ConstEigenMatrixMapRowMajor<float>& raw_data, int64_t row_num, int64_t num_blocks,
int64_t block_slice, int64_t inter_block_offset, const unsigned k,
bool sort_top_k) {
// create a data holder and insert elements
vector<pair<float, int64_t>> data_holder;
data_holder.reserve(num_blocks);
for (int64_t l = 0; l < num_blocks; ++l) {
data_holder.push_back({raw_data(row_num, l * block_slice + inter_block_offset), l});
}
if (k == 0) {
vector<int64_t> out_dims = in_dims;
out_dims[axis_parsed] = 0;
p_op_kernel_context->Output(0, out_dims);
p_op_kernel_context->Output(1, out_dims);
return Status::OK();
// find the top k (largest or smallest) elements in the data holder - O(n)
nth_element(data_holder.begin(), data_holder.begin() + (k - 1), data_holder.end(), Comparator());
// sort the top k elements if needed - O (k log k)
if (sort_top_k) {
std::sort(data_holder.begin(), data_holder.begin() + k, Comparator());
}
const int64_t rows = SizeToDim(axis_parsed, in_dims);
const int64_t cols = X->Shape().Size() / rows;
auto input_map = ConstEigenMatrixMapRowMajor<float>(
static_cast<const float*>(X->template Data<float>()),
rows,
cols);
// the data_holder now contains the top k elements in the first k indices
return data_holder;
}
// Resize output tensors to be the same shape as the input except
// for the specified dimension ((i.e.) axis_parsed), which will be of size k. E.x. for an input tensor
// of shape [3, 4, 5] and k=2 with axis_parsed=1, both of these will be shape [3, 2, 5]
vector<int64_t> output_linear_shape = in_dims;
output_linear_shape[axis_parsed] = k;
auto* Values = p_op_kernel_context->Output(0, output_linear_shape);
auto* Indices = p_op_kernel_context->Output(1, output_linear_shape);
// Given an input tensor 'input' and metadata values - 'k' and 'axis_parsed',
// this method will extract the sorted top k largest/smallest elements and place them in the output tensor 'values'
// along with the metadata output 'indices'
template <bool largest, bool sorted, class Comparator>
static void extract_top_k_elements(const Tensor* input, const TensorShape& input_shape, Tensor* values,
Tensor* indices, const TensorShape& output_shape, const unsigned k,
const unsigned axis_parsed) {
// Cache some values that will be used in the implementation below
const int64_t rows = input_shape.SizeToDimension(static_cast<size_t>(axis_parsed));
const int64_t cols = input->Shape().Size() / rows;
auto input_map =
ConstEigenMatrixMapRowMajor<float>(static_cast<const float*>(input->template Data<float>()), rows, cols);
// Use Eigen maps to allow indexing into the 2d tensors like Values_map(i,j)
const int64_t reduced_cols = SizeFromDim(axis_parsed, output_linear_shape);
auto Values_map = EigenMatrixMapRowMajor<float>(
Values->template MutableData<float>(), rows, reduced_cols);
auto Indices_map = EigenMatrixMapRowMajor<int64_t>(
Indices->template MutableData<int64_t>(), rows, reduced_cols);
const int64_t reduced_cols = output_shape.SizeFromDimension(static_cast<size_t>(axis_parsed));
auto values_map = EigenMatrixMapRowMajor<float>(values->template MutableData<float>(), rows, reduced_cols);
auto indices_map = EigenMatrixMapRowMajor<int64_t>(indices->template MutableData<int64_t>(), rows, reduced_cols);
// This is basically the number of elements within each of the "k" rows
const int64_t block_slice = reduced_cols / k;
// Sort preserving Indices
const int64_t num_blocks = input_shape[axis_parsed];
for (int64_t i = 0; i < rows; ++i) {
for (int64_t j = 0; j < block_slice; ++j) {
// Build a min-heap, the heap element is pair of (value, idx)
// the top of the heap is the smallest value
priority_queue<
pair<float, int64_t>,
vector<pair<float, int64_t>>,
ValueCmp<float>>
min_heap;
// Maintain the size of heap to be less or equal to k_, so the
// heap will hold the k largest Values
for (int64_t l = 0; l < in_dims[axis_parsed]; ++l) {
const auto value = input_map(i, l * block_slice + j);
if (min_heap.size() < k || value > min_heap.top().first) {
min_heap.push({value, l});
// Since sorted == true, we will use a Heap to hold the top K values in sorted fashion
if (sorted) { // The optimizer will clean-up the redundant condition based on the template parameter 'sorted'
auto n_casted = static_cast<double>(num_blocks);
auto k_casted = static_cast<double>(k);
if ((n_casted + k_casted * log(k_casted)) < (n_casted * log(k_casted))) {
// Select first - O(n), then sort O(k * ln(k))
// Overall complexity = O (n + k * ln(k))
const auto& data_holder = select_top_k<Comparator>(input_map, i, num_blocks, block_slice, j, k, true);
for (int64_t l = 0; l < k; ++l) {
const auto& elem = data_holder[l];
auto col_index = l * block_slice + j;
values_map(i, col_index) = elem.first;
indices_map(i, col_index) = elem.second;
}
} else {
// Perform sorted selection by passing 'n' elements over a heap of size 'k'
// overall complexity = O (n * ln(k))
// Build a min-heap/max-heap, the heap element is pair of (value, idx)
// The top of the heap is the smallest/largest value depending on whether it is a min-heap/max-heap
// This is a min-heap if largest == true, this is a max-heap if largest == false
priority_queue<pair<float, int64_t>, vector<pair<float, int64_t>>, Comparator> heap;
// Maintain the size of heap to be less or equal to k, so the
// heap will hold the k largest/smallest values
for (int64_t l = 0; l < num_blocks; ++l) {
const auto value = input_map(i, l * block_slice + j);
// largest == true: insert into the min-heap if the size is < k or if the new
// element is greater than the min element in the min-heap
// largest == false: insert into the min-heap if the size is < k or if the new
// element is lesser than the max element in the max-heap
if ((heap.size() < k) || (largest && value > heap.top().first) ||
(!largest && value < heap.top().first)) { // the optimizer will clean-up the redundant condition based
// on the template parameter 'largest'
heap.push({value, l});
}
if (heap.size() > k) {
heap.pop();
}
}
// Extract these k elements and place them in the results placeholder
for (int64_t l = 0; l < k; ++l) {
const auto& elem = heap.top();
auto col_index = (k - l - 1) * block_slice + j;
values_map(i, col_index) = elem.first;
indices_map(i, col_index) = elem.second;
heap.pop();
}
}
if (min_heap.size() > k) {
min_heap.pop();
} else { // sorted == false
// The optimizer will clean-up the redundant condition based on the template parameter 'sorted'
// If the top K values are not required to be sorted, we use a more optimal selection algorithm
// Average - O(n). Worst - O(n * ln(n)) or O(n^2) depending on the implementation, where 'n' is the number of input
const auto& data_holder = select_top_k<Comparator>(input_map, i, num_blocks, block_slice, j, k, false);
// Insert the top 'k' (largest or smallest) elements into the final output buffers
for (int64_t l = 0; l < k; ++l) {
const auto& elem = data_holder[l];
auto col_index = l * block_slice + j;
values_map(i, col_index) = elem.first;
indices_map(i, col_index) = elem.second;
}
}
// Extract these k elements and place them in the results placeholder
for (int64_t l = 0; l < k; ++l) {
auto& pqElem = min_heap.top();
auto col_index = (k - l - 1) * block_slice + j;
Values_map(i, col_index) = pqElem.first;
Indices_map(i, col_index) = pqElem.second;
min_heap.pop();
}
}
}
}
// Wrapper over core TopK implementation
static Status TopKImpl(OpKernelContext* p_op_kernel_context, const Tensor* input, const int axis, const unsigned k,
bool largest = true, bool sorted = true) {
const TensorShape& input_shape = input->Shape();
// Will return axis_ as is if positive or fixes it in case it is negative
const auto axis_parsed = HandleNegativeAxis(axis, static_cast<int64_t>(input_shape.NumDimensions()));
// Check to ensure k is within the bounds of what is available in that specific axis
if (input_shape[axis_parsed] < k) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "k argument [", k,
"] should not be greater than specified axis dim value [", input_shape[axis_parsed], "]");
}
// Resize output tensors to be the same shape as the input except
// for the specified dimension ((i.e.) axis_parsed), which will be of size k. E.x. for an input tensor
// of shape [3, 4, 5] and k=2 with axis_parsed=1, both of the outputs will be shape [3, 2, 5]
TensorShape output_shape = input_shape;
output_shape[axis_parsed] = k;
auto* values = p_op_kernel_context->Output(0, output_shape);
auto* indices = p_op_kernel_context->Output(1, output_shape);
if (values == nullptr || indices == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
"output count mismatch, expected 2 outputs to be present for TopK operator");
}
// no-op - no output buffers to fill - return silently
if (k == 0) {
return Status::OK();
}
if (sorted && largest) {
// extract sorted largest TopK elements
extract_top_k_elements<true, true, GreaterValueCmp<float>>(input, input_shape, values, indices, output_shape, k,
gsl::narrow_cast<unsigned>(axis_parsed));
} else if (sorted && !largest) {
// extract sorted smallest TopK elements
extract_top_k_elements<false, true, LesserValueCmp<float>>(input, input_shape, values, indices, output_shape, k,
gsl::narrow_cast<unsigned>(axis_parsed));
} else if (largest) {
// extract unsorted (order undefined) largest TopK elements
extract_top_k_elements<true, false, GreaterValueCmp<float>>(input, input_shape, values, indices, output_shape, k,
gsl::narrow_cast<unsigned>(axis_parsed));
} else {
// extract unsorted (order undefined) smallest TopK elements
extract_top_k_elements<false, false, LesserValueCmp<float>>(input, input_shape, values, indices, output_shape, k,
gsl::narrow_cast<unsigned>(axis_parsed));
}
return Status::OK();
}
@ -152,8 +236,10 @@ TopK<9, float>::TopK(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_in
template <>
Status TopK<9, float>::Compute(OpKernelContext* p_op_kernel_context) const {
const auto* X = p_op_kernel_context->Input<Tensor>(0);
if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL,
"input count mismatch, expected 1 input - the tensor to be processed");
if (X == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "input count mismatch, expected 1 input - the tensor to be processed");
}
return TopKImpl(p_op_kernel_context, X, axis_, k_);
}
@ -170,28 +256,83 @@ template <>
Status TopK<10, float>::Compute(OpKernelContext* p_op_kernel_context) const {
const auto* X = p_op_kernel_context->Input<Tensor>(0);
const auto* Y = p_op_kernel_context->Input<Tensor>(1);
if (X == nullptr || Y == nullptr) return Status(common::ONNXRUNTIME, common::FAIL,
"input count mismatch, expected 2 inputs - "
"the tensor to be processed and a tensor containing k value");
if (X == nullptr || Y == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
"input count mismatch, expected 2 inputs - "
"the tensor to be processed and a tensor containing k value");
}
const vector<int64_t>& y_shape = Y->Shape().GetDims();
if (y_shape.size() != 1 || y_shape[0] != 1) return Status(common::ONNXRUNTIME, common::FAIL, "k tensor should be a 1D tensor of size 1");
if (y_shape.size() != 1 || y_shape[0] != 1) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "k tensor should be a 1D tensor of size 1");
}
auto parsed_input_k = Y->template Data<int64_t>()[0];
if (parsed_input_k < 0) return Status(common::ONNXRUNTIME, common::FAIL, "value of k must not be negative");
if (parsed_input_k < 0) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "value of k must not be negative");
}
return TopKImpl(p_op_kernel_context, X, axis_, gsl::narrow_cast<unsigned>(parsed_input_k));
}
// Opset ver - 11
template <>
TopK<11, float>::TopK(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) {
int64_t axis_temp;
ORT_ENFORCE(op_kernel_info.GetAttr<int64_t>("axis", &axis_temp).IsOK());
axis_ = gsl::narrow_cast<int>(axis_temp);
int64_t largest_temp;
ORT_ENFORCE(op_kernel_info.GetAttr<int64_t>("largest", &largest_temp).IsOK());
largest_ = largest_temp == 1 ? true : false;
int64_t sorted_temp;
ORT_ENFORCE(op_kernel_info.GetAttr<int64_t>("sorted", &sorted_temp).IsOK());
sorted_ = sorted_temp == 1 ? true : false;
}
// Opset ver - 11
template <>
Status TopK<11, float>::Compute(OpKernelContext* p_op_kernel_context) const {
const auto* X = p_op_kernel_context->Input<Tensor>(0);
const auto* Y = p_op_kernel_context->Input<Tensor>(1);
if (X == nullptr || Y == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
"input count mismatch, expected 2 inputs - "
"the tensor to be processed and a tensor containing k value");
}
const vector<int64_t>& y_shape = Y->Shape().GetDims();
if (y_shape.size() != 1 || y_shape[0] != 1) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "k tensor should be a 1D tensor of size 1");
}
auto parsed_input_k = Y->template Data<int64_t>()[0];
if (parsed_input_k < 0) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "value of k must not be negative");
}
return TopKImpl(p_op_kernel_context, X, axis_, gsl::narrow_cast<unsigned>(parsed_input_k), largest_, sorted_);
}
// Register necessary kernels
// spec https://github.com/onnx/onnx/blob/master/docs/Operators.md#TopK
ONNX_CPU_OPERATOR_VERSIONED_KERNEL(
TopK,
1, 9,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()).TypeConstraint("I", DataTypeImpl::GetTensorType<int64_t>()),
TopK<9, float>);
ONNX_CPU_OPERATOR_VERSIONED_KERNEL(TopK, 1, 9,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<float>())
.TypeConstraint("I", DataTypeImpl::GetTensorType<int64_t>()),
TopK<9, float>);
ONNX_CPU_OPERATOR_KERNEL(
TopK,
10,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()).TypeConstraint("I", DataTypeImpl::GetTensorType<int64_t>()),
TopK<10, float>);
ONNX_CPU_OPERATOR_VERSIONED_KERNEL(TopK, 10, 10,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<float>())
.TypeConstraint("I", DataTypeImpl::GetTensorType<int64_t>()),
TopK<10, float>);
} // namespace onnxruntime
ONNX_CPU_OPERATOR_KERNEL(TopK, 11,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<float>())
.TypeConstraint("I", DataTypeImpl::GetTensorType<int64_t>()),
TopK<11, float>);
} // namespace onnxruntime

6
onnxruntime/core/providers/cpu/math/top_k.h
View file

@ -14,7 +14,9 @@ class TopK final : public OpKernel {
Status Compute(OpKernelContext* p_op_kernel_context) const override;
private:
int axis_;
unsigned k_;
int axis_; // used by all opset versions
unsigned k_; // opset-9 only
bool largest_; // opset-11 only
bool sorted_; // opset-11 only
};
} // namespace onnxruntime

49
onnxruntime/test/onnx/main.cc
View file

@ -38,11 +38,13 @@ void usage() {
"\t-r [repeat]: Specifies the number of times to repeat\n"
"\t-v: verbose\n"
"\t-n [test_case_name]: Specifies a single test case to run.\n"
"\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'mkldnn', 'tensorrt', 'ngraph', 'openvino' or 'nuphar'. "
"\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'mkldnn', 'tensorrt', 'ngraph', "
"'openvino' or 'nuphar'. "
"Default: 'cpu'.\n"
"\t-x: Use parallel executor, default (without -x): sequential executor.\n"
"\t-o [optimization level]: Default is 1. Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).\n"
"\t\tPlease see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels. \n"
"\t\tPlease see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels. "
"\n"
"\t-h: help\n"
"\n"
"onnxruntime version: %s\n",
@ -291,7 +293,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
return -1;
#endif
}
if (enable_ngraph) { //TODO: Re-order the priority?
if (enable_ngraph) { // TODO: Re-order the priority?
#ifdef USE_NGRAPH
ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_NGraph(sf, "CPU"));
#else
@ -312,25 +314,26 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
sf.SetGraphOptimizationLevel(graph_optimization_level);
}
std::unordered_set<std::string> cuda_flaky_tests = {
"fp16_inception_v1", "fp16_shufflenet", "fp16_tiny_yolov2"};
std::unordered_set<std::string> cuda_flaky_tests = {"fp16_inception_v1", "fp16_shufflenet", "fp16_tiny_yolov2"};
#if (defined(_WIN32) && !defined(_WIN64)) || (defined(__GNUG__) && !defined(__LP64__))
//Minimize mem consumption
LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance, [&stat, &sf, enable_cuda, &cuda_flaky_tests, &env](ITestCase* l) {
std::unique_ptr<ITestCase> test_case_ptr(l);
if (enable_cuda && cuda_flaky_tests.find(l->GetTestCaseName()) != cuda_flaky_tests.end()) {
return;
}
TestResultStat per_case_stat;
std::vector<ITestCase*> per_case_tests = {l};
TestEnv per_case_args(per_case_tests, per_case_stat, env, sf);
RunTests(per_case_args, 1, 1, 1, GetDefaultThreadPool(Env::Default()));
stat += per_case_stat;
});
// Minimize mem consumption
LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance,
[&stat, &sf, enable_cuda, &cuda_flaky_tests, &env](ITestCase* l) {
std::unique_ptr<ITestCase> test_case_ptr(l);
if (enable_cuda && cuda_flaky_tests.find(l->GetTestCaseName()) != cuda_flaky_tests.end()) {
return;
}
TestResultStat per_case_stat;
std::vector<ITestCase*> per_case_tests = {l};
TestEnv per_case_args(per_case_tests, per_case_stat, env, sf);
RunTests(per_case_args, 1, 1, 1, GetDefaultThreadPool(Env::Default()));
stat += per_case_stat;
});
#else
std::vector<ITestCase*> tests;
LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance, [&tests](ITestCase* l) { tests.push_back(l); });
LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance,
[&tests](ITestCase* l) { tests.push_back(l); });
if (enable_cuda) {
for (auto it = tests.begin(); it != tests.end();) {
auto iter = cuda_flaky_tests.find((*it)->GetTestCaseName());
@ -389,9 +392,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
{"maxpool_with_argmax_2d_precomputed_strides", "ShapeInferenceError"},
{"tf_inception_v2", "result mismatch"},
{"mxnet_arcface", "result mismatch"},
{"top_k", "not implemented yet for opset 11"},
{"top_k_smallest", "not implemented yet for opset 11"},
{"top_k_negative_axis", "TopK(11) not implemented yet"},
{"unique_not_sorted_without_axis", "Expected data for 'Y' is incorrect and in sorted order."},
{"cumsum_1d_reverse_exclusive", "only failing linux GPU CI. Likely build error."},
{"det_2d", "not implemented yet"},
@ -545,7 +545,8 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
#endif
#if defined(__GNUG__) && !defined(__LP64__)
broken_tests.insert({"nonzero_example", "failed: type mismatch", {"onnx123", "onnx130", "onnx141", "onnx150", "onnxtip"}});
broken_tests.insert(
{"nonzero_example", "failed: type mismatch", {"onnx123", "onnx130", "onnx141", "onnx150", "onnxtip"}});
broken_tests.insert({"slice_neg_steps", "failed: type mismatch"});
broken_tests.insert({"mod_float_mixed_sign_example", "failed: type mismatch"});
#endif
@ -594,8 +595,8 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
for (const auto& p : stat.GetFailedTest()) {
BrokenTest t = {p.first, ""};
auto iter = broken_tests.find(t);
if (iter == broken_tests.end() ||
(p.second != TestModelInfo::unknown_version && !iter->broken_versions_.empty() && iter->broken_versions_.find(p.second) == iter->broken_versions_.end())) {
if (iter == broken_tests.end() || (p.second != TestModelInfo::unknown_version && !iter->broken_versions_.empty() &&
iter->broken_versions_.find(p.second) == iter->broken_versions_.end())) {
fprintf(stderr, "test %s failed, please fix it\n", p.first.c_str());
result = -1;
}

246
onnxruntime/test/providers/cpu/math/topk_op_test.cc
View file

@ -17,6 +17,8 @@ static void RunTest(int op_set,
const std::vector<int64_t>& expected_dimensions,
bool is_tensorrt_supported = true,
int64_t axis = -1,
int64_t largest = 1,
int64_t sorted = 1,
OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess,
const std::string& expected_err_str = "") {
OpTester test("TopK", op_set);
@ -26,20 +28,29 @@ static void RunTest(int op_set,
test.AddAttribute("axis", axis);
if (op_set <= 9)
test.AddAttribute("k", k);
if (op_set == 11 && largest != 1)
test.AddAttribute("largest", largest);
if (op_set == 11 && sorted != 1)
test.AddAttribute("sorted", sorted);
// Inputs
test.AddInput<float>("X", input_dimensions, input_vals);
if (op_set == 10)
if (op_set >= 10)
test.AddInput<int64_t>("K", {1}, {k});
// Outputs
test.AddOutput<float>("Values", expected_dimensions, expected_vals);
test.AddOutput<int64_t>("Indices", expected_dimensions, expected_indices);
if (sorted == 1) {
test.AddOutput<float>("Values", expected_dimensions, expected_vals);
test.AddOutput<int64_t>("Indices", expected_dimensions, expected_indices);
} else {
test.AddOutput<float>("Values", expected_dimensions, expected_vals, true);
test.AddOutput<int64_t>("Indices", expected_dimensions, expected_indices, true);
}
// Run test and check results
std::unordered_set<std::string> excluded_providers;
if (!is_tensorrt_supported) {
excluded_providers.insert(kTensorrtExecutionProvider);//Disable TensorRT because of unsupported data types
excluded_providers.insert(kTensorrtExecutionProvider); //Disable TensorRT because of unsupported data types
}
test.Run(expect_result, expected_err_str, excluded_providers);
}
@ -155,113 +166,294 @@ TEST(TopKOperator, InvalidKOpset9) {
expected_dimensions,
true,
1,
1,
1,
OpTester::ExpectResult::kExpectFailure,
"Invalid value for attribute k");
}
TEST(TopKOperator, Top0DefaultAxisOpset10) {
static void top_0_default_axis(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f};
std::vector<int64_t> input_dimensions = {2, 4};
std::vector<float> expected_vals = {};
std::vector<int64_t> expected_indices = {};
std::vector<int64_t> expected_dimensions = {2,0};
RunTest(10, 0, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false);
std::vector<int64_t> expected_dimensions = {2, 0};
RunTest(opset_version, 0, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, -1, 1, sorted);
}
TEST(TopKOperator, Top1DefaultAxisOpset10) {
TEST(TopKOperator, Top0DefaultAxisLargestElements) {
top_0_default_axis(10);
top_0_default_axis(11);
top_0_default_axis(11, 0); // unsorted
}
static void top_1_default_axis(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f};
std::vector<int64_t> input_dimensions = {2, 4};
std::vector<float> expected_vals = {0.4f, 0.3f};
std::vector<int64_t> expected_indices = {3, 1};
std::vector<int64_t> expected_dimensions = {2, 1};
RunTest(10, 1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false);
int64_t axis = -1;
RunTest(opset_version, 1, input_vals, input_dimensions, expected_vals, expected_indices,
expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, Top2DefaultAxisOpset10) {
TEST(TopKOperator, Top1DefaultAxisLargestElements) {
top_1_default_axis(10);
top_1_default_axis(11);
top_1_default_axis(11, 0); // unsorted
}
static void top_2_default_axis(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.4f, 0.2f};
std::vector<int64_t> input_dimensions = {2, 4};
std::vector<float> expected_vals = {0.4f, 0.3f, 0.4f, 0.3f};
std::vector<int64_t> expected_indices = {3, 1, 2, 1};
std::vector<int64_t> expected_dimensions = {2, 2};
RunTest(10, 2, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false);
int64_t axis = -1;
RunTest(opset_version, 2, input_vals, input_dimensions, expected_vals, expected_indices,
expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, Top3DefaultAxisOpset10) {
TEST(TopKOperator, Top2DefaultAxisLargestElements) {
top_2_default_axis(10);
top_2_default_axis(11);
top_2_default_axis(11, 0); // unsorted
}
static void top_3_default_axis(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.4f, 0.2f};
std::vector<int64_t> input_dimensions = {2, 4};
std::vector<float> expected_vals = {0.4f, 0.3f, 0.2f, 0.4f, 0.3f, 0.2f};
std::vector<int64_t> expected_indices = {3, 1, 2, 2, 1, 3};
std::vector<int64_t> expected_dimensions = {2, 3};
RunTest(10, 3, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false);
int64_t axis = -1;
RunTest(opset_version, 3, input_vals, input_dimensions, expected_vals, expected_indices,
expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, TopAllDefaultAxisOpset10) {
TEST(TopKOperator, Top3DefaultAxisLargestElements) {
top_3_default_axis(10);
top_3_default_axis(11);
top_3_default_axis(11, 0); //unsorted
}
static void top_all_default_axis(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f};
std::vector<int64_t> input_dimensions = {2, 4};
std::vector<float> expected_vals = {0.4f, 0.3f, 0.2f, 0.1f, 0.3f, 0.3f, 0.2f, 0.1f};
std::vector<int64_t> expected_indices = {3, 1, 2, 0, 1, 2, 3, 0};
std::vector<int64_t> expected_dimensions = {2, 4};
RunTest(10, 4, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false);
int64_t axis = -1;
RunTest(opset_version, 4, input_vals, input_dimensions, expected_vals, expected_indices,
expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, Top1ExplicitAxisOpset10) {
TEST(TopKOperator, TopAllDefaultAxisLargestElements) {
top_all_default_axis(10);
top_all_default_axis(11);
top_all_default_axis(11, 0); // unsorted
}
static void top_1_explicit_axis(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f};
std::vector<int64_t> input_dimensions = {4, 2};
std::vector<float> expected_vals = {0.3f, 0.4f};
std::vector<int64_t> expected_indices = {3, 1};
std::vector<int64_t> expected_dimensions = {1, 2};
int64_t axis = 0;
RunTest(10, 1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis);
RunTest(opset_version, 1, input_vals, input_dimensions, expected_vals, expected_indices,
expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, Top2ExplicitAxisOpset10) {
TEST(TopKOperator, Top1ExplicitAxisLargestElements) {
top_1_explicit_axis(10);
top_1_explicit_axis(11);
top_1_explicit_axis(11, 0); // unsorted
}
static void top_2_explicit_axis(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.0f, 1.0f, 2.0f, 11.0f, 08.0f, 5.0f, 6.0f, 7.0f, 4.0f, 9.0f, 10.0f, 3.0f};
std::vector<int64_t> input_dimensions = {3, 4};
std::vector<float> expected_vals = {8.0f, 9.0f, 10.0f, 11.0f, 4.0f, 5.0f, 6.0f, 7.0f};
std::vector<int64_t> expected_indices = {1, 2, 2, 0, 2, 1, 1, 1};
std::vector<int64_t> expected_dimensions = {2, 4};
int64_t axis = 0;
RunTest(10, 2, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis);
RunTest(opset_version, 2, input_vals, input_dimensions, expected_vals, expected_indices,
expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, Top3ExplicitAxisOpset10) {
TEST(TopKOperator, Top2ExplicitAxisLargestElements) {
top_2_explicit_axis(10);
top_2_explicit_axis(11);
top_2_explicit_axis(11, 0); //unsorted
}
static void top_3_explicit_axis(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f};
std::vector<int64_t> input_dimensions = {4, 2};
std::vector<float> expected_vals = {0.3f, 0.4f, 0.2f, 0.3f, 0.1f, 0.3f};
std::vector<int64_t> expected_indices = {3, 1, 1, 0, 0, 2};
std::vector<int64_t> expected_dimensions = {3, 2};
int64_t axis = 0;
RunTest(10, 3, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis);
RunTest(opset_version, 3, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, TopAllExplicitAxisOpset10) {
TEST(TopKOperator, Top3ExplicitAxisLargestElements) {
top_3_explicit_axis(10);
top_3_explicit_axis(11);
top_3_explicit_axis(11, 0); //unsorted
}
static void top_all_explicit_axis(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f};
std::vector<int64_t> input_dimensions = {4, 2};
std::vector<float> expected_vals = {0.3f, 0.4f, 0.2f, 0.3f, 0.1f, 0.3f, 0.1f, 0.2f};
std::vector<int64_t> expected_indices = {3, 1, 1, 0, 0, 2, 2, 3};
std::vector<int64_t> expected_dimensions = {4, 2};
int64_t axis = 0;
RunTest(10, 4, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis);
RunTest(opset_version, 4, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, TopAllExplicitAxis1DInputOpset10) {
TEST(TopKOperator, TopAllExplicitAxisLargestElements) {
top_all_explicit_axis(10);
top_all_explicit_axis(11);
top_all_explicit_axis(11, 0); // unsorted
}
static void top_all_explicit_axis_1D_input(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, 285.0f, 527.0f, 862.0f};
std::vector<int64_t> input_dimensions = {13};
std::vector<float> expected_vals = {983.0f, 978.0f, 971.0f, 862.0f, 723.0f, 695.0f, 531.0f, 527.0f, 483.0f, 285.0f, 247.0f, 242.0f, 93.0f};
std::vector<int64_t> expected_indices = {7, 3, 2, 12, 9, 1, 8, 11, 4, 10, 5, 6, 0};
std::vector<int64_t> expected_dimensions = {13};
int64_t axis = 0;
RunTest(10, 13, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis);
RunTest(opset_version, 13, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, Top1ExplicitAxisMultiDInputOpset10) {
TEST(TopKOperator, TopAllExplicitAxis1DInputLargestElements) {
top_all_explicit_axis_1D_input(10);
top_all_explicit_axis_1D_input(11);
top_all_explicit_axis_1D_input(11, 0); // unsorted
}
static void top_2_explicit_axis_1D_large_input(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f,
93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f,
93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f,
93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f,
93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f,
93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f,
93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f,
93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f,
93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f,
93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f};
std::vector<int64_t> input_dimensions = {100};
std::vector<float> expected_vals = {983.0f, 983.0f};
std::vector<int64_t> expected_indices = {7, 17};
std::vector<int64_t> expected_dimensions = {2};
int64_t axis = 0;
RunTest(opset_version, 2, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, TopAllExplicitAxis1DLargeInputLargestElements) {
top_2_explicit_axis_1D_large_input(10);
top_2_explicit_axis_1D_large_input(11);
top_2_explicit_axis_1D_large_input(11, 0); // unsorted
}
static void top_1_explicit_axis_MultiD_input(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f};
std::vector<int64_t> input_dimensions = {2, 2, 2};
std::vector<float> expected_vals = {3, 4, 7, 8};
std::vector<int64_t> expected_indices = {1, 1, 1, 1};
std::vector<int64_t> expected_dimensions = {2, 1, 2};
int64_t axis = 1;
RunTest(10, 1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis);
RunTest(opset_version, 1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 1, sorted);
}
TEST(TopKOperator, Top1ExplicitAxisMultiDInputLargestElements) {
top_1_explicit_axis_MultiD_input(10);
top_1_explicit_axis_MultiD_input(11);
top_1_explicit_axis_MultiD_input(11, 0); // unsorted
}
static void top_2_default_axis_smallest(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.4f, 0.2f};
std::vector<int64_t> input_dimensions = {2, 4};
std::vector<float> expected_vals = {0.1f, 0.2f, 0.1f, 0.2f};
std::vector<int64_t> expected_indices = {0, 2, 0, 3};
std::vector<int64_t> expected_dimensions = {2, 2};
int64_t axis = -1;
RunTest(opset_version, 2, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 0, sorted);
}
TEST(TopKOperator, Top2DefaultAxisSmallestElements) {
top_2_default_axis_smallest(11);
top_2_default_axis_smallest(11, 0); // unsorted
}
static void top_3_explicit_axis_smallest(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f};
std::vector<int64_t> input_dimensions = {4, 2};
std::vector<float> expected_vals = {0.1f, 0.2f, 0.1f, 0.3f, 0.2f, 0.3f};
std::vector<int64_t> expected_indices = {0, 3, 2, 0, 1, 2};
std::vector<int64_t> expected_dimensions = {3, 2};
int64_t axis = 0;
RunTest(opset_version, 3, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 0, sorted);
}
TEST(TopKOperator, Top3ExplicitAxisSmallestElements) {
top_3_explicit_axis_smallest(11);
top_3_explicit_axis_smallest(11, 0); //unsorted
}
static void top_1_explicit_axis_MultiD_input_smallest(int opset_version, int64_t sorted = 1) {
std::vector<float> input_vals = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f};
std::vector<int64_t> input_dimensions = {2, 2, 2};
std::vector<float> expected_vals = {1, 2, 5, 6};
std::vector<int64_t> expected_indices = {0, 0, 0, 0};
std::vector<int64_t> expected_dimensions = {2, 1, 2};
int64_t axis = 1;
RunTest(opset_version, 1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 0, sorted);
}
TEST(TopKOperator, Top1ExplicitAxisMultiDInputSmallestElements) {
top_1_explicit_axis_MultiD_input_smallest(11);
top_1_explicit_axis_MultiD_input_smallest(11, 0); //unsorted
}
TEST(TopKOperator, SelectFirstSortNext) {
// in this test, we will select the top 5 elements first then sort the chosen 5 elements
// Select + Sort = O(n + k * ln(k)) = 50 + 5 * ln(5) = 58.047
// Sorted selection: O(n * ln(k)) = 50 * ln(5) = 80.47
// The algorithm used will be Select + Sort
std::vector<float> input_vals = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0,
11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f, 17.0f, 18.0f, 19.0f, 20.0,
21.0f, 22.0f, 23.0f, 24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0,
31.0f, 32.0f, 33.0f, 34.0f, 35.0f, 36.0f, 37.0f, 38.0f, 39.0f, 40.0,
41.0f, 42.0f, 43.0f, 44.0f, 45.0f, 46.0f, 47.0f, 48.0f, 49.0f, 50.0};
std::vector<int64_t> input_dimensions = {50};
std::vector<float> expected_vals = {50.0f, 49.0f, 48.0f, 47.0f, 46.0f};
std::vector<int64_t> expected_indices = {49, 48, 47, 46, 45};
std::vector<int64_t> expected_dimensions = {5};
int64_t axis = 0;
RunTest(11, 5, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis); // largest values
}
TEST(TopKOperator, SortedSelection) {
// in this test, we will use sorted selection (using heap)
// Select + Sort = O(n + k * ln(k)) = 10 + 5 * ln(5) = 18.04
// Sorted selection: O(n * ln(k)) = 10 * ln(5) = 16.09
// The algorithm used will be Sorted selection
std::vector<float> input_vals = {10.0f, 8.0f, 7.0f, 4.0f, 5.0f, 6.0f, 1.0f, 2.0f, 9.0f, 3.0};
std::vector<int64_t> input_dimensions = {10};
std::vector<float> expected_vals = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f};
std::vector<int64_t> expected_indices = {6, 7, 9, 3, 4};
std::vector<int64_t> expected_dimensions = {5};
int64_t axis = 0;
RunTest(11, 5, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 0); // smallest values
}
} // namespace test

110
onnxruntime/test/providers/provider_test_utils.cc
View file

@ -25,6 +25,18 @@ namespace onnxruntime {
namespace test {
// Check functions for tensor types
template <typename T>
void sort_expected_and_actual_buffers(const T* expected, const T* actual, int64_t size) {
std::sort(const_cast<T*>(expected), const_cast<T*>(expected + size));
std::sort(const_cast<T*>(actual), const_cast<T*>(actual + size));
}
// Check functions for tensor types
template <typename T>
void sort_expected_and_actual_buffers(std::vector<T> expected, std::vector<T> actual) {
ORT_ENFORCE(expected.size() == actual.size(), "The 2 containers contain different number of elements");
sort_expected_and_actual_buffers(expected.data(), actual.data(), expected.size());
}
// The default implementation compares for equality, specialized versions for other types are below
template <typename T>
@ -34,13 +46,21 @@ void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, con
auto* output = output_tensor.template Data<T>();
auto size = output_tensor.Shape().Size();
if (expected_data.sort_output_) {
// if order can be jumbled in the output of an operator, sort both the expected and output buffers prior to
// comparison this is a "best-effort" algo and should satisfy the requirement for the few ops that do require this
// support without investing in a more sophisticated infrastructure for the same
sort_expected_and_actual_buffers<T>(expected, output, size);
}
for (int i = 0; i < size; ++i) {
EXPECT_EQ(expected[i], output[i]) << "i:" << i << ", provider_type: " << provider_type;
}
}
template <>
void Check<double>(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) {
void Check<double>(const OpTester::Data& expected_data, const Tensor& output_tensor,
const std::string& provider_type) {
auto& expected_tensor = expected_data.data_.Get<Tensor>();
auto* expected = expected_tensor.template Data<double>();
auto* output = output_tensor.template Data<double>();
@ -49,6 +69,11 @@ void Check<double>(const OpTester::Data& expected_data, const Tensor& output_ten
bool has_abs_err = expected_data.absolute_error_.has_value();
bool has_rel_err = expected_data.relative_error_.has_value();
// deal with rare cases in which order of output data from a kernel MAY be undefined
if (expected_data.sort_output_) {
sort_expected_and_actual_buffers<double>(expected, output, size);
}
double threshold = 0.001;
#ifdef USE_CUDA
threshold = 0.005;
@ -87,6 +112,11 @@ void Check<float>(const OpTester::Data& expected_data, const Tensor& output_tens
bool has_abs_err = expected_data.absolute_error_.has_value();
bool has_rel_err = expected_data.relative_error_.has_value();
// deal with rare cases in which order of output data from a kernel MAY be undefined
if (expected_data.sort_output_) {
sort_expected_and_actual_buffers<float>(expected, output, size);
}
float threshold = 0.001f;
#ifdef USE_CUDA
threshold = 0.005f;
@ -116,7 +146,8 @@ void Check<float>(const OpTester::Data& expected_data, const Tensor& output_tens
}
template <>
void Check<MLFloat16>(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) {
void Check<MLFloat16>(const OpTester::Data& expected_data, const Tensor& output_tensor,
const std::string& provider_type) {
auto& expected_tensor = expected_data.data_.Get<Tensor>();
auto* expected = expected_tensor.template Data<MLFloat16>();
auto* output = output_tensor.template Data<MLFloat16>();
@ -127,6 +158,11 @@ void Check<MLFloat16>(const OpTester::Data& expected_data, const Tensor& output_
ConvertMLFloat16ToFloat(expected, f_expected.data(), static_cast<int>(size));
ConvertMLFloat16ToFloat(output, f_output.data(), static_cast<int>(size));
// deal with rare cases in which order of output data from a kernel MAY be undefined
if (expected_data.sort_output_) {
sort_expected_and_actual_buffers<float>(f_expected, f_output);
}
float threshold = 0.001f;
for (int i = 0; i < size; ++i) {
if (std::isinf(f_expected[i])) // Test infinity for equality
@ -139,7 +175,8 @@ void Check<MLFloat16>(const OpTester::Data& expected_data, const Tensor& output_
}
template <>
void Check<BFloat16>(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) {
void Check<BFloat16>(const OpTester::Data& expected_data, const Tensor& output_tensor,
const std::string& provider_type) {
auto& expected_tensor = expected_data.data_.Get<Tensor>();
auto* expected = expected_tensor.template Data<BFloat16>();
auto* output = output_tensor.template Data<BFloat16>();
@ -150,6 +187,11 @@ void Check<BFloat16>(const OpTester::Data& expected_data, const Tensor& output_t
BFloat16ToFloat(expected, f_expected.data(), static_cast<size_t>(size));
BFloat16ToFloat(output, f_output.data(), static_cast<size_t>(size));
// deal with rare cases in which order of output data from a kernel MAY be undefined
if (expected_data.sort_output_) {
sort_expected_and_actual_buffers<float>(f_expected, f_output);
}
/// XXX: May need to adjust threshold as BFloat is coarse
float threshold = 0.001f;
for (int i = 0; i < size; ++i) {
@ -163,7 +205,8 @@ void Check<BFloat16>(const OpTester::Data& expected_data, const Tensor& output_t
}
template <typename Type>
void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) {
void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const Tensor& output_tensor,
const std::string& provider_type) {
if (type == DataTypeImpl::GetType<Type>())
Check<Type>(expected_data, output_tensor, provider_type);
else
@ -171,7 +214,8 @@ void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const T
}
template <typename Type, typename Next, typename... Types>
void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) {
void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const Tensor& output_tensor,
const std::string& provider_type) {
if (type == DataTypeImpl::GetType<Type>())
Check<Type>(expected_data, output_tensor, provider_type);
else
@ -181,10 +225,12 @@ void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const T
void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) {
ORT_ENFORCE(expected_data.data_.Get<Tensor>().Shape() == output_tensor.Shape(),
"Expected output shape [" + expected_data.data_.Get<Tensor>().Shape().ToString() +
"] did not match run output shape [" +
output_tensor.Shape().ToString() + "] for " + expected_data.def_.Name());
"] did not match run output shape [" + output_tensor.Shape().ToString() + "] for " +
expected_data.def_.Name());
CheckDispatch<bool, float, double, uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t, std::string, MLFloat16, BFloat16>(output_tensor.DataType(), expected_data, output_tensor, provider_type);
CheckDispatch<bool, float, double, uint8_t, uint16_t, uint32_t, uint64_t, int8_t, int16_t, int32_t, int64_t,
std::string, MLFloat16, BFloat16>(output_tensor.DataType(), expected_data, output_tensor,
provider_type);
}
// Check for non tensor types
@ -242,41 +288,32 @@ OpTester::~OpTester() {
void OpTester::FillFeedsAndOutputNames(std::unordered_map<std::string, OrtValue>& feeds,
std::vector<std::string>& output_names) {
for (auto& output : output_data_) {
if (output.def_.Exists())
output_names.push_back(output.def_.Name());
if (output.def_.Exists()) output_names.push_back(output.def_.Name());
}
for (size_t i = 0; i < input_data_.size(); ++i) {
if (std::find(initializer_index_.begin(), initializer_index_.end(), i) == initializer_index_.end() && input_data_[i].def_.Exists()) {
if (std::find(initializer_index_.begin(), initializer_index_.end(), i) == initializer_index_.end() &&
input_data_[i].def_.Exists()) {
feeds[input_data_[i].def_.Name()] = input_data_[i].data_;
}
}
}
void OpTester::SetOutputAbsErr(const char* name, float v) {
auto it = std::find_if(
output_data_.begin(),
output_data_.end(),
[name](Data& data) {
return (data.def_.Name() == name);
});
auto it = std::find_if(output_data_.begin(), output_data_.end(),
[name](Data& data) { return (data.def_.Name() == name); });
ORT_ENFORCE(it != output_data_.end());
it->absolute_error_ = optional<float>(v);
}
void OpTester::SetOutputRelErr(const char* name, float v) {
auto it = std::find_if(
output_data_.begin(),
output_data_.end(),
[name](Data& data) {
return (data.def_.Name() == name);
});
auto it = std::find_if(output_data_.begin(), output_data_.end(),
[name](Data& data) { return (data.def_.Name() == name); });
ORT_ENFORCE(it != output_data_.end());
it->relative_error_ = optional<float>(v);
}
void OpTester::AddNodes(onnxruntime::Graph& graph,
std::vector<onnxruntime::NodeArg*>& graph_input_defs,
void OpTester::AddNodes(onnxruntime::Graph& graph, std::vector<onnxruntime::NodeArg*>& graph_input_defs,
std::vector<onnxruntime::NodeArg*>& graph_output_defs,
std::vector<std::function<void(onnxruntime::Node& node)>>& add_attribute_funcs) {
// default behavior is to create a single Node for the op being tested, with node inputs/outputs
@ -284,8 +321,7 @@ void OpTester::AddNodes(onnxruntime::Graph& graph,
auto& node = graph.AddNode("node1", op_, op_, graph_input_defs, graph_output_defs, nullptr, domain_);
// Add the attributes if any
for (auto& add_attribute_fn : add_attribute_funcs)
add_attribute_fn(node);
for (auto& add_attribute_fn : add_attribute_funcs) add_attribute_fn(node);
}
void OpTester::AddInitializers(onnxruntime::Graph& graph) {
@ -293,14 +329,14 @@ void OpTester::AddInitializers(onnxruntime::Graph& graph) {
auto& data = input_data_[index];
auto& tensor = data.data_.Get<Tensor>();
ONNX_NAMESPACE::TensorProto tensor_proto;
//1. set dimension
// 1. set dimension
auto& shape = tensor.Shape();
for (auto& dim : shape.GetDims()) {
tensor_proto.add_dims(dim);
}
//2. set type
// 2. set type
tensor_proto.set_data_type(data.def_.TypeAsProto()->tensor_type().elem_type());
//3. data
// 3. data
if (data.def_.TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType_STRING) {
const std::string* string_data = tensor.Data<std::string>();
for (auto i = 0; i < shape.Size(); i++) {
@ -310,7 +346,7 @@ void OpTester::AddInitializers(onnxruntime::Graph& graph) {
auto buffer_size = tensor.DataType()->Size() * shape.Size();
tensor_proto.set_raw_data(tensor.DataRaw(), buffer_size);
}
//4. name
// 4. name
tensor_proto.set_name(data.def_.Name());
graph.AddInitializedTensor(tensor_proto);
}
@ -337,7 +373,7 @@ std::unique_ptr<onnxruntime::Model> OpTester::BuildGraph() {
onnxruntime::Graph& graph = p_model->MainGraph();
AddNodes(graph, node_input_defs, output_defs, add_attribute_funcs_);
//Add Initializer
// Add Initializer
AddInitializers(graph);
return p_model;
}
@ -520,8 +556,8 @@ void OpTester::Run(const SessionOptions& so,
EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(entry)).IsOK());
}
ExecuteModel(*p_model, session_object, expect_result, expected_failure_string, run_options,
feeds, output_names, provider_types);
ExecuteModel(*p_model, session_object, expect_result, expected_failure_string, run_options, feeds, output_names,
provider_types);
} else {
for (const std::string& provider_type : all_provider_types) {
if (excluded_provider_types.count(provider_type) > 0)
@ -562,7 +598,7 @@ void OpTester::Run(const SessionOptions& so,
if (node.OpType() == kConstant)
continue;
//if node is not registered for the provider, skip
// if node is not registered for the provider, skip
node.SetExecutionProviderType(provider_type);
if (provider_type == onnxruntime::kNGraphExecutionProvider ||
provider_type == onnxruntime::kTensorrtExecutionProvider ||
@ -592,8 +628,8 @@ void OpTester::Run(const SessionOptions& so,
EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
ExecuteModel(*p_model, session_object, expect_result, expected_failure_string, run_options,
feeds, output_names, provider_type);
ExecuteModel(*p_model, session_object, expect_result, expected_failure_string, run_options, feeds,
output_names, provider_type);
}
EXPECT_TRUE(has_run) << "No registered execution providers were able to run the model.";

159
onnxruntime/test/providers/provider_test_utils.h
View file

@ -64,46 +64,74 @@ template <typename T>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType();
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<float>() { return ONNX_NAMESPACE::TensorProto_DataType_FLOAT; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<float>() {
return ONNX_NAMESPACE::TensorProto_DataType_FLOAT;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<double>() { return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<double>() {
return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<int32_t>() { return ONNX_NAMESPACE::TensorProto_DataType_INT32; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<int32_t>() {
return ONNX_NAMESPACE::TensorProto_DataType_INT32;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<int64_t>() { return ONNX_NAMESPACE::TensorProto_DataType_INT64; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<int64_t>() {
return ONNX_NAMESPACE::TensorProto_DataType_INT64;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<bool>() { return ONNX_NAMESPACE::TensorProto_DataType_BOOL; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<bool>() {
return ONNX_NAMESPACE::TensorProto_DataType_BOOL;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<int8_t>() { return ONNX_NAMESPACE::TensorProto_DataType_INT8; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<int8_t>() {
return ONNX_NAMESPACE::TensorProto_DataType_INT8;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<int16_t>() { return ONNX_NAMESPACE::TensorProto_DataType_INT16; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<int16_t>() {
return ONNX_NAMESPACE::TensorProto_DataType_INT16;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<uint8_t>() { return ONNX_NAMESPACE::TensorProto_DataType_UINT8; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<uint8_t>() {
return ONNX_NAMESPACE::TensorProto_DataType_UINT8;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<uint16_t>() { return ONNX_NAMESPACE::TensorProto_DataType_UINT16; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<uint16_t>() {
return ONNX_NAMESPACE::TensorProto_DataType_UINT16;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<uint32_t>() { return ONNX_NAMESPACE::TensorProto_DataType_UINT32; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<uint32_t>() {
return ONNX_NAMESPACE::TensorProto_DataType_UINT32;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<uint64_t>() { return ONNX_NAMESPACE::TensorProto_DataType_UINT64; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<uint64_t>() {
return ONNX_NAMESPACE::TensorProto_DataType_UINT64;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<std::string>() { return ONNX_NAMESPACE::TensorProto_DataType_STRING; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<std::string>() {
return ONNX_NAMESPACE::TensorProto_DataType_STRING;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<MLFloat16>() { return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<MLFloat16>() {
return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16;
}
template <>
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<BFloat16>() { return ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16; }
constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType<BFloat16>() {
return ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16;
}
template <typename T>
struct TTypeProto : ONNX_NAMESPACE::TypeProto {
@ -132,7 +160,7 @@ struct TTensorType {
template <typename T>
const TTypeProto<T> TTensorType<T>::s_type_proto;
//TypeProto for map<TKey, TVal>
// TypeProto for map<TKey, TVal>
template <typename TKey, typename TVal>
struct MTypeProto : ONNX_NAMESPACE::TypeProto {
MTypeProto() {
@ -150,7 +178,7 @@ struct MMapType {
template <typename TKey, typename TVal>
const MTypeProto<TKey, TVal> MMapType<TKey, TVal>::s_map_type_proto;
//TypeProto for vector<map<TKey, TVal>>
// TypeProto for vector<map<TKey, TVal>>
template <typename TKey, typename TVal>
struct VectorOfMapTypeProto : ONNX_NAMESPACE::TypeProto {
VectorOfMapTypeProto() {
@ -194,9 +222,9 @@ const SequenceTensorTypeProto<ElemType> SequenceTensorType<ElemType>::s_sequence
// 3. Call AddInput for all the inputs
// 4. Call AddOutput with all expected outputs
// 5. Call Run
// Not all tensor types and output types are added, if a new input type is used, add it to the TypeToDataType list above
// for new output types, add a new specialization for Check<>
// See current usage for an example, should be self explanatory
// Not all tensor types and output types are added, if a new input type is used, add it to the TypeToDataType list
// above for new output types, add a new specialization for Check<> See current usage for an example, should be self
// explanatory
class OpTester {
public:
explicit OpTester(const char* op, int opset_version = 7, const char* domain = onnxruntime::kOnnxDomain)
@ -217,12 +245,14 @@ class OpTester {
// We have an initializer_list and vector version of the Add functions because std::vector is specialized for
// bool and we can't get the raw data out. So those cases must use an initializer_list
template <typename T>
void AddInput(const char* name, const std::vector<int64_t>& dims, const std::initializer_list<T>& values, bool is_initializer = false) {
void AddInput(const char* name, const std::vector<int64_t>& dims, const std::initializer_list<T>& values,
bool is_initializer = false) {
AddData(input_data_, name, dims, values.begin(), values.size(), is_initializer);
}
template <typename T>
void AddInput(const char* name, const std::vector<int64_t>& dims, const std::vector<T>& values, bool is_initializer = false) {
void AddInput(const char* name, const std::vector<int64_t>& dims, const std::vector<T>& values,
bool is_initializer = false) {
AddData(input_data_, name, dims, values.data(), values.size(), is_initializer);
}
@ -235,7 +265,8 @@ class OpTester {
OrtValue value;
value.Init(ptr.get(), mltype, mltype->GetDeleteFunc());
ptr.release();
input_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional<float>(), optional<float>()));
input_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional<float>(),
optional<float>()));
}
template <typename T>
@ -246,7 +277,8 @@ class OpTester {
OrtValue value;
value.Init(ptr.get(), mltype, mltype->GetDeleteFunc());
ptr.release();
input_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional<float>(), optional<float>()));
input_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional<float>(),
optional<float>()));
}
template <typename T>
@ -286,32 +318,36 @@ class OpTester {
void AddInput(const char* name, const std::map<TKey, TVal>& val) {
std::unique_ptr<std::map<TKey, TVal>> ptr = onnxruntime::make_unique<std::map<TKey, TVal>>(val);
OrtValue value;
value.Init(ptr.release(),
DataTypeImpl::GetType<std::map<TKey, TVal>>(),
value.Init(ptr.release(), DataTypeImpl::GetType<std::map<TKey, TVal>>(),
DataTypeImpl::GetType<std::map<TKey, TVal>>()->GetDeleteFunc());
input_data_.push_back(Data(NodeArg(name, &MMapType<TKey, TVal>::s_map_type_proto), std::move(value), optional<float>(), optional<float>()));
input_data_.push_back(Data(NodeArg(name, &MMapType<TKey, TVal>::s_map_type_proto), std::move(value),
optional<float>(), optional<float>()));
}
template <typename T>
void AddMissingOptionalInput() {
std::string name; // empty == input doesn't exist
input_data_.push_back(Data(NodeArg(name, &TTensorType<T>::s_type_proto), OrtValue(), optional<float>(), optional<float>()));
input_data_.push_back(Data(NodeArg(name, &TTensorType<T>::s_type_proto), OrtValue(), optional<float>(),
optional<float>()));
}
template <typename T>
void AddOutput(const char* name, const std::vector<int64_t>& dims, const std::initializer_list<T>& expected_values) {
AddData(output_data_, name, dims, expected_values.begin(), expected_values.size());
void AddOutput(const char* name, const std::vector<int64_t>& dims, const std::initializer_list<T>& expected_values,
bool sort_output = false) {
AddData(output_data_, name, dims, expected_values.begin(), expected_values.size(), false, sort_output);
}
template <typename T>
void AddOutput(const char* name, const std::vector<int64_t>& dims, const std::vector<T>& expected_values) {
AddData(output_data_, name, dims, expected_values.data(), expected_values.size());
void AddOutput(const char* name, const std::vector<int64_t>& dims, const std::vector<T>& expected_values,
bool sort_output = false) {
AddData(output_data_, name, dims, expected_values.data(), expected_values.size(), false, sort_output);
}
template <typename T>
void AddMissingOptionalOutput() {
std::string name; // empty == input doesn't exist
output_data_.push_back(Data(NodeArg(name, &TTensorType<T>::s_type_proto), OrtValue(), optional<float>(), optional<float>()));
output_data_.push_back(Data(NodeArg(name, &TTensorType<T>::s_type_proto), OrtValue(), optional<float>(),
optional<float>()));
}
// Add other registered types, possibly experimental
@ -323,7 +359,8 @@ class OpTester {
OrtValue value;
value.Init(ptr.get(), mltype, mltype->GetDeleteFunc());
ptr.release();
output_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional<float>(), optional<float>()));
output_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional<float>(),
optional<float>()));
}
template <typename T>
@ -334,7 +371,8 @@ class OpTester {
OrtValue value;
value.Init(ptr.get(), mltype, mltype->GetDeleteFunc());
ptr.release();
output_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional<float>(), optional<float>()));
output_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional<float>(),
optional<float>()));
}
// Add non tensor output
@ -342,10 +380,10 @@ class OpTester {
void AddOutput(const char* name, const std::vector<std::map<TKey, TVal>>& val) {
auto ptr = onnxruntime::make_unique<std::vector<std::map<TKey, TVal>>>(val);
OrtValue ml_value;
ml_value.Init(ptr.release(),
DataTypeImpl::GetType<std::vector<std::map<TKey, TVal>>>(),
ml_value.Init(ptr.release(), DataTypeImpl::GetType<std::vector<std::map<TKey, TVal>>>(),
DataTypeImpl::GetType<std::vector<std::map<TKey, TVal>>>()->GetDeleteFunc());
output_data_.push_back(Data(NodeArg(name, &VectorOfMapType<TKey, TVal>::s_vec_map_type_proto), std::move(ml_value), optional<float>(), optional<float>()));
output_data_.push_back(Data(NodeArg(name, &VectorOfMapType<TKey, TVal>::s_vec_map_type_proto), std::move(ml_value),
optional<float>(), optional<float>()));
}
void AddCustomOpRegistry(std::shared_ptr<CustomRegistry> registry) {
@ -366,14 +404,13 @@ class OpTester {
template <typename T>
void AddAttribute(std::string name, T value) {
// Generate a the proper AddAttribute call for later
add_attribute_funcs_.emplace_back(
[name = std::move(name), value = std::move(value)](onnxruntime::Node& node) { node.AddAttribute(name, value); });
add_attribute_funcs_.emplace_back([name = std::move(name), value = std::move(value)](onnxruntime::Node& node) {
node.AddAttribute(name, value);
});
}
enum class ExpectResult {
kExpectSuccess,
kExpectFailure
};
enum class ExpectResult { kExpectSuccess,
kExpectFailure };
void Run(ExpectResult expect_result = ExpectResult::kExpectSuccess, const std::string& expected_failure_string = "",
const std::unordered_set<std::string>& excluded_provider_types = {},
@ -393,14 +430,20 @@ class OpTester {
OrtValue data_;
optional<float> relative_error_;
optional<float> absolute_error_;
Data(onnxruntime::NodeArg&& def, OrtValue&& data, optional<float>&& rel, optional<float>&& abs) : def_(std::move(def)), data_(std::move(data)), relative_error_(std::move(rel)), absolute_error_(abs) {}
bool sort_output_;
Data(onnxruntime::NodeArg&& def, OrtValue&& data, optional<float>&& rel, optional<float>&& abs,
bool sort_output = false)
: def_(std::move(def)),
data_(std::move(data)),
relative_error_(std::move(rel)),
absolute_error_(abs),
sort_output_(sort_output) {}
Data(Data&&) = default;
Data& operator=(Data&&) = default;
};
protected:
virtual void AddNodes(onnxruntime::Graph& graph,
std::vector<onnxruntime::NodeArg*>& graph_input_defs,
virtual void AddNodes(onnxruntime::Graph& graph, std::vector<onnxruntime::NodeArg*>& graph_input_defs,
std::vector<onnxruntime::NodeArg*>& graph_output_defs,
std::vector<std::function<void(onnxruntime::Node& node)>>& add_attribute_funcs);
@ -419,18 +462,15 @@ class OpTester {
private:
template <typename T>
void AddData(std::vector<Data>& data, const char* name,
const std::vector<int64_t>& dims, const T* values,
int64_t values_count, bool is_initializer = false) {
void AddData(std::vector<Data>& data, const char* name, const std::vector<int64_t>& dims, const T* values,
int64_t values_count, bool is_initializer = false, bool sort_output = false) {
try {
TensorShape shape{dims};
ORT_ENFORCE(shape.Size() == values_count, values_count,
" input values doesn't match tensor size of ", shape.Size());
ORT_ENFORCE(shape.Size() == values_count, values_count, " input values doesn't match tensor size of ",
shape.Size());
auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU);
auto p_tensor = onnxruntime::make_unique<Tensor>(DataTypeImpl::GetType<T>(),
shape,
allocator);
auto p_tensor = onnxruntime::make_unique<Tensor>(DataTypeImpl::GetType<T>(), shape, allocator);
auto* data_ptr = p_tensor->template MutableData<T>();
for (int64_t i = 0; i < values_count; i++) {
@ -445,10 +485,10 @@ class OpTester {
TTypeProto<T> type_proto(add_shape_to_tensor_data_ ? &dims_for_proto : nullptr);
OrtValue value;
value.Init(p_tensor.release(), DataTypeImpl::GetType<Tensor>(), DataTypeImpl::GetType<Tensor>()->GetDeleteFunc());
data.push_back(Data(NodeArg(name, &type_proto), std::move(value), optional<float>(), optional<float>()));
if (is_initializer)
initializer_index_.push_back(data.size() - 1);
value.Init(p_tensor.release(), DataTypeImpl::GetType<Tensor>(),
DataTypeImpl::GetType<Tensor>()->GetDeleteFunc());
data.push_back(Data(NodeArg(name, &type_proto), std::move(value), optional<float>(), optional<float>(), sort_output));
if (is_initializer) initializer_index_.push_back(data.size() - 1);
} catch (const std::exception& ex) {
std::cerr << "AddData for '" << name << "' threw: " << ex.what();
throw;
@ -499,7 +539,8 @@ inline void ConvertFloatToMLFloat16(const float* f_datat, MLFloat16* h_data, int
#endif
inline void ConvertMLFloat16ToFloat(const MLFloat16* h_data, float* f_data, int input_size) {
auto in_vector = ConstEigenVectorMap<Eigen::half>(static_cast<const Eigen::half*>(static_cast<const void*>(h_data)), input_size);
auto in_vector =
ConstEigenVectorMap<Eigen::half>(static_cast<const Eigen::half*>(static_cast<const void*>(h_data)), input_size);
auto output_vector = EigenVectorMap<float>(f_data, input_size);
output_vector = in_vector.template cast<float>();
}