From d186c19c454279ae0536e6361bcc5e649a5a1c17 Mon Sep 17 00:00:00 2001 From: Hariharan Seshadri Date: Wed, 9 Oct 2019 19:09:30 -0700 Subject: [PATCH] Add opset-11 TopK CPU kernel (#1912) * initial commit * Update * Update top_k.cc * PR comments * Add more tests * Update * Add another test case * Update * Resolve conflicts * Update * Nits * Nits * Nits * Pick sorted content using 2 different approaches * Update to logic * PR comments * PR feedback * Update * Fix build * Fix build * Update --- .../providers/cpu/cpu_execution_provider.cc | 12 +- onnxruntime/core/providers/cpu/math/top_k.cc | 373 ++++++++++++------ onnxruntime/core/providers/cpu/math/top_k.h | 6 +- onnxruntime/test/onnx/main.cc | 49 +-- .../test/providers/cpu/math/topk_op_test.cc | 246 ++++++++++-- .../test/providers/provider_test_utils.cc | 110 ++++-- .../test/providers/provider_test_utils.h | 159 +++++--- 7 files changed, 686 insertions(+), 269 deletions(-) diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 4f1de06a03..d61b19b603 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -278,7 +278,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, // Opset 10 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, StringNormalizer); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, TopK); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, TopK); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, MaxPool); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, AveragePool); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, Mod); @@ -401,6 +401,7 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Pa class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, GatherND); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Range); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Unique); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, TopK); void RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { static const BuildKernelCreateInfoFn function_table[] = { @@ -874,7 +875,8 @@ void RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { // Opset 10 BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo}; + BuildKernelCreateInfo, + BuildKernelCreateInfo, + }; for (auto& function_table_entry : function_table) { kernel_registry.Register(function_table_entry()); } -} +} // namespace onnxruntime // Forward declarations of ml op kernels namespace ml { diff --git a/onnxruntime/core/providers/cpu/math/top_k.cc b/onnxruntime/core/providers/cpu/math/top_k.cc index 8dc481d1db..5e8264ef93 100644 --- a/onnxruntime/core/providers/cpu/math/top_k.cc +++ b/onnxruntime/core/providers/cpu/math/top_k.cc @@ -1,18 +1,18 @@ /** -* Copyright (c) 2016-present, Facebook, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Copyright (c) 2016-present, Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "core/providers/cpu/math/top_k.h" #include "core/providers/common.h" @@ -22,115 +22,199 @@ #include "core/framework/tensor.h" #include "core/util/math_cpuonly.h" #include +#include +#include + using namespace std; namespace onnxruntime { -// Helper methods -static int64_t SizeToDim(size_t k, const vector& dims) { - ORT_ENFORCE(k <= dims.size()); - int64_t r = 1; - for (size_t i = 0; i < k; ++i) { - r *= dims[i]; - } - return r; -} - -static int64_t SizeFromDim(size_t k, const vector& dims) { - ORT_ENFORCE(k <= dims.size()); - int64_t r = 1; - for (size_t i = k; i < dims.size(); ++i) { - r *= dims[i]; - } - return r; -} - template -struct ValueCmp { - bool operator()( - const pair& lhs, - const pair& rhs) { - return ( - lhs.first > rhs.first || - (lhs.first == rhs.first && lhs.second < rhs.second)); +struct GreaterValueCmp { + bool operator()(const pair& lhs, const pair& rhs) { + return (lhs.first > rhs.first || + // when values are equal, we want lhs to get higher "priority" + // if its corresponding index comes first (i.e.) is lower + (lhs.first == rhs.first && lhs.second < rhs.second)); } }; -// Core TopK implementation -Status TopKImpl(OpKernelContext* p_op_kernel_context, const Tensor* X, const int axis, const unsigned k) { +template +struct LesserValueCmp { + bool operator()(const pair& lhs, const pair& rhs) { + return (lhs.first < rhs.first || + // when values are equal, we want lhs to get higher "priority" + // if its corresponding index comes first (i.e.) is lower + (lhs.first == rhs.first && lhs.second < rhs.second)); + } +}; - const vector& in_dims = X->Shape().GetDims(); - // Will return axis_ as is if positive or fixes it in case it is negative - auto axis_parsed = HandleNegativeAxis(axis, in_dims.size()); - // Check to ensure k is within the bounds of what is available in that specific axis - if (in_dims.at(axis_parsed) < k) { - ostringstream err_msg; - err_msg << "k argment [" << k << "] should not be greater than specified axis dim value [" << in_dims.at(axis_parsed) << "]"; - return Status(common::ONNXRUNTIME, common::FAIL, err_msg.str()); +// Static helpers that implement the core logic for each of the 'TopK' operator flavor + +// Selects the top k elements (largest or smallest based on template parameter) +template +static vector> select_top_k(const ConstEigenMatrixMapRowMajor& raw_data, int64_t row_num, int64_t num_blocks, + int64_t block_slice, int64_t inter_block_offset, const unsigned k, + bool sort_top_k) { + // create a data holder and insert elements + vector> data_holder; + data_holder.reserve(num_blocks); + for (int64_t l = 0; l < num_blocks; ++l) { + data_holder.push_back({raw_data(row_num, l * block_slice + inter_block_offset), l}); } - if (k == 0) { - vector out_dims = in_dims; - out_dims[axis_parsed] = 0; - p_op_kernel_context->Output(0, out_dims); - p_op_kernel_context->Output(1, out_dims); - return Status::OK(); + // find the top k (largest or smallest) elements in the data holder - O(n) + nth_element(data_holder.begin(), data_holder.begin() + (k - 1), data_holder.end(), Comparator()); + + // sort the top k elements if needed - O (k log k) + if (sort_top_k) { + std::sort(data_holder.begin(), data_holder.begin() + k, Comparator()); } - const int64_t rows = SizeToDim(axis_parsed, in_dims); - const int64_t cols = X->Shape().Size() / rows; - auto input_map = ConstEigenMatrixMapRowMajor( - static_cast(X->template Data()), - rows, - cols); + // the data_holder now contains the top k elements in the first k indices + return data_holder; +} - // Resize output tensors to be the same shape as the input except - // for the specified dimension ((i.e.) axis_parsed), which will be of size k. E.x. for an input tensor - // of shape [3, 4, 5] and k=2 with axis_parsed=1, both of these will be shape [3, 2, 5] - vector output_linear_shape = in_dims; - output_linear_shape[axis_parsed] = k; - auto* Values = p_op_kernel_context->Output(0, output_linear_shape); - auto* Indices = p_op_kernel_context->Output(1, output_linear_shape); +// Given an input tensor 'input' and metadata values - 'k' and 'axis_parsed', +// this method will extract the sorted top k largest/smallest elements and place them in the output tensor 'values' +// along with the metadata output 'indices' +template +static void extract_top_k_elements(const Tensor* input, const TensorShape& input_shape, Tensor* values, + Tensor* indices, const TensorShape& output_shape, const unsigned k, + const unsigned axis_parsed) { + // Cache some values that will be used in the implementation below + const int64_t rows = input_shape.SizeToDimension(static_cast(axis_parsed)); + const int64_t cols = input->Shape().Size() / rows; + auto input_map = + ConstEigenMatrixMapRowMajor(static_cast(input->template Data()), rows, cols); // Use Eigen maps to allow indexing into the 2d tensors like Values_map(i,j) - const int64_t reduced_cols = SizeFromDim(axis_parsed, output_linear_shape); - auto Values_map = EigenMatrixMapRowMajor( - Values->template MutableData(), rows, reduced_cols); - auto Indices_map = EigenMatrixMapRowMajor( - Indices->template MutableData(), rows, reduced_cols); + const int64_t reduced_cols = output_shape.SizeFromDimension(static_cast(axis_parsed)); + auto values_map = EigenMatrixMapRowMajor(values->template MutableData(), rows, reduced_cols); + auto indices_map = EigenMatrixMapRowMajor(indices->template MutableData(), rows, reduced_cols); // This is basically the number of elements within each of the "k" rows const int64_t block_slice = reduced_cols / k; - // Sort preserving Indices + const int64_t num_blocks = input_shape[axis_parsed]; + for (int64_t i = 0; i < rows; ++i) { for (int64_t j = 0; j < block_slice; ++j) { - // Build a min-heap, the heap element is pair of (value, idx) - // the top of the heap is the smallest value - priority_queue< - pair, - vector>, - ValueCmp> - min_heap; - // Maintain the size of heap to be less or equal to k_, so the - // heap will hold the k largest Values - for (int64_t l = 0; l < in_dims[axis_parsed]; ++l) { - const auto value = input_map(i, l * block_slice + j); - if (min_heap.size() < k || value > min_heap.top().first) { - min_heap.push({value, l}); + // Since sorted == true, we will use a Heap to hold the top K values in sorted fashion + if (sorted) { // The optimizer will clean-up the redundant condition based on the template parameter 'sorted' + auto n_casted = static_cast(num_blocks); + auto k_casted = static_cast(k); + if ((n_casted + k_casted * log(k_casted)) < (n_casted * log(k_casted))) { + // Select first - O(n), then sort O(k * ln(k)) + // Overall complexity = O (n + k * ln(k)) + const auto& data_holder = select_top_k(input_map, i, num_blocks, block_slice, j, k, true); + for (int64_t l = 0; l < k; ++l) { + const auto& elem = data_holder[l]; + auto col_index = l * block_slice + j; + values_map(i, col_index) = elem.first; + indices_map(i, col_index) = elem.second; + } + } else { + // Perform sorted selection by passing 'n' elements over a heap of size 'k' + // overall complexity = O (n * ln(k)) + + // Build a min-heap/max-heap, the heap element is pair of (value, idx) + // The top of the heap is the smallest/largest value depending on whether it is a min-heap/max-heap + // This is a min-heap if largest == true, this is a max-heap if largest == false + priority_queue, vector>, Comparator> heap; + + // Maintain the size of heap to be less or equal to k, so the + // heap will hold the k largest/smallest values + for (int64_t l = 0; l < num_blocks; ++l) { + const auto value = input_map(i, l * block_slice + j); + // largest == true: insert into the min-heap if the size is < k or if the new + // element is greater than the min element in the min-heap + + // largest == false: insert into the min-heap if the size is < k or if the new + // element is lesser than the max element in the max-heap + if ((heap.size() < k) || (largest && value > heap.top().first) || + (!largest && value < heap.top().first)) { // the optimizer will clean-up the redundant condition based + // on the template parameter 'largest' + heap.push({value, l}); + } + if (heap.size() > k) { + heap.pop(); + } + } + // Extract these k elements and place them in the results placeholder + for (int64_t l = 0; l < k; ++l) { + const auto& elem = heap.top(); + auto col_index = (k - l - 1) * block_slice + j; + values_map(i, col_index) = elem.first; + indices_map(i, col_index) = elem.second; + heap.pop(); + } } - if (min_heap.size() > k) { - min_heap.pop(); + } else { // sorted == false + // The optimizer will clean-up the redundant condition based on the template parameter 'sorted' + + // If the top K values are not required to be sorted, we use a more optimal selection algorithm + // Average - O(n). Worst - O(n * ln(n)) or O(n^2) depending on the implementation, where 'n' is the number of input + + const auto& data_holder = select_top_k(input_map, i, num_blocks, block_slice, j, k, false); + + // Insert the top 'k' (largest or smallest) elements into the final output buffers + for (int64_t l = 0; l < k; ++l) { + const auto& elem = data_holder[l]; + auto col_index = l * block_slice + j; + values_map(i, col_index) = elem.first; + indices_map(i, col_index) = elem.second; } } - // Extract these k elements and place them in the results placeholder - for (int64_t l = 0; l < k; ++l) { - auto& pqElem = min_heap.top(); - auto col_index = (k - l - 1) * block_slice + j; - Values_map(i, col_index) = pqElem.first; - Indices_map(i, col_index) = pqElem.second; - min_heap.pop(); - } } } +} + +// Wrapper over core TopK implementation +static Status TopKImpl(OpKernelContext* p_op_kernel_context, const Tensor* input, const int axis, const unsigned k, + bool largest = true, bool sorted = true) { + const TensorShape& input_shape = input->Shape(); + // Will return axis_ as is if positive or fixes it in case it is negative + const auto axis_parsed = HandleNegativeAxis(axis, static_cast(input_shape.NumDimensions())); + // Check to ensure k is within the bounds of what is available in that specific axis + if (input_shape[axis_parsed] < k) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "k argument [", k, + "] should not be greater than specified axis dim value [", input_shape[axis_parsed], "]"); + } + + // Resize output tensors to be the same shape as the input except + // for the specified dimension ((i.e.) axis_parsed), which will be of size k. E.x. for an input tensor + // of shape [3, 4, 5] and k=2 with axis_parsed=1, both of the outputs will be shape [3, 2, 5] + TensorShape output_shape = input_shape; + output_shape[axis_parsed] = k; + auto* values = p_op_kernel_context->Output(0, output_shape); + auto* indices = p_op_kernel_context->Output(1, output_shape); + + if (values == nullptr || indices == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "output count mismatch, expected 2 outputs to be present for TopK operator"); + } + + // no-op - no output buffers to fill - return silently + if (k == 0) { + return Status::OK(); + } + + if (sorted && largest) { + // extract sorted largest TopK elements + extract_top_k_elements>(input, input_shape, values, indices, output_shape, k, + gsl::narrow_cast(axis_parsed)); + } else if (sorted && !largest) { + // extract sorted smallest TopK elements + extract_top_k_elements>(input, input_shape, values, indices, output_shape, k, + gsl::narrow_cast(axis_parsed)); + } else if (largest) { + // extract unsorted (order undefined) largest TopK elements + extract_top_k_elements>(input, input_shape, values, indices, output_shape, k, + gsl::narrow_cast(axis_parsed)); + } else { + // extract unsorted (order undefined) smallest TopK elements + extract_top_k_elements>(input, input_shape, values, indices, output_shape, k, + gsl::narrow_cast(axis_parsed)); + } return Status::OK(); } @@ -152,8 +236,10 @@ TopK<9, float>::TopK(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_in template <> Status TopK<9, float>::Compute(OpKernelContext* p_op_kernel_context) const { const auto* X = p_op_kernel_context->Input(0); - if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, - "input count mismatch, expected 1 input - the tensor to be processed"); + if (X == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "input count mismatch, expected 1 input - the tensor to be processed"); + } + return TopKImpl(p_op_kernel_context, X, axis_, k_); } @@ -170,28 +256,83 @@ template <> Status TopK<10, float>::Compute(OpKernelContext* p_op_kernel_context) const { const auto* X = p_op_kernel_context->Input(0); const auto* Y = p_op_kernel_context->Input(1); - if (X == nullptr || Y == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, - "input count mismatch, expected 2 inputs - " - "the tensor to be processed and a tensor containing k value"); + if (X == nullptr || Y == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "input count mismatch, expected 2 inputs - " + "the tensor to be processed and a tensor containing k value"); + } + const vector& y_shape = Y->Shape().GetDims(); - if (y_shape.size() != 1 || y_shape[0] != 1) return Status(common::ONNXRUNTIME, common::FAIL, "k tensor should be a 1D tensor of size 1"); + if (y_shape.size() != 1 || y_shape[0] != 1) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "k tensor should be a 1D tensor of size 1"); + } + auto parsed_input_k = Y->template Data()[0]; - if (parsed_input_k < 0) return Status(common::ONNXRUNTIME, common::FAIL, "value of k must not be negative"); + if (parsed_input_k < 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "value of k must not be negative"); + } + return TopKImpl(p_op_kernel_context, X, axis_, gsl::narrow_cast(parsed_input_k)); } +// Opset ver - 11 +template <> +TopK<11, float>::TopK(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) { + int64_t axis_temp; + ORT_ENFORCE(op_kernel_info.GetAttr("axis", &axis_temp).IsOK()); + axis_ = gsl::narrow_cast(axis_temp); + + int64_t largest_temp; + ORT_ENFORCE(op_kernel_info.GetAttr("largest", &largest_temp).IsOK()); + largest_ = largest_temp == 1 ? true : false; + + int64_t sorted_temp; + ORT_ENFORCE(op_kernel_info.GetAttr("sorted", &sorted_temp).IsOK()); + sorted_ = sorted_temp == 1 ? true : false; +} + +// Opset ver - 11 +template <> +Status TopK<11, float>::Compute(OpKernelContext* p_op_kernel_context) const { + const auto* X = p_op_kernel_context->Input(0); + const auto* Y = p_op_kernel_context->Input(1); + if (X == nullptr || Y == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "input count mismatch, expected 2 inputs - " + "the tensor to be processed and a tensor containing k value"); + } + + const vector& y_shape = Y->Shape().GetDims(); + if (y_shape.size() != 1 || y_shape[0] != 1) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "k tensor should be a 1D tensor of size 1"); + } + + auto parsed_input_k = Y->template Data()[0]; + if (parsed_input_k < 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "value of k must not be negative"); + } + + return TopKImpl(p_op_kernel_context, X, axis_, gsl::narrow_cast(parsed_input_k), largest_, sorted_); +} + // Register necessary kernels // spec https://github.com/onnx/onnx/blob/master/docs/Operators.md#TopK -ONNX_CPU_OPERATOR_VERSIONED_KERNEL( - TopK, - 1, 9, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).TypeConstraint("I", DataTypeImpl::GetTensorType()), - TopK<9, float>); +ONNX_CPU_OPERATOR_VERSIONED_KERNEL(TopK, 1, 9, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("I", DataTypeImpl::GetTensorType()), + TopK<9, float>); -ONNX_CPU_OPERATOR_KERNEL( - TopK, - 10, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).TypeConstraint("I", DataTypeImpl::GetTensorType()), - TopK<10, float>); +ONNX_CPU_OPERATOR_VERSIONED_KERNEL(TopK, 10, 10, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("I", DataTypeImpl::GetTensorType()), + TopK<10, float>); -} // namespace onnxruntime \ No newline at end of file +ONNX_CPU_OPERATOR_KERNEL(TopK, 11, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("I", DataTypeImpl::GetTensorType()), + TopK<11, float>); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/top_k.h b/onnxruntime/core/providers/cpu/math/top_k.h index b973020be7..6cea1c7d6e 100644 --- a/onnxruntime/core/providers/cpu/math/top_k.h +++ b/onnxruntime/core/providers/cpu/math/top_k.h @@ -14,7 +14,9 @@ class TopK final : public OpKernel { Status Compute(OpKernelContext* p_op_kernel_context) const override; private: - int axis_; - unsigned k_; + int axis_; // used by all opset versions + unsigned k_; // opset-9 only + bool largest_; // opset-11 only + bool sorted_; // opset-11 only }; } // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 8036590aa1..f9efbf5e1b 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -38,11 +38,13 @@ void usage() { "\t-r [repeat]: Specifies the number of times to repeat\n" "\t-v: verbose\n" "\t-n [test_case_name]: Specifies a single test case to run.\n" - "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'mkldnn', 'tensorrt', 'ngraph', 'openvino' or 'nuphar'. " + "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'mkldnn', 'tensorrt', 'ngraph', " + "'openvino' or 'nuphar'. " "Default: 'cpu'.\n" "\t-x: Use parallel executor, default (without -x): sequential executor.\n" "\t-o [optimization level]: Default is 1. Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).\n" - "\t\tPlease see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels. \n" + "\t\tPlease see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels. " + "\n" "\t-h: help\n" "\n" "onnxruntime version: %s\n", @@ -291,7 +293,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) { return -1; #endif } - if (enable_ngraph) { //TODO: Re-order the priority? + if (enable_ngraph) { // TODO: Re-order the priority? #ifdef USE_NGRAPH ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_NGraph(sf, "CPU")); #else @@ -312,25 +314,26 @@ int real_main(int argc, char* argv[], Ort::Env& env) { sf.SetGraphOptimizationLevel(graph_optimization_level); } - std::unordered_set cuda_flaky_tests = { - "fp16_inception_v1", "fp16_shufflenet", "fp16_tiny_yolov2"}; + std::unordered_set cuda_flaky_tests = {"fp16_inception_v1", "fp16_shufflenet", "fp16_tiny_yolov2"}; #if (defined(_WIN32) && !defined(_WIN64)) || (defined(__GNUG__) && !defined(__LP64__)) - //Minimize mem consumption - LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance, [&stat, &sf, enable_cuda, &cuda_flaky_tests, &env](ITestCase* l) { - std::unique_ptr test_case_ptr(l); - if (enable_cuda && cuda_flaky_tests.find(l->GetTestCaseName()) != cuda_flaky_tests.end()) { - return; - } - TestResultStat per_case_stat; - std::vector per_case_tests = {l}; - TestEnv per_case_args(per_case_tests, per_case_stat, env, sf); - RunTests(per_case_args, 1, 1, 1, GetDefaultThreadPool(Env::Default())); - stat += per_case_stat; - }); + // Minimize mem consumption + LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance, + [&stat, &sf, enable_cuda, &cuda_flaky_tests, &env](ITestCase* l) { + std::unique_ptr test_case_ptr(l); + if (enable_cuda && cuda_flaky_tests.find(l->GetTestCaseName()) != cuda_flaky_tests.end()) { + return; + } + TestResultStat per_case_stat; + std::vector per_case_tests = {l}; + TestEnv per_case_args(per_case_tests, per_case_stat, env, sf); + RunTests(per_case_args, 1, 1, 1, GetDefaultThreadPool(Env::Default())); + stat += per_case_stat; + }); #else std::vector tests; - LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance, [&tests](ITestCase* l) { tests.push_back(l); }); + LoadTests(data_dirs, whitelisted_test_cases, per_sample_tolerance, relative_per_sample_tolerance, + [&tests](ITestCase* l) { tests.push_back(l); }); if (enable_cuda) { for (auto it = tests.begin(); it != tests.end();) { auto iter = cuda_flaky_tests.find((*it)->GetTestCaseName()); @@ -389,9 +392,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) { {"maxpool_with_argmax_2d_precomputed_strides", "ShapeInferenceError"}, {"tf_inception_v2", "result mismatch"}, {"mxnet_arcface", "result mismatch"}, - {"top_k", "not implemented yet for opset 11"}, - {"top_k_smallest", "not implemented yet for opset 11"}, - {"top_k_negative_axis", "TopK(11) not implemented yet"}, {"unique_not_sorted_without_axis", "Expected data for 'Y' is incorrect and in sorted order."}, {"cumsum_1d_reverse_exclusive", "only failing linux GPU CI. Likely build error."}, {"det_2d", "not implemented yet"}, @@ -545,7 +545,8 @@ int real_main(int argc, char* argv[], Ort::Env& env) { #endif #if defined(__GNUG__) && !defined(__LP64__) - broken_tests.insert({"nonzero_example", "failed: type mismatch", {"onnx123", "onnx130", "onnx141", "onnx150", "onnxtip"}}); + broken_tests.insert( + {"nonzero_example", "failed: type mismatch", {"onnx123", "onnx130", "onnx141", "onnx150", "onnxtip"}}); broken_tests.insert({"slice_neg_steps", "failed: type mismatch"}); broken_tests.insert({"mod_float_mixed_sign_example", "failed: type mismatch"}); #endif @@ -594,8 +595,8 @@ int real_main(int argc, char* argv[], Ort::Env& env) { for (const auto& p : stat.GetFailedTest()) { BrokenTest t = {p.first, ""}; auto iter = broken_tests.find(t); - if (iter == broken_tests.end() || - (p.second != TestModelInfo::unknown_version && !iter->broken_versions_.empty() && iter->broken_versions_.find(p.second) == iter->broken_versions_.end())) { + if (iter == broken_tests.end() || (p.second != TestModelInfo::unknown_version && !iter->broken_versions_.empty() && + iter->broken_versions_.find(p.second) == iter->broken_versions_.end())) { fprintf(stderr, "test %s failed, please fix it\n", p.first.c_str()); result = -1; } diff --git a/onnxruntime/test/providers/cpu/math/topk_op_test.cc b/onnxruntime/test/providers/cpu/math/topk_op_test.cc index 02aeb68284..26e49dab6c 100644 --- a/onnxruntime/test/providers/cpu/math/topk_op_test.cc +++ b/onnxruntime/test/providers/cpu/math/topk_op_test.cc @@ -17,6 +17,8 @@ static void RunTest(int op_set, const std::vector& expected_dimensions, bool is_tensorrt_supported = true, int64_t axis = -1, + int64_t largest = 1, + int64_t sorted = 1, OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, const std::string& expected_err_str = "") { OpTester test("TopK", op_set); @@ -26,20 +28,29 @@ static void RunTest(int op_set, test.AddAttribute("axis", axis); if (op_set <= 9) test.AddAttribute("k", k); + if (op_set == 11 && largest != 1) + test.AddAttribute("largest", largest); + if (op_set == 11 && sorted != 1) + test.AddAttribute("sorted", sorted); // Inputs test.AddInput("X", input_dimensions, input_vals); - if (op_set == 10) + if (op_set >= 10) test.AddInput("K", {1}, {k}); // Outputs - test.AddOutput("Values", expected_dimensions, expected_vals); - test.AddOutput("Indices", expected_dimensions, expected_indices); + if (sorted == 1) { + test.AddOutput("Values", expected_dimensions, expected_vals); + test.AddOutput("Indices", expected_dimensions, expected_indices); + } else { + test.AddOutput("Values", expected_dimensions, expected_vals, true); + test.AddOutput("Indices", expected_dimensions, expected_indices, true); + } // Run test and check results std::unordered_set excluded_providers; if (!is_tensorrt_supported) { - excluded_providers.insert(kTensorrtExecutionProvider);//Disable TensorRT because of unsupported data types + excluded_providers.insert(kTensorrtExecutionProvider); //Disable TensorRT because of unsupported data types } test.Run(expect_result, expected_err_str, excluded_providers); } @@ -155,113 +166,294 @@ TEST(TopKOperator, InvalidKOpset9) { expected_dimensions, true, 1, + 1, + 1, OpTester::ExpectResult::kExpectFailure, "Invalid value for attribute k"); } -TEST(TopKOperator, Top0DefaultAxisOpset10) { +static void top_0_default_axis(int opset_version, int64_t sorted = 1) { std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f}; std::vector input_dimensions = {2, 4}; std::vector expected_vals = {}; std::vector expected_indices = {}; - std::vector expected_dimensions = {2,0}; - RunTest(10, 0, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false); + std::vector expected_dimensions = {2, 0}; + RunTest(opset_version, 0, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, -1, 1, sorted); } -TEST(TopKOperator, Top1DefaultAxisOpset10) { +TEST(TopKOperator, Top0DefaultAxisLargestElements) { + top_0_default_axis(10); + top_0_default_axis(11); + top_0_default_axis(11, 0); // unsorted +} + +static void top_1_default_axis(int opset_version, int64_t sorted = 1) { std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f}; std::vector input_dimensions = {2, 4}; std::vector expected_vals = {0.4f, 0.3f}; std::vector expected_indices = {3, 1}; std::vector expected_dimensions = {2, 1}; - RunTest(10, 1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false); + int64_t axis = -1; + RunTest(opset_version, 1, input_vals, input_dimensions, expected_vals, expected_indices, + expected_dimensions, false, axis, 1, sorted); } -TEST(TopKOperator, Top2DefaultAxisOpset10) { +TEST(TopKOperator, Top1DefaultAxisLargestElements) { + top_1_default_axis(10); + top_1_default_axis(11); + top_1_default_axis(11, 0); // unsorted +} + +static void top_2_default_axis(int opset_version, int64_t sorted = 1) { std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.4f, 0.2f}; std::vector input_dimensions = {2, 4}; std::vector expected_vals = {0.4f, 0.3f, 0.4f, 0.3f}; std::vector expected_indices = {3, 1, 2, 1}; std::vector expected_dimensions = {2, 2}; - RunTest(10, 2, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false); + int64_t axis = -1; + RunTest(opset_version, 2, input_vals, input_dimensions, expected_vals, expected_indices, + expected_dimensions, false, axis, 1, sorted); } -TEST(TopKOperator, Top3DefaultAxisOpset10) { +TEST(TopKOperator, Top2DefaultAxisLargestElements) { + top_2_default_axis(10); + top_2_default_axis(11); + top_2_default_axis(11, 0); // unsorted +} + +static void top_3_default_axis(int opset_version, int64_t sorted = 1) { std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.4f, 0.2f}; std::vector input_dimensions = {2, 4}; std::vector expected_vals = {0.4f, 0.3f, 0.2f, 0.4f, 0.3f, 0.2f}; std::vector expected_indices = {3, 1, 2, 2, 1, 3}; std::vector expected_dimensions = {2, 3}; - RunTest(10, 3, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false); + int64_t axis = -1; + RunTest(opset_version, 3, input_vals, input_dimensions, expected_vals, expected_indices, + expected_dimensions, false, axis, 1, sorted); } -TEST(TopKOperator, TopAllDefaultAxisOpset10) { +TEST(TopKOperator, Top3DefaultAxisLargestElements) { + top_3_default_axis(10); + top_3_default_axis(11); + top_3_default_axis(11, 0); //unsorted +} + +static void top_all_default_axis(int opset_version, int64_t sorted = 1) { std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f}; std::vector input_dimensions = {2, 4}; std::vector expected_vals = {0.4f, 0.3f, 0.2f, 0.1f, 0.3f, 0.3f, 0.2f, 0.1f}; std::vector expected_indices = {3, 1, 2, 0, 1, 2, 3, 0}; std::vector expected_dimensions = {2, 4}; - RunTest(10, 4, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false); + int64_t axis = -1; + RunTest(opset_version, 4, input_vals, input_dimensions, expected_vals, expected_indices, + expected_dimensions, false, axis, 1, sorted); } -TEST(TopKOperator, Top1ExplicitAxisOpset10) { +TEST(TopKOperator, TopAllDefaultAxisLargestElements) { + top_all_default_axis(10); + top_all_default_axis(11); + top_all_default_axis(11, 0); // unsorted +} + +static void top_1_explicit_axis(int opset_version, int64_t sorted = 1) { std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f}; std::vector input_dimensions = {4, 2}; std::vector expected_vals = {0.3f, 0.4f}; std::vector expected_indices = {3, 1}; std::vector expected_dimensions = {1, 2}; int64_t axis = 0; - RunTest(10, 1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis); + RunTest(opset_version, 1, input_vals, input_dimensions, expected_vals, expected_indices, + expected_dimensions, false, axis, 1, sorted); } -TEST(TopKOperator, Top2ExplicitAxisOpset10) { +TEST(TopKOperator, Top1ExplicitAxisLargestElements) { + top_1_explicit_axis(10); + top_1_explicit_axis(11); + top_1_explicit_axis(11, 0); // unsorted +} + +static void top_2_explicit_axis(int opset_version, int64_t sorted = 1) { std::vector input_vals = {0.0f, 1.0f, 2.0f, 11.0f, 08.0f, 5.0f, 6.0f, 7.0f, 4.0f, 9.0f, 10.0f, 3.0f}; std::vector input_dimensions = {3, 4}; std::vector expected_vals = {8.0f, 9.0f, 10.0f, 11.0f, 4.0f, 5.0f, 6.0f, 7.0f}; std::vector expected_indices = {1, 2, 2, 0, 2, 1, 1, 1}; std::vector expected_dimensions = {2, 4}; int64_t axis = 0; - RunTest(10, 2, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis); + RunTest(opset_version, 2, input_vals, input_dimensions, expected_vals, expected_indices, + expected_dimensions, false, axis, 1, sorted); } -TEST(TopKOperator, Top3ExplicitAxisOpset10) { +TEST(TopKOperator, Top2ExplicitAxisLargestElements) { + top_2_explicit_axis(10); + top_2_explicit_axis(11); + top_2_explicit_axis(11, 0); //unsorted +} + +static void top_3_explicit_axis(int opset_version, int64_t sorted = 1) { std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f}; std::vector input_dimensions = {4, 2}; std::vector expected_vals = {0.3f, 0.4f, 0.2f, 0.3f, 0.1f, 0.3f}; std::vector expected_indices = {3, 1, 1, 0, 0, 2}; std::vector expected_dimensions = {3, 2}; int64_t axis = 0; - RunTest(10, 3, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis); + RunTest(opset_version, 3, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 1, sorted); } -TEST(TopKOperator, TopAllExplicitAxisOpset10) { +TEST(TopKOperator, Top3ExplicitAxisLargestElements) { + top_3_explicit_axis(10); + top_3_explicit_axis(11); + top_3_explicit_axis(11, 0); //unsorted +} + +static void top_all_explicit_axis(int opset_version, int64_t sorted = 1) { std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f}; std::vector input_dimensions = {4, 2}; std::vector expected_vals = {0.3f, 0.4f, 0.2f, 0.3f, 0.1f, 0.3f, 0.1f, 0.2f}; std::vector expected_indices = {3, 1, 1, 0, 0, 2, 2, 3}; std::vector expected_dimensions = {4, 2}; int64_t axis = 0; - RunTest(10, 4, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis); + RunTest(opset_version, 4, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 1, sorted); } -TEST(TopKOperator, TopAllExplicitAxis1DInputOpset10) { +TEST(TopKOperator, TopAllExplicitAxisLargestElements) { + top_all_explicit_axis(10); + top_all_explicit_axis(11); + top_all_explicit_axis(11, 0); // unsorted +} + +static void top_all_explicit_axis_1D_input(int opset_version, int64_t sorted = 1) { std::vector input_vals = {93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, 285.0f, 527.0f, 862.0f}; std::vector input_dimensions = {13}; std::vector expected_vals = {983.0f, 978.0f, 971.0f, 862.0f, 723.0f, 695.0f, 531.0f, 527.0f, 483.0f, 285.0f, 247.0f, 242.0f, 93.0f}; std::vector expected_indices = {7, 3, 2, 12, 9, 1, 8, 11, 4, 10, 5, 6, 0}; std::vector expected_dimensions = {13}; int64_t axis = 0; - RunTest(10, 13, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis); + RunTest(opset_version, 13, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 1, sorted); } -TEST(TopKOperator, Top1ExplicitAxisMultiDInputOpset10) { +TEST(TopKOperator, TopAllExplicitAxis1DInputLargestElements) { + top_all_explicit_axis_1D_input(10); + top_all_explicit_axis_1D_input(11); + top_all_explicit_axis_1D_input(11, 0); // unsorted +} + +static void top_2_explicit_axis_1D_large_input(int opset_version, int64_t sorted = 1) { + std::vector input_vals = {93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, + 93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, + 93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, + 93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, + 93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, + 93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, + 93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, + 93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, + 93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f, + 93.0f, 695.0f, 971.0f, 978.0f, 483.0f, 247.0f, 242.0f, 983.0f, 531.0f, 723.0f}; + + std::vector input_dimensions = {100}; + std::vector expected_vals = {983.0f, 983.0f}; + std::vector expected_indices = {7, 17}; + std::vector expected_dimensions = {2}; + int64_t axis = 0; + RunTest(opset_version, 2, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 1, sorted); +} + +TEST(TopKOperator, TopAllExplicitAxis1DLargeInputLargestElements) { + top_2_explicit_axis_1D_large_input(10); + top_2_explicit_axis_1D_large_input(11); + top_2_explicit_axis_1D_large_input(11, 0); // unsorted +} + +static void top_1_explicit_axis_MultiD_input(int opset_version, int64_t sorted = 1) { std::vector input_vals = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}; std::vector input_dimensions = {2, 2, 2}; std::vector expected_vals = {3, 4, 7, 8}; std::vector expected_indices = {1, 1, 1, 1}; std::vector expected_dimensions = {2, 1, 2}; int64_t axis = 1; - RunTest(10, 1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis); + RunTest(opset_version, 1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 1, sorted); +} + +TEST(TopKOperator, Top1ExplicitAxisMultiDInputLargestElements) { + top_1_explicit_axis_MultiD_input(10); + top_1_explicit_axis_MultiD_input(11); + top_1_explicit_axis_MultiD_input(11, 0); // unsorted +} + +static void top_2_default_axis_smallest(int opset_version, int64_t sorted = 1) { + std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.4f, 0.2f}; + std::vector input_dimensions = {2, 4}; + std::vector expected_vals = {0.1f, 0.2f, 0.1f, 0.2f}; + std::vector expected_indices = {0, 2, 0, 3}; + std::vector expected_dimensions = {2, 2}; + int64_t axis = -1; + RunTest(opset_version, 2, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 0, sorted); +} + +TEST(TopKOperator, Top2DefaultAxisSmallestElements) { + top_2_default_axis_smallest(11); + top_2_default_axis_smallest(11, 0); // unsorted +} + +static void top_3_explicit_axis_smallest(int opset_version, int64_t sorted = 1) { + std::vector input_vals = {0.1f, 0.3f, 0.2f, 0.4f, 0.1f, 0.3f, 0.3f, 0.2f}; + std::vector input_dimensions = {4, 2}; + std::vector expected_vals = {0.1f, 0.2f, 0.1f, 0.3f, 0.2f, 0.3f}; + std::vector expected_indices = {0, 3, 2, 0, 1, 2}; + std::vector expected_dimensions = {3, 2}; + int64_t axis = 0; + RunTest(opset_version, 3, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 0, sorted); +} + +TEST(TopKOperator, Top3ExplicitAxisSmallestElements) { + top_3_explicit_axis_smallest(11); + top_3_explicit_axis_smallest(11, 0); //unsorted +} + +static void top_1_explicit_axis_MultiD_input_smallest(int opset_version, int64_t sorted = 1) { + std::vector input_vals = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}; + std::vector input_dimensions = {2, 2, 2}; + std::vector expected_vals = {1, 2, 5, 6}; + std::vector expected_indices = {0, 0, 0, 0}; + std::vector expected_dimensions = {2, 1, 2}; + int64_t axis = 1; + RunTest(opset_version, 1, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 0, sorted); +} + +TEST(TopKOperator, Top1ExplicitAxisMultiDInputSmallestElements) { + top_1_explicit_axis_MultiD_input_smallest(11); + top_1_explicit_axis_MultiD_input_smallest(11, 0); //unsorted +} + +TEST(TopKOperator, SelectFirstSortNext) { + // in this test, we will select the top 5 elements first then sort the chosen 5 elements + // Select + Sort = O(n + k * ln(k)) = 50 + 5 * ln(5) = 58.047 + // Sorted selection: O(n * ln(k)) = 50 * ln(5) = 80.47 + // The algorithm used will be Select + Sort + std::vector input_vals = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0, + 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f, 17.0f, 18.0f, 19.0f, 20.0, + 21.0f, 22.0f, 23.0f, 24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0, + 31.0f, 32.0f, 33.0f, 34.0f, 35.0f, 36.0f, 37.0f, 38.0f, 39.0f, 40.0, + 41.0f, 42.0f, 43.0f, 44.0f, 45.0f, 46.0f, 47.0f, 48.0f, 49.0f, 50.0}; + std::vector input_dimensions = {50}; + std::vector expected_vals = {50.0f, 49.0f, 48.0f, 47.0f, 46.0f}; + std::vector expected_indices = {49, 48, 47, 46, 45}; + std::vector expected_dimensions = {5}; + int64_t axis = 0; + RunTest(11, 5, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis); // largest values +} + +TEST(TopKOperator, SortedSelection) { + // in this test, we will use sorted selection (using heap) + // Select + Sort = O(n + k * ln(k)) = 10 + 5 * ln(5) = 18.04 + // Sorted selection: O(n * ln(k)) = 10 * ln(5) = 16.09 + // The algorithm used will be Sorted selection + std::vector input_vals = {10.0f, 8.0f, 7.0f, 4.0f, 5.0f, 6.0f, 1.0f, 2.0f, 9.0f, 3.0}; + std::vector input_dimensions = {10}; + std::vector expected_vals = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f}; + std::vector expected_indices = {6, 7, 9, 3, 4}; + std::vector expected_dimensions = {5}; + int64_t axis = 0; + RunTest(11, 5, input_vals, input_dimensions, expected_vals, expected_indices, expected_dimensions, false, axis, 0); // smallest values } } // namespace test diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index cc37ec8620..7b5143cd4d 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -25,6 +25,18 @@ namespace onnxruntime { namespace test { // Check functions for tensor types +template +void sort_expected_and_actual_buffers(const T* expected, const T* actual, int64_t size) { + std::sort(const_cast(expected), const_cast(expected + size)); + std::sort(const_cast(actual), const_cast(actual + size)); +} + +// Check functions for tensor types +template +void sort_expected_and_actual_buffers(std::vector expected, std::vector actual) { + ORT_ENFORCE(expected.size() == actual.size(), "The 2 containers contain different number of elements"); + sort_expected_and_actual_buffers(expected.data(), actual.data(), expected.size()); +} // The default implementation compares for equality, specialized versions for other types are below template @@ -34,13 +46,21 @@ void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, con auto* output = output_tensor.template Data(); auto size = output_tensor.Shape().Size(); + if (expected_data.sort_output_) { + // if order can be jumbled in the output of an operator, sort both the expected and output buffers prior to + // comparison this is a "best-effort" algo and should satisfy the requirement for the few ops that do require this + // support without investing in a more sophisticated infrastructure for the same + sort_expected_and_actual_buffers(expected, output, size); + } + for (int i = 0; i < size; ++i) { EXPECT_EQ(expected[i], output[i]) << "i:" << i << ", provider_type: " << provider_type; } } template <> -void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { +void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, + const std::string& provider_type) { auto& expected_tensor = expected_data.data_.Get(); auto* expected = expected_tensor.template Data(); auto* output = output_tensor.template Data(); @@ -49,6 +69,11 @@ void Check(const OpTester::Data& expected_data, const Tensor& output_ten bool has_abs_err = expected_data.absolute_error_.has_value(); bool has_rel_err = expected_data.relative_error_.has_value(); + // deal with rare cases in which order of output data from a kernel MAY be undefined + if (expected_data.sort_output_) { + sort_expected_and_actual_buffers(expected, output, size); + } + double threshold = 0.001; #ifdef USE_CUDA threshold = 0.005; @@ -87,6 +112,11 @@ void Check(const OpTester::Data& expected_data, const Tensor& output_tens bool has_abs_err = expected_data.absolute_error_.has_value(); bool has_rel_err = expected_data.relative_error_.has_value(); + // deal with rare cases in which order of output data from a kernel MAY be undefined + if (expected_data.sort_output_) { + sort_expected_and_actual_buffers(expected, output, size); + } + float threshold = 0.001f; #ifdef USE_CUDA threshold = 0.005f; @@ -116,7 +146,8 @@ void Check(const OpTester::Data& expected_data, const Tensor& output_tens } template <> -void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { +void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, + const std::string& provider_type) { auto& expected_tensor = expected_data.data_.Get(); auto* expected = expected_tensor.template Data(); auto* output = output_tensor.template Data(); @@ -127,6 +158,11 @@ void Check(const OpTester::Data& expected_data, const Tensor& output_ ConvertMLFloat16ToFloat(expected, f_expected.data(), static_cast(size)); ConvertMLFloat16ToFloat(output, f_output.data(), static_cast(size)); + // deal with rare cases in which order of output data from a kernel MAY be undefined + if (expected_data.sort_output_) { + sort_expected_and_actual_buffers(f_expected, f_output); + } + float threshold = 0.001f; for (int i = 0; i < size; ++i) { if (std::isinf(f_expected[i])) // Test infinity for equality @@ -139,7 +175,8 @@ void Check(const OpTester::Data& expected_data, const Tensor& output_ } template <> -void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { +void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, + const std::string& provider_type) { auto& expected_tensor = expected_data.data_.Get(); auto* expected = expected_tensor.template Data(); auto* output = output_tensor.template Data(); @@ -150,6 +187,11 @@ void Check(const OpTester::Data& expected_data, const Tensor& output_t BFloat16ToFloat(expected, f_expected.data(), static_cast(size)); BFloat16ToFloat(output, f_output.data(), static_cast(size)); + // deal with rare cases in which order of output data from a kernel MAY be undefined + if (expected_data.sort_output_) { + sort_expected_and_actual_buffers(f_expected, f_output); + } + /// XXX: May need to adjust threshold as BFloat is coarse float threshold = 0.001f; for (int i = 0; i < size; ++i) { @@ -163,7 +205,8 @@ void Check(const OpTester::Data& expected_data, const Tensor& output_t } template -void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { +void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const Tensor& output_tensor, + const std::string& provider_type) { if (type == DataTypeImpl::GetType()) Check(expected_data, output_tensor, provider_type); else @@ -171,7 +214,8 @@ void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const T } template -void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { +void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const Tensor& output_tensor, + const std::string& provider_type) { if (type == DataTypeImpl::GetType()) Check(expected_data, output_tensor, provider_type); else @@ -181,10 +225,12 @@ void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, const T void Check(const OpTester::Data& expected_data, const Tensor& output_tensor, const std::string& provider_type) { ORT_ENFORCE(expected_data.data_.Get().Shape() == output_tensor.Shape(), "Expected output shape [" + expected_data.data_.Get().Shape().ToString() + - "] did not match run output shape [" + - output_tensor.Shape().ToString() + "] for " + expected_data.def_.Name()); + "] did not match run output shape [" + output_tensor.Shape().ToString() + "] for " + + expected_data.def_.Name()); - CheckDispatch(output_tensor.DataType(), expected_data, output_tensor, provider_type); + CheckDispatch(output_tensor.DataType(), expected_data, output_tensor, + provider_type); } // Check for non tensor types @@ -242,41 +288,32 @@ OpTester::~OpTester() { void OpTester::FillFeedsAndOutputNames(std::unordered_map& feeds, std::vector& output_names) { for (auto& output : output_data_) { - if (output.def_.Exists()) - output_names.push_back(output.def_.Name()); + if (output.def_.Exists()) output_names.push_back(output.def_.Name()); } for (size_t i = 0; i < input_data_.size(); ++i) { - if (std::find(initializer_index_.begin(), initializer_index_.end(), i) == initializer_index_.end() && input_data_[i].def_.Exists()) { + if (std::find(initializer_index_.begin(), initializer_index_.end(), i) == initializer_index_.end() && + input_data_[i].def_.Exists()) { feeds[input_data_[i].def_.Name()] = input_data_[i].data_; } } } void OpTester::SetOutputAbsErr(const char* name, float v) { - auto it = std::find_if( - output_data_.begin(), - output_data_.end(), - [name](Data& data) { - return (data.def_.Name() == name); - }); + auto it = std::find_if(output_data_.begin(), output_data_.end(), + [name](Data& data) { return (data.def_.Name() == name); }); ORT_ENFORCE(it != output_data_.end()); it->absolute_error_ = optional(v); } void OpTester::SetOutputRelErr(const char* name, float v) { - auto it = std::find_if( - output_data_.begin(), - output_data_.end(), - [name](Data& data) { - return (data.def_.Name() == name); - }); + auto it = std::find_if(output_data_.begin(), output_data_.end(), + [name](Data& data) { return (data.def_.Name() == name); }); ORT_ENFORCE(it != output_data_.end()); it->relative_error_ = optional(v); } -void OpTester::AddNodes(onnxruntime::Graph& graph, - std::vector& graph_input_defs, +void OpTester::AddNodes(onnxruntime::Graph& graph, std::vector& graph_input_defs, std::vector& graph_output_defs, std::vector>& add_attribute_funcs) { // default behavior is to create a single Node for the op being tested, with node inputs/outputs @@ -284,8 +321,7 @@ void OpTester::AddNodes(onnxruntime::Graph& graph, auto& node = graph.AddNode("node1", op_, op_, graph_input_defs, graph_output_defs, nullptr, domain_); // Add the attributes if any - for (auto& add_attribute_fn : add_attribute_funcs) - add_attribute_fn(node); + for (auto& add_attribute_fn : add_attribute_funcs) add_attribute_fn(node); } void OpTester::AddInitializers(onnxruntime::Graph& graph) { @@ -293,14 +329,14 @@ void OpTester::AddInitializers(onnxruntime::Graph& graph) { auto& data = input_data_[index]; auto& tensor = data.data_.Get(); ONNX_NAMESPACE::TensorProto tensor_proto; - //1. set dimension + // 1. set dimension auto& shape = tensor.Shape(); for (auto& dim : shape.GetDims()) { tensor_proto.add_dims(dim); } - //2. set type + // 2. set type tensor_proto.set_data_type(data.def_.TypeAsProto()->tensor_type().elem_type()); - //3. data + // 3. data if (data.def_.TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType_STRING) { const std::string* string_data = tensor.Data(); for (auto i = 0; i < shape.Size(); i++) { @@ -310,7 +346,7 @@ void OpTester::AddInitializers(onnxruntime::Graph& graph) { auto buffer_size = tensor.DataType()->Size() * shape.Size(); tensor_proto.set_raw_data(tensor.DataRaw(), buffer_size); } - //4. name + // 4. name tensor_proto.set_name(data.def_.Name()); graph.AddInitializedTensor(tensor_proto); } @@ -337,7 +373,7 @@ std::unique_ptr OpTester::BuildGraph() { onnxruntime::Graph& graph = p_model->MainGraph(); AddNodes(graph, node_input_defs, output_defs, add_attribute_funcs_); - //Add Initializer + // Add Initializer AddInitializers(graph); return p_model; } @@ -520,8 +556,8 @@ void OpTester::Run(const SessionOptions& so, EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(entry)).IsOK()); } - ExecuteModel(*p_model, session_object, expect_result, expected_failure_string, run_options, - feeds, output_names, provider_types); + ExecuteModel(*p_model, session_object, expect_result, expected_failure_string, run_options, feeds, output_names, + provider_types); } else { for (const std::string& provider_type : all_provider_types) { if (excluded_provider_types.count(provider_type) > 0) @@ -562,7 +598,7 @@ void OpTester::Run(const SessionOptions& so, if (node.OpType() == kConstant) continue; - //if node is not registered for the provider, skip + // if node is not registered for the provider, skip node.SetExecutionProviderType(provider_type); if (provider_type == onnxruntime::kNGraphExecutionProvider || provider_type == onnxruntime::kTensorrtExecutionProvider || @@ -592,8 +628,8 @@ void OpTester::Run(const SessionOptions& so, EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); - ExecuteModel(*p_model, session_object, expect_result, expected_failure_string, run_options, - feeds, output_names, provider_type); + ExecuteModel(*p_model, session_object, expect_result, expected_failure_string, run_options, feeds, + output_names, provider_type); } EXPECT_TRUE(has_run) << "No registered execution providers were able to run the model."; diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h index 27c84fcffc..984eb3a08a 100644 --- a/onnxruntime/test/providers/provider_test_utils.h +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -64,46 +64,74 @@ template constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType(); template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_FLOAT; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_FLOAT; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_INT32; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_INT32; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_INT64; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_INT64; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_BOOL; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_BOOL; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_INT8; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_INT8; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_INT16; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_INT16; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_UINT8; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_UINT8; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_UINT16; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_UINT16; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_UINT32; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_UINT32; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_UINT64; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_UINT64; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_STRING; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_STRING; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16; +} template <> -constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { return ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16; } +constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { + return ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16; +} template struct TTypeProto : ONNX_NAMESPACE::TypeProto { @@ -132,7 +160,7 @@ struct TTensorType { template const TTypeProto TTensorType::s_type_proto; -//TypeProto for map +// TypeProto for map template struct MTypeProto : ONNX_NAMESPACE::TypeProto { MTypeProto() { @@ -150,7 +178,7 @@ struct MMapType { template const MTypeProto MMapType::s_map_type_proto; -//TypeProto for vector> +// TypeProto for vector> template struct VectorOfMapTypeProto : ONNX_NAMESPACE::TypeProto { VectorOfMapTypeProto() { @@ -194,9 +222,9 @@ const SequenceTensorTypeProto SequenceTensorType::s_sequence // 3. Call AddInput for all the inputs // 4. Call AddOutput with all expected outputs // 5. Call Run -// Not all tensor types and output types are added, if a new input type is used, add it to the TypeToDataType list above -// for new output types, add a new specialization for Check<> -// See current usage for an example, should be self explanatory +// Not all tensor types and output types are added, if a new input type is used, add it to the TypeToDataType list +// above for new output types, add a new specialization for Check<> See current usage for an example, should be self +// explanatory class OpTester { public: explicit OpTester(const char* op, int opset_version = 7, const char* domain = onnxruntime::kOnnxDomain) @@ -217,12 +245,14 @@ class OpTester { // We have an initializer_list and vector version of the Add functions because std::vector is specialized for // bool and we can't get the raw data out. So those cases must use an initializer_list template - void AddInput(const char* name, const std::vector& dims, const std::initializer_list& values, bool is_initializer = false) { + void AddInput(const char* name, const std::vector& dims, const std::initializer_list& values, + bool is_initializer = false) { AddData(input_data_, name, dims, values.begin(), values.size(), is_initializer); } template - void AddInput(const char* name, const std::vector& dims, const std::vector& values, bool is_initializer = false) { + void AddInput(const char* name, const std::vector& dims, const std::vector& values, + bool is_initializer = false) { AddData(input_data_, name, dims, values.data(), values.size(), is_initializer); } @@ -235,7 +265,8 @@ class OpTester { OrtValue value; value.Init(ptr.get(), mltype, mltype->GetDeleteFunc()); ptr.release(); - input_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional(), optional())); + input_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional(), + optional())); } template @@ -246,7 +277,8 @@ class OpTester { OrtValue value; value.Init(ptr.get(), mltype, mltype->GetDeleteFunc()); ptr.release(); - input_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional(), optional())); + input_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional(), + optional())); } template @@ -286,32 +318,36 @@ class OpTester { void AddInput(const char* name, const std::map& val) { std::unique_ptr> ptr = onnxruntime::make_unique>(val); OrtValue value; - value.Init(ptr.release(), - DataTypeImpl::GetType>(), + value.Init(ptr.release(), DataTypeImpl::GetType>(), DataTypeImpl::GetType>()->GetDeleteFunc()); - input_data_.push_back(Data(NodeArg(name, &MMapType::s_map_type_proto), std::move(value), optional(), optional())); + input_data_.push_back(Data(NodeArg(name, &MMapType::s_map_type_proto), std::move(value), + optional(), optional())); } template void AddMissingOptionalInput() { std::string name; // empty == input doesn't exist - input_data_.push_back(Data(NodeArg(name, &TTensorType::s_type_proto), OrtValue(), optional(), optional())); + input_data_.push_back(Data(NodeArg(name, &TTensorType::s_type_proto), OrtValue(), optional(), + optional())); } template - void AddOutput(const char* name, const std::vector& dims, const std::initializer_list& expected_values) { - AddData(output_data_, name, dims, expected_values.begin(), expected_values.size()); + void AddOutput(const char* name, const std::vector& dims, const std::initializer_list& expected_values, + bool sort_output = false) { + AddData(output_data_, name, dims, expected_values.begin(), expected_values.size(), false, sort_output); } template - void AddOutput(const char* name, const std::vector& dims, const std::vector& expected_values) { - AddData(output_data_, name, dims, expected_values.data(), expected_values.size()); + void AddOutput(const char* name, const std::vector& dims, const std::vector& expected_values, + bool sort_output = false) { + AddData(output_data_, name, dims, expected_values.data(), expected_values.size(), false, sort_output); } template void AddMissingOptionalOutput() { std::string name; // empty == input doesn't exist - output_data_.push_back(Data(NodeArg(name, &TTensorType::s_type_proto), OrtValue(), optional(), optional())); + output_data_.push_back(Data(NodeArg(name, &TTensorType::s_type_proto), OrtValue(), optional(), + optional())); } // Add other registered types, possibly experimental @@ -323,7 +359,8 @@ class OpTester { OrtValue value; value.Init(ptr.get(), mltype, mltype->GetDeleteFunc()); ptr.release(); - output_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional(), optional())); + output_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional(), + optional())); } template @@ -334,7 +371,8 @@ class OpTester { OrtValue value; value.Init(ptr.get(), mltype, mltype->GetDeleteFunc()); ptr.release(); - output_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional(), optional())); + output_data_.push_back(Data(NodeArg(name, mltype->GetTypeProto()), std::move(value), optional(), + optional())); } // Add non tensor output @@ -342,10 +380,10 @@ class OpTester { void AddOutput(const char* name, const std::vector>& val) { auto ptr = onnxruntime::make_unique>>(val); OrtValue ml_value; - ml_value.Init(ptr.release(), - DataTypeImpl::GetType>>(), + ml_value.Init(ptr.release(), DataTypeImpl::GetType>>(), DataTypeImpl::GetType>>()->GetDeleteFunc()); - output_data_.push_back(Data(NodeArg(name, &VectorOfMapType::s_vec_map_type_proto), std::move(ml_value), optional(), optional())); + output_data_.push_back(Data(NodeArg(name, &VectorOfMapType::s_vec_map_type_proto), std::move(ml_value), + optional(), optional())); } void AddCustomOpRegistry(std::shared_ptr registry) { @@ -366,14 +404,13 @@ class OpTester { template void AddAttribute(std::string name, T value) { // Generate a the proper AddAttribute call for later - add_attribute_funcs_.emplace_back( - [name = std::move(name), value = std::move(value)](onnxruntime::Node& node) { node.AddAttribute(name, value); }); + add_attribute_funcs_.emplace_back([name = std::move(name), value = std::move(value)](onnxruntime::Node& node) { + node.AddAttribute(name, value); + }); } - enum class ExpectResult { - kExpectSuccess, - kExpectFailure - }; + enum class ExpectResult { kExpectSuccess, + kExpectFailure }; void Run(ExpectResult expect_result = ExpectResult::kExpectSuccess, const std::string& expected_failure_string = "", const std::unordered_set& excluded_provider_types = {}, @@ -393,14 +430,20 @@ class OpTester { OrtValue data_; optional relative_error_; optional absolute_error_; - Data(onnxruntime::NodeArg&& def, OrtValue&& data, optional&& rel, optional&& abs) : def_(std::move(def)), data_(std::move(data)), relative_error_(std::move(rel)), absolute_error_(abs) {} + bool sort_output_; + Data(onnxruntime::NodeArg&& def, OrtValue&& data, optional&& rel, optional&& abs, + bool sort_output = false) + : def_(std::move(def)), + data_(std::move(data)), + relative_error_(std::move(rel)), + absolute_error_(abs), + sort_output_(sort_output) {} Data(Data&&) = default; Data& operator=(Data&&) = default; }; protected: - virtual void AddNodes(onnxruntime::Graph& graph, - std::vector& graph_input_defs, + virtual void AddNodes(onnxruntime::Graph& graph, std::vector& graph_input_defs, std::vector& graph_output_defs, std::vector>& add_attribute_funcs); @@ -419,18 +462,15 @@ class OpTester { private: template - void AddData(std::vector& data, const char* name, - const std::vector& dims, const T* values, - int64_t values_count, bool is_initializer = false) { + void AddData(std::vector& data, const char* name, const std::vector& dims, const T* values, + int64_t values_count, bool is_initializer = false, bool sort_output = false) { try { TensorShape shape{dims}; - ORT_ENFORCE(shape.Size() == values_count, values_count, - " input values doesn't match tensor size of ", shape.Size()); + ORT_ENFORCE(shape.Size() == values_count, values_count, " input values doesn't match tensor size of ", + shape.Size()); auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU); - auto p_tensor = onnxruntime::make_unique(DataTypeImpl::GetType(), - shape, - allocator); + auto p_tensor = onnxruntime::make_unique(DataTypeImpl::GetType(), shape, allocator); auto* data_ptr = p_tensor->template MutableData(); for (int64_t i = 0; i < values_count; i++) { @@ -445,10 +485,10 @@ class OpTester { TTypeProto type_proto(add_shape_to_tensor_data_ ? &dims_for_proto : nullptr); OrtValue value; - value.Init(p_tensor.release(), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); - data.push_back(Data(NodeArg(name, &type_proto), std::move(value), optional(), optional())); - if (is_initializer) - initializer_index_.push_back(data.size() - 1); + value.Init(p_tensor.release(), DataTypeImpl::GetType(), + DataTypeImpl::GetType()->GetDeleteFunc()); + data.push_back(Data(NodeArg(name, &type_proto), std::move(value), optional(), optional(), sort_output)); + if (is_initializer) initializer_index_.push_back(data.size() - 1); } catch (const std::exception& ex) { std::cerr << "AddData for '" << name << "' threw: " << ex.what(); throw; @@ -499,7 +539,8 @@ inline void ConvertFloatToMLFloat16(const float* f_datat, MLFloat16* h_data, int #endif inline void ConvertMLFloat16ToFloat(const MLFloat16* h_data, float* f_data, int input_size) { - auto in_vector = ConstEigenVectorMap(static_cast(static_cast(h_data)), input_size); + auto in_vector = + ConstEigenVectorMap(static_cast(static_cast(h_data)), input_size); auto output_vector = EigenVectorMap(f_data, input_size); output_vector = in_vector.template cast(); }