From 33dd2f8f5e517f09871975f64db974acee3376b2 Mon Sep 17 00:00:00 2001 From: Chen Fu <1316708+chenfucn@users.noreply.github.com> Date: Tue, 18 Jan 2022 08:09:27 -0800 Subject: [PATCH 01/59] fix mac compilation error (#10268) Fix Mac compilation error in new cpuinfo changes --- onnxruntime/core/common/cpuid_info.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/common/cpuid_info.cc b/onnxruntime/core/common/cpuid_info.cc index 842e260c8b..b2590ccc3d 100644 --- a/onnxruntime/core/common/cpuid_info.cc +++ b/onnxruntime/core/common/cpuid_info.cc @@ -18,7 +18,7 @@ #endif #endif -#if defined(CPUIDINFO_ARCH_ARM) && defined(CPUINFO_SUPPORTED) +#if defined(CPUIDINFO_ARCH_ARM) && defined(CPUINFO_SUPPORTED) && defined(__linux__) #include #include // N.B. Support building with older versions of asm/hwcap.h that do not define @@ -109,7 +109,7 @@ CPUIDInfo::CPUIDInfo() { #endif #if defined(CPUIDINFO_ARCH_ARM) -#ifdef CPUINFO_SUPPORTED +#ifdef HWCAP_ASIMDDP // only works on ARM linux or android, does not work on Windows if (pytorch_cpuinfo_init_) { From 6ae22d562b0030190e1ee662118cbed49e9b7fdc Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Tue, 18 Jan 2022 12:54:58 -0800 Subject: [PATCH 02/59] [QDQ] Move NNAPI EP to use NodeUnitIODef for non-QDQ ops (#10237) --- .../nnapi/nnapi_builtin/builders/helper.cc | 207 ++-- .../nnapi/nnapi_builtin/builders/helper.h | 39 +- .../nnapi_builtin/builders/model_builder.cc | 175 ++-- .../nnapi_builtin/builders/model_builder.h | 98 +- .../nnapi_builtin/builders/op_builder.cc | 948 ++++++++---------- .../nnapi/nnapi_builtin/builders/op_builder.h | 11 +- .../builders/op_support_checker.cc | 461 ++++----- .../nnapi/nnapi_builtin/builders/shaper.cc | 23 +- .../nnapi/nnapi_builtin/builders/shaper.h | 161 ++- .../providers/nnapi/nnapi_builtin/model.cc | 4 +- .../providers/nnapi/nnapi_builtin/model.h | 15 +- .../nnapi_builtin/nnapi_execution_provider.cc | 10 - .../providers/shared/node_unit/node_unit.cc | 180 +++- .../providers/shared/node_unit/node_unit.h | 45 +- .../core/providers/shared/utils/utils.cc | 4 + .../core/providers/shared/utils/utils.h | 8 +- 16 files changed, 1202 insertions(+), 1187 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index 2fc1afd435..af93017649 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -5,22 +5,18 @@ #include #include -#include -#include -#include -#include -#include -#include +#include "helper.h" +#include "core/common/safeint.h" +#include "core/common/logging/logging.h" +#include "core/framework/tensorprotoutils.h" +#include "core/graph/graph.h" +#include "core/graph/graph_viewer.h" +#include "core/providers/common.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/shared/utils/utils.h" -#include "helper.h" #include "op_support_checker.h" -using onnxruntime::NodeUnit; -using std::string; -using std::vector; - namespace onnxruntime { namespace nnapi { @@ -72,16 +68,11 @@ QLinearOpType GetQLinearOpType(const onnxruntime::Node& node) { return QLinearOpType::Unknown; } -ConvType GetConvType(const onnxruntime::Node& node, const InitializedTensorSet& initializers) { - const auto& op_type = node.OpType(); - bool is_qlinear_conv = (op_type == "QLinearConv"); - ORT_ENFORCE(op_type == "Conv" || is_qlinear_conv); - - NodeAttrHelper helper(node); +ConvType GetConvType(const NodeUnit& node_unit, const InitializedTensorSet& initializers) { + NodeAttrHelper helper(node_unit); const auto group = helper.Get("group", 1); - size_t w_idx = is_qlinear_conv ? 3 : 1; - const auto& weight = node.InputDefs()[w_idx]->Name(); + const auto& weight = node_unit.Inputs()[1].node_arg.Name(); const auto& weight_tensor = *initializers.at(weight); // For ONNX we only have 1 conv ops @@ -104,13 +95,13 @@ bool IsQLinearBinaryOp(QLinearOpType qlinear_op_type) { qlinear_op_type == QLinearOpType::QLinearAdd; } -bool HasValidUnaryOpQuantizedInputs(const Node& node) { +bool HasValidUnaryOpQuantizedInputs(const NodeUnit& node_unit) { int32_t input_type; - if (!GetType(*node.InputDefs()[0], input_type)) + if (!GetType(node_unit.Inputs()[0].node_arg, input_type)) return false; if (input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { - LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() + LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() << "] Input type: [" << input_type << "] is not supported for now"; return false; @@ -119,18 +110,18 @@ bool HasValidUnaryOpQuantizedInputs(const Node& node) { return true; } -bool HasValidBinaryOpQuantizedInputs(const Node& node) { - auto op_type = GetQLinearOpType(node); +bool HasValidBinaryOpQuantizedInputs(const NodeUnit& node_unit) { + auto op_type = GetQLinearOpType(node_unit.GetNode()); int32_t a_input_type, b_input_type; if (!IsQLinearBinaryOp(op_type)) { - LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() << "] is not a binary qlinear op"; + LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() << "] is not a binary qlinear op"; return false; } - const auto input_defs(node.InputDefs()); - if (!GetType(*input_defs[0], a_input_type)) + const auto& inputs = node_unit.Inputs(); + if (!GetType(inputs[0].node_arg, a_input_type)) return false; - if (!GetType(*input_defs[3], b_input_type)) + if (!GetType(inputs[1].node_arg, b_input_type)) return false; // QlinearConv supports u8u8 or u8s8 @@ -143,7 +134,7 @@ bool HasValidBinaryOpQuantizedInputs(const Node& node) { if (a_input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8 || (!is_qlinear_conv && a_input_type != b_input_type) || (is_qlinear_conv && !has_valid_qlinear_conv_weight)) { - LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() + LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() << "] A Input type: [" << a_input_type << "] B Input type: [" << b_input_type << "] is not supported for now"; @@ -153,32 +144,41 @@ bool HasValidBinaryOpQuantizedInputs(const Node& node) { return true; } -bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const Node& node, - const std::vector& indices, const OpSupportCheckParams& params) { - const auto& op_type = node.OpType(); - auto qlinear_op_type = GetQLinearOpType(node); +bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const std::vector& indices, const OpSupportCheckParams& params, bool is_input) { + const auto& op_type = node_unit.OpType(); + auto qlinear_op_type = GetQLinearOpType(node_unit.GetNode()); bool is_qlinear_conv = (qlinear_op_type == QLinearOpType::QLinearConv); bool is_qlinear_matmul = (qlinear_op_type == QLinearOpType::QLinearMatMul); - const auto input_defs(node.InputDefs()); + const auto& io_defs = is_input ? node_unit.Inputs() : node_unit.Outputs(); for (const auto idx : indices) { - if (idx >= input_defs.size()) { - LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationScales, Input index, " << idx - << " >= input number, " << input_defs.size(); + if (idx >= io_defs.size()) { + LOGS_DEFAULT(VERBOSE) << (is_input ? "Input" : "Output") << " index, " << idx + << " >= size, " << io_defs.size() + << " of NodeUnit: " << node_unit.Name(); return false; } - const auto scale_name = input_defs[idx]->Name(); + const auto& io_def = io_defs[idx]; + if (!io_def.quant_param.has_value()) { + LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, Input index, " << idx + << " has no quant_param"; + return false; + } + + const auto scale_name = io_def.quant_param->scale.Name(); + if (!Contains(initializers, scale_name)) { LOGS_DEFAULT(VERBOSE) << "The scale of " << op_type << " must be an initializer tensor"; return false; } // If this op is Qlinear[Conv/MatMul], we want to check u8s8 support for weight tensor (or B tensor for QlinearMatMul) - bool is_conv_matmul_weight = (is_qlinear_conv || is_qlinear_matmul) && idx == 4; + bool is_conv_matmul_weight = is_input && (is_qlinear_conv || is_qlinear_matmul) && idx == 1; bool is_conv_matmul_u8s8_weight = false; if (is_conv_matmul_weight) { - const auto& weight_tensor = *initializers.at(node.InputDefs()[3]->Name()); + const auto& weight_tensor = *initializers.at(io_def.node_arg.Name()); is_conv_matmul_u8s8_weight = weight_tensor.data_type() == ONNX_NAMESPACE::TensorProto_DataType_INT8; } @@ -205,7 +205,7 @@ bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const return false; } - const auto& weight_tensor = *initializers.at(node.InputDefs()[3]->Name()); + const auto& weight_tensor = *initializers.at(io_def.node_arg.Name()); if (weight_tensor.dims()[0] != scales_dim) { LOGS_DEFAULT(VERBOSE) << op_type << " mismatch int8 per-channel quantization weight," << " weight dimension[0] " << weight_tensor.dims()[0] @@ -218,30 +218,44 @@ bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const return true; } -bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const Node& node, - const std::vector& indices) { - const auto& op_type = node.OpType(); - auto qlinear_op_type = GetQLinearOpType(node); +bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const std::vector& indices, bool is_input) { + const auto& op_type = node_unit.OpType(); + auto qlinear_op_type = GetQLinearOpType(node_unit.GetNode()); bool is_qlinear_conv = (qlinear_op_type == QLinearOpType::QLinearConv); bool is_qlinear_matmul = (qlinear_op_type == QLinearOpType::QLinearMatMul); - const auto input_defs(node.InputDefs()); + + const auto& io_defs = is_input ? node_unit.Inputs() : node_unit.Outputs(); for (const auto idx : indices) { - if (idx >= input_defs.size()) { - LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, Input index, " << idx - << " >= input number, " << input_defs.size(); + if (idx >= io_defs.size()) { + LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, " + << (is_input ? "Input" : "Output") << " index, " << idx + << " >= size, " << io_defs.size(); return false; } - const auto zero_point_name = input_defs[idx]->Name(); + const auto& io_def = io_defs[idx]; + if (!io_def.quant_param.has_value()) { + LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, Input index, " << idx + << " has no quant_param"; + return false; + } + + // zero point is optional here + if (!io_def.quant_param->zero_point) + return true; + + const auto& zero_point_name = io_def.quant_param->zero_point->Name(); if (!Contains(initializers, zero_point_name)) { LOGS_DEFAULT(VERBOSE) << "The zero point of " << op_type << " must be an initializer tensor"; return false; } - bool is_conv_matmul_weight = is_qlinear_conv && idx == 5; + bool is_conv_matmul_weight = is_input && (is_qlinear_conv || is_qlinear_matmul) && idx == 1; bool is_conv_matmul_u8s8_weight = false; + if (is_conv_matmul_weight) { - const auto& weight_tensor = *initializers.at(node.InputDefs()[3]->Name()); + const auto& weight_tensor = *initializers.at(io_def.node_arg.Name()); is_conv_matmul_u8s8_weight = weight_tensor.data_type() == ONNX_NAMESPACE::TensorProto_DataType_INT8; } @@ -275,7 +289,7 @@ bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, co // or a tensor with same channel as weight, for NNAPI we only support it be // 0 (scalar) or all 0 (tensor), NNAPI will assume the zero point for per-channel // quantization is 0 there is no input for it - const auto& weight_tensor = *initializers.at(node.InputDefs()[3]->Name()); + const auto& weight_tensor = *initializers.at(io_def.node_arg.Name()); if (weight_tensor.dims()[0] != zero_dim && zero_dim != 1) { LOGS_DEFAULT(VERBOSE) << op_type << " mismatch int8 per-channel quantization weight," << " weight dimension[0] " << weight_tensor.dims()[0] @@ -284,7 +298,7 @@ bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, co } std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(zero_tensor, node.ModelPath(), unpacked_tensor); + auto status = onnxruntime::utils::UnpackInitializerData(zero_tensor, node_unit.ModelPath(), unpacked_tensor); if (!status.IsOK()) { LOGS_DEFAULT(ERROR) << "Qlinear[Conv/MatMul] error when unpack zero tensor: " << zero_point_name << ", error msg: " << status.ErrorMessage(); @@ -306,33 +320,61 @@ bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, co return true; } -common::Status GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node, - size_t idx, float& scale) { - std::vector unpacked_tensor; - const auto& name = node.InputDefs()[idx]->Name(); - const auto& scale_tensor = *initializers.at(name); - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(scale_tensor, node.ModelPath(), unpacked_tensor)); +common::Status GetQuantizationScaleAndZeroPoint( + const InitializedTensorSet& initializers, const NodeUnitIODef& io_def, const Path& model_path, + float& scale, int32_t& zero_point) { + scale = 0.0f; + zero_point = 0; + + if (!io_def.quant_param) { // Not a quantized IO + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "NodeArg: ", io_def.node_arg.Name(), " is not quantized"); + } + + const auto unpack_tensor = [&model_path](const InitializedTensorSet& initializers, + const std::string& name, std::vector& unpacked_tensor) { + const auto& tensor = *initializers.at(name); + ORT_RETURN_IF_ERROR( + onnxruntime::utils::UnpackInitializerData(tensor, model_path, unpacked_tensor)); + return Status::OK(); + }; + + const auto& quant_param = *io_def.quant_param; + { // get the scale + std::vector unpacked_tensor; + const auto& name = quant_param.scale.Name(); + ORT_RETURN_IF_ERROR(unpack_tensor(initializers, name, unpacked_tensor)); + // The scale should be one or more floats + ORT_RETURN_IF(unpacked_tensor.size() < 4, + "The initializer [", name, "] should have one or more floats ", + "with size no less than 4, actual size: ", unpacked_tensor.size()); + scale = reinterpret_cast(unpacked_tensor.data())[0]; + } + + if (quant_param.zero_point) { // get the zero point if it's there + std::vector unpacked_tensor; + const auto& name = quant_param.zero_point->Name(); + ORT_RETURN_IF_ERROR(unpack_tensor(initializers, name, unpacked_tensor)); + ORT_RETURN_IF(unpacked_tensor.empty(), "The initializer [", name, "] is empty"); + // Onnx quantization uses uint8 [int8 not yet supported], need to cast to int32_t used by NNAPI + zero_point = static_cast(unpacked_tensor[0]); + } - // The scale should be one or more floats - ORT_RETURN_IF(unpacked_tensor.size() < 4, "The initializer [", name, "] should have one or more floats ", - "with size no less than 4, actual size: ", unpacked_tensor.size()); - scale = reinterpret_cast(unpacked_tensor.data())[0]; return Status::OK(); } -common::Status GetQuantizationZeroPoint(const InitializedTensorSet& initializers, - const Node& node, size_t idx, int32_t& zero_point) { - std::vector unpacked_tensor; - const auto& name = node.InputDefs()[idx]->Name(); - const auto& zero_point_tensor = *initializers.at(name); - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(zero_point_tensor, node.ModelPath(), unpacked_tensor)); +common::Status GetQuantizationScaleAndZeroPoint( + const InitializedTensorSet& initializers, const NodeUnit& node_unit, const std::string& name, + float& scale, int32_t& zero_point, bool is_input) { + const auto& io_defs = is_input ? node_unit.Inputs() : node_unit.Outputs(); + for (const auto& io_def : io_defs) { + if (io_def.node_arg.Name() == name) + return GetQuantizationScaleAndZeroPoint(initializers, io_def, node_unit.ModelPath(), + scale, zero_point); + } - ORT_RETURN_IF(unpacked_tensor.empty(), "The initializer [", name, "] is empty"); - // Onnx quantization uses uint8 [int8 not yet supported], need to cast to int32_t used by NNAPI - zero_point = static_cast(unpacked_tensor[0]); - return Status::OK(); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Unknown input: ", name, ", for NodeUnit with node index: ", node_unit.Index()); } bool GetShape(const NodeArg& node_arg, Shape& shape) { @@ -363,9 +405,9 @@ bool GetType(const NodeArg& node_arg, int32_t& type) { return true; } -void GetFlattenOutputShape(const Node& node, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2) { +void GetFlattenOutputShape(const NodeUnit& node_unit, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2) { int32_t rank = static_cast(input_shape.size()); - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); int32_t axis = helper.Get("axis", 1); // axis == rank is a valid input, but invalid for HandleNegativeAxis // Skip non-negative axis here @@ -491,10 +533,11 @@ std::string Shape2String(const std::vector& shape) { return os.str(); } -bool CheckIsInitializer(const InitializedTensorSet& initializers, const Node& node, - size_t input_idx, const char* input_name) { - if (!Contains(initializers, node.InputDefs()[input_idx]->Name())) { - LOGS_DEFAULT(VERBOSE) << input_name << " of " << node.OpType() << " must be an initializer tensor"; +bool CheckIsInitializer(const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const std::string& input_name, const char* input_description) { + if (!Contains(initializers, input_name)) { + LOGS_DEFAULT(VERBOSE) << input_description << " of " << node_unit.Name() << "of type [" + << node_unit.OpType() << "] must be an initializer tensor"; return false; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index d8d89269c9..c3729fb1c8 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -26,10 +26,13 @@ namespace onnxruntime { using Shape = std::vector; using InitializerMap = std::unordered_map; +class GraphViewer; class Node; class NodeArg; class NodeUnit; -class GraphViewer; +class Path; + +struct NodeUnitIODef; namespace nnapi { @@ -94,28 +97,32 @@ QLinearOpType GetQLinearOpType(const onnxruntime::Node& node); // Return the type of the conv ops, // This function assumes the input is a 2d conv node -ConvType GetConvType(const onnxruntime::Node& node, const InitializedTensorSet& initializers); +ConvType GetConvType(const NodeUnit& node_unit, const InitializedTensorSet& initializers); // This qlinear op is an operator takes 2 inputs and produces 1 output // Such as QLinearConv, QLinearMatMul, QLinearAdd, ... bool IsQLinearBinaryOp(QLinearOpType qlinear_op_type); // Check if a qlinear unary op has valid inputs, Qlinear[Sigmoid/AveragePool] -bool HasValidUnaryOpQuantizedInputs(const Node& node); +bool HasValidUnaryOpQuantizedInputs(const NodeUnit& node_unit); // Check if a qlinear binary op has valid inputs, Qlinear[Conv/MatMul/Add] -bool HasValidBinaryOpQuantizedInputs(const Node& node); +bool HasValidBinaryOpQuantizedInputs(const NodeUnit& node_unit); + // Check if a qlinear op has valid scales for given indices -bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const Node& node, - const std::vector& indices, const OpSupportCheckParams& params); +bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const std::vector& indices, const OpSupportCheckParams& params, bool is_input); + // Check if a qlinear op has valid zero points for given indices -bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const Node& node, - const std::vector& indices); +bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const std::vector& indices, bool is_input); -common::Status GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node, - size_t idx, float& scale); +common::Status GetQuantizationScaleAndZeroPoint( + const InitializedTensorSet& initializers, const NodeUnitIODef& io_def, const Path& model_path, + float& scale, int32_t& zero_point); -common::Status GetQuantizationZeroPoint(const InitializedTensorSet& initializers, - const Node& node, size_t idx, int32_t& zero_point) ORT_MUST_USE_RESULT; +common::Status GetQuantizationScaleAndZeroPoint( + const InitializedTensorSet& initializers, const NodeUnit& node_unit, const std::string& name, + float& scale, int32_t& zero_point, bool is_input = true); // Get Shape/Type of a NodeArg // TODO, move to shared_utils @@ -123,7 +130,7 @@ bool GetShape(const NodeArg& node_arg, Shape& shape); bool GetType(const NodeArg& node_arg, int32_t& type); // Get the output shape of Flatten Op -void GetFlattenOutputShape(const Node& node, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2); +void GetFlattenOutputShape(const NodeUnit& node_unit, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2); // If a node is supported by NNAPI bool IsNodeSupported(const NodeUnit& node_unit, const GraphViewer& graph_viewer, const OpSupportCheckParams& params); @@ -144,8 +151,10 @@ bool IsValidSupportedNodeGroup(const std::vector& supported_node_gr std::string Shape2String(const std::vector& shape); // Check the given input is an initializer tensor -bool CheckIsInitializer(const InitializedTensorSet& initializers, const Node& node, - size_t index, const char* input_name) ORT_MUST_USE_RESULT; +// input_name is the name of the initializer +// input_description is the string describing the input in the output message (if any) +bool CheckIsInitializer(const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const std::string& input_name, const char* input_description); } // namespace nnapi } // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 645ab23a85..fe6eade431 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -1,22 +1,23 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include -#include -#include +#include "model_builder.h" +#include "core/common/logging/logging.h" +#include "core/common/safeint.h" +#include "core/common/status.h" +#include "core/framework/tensorprotoutils.h" +#include "core/graph/graph_viewer.h" #include "core/providers/common.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/shared/utils/utils.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" + #include "helper.h" -#include "model_builder.h" #include "op_builder.h" #include "op_support_checker.h" -using onnxruntime::NodeUnit; using namespace android::nn::wrapper; -using std::vector; namespace onnxruntime { namespace nnapi { @@ -31,7 +32,7 @@ int32_t ModelBuilder::GetNNAPIFeatureLevel() const { // Scalar operand is copied into the model, no need to persist #define DEFINE_ADD_OPERAND_FROM_SCALAR(scalar_type, op_type) \ Status ModelBuilder::AddOperandFromScalar(scalar_type value, uint32_t& index) { \ - OperandType operandType(Type::op_type, vector{}); \ + OperandType operandType(Type::op_type, std::vector{}); \ ORT_RETURN_IF_ERROR(AddNewNNAPIOperand(operandType, index)); \ RETURN_STATUS_ON_ERROR_WITH_NOTE( \ nnapi_->ANeuralNetworksModel_setOperandValue( \ @@ -50,13 +51,12 @@ void ModelBuilder::AddInitializerToSkip(const std::string& tensor_name) { skipped_initializers_.insert(tensor_name); } -static std::unordered_map> GetAllQuantizedOpInputs(const GraphViewer& graph_viewer); - Status ModelBuilder::Prepare() { nnapi_model_ = std::unique_ptr(new Model()); RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksModel_create(&nnapi_model_->model_)); ORT_RETURN_IF_ERROR(GetTargetDevices()); - all_quantized_op_inputs_ = GetAllQuantizedOpInputs(graph_viewer_); + PreprocessNodeUnits(); + GetAllQuantizedOpInputs(); PreprocessInitializers(); PreprocessActivations(); ORT_RETURN_IF_ERROR(RegisterInitializers()); @@ -118,74 +118,87 @@ Status ModelBuilder::GetTargetDevices() { } void ModelBuilder::PreprocessInitializers() { - const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder(); - for (size_t i = 0; i < node_indices.size(); i++) { - const auto* node(graph_viewer_.GetNode(node_indices[i])); - if (const auto* op_builder = GetOpBuilder(*node)) { - const NodeUnit node_unit(*node); - op_builder->AddInitializersToSkip(*this, node_unit); + for (const auto& node_unit : node_unit_holder_) { + if (const auto* op_builder = GetOpBuilder(*node_unit)) { + op_builder->AddInitializersToSkip(*this, *node_unit); } } } void ModelBuilder::PreprocessActivations() { - const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder(); - for (size_t i = 0; i < node_indices.size(); i++) { - const auto* node(graph_viewer_.GetNode(node_indices[i])); - const auto& op_type(node->OpType()); - + for (const auto& node_unit : node_unit_holder_) { + const auto& node = node_unit->GetNode(); + const auto& op_type(node.OpType()); if (op_type == "Relu") { - activation_nodes_.emplace(node->Index(), ANEURALNETWORKS_FUSED_RELU); + activation_node_units_.emplace(node_unit.get(), ANEURALNETWORKS_FUSED_RELU); } else if (op_type == "Clip") { // Relu1 or Relu6 float min, max; - if (!GetClipMinMax(GetInitializerTensors(), *node, min, max, logging::LoggingManager::DefaultLogger())) + if (!GetClipMinMax(GetInitializerTensors(), node, min, max, logging::LoggingManager::DefaultLogger())) continue; if (min == -1.0f && max == 1.0f) { - activation_nodes_.emplace(node->Index(), ANEURALNETWORKS_FUSED_RELU1); + activation_node_units_.emplace(node_unit.get(), ANEURALNETWORKS_FUSED_RELU1); } else if (min == 0.0f && max == 6.0f) { - activation_nodes_.emplace(node->Index(), ANEURALNETWORKS_FUSED_RELU6); + activation_node_units_.emplace(node_unit.get(), ANEURALNETWORKS_FUSED_RELU6); } } } } -// Help to get all quantized operators' input and the node(s) using the input -static std::unordered_map> GetAllQuantizedOpInputs(const GraphViewer& graph_viewer) { - std::unordered_map> all_quantized_op_inputs; - const auto& node_indices = graph_viewer.GetNodesInTopologicalOrder(); - for (const auto& node_idx : node_indices) { - const auto* node(graph_viewer.GetNode(node_idx)); - auto qlinear_op_type = GetQLinearOpType(*node); +const NodeUnit& ModelBuilder::GetNodeUnit(const Node* node) const { + // In theory, if node_unit_map_ is generated correctly, see PreprocessNodeUnits(), a NodeUnit can be + // found for any single node in the graph_viewer_, unless the given node is not from graph_viewer_ + return *node_unit_map_.at(node); +} + +void ModelBuilder::PreprocessNodeUnits() { + // TODO, hookup shared QDQ selectors here to identify all the qdq NodeUnit in the graph + const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder(); + for (size_t i = 0; i < node_indices.size(); i++) { + const auto node_idx = node_indices[i]; + // TODO, check if the node is already part of a qdq group + const auto* node(graph_viewer_.GetNode(node_idx)); + auto node_unit = std::make_unique(*node); + node_unit_map_.insert({node, node_unit.get()}); + node_unit_holder_.push_back(std::move(node_unit)); + } +} + +// Help to get all quantized operators' input and the NodeUnit(s) using the input +void ModelBuilder::GetAllQuantizedOpInputs() { + for (const auto& node_unit : node_unit_holder_) { + // TODO, hookup getting quantized inputs with QDQ NodeUnits and remove the ORT_ENFORCE + ORT_ENFORCE(node_unit->UnitType() == NodeUnit::Type::SingleNode, "QDQ NodeUnit is not yet implemented"); + + auto qlinear_op_type = GetQLinearOpType(node_unit->GetNode()); // Not a qlinear op + // TODO, add handling for QDQ NodeUnit if (qlinear_op_type == QLinearOpType::Unknown) continue; + const auto add_quantized_input = + [&all_quantized_op_inputs = all_quantized_op_inputs_](const NodeUnit& node_unit, size_t input_idx) { + const auto& input_name = node_unit.Inputs()[input_idx].node_arg.Name(); + all_quantized_op_inputs[input_name].push_back(&node_unit); + }; + // All qlinear ops EXCEPT QuantizeLinear has quantized input if (qlinear_op_type != QLinearOpType::QuantizeLinear) { - const auto& input_name = node->InputDefs()[0]->Name(); - if (Contains(all_quantized_op_inputs, input_name)) - all_quantized_op_inputs.at(input_name).push_back(node); - else - all_quantized_op_inputs.emplace(input_name, vector{node}); + add_quantized_input(*node_unit, 0); } if (IsQLinearBinaryOp(qlinear_op_type)) { - const auto& input_name = node->InputDefs()[3]->Name(); - if (Contains(all_quantized_op_inputs, input_name)) - all_quantized_op_inputs.at(input_name).push_back(node); - else - all_quantized_op_inputs.emplace(input_name, vector{node}); + add_quantized_input(*node_unit, 1); } - } - return all_quantized_op_inputs; + // TODO, add handling for varidiac nodes such as QLinearConcat + } } static Status GetInputDataType( const InitializedTensorSet& initializers, - const std::unordered_map>& all_quantized_op_inputs, + const std::unordered_map>& all_quantized_op_inputs, const std::string& name, int32_t data_type, const Shape& shape, OperandType& operand_type) { Type type = Type::TENSOR_FLOAT32; @@ -208,10 +221,9 @@ static Status GetInputDataType( } // TODO, verify the scale and zero point match if there are multiple op using same input - const auto* node = all_quantized_op_inputs.at(name)[0]; - const NodeUnit node_unit(*node); - ORT_RETURN_IF_ERROR(GetQuantizedInputScaleAndZeroPoint( - initializers, node_unit, name, scale, zero_point)); + const auto* node_unit = all_quantized_op_inputs.at(name)[0]; + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + initializers, *node_unit, name, scale, zero_point, true /* is_input */)); break; } // case ONNX_NAMESPACE::TensorProto_DataType_INT8: @@ -491,15 +503,23 @@ Status ModelBuilder::AddOperandFromPersistMemoryBuffer( Status ModelBuilder::AddOperations() { const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder(); + std::unordered_set processed_node_units; for (size_t i = 0; i < node_indices.size(); i++) { const auto* node(graph_viewer_.GetNode(node_indices[i])); - if (const auto* op_builder = GetOpBuilder(*node)) { - const NodeUnit node_unit(*node); + const NodeUnit& node_unit = GetNodeUnit(node); + + // Since a NodeUnit may contain multiple nodes, avoid processing the same NodeUnit multiple times + if (Contains(processed_node_units, &node_unit)) + continue; + + if (const auto* op_builder = GetOpBuilder(node_unit)) { ORT_RETURN_IF_ERROR(op_builder->AddToModelBuilder(*this, node_unit)); } else { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "Node [", node->Name(), "], type [", node->OpType(), "] is not supported"); + "Node [", node_unit.Name(), "], type [", node_unit.OpType(), "] is not supported"); } + + processed_node_units.insert(&node_unit); } return Status::OK(); @@ -605,20 +625,40 @@ Status ModelBuilder::Compile(std::unique_ptr& model) { return Status::OK(); } -int32_t ModelBuilder::FindActivation(const Node& node, const NodeArg& output) { +int32_t ModelBuilder::FindActivation(const NodeUnit& node_unit) { int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE; + const auto& output_nodes = node_unit.GetOutputNodes(); + if (node_unit.GetOutputNodes().size() != 1) { + LOGS_DEFAULT(VERBOSE) << "FindActivation does not support, NodeUnit [" << node_unit.Name() + << "] type [" << node_unit.OpType() + << "], with " << output_nodes.size() << " output nodes"; + return fuse_code; + } + const auto& outputs = node_unit.Outputs(); + if (outputs.size() != 1) { + LOGS_DEFAULT(VERBOSE) << "FindActivation does not support, NodeUnit [" << node_unit.Name() + << "] type [" << node_unit.OpType() + << "], with " << outputs.size() << " outputs"; + return fuse_code; + } + + const NodeArg& output = outputs[0].node_arg; + const auto& output_node = *output_nodes[0]; + + // TODO, add support of activation fusion for quantized node group (qdq or qlinear) // We do not support activation fusion for quantized operators for now - auto qlinear_op_type = GetQLinearOpType(node); + auto qlinear_op_type = GetQLinearOpType(node_unit.GetNode()); if (qlinear_op_type != QLinearOpType::Unknown) return fuse_code; - for (auto it = node.OutputEdgesBegin(), end = node.OutputEdgesEnd(); it != end; ++it) { + for (auto it = output_node.OutputEdgesBegin(), end = output_node.OutputEdgesEnd(); it != end; ++it) { const auto& dst_node = it->GetNode(); const auto* dst_input = dst_node.InputDefs()[it->GetDstArgIndex()]; - if (Contains(activation_nodes_, dst_node.Index())) { + const auto& dst_node_unit = GetNodeUnit(&dst_node); + if (Contains(activation_node_units_, &dst_node_unit)) { if (&output == dst_input) { - fuse_code = activation_nodes_.at(dst_node.Index()); + fuse_code = activation_node_units_.at(&dst_node_unit); } } else { // if there is any other non-relu node using the output @@ -628,14 +668,14 @@ int32_t ModelBuilder::FindActivation(const Node& node, const NodeArg& output) { } } - // if output is a graph output, will add relu separately + // if output is a graph output, will add activation separately if (fuse_code != ANEURALNETWORKS_FUSED_NONE) { - for (const auto* graph_output : graph_viewer_.GetOutputs()) { - if (&output == graph_output) - return ANEURALNETWORKS_FUSED_NONE; + const auto& graph_outputs = graph_viewer_.GetOutputs(); + if (std::find(graph_outputs.cbegin(), graph_outputs.cend(), &output) != graph_outputs.cend()) { + return ANEURALNETWORKS_FUSED_NONE; } - LOGS_DEFAULT(VERBOSE) << "Node [" << node.Name() << "] type [" << node.OpType() + LOGS_DEFAULT(VERBOSE) << "Node [" << node_unit.Name() << "] type [" << node_unit.OpType() << "], fused the output [" << output.Name() << "]"; fused_activations_.insert(output.Name()); @@ -644,12 +684,13 @@ int32_t ModelBuilder::FindActivation(const Node& node, const NodeArg& output) { return fuse_code; } -/* static */ const IOpBuilder* ModelBuilder::GetOpBuilder(const Node& node) { +/* static */ const IOpBuilder* ModelBuilder::GetOpBuilder(const NodeUnit& node_unit) { const auto& op_builders = GetOpBuilders(); - if (!Contains(op_builders, node.OpType())) + const auto& op_type = node_unit.GetNode().OpType(); + if (!Contains(op_builders, op_type)) return nullptr; - return op_builders.at(node.OpType()); + return op_builders.at(op_type); } std::string ModelBuilder::GetUniqueName(const std::string& base_name) { @@ -663,6 +704,10 @@ std::string ModelBuilder::GetUniqueName(const std::string& base_name) { return unique_name; } +const InitializedTensorSet& ModelBuilder::GetInitializerTensors() const { + return graph_viewer_.GetAllInitializedTensors(); +} + void ModelBuilder::RegisterNHWCOperand(const std::string& name) { nhwc_operands_.insert(name); } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index d7dfd78ac0..2269c986f6 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -5,16 +5,22 @@ #include #include -#include +#include "core/graph/basic_types.h" #include "core/providers/nnapi/nnapi_builtin/model.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h" -#include "op_support_checker.h" #include "shaper.h" namespace onnxruntime { + +class GraphViewer; +class NodeUnit; +class Node; +class NodeArg; + namespace nnapi { class IOpBuilder; +class IOpSupportChecker; class ModelBuilder { public: @@ -33,30 +39,30 @@ class ModelBuilder { }; ModelBuilder(const GraphViewer& graph_viewer); - ~ModelBuilder() = default; - Status Compile(std::unique_ptr& model) ORT_MUST_USE_RESULT; + common::Status Compile(std::unique_ptr& model); int32_t GetNNAPIFeatureLevel() const; // Add an NNAPI operation (operator) - Status AddOperation(int op, const std::vector& input_indices, - const std::vector& output_names, - const std::vector& types, - const std::vector& is_nhwc_vec) ORT_MUST_USE_RESULT; + common::Status AddOperation(int op, const std::vector& input_indices, + const std::vector& output_names, + const std::vector& types, + const std::vector& is_nhwc_vec); - // Find if an output has a fuseable activation (Relu) - int32_t FindActivation(const Node& node, const NodeArg& output); + // Find if the given node_unit has a fuseable activation (Relu/Relu1/Relu6) + // For now we only support node_unit with a single output + int32_t FindActivation(const NodeUnit& node_unit); // Add an NNAPI scalar operand - Status AddOperandFromScalar(bool value, uint32_t& index) ORT_MUST_USE_RESULT; - Status AddOperandFromScalar(float value, uint32_t& index) ORT_MUST_USE_RESULT; - Status AddOperandFromScalar(int32_t value, uint32_t& index) ORT_MUST_USE_RESULT; + common::Status AddOperandFromScalar(bool value, uint32_t& index); + common::Status AddOperandFromScalar(float value, uint32_t& index); + common::Status AddOperandFromScalar(int32_t value, uint32_t& index); // Add an NNAPI tensor operand (and allocate persist buffer) - Status AddOperandFromPersistMemoryBuffer( + common::Status AddOperandFromPersistMemoryBuffer( const std::string& name, const void* buffer, - const android::nn::wrapper::OperandType& operand_type) ORT_MUST_USE_RESULT; + const android::nn::wrapper::OperandType& operand_type); // The initializer will be processed separately, skip it as an initializer void AddInitializerToSkip(const std::string& tensor_name); @@ -96,7 +102,7 @@ class ModelBuilder { const std::unordered_set& GetFusedActivations() const { return fused_activations_; } - const InitializedTensorSet& GetInitializerTensors() const { return graph_viewer_.GetAllInitializedTensors(); } + const InitializedTensorSet& GetInitializerTensors() const; const GraphViewer& GetGraphViewer() const { return graph_viewer_; } @@ -107,10 +113,13 @@ class ModelBuilder { bool GetNCHWOperand(const std::string& nhwc_name, std::string& nchw_name); bool GetNHWCOperand(const std::string& nchw_name, std::string& nhwc_name); - Status SetNHWCToNCHWOperandMap(const std::string& nhwc_name, - const std::string& nchw_name) ORT_MUST_USE_RESULT; - Status SetNCHWToNHWCOperandMap(const std::string& nchw_name, - const std::string& nhwc_name) ORT_MUST_USE_RESULT; + // Get the NodeUnit which contains the given node + const NodeUnit& GetNodeUnit(const Node* node) const; + + common::Status SetNHWCToNCHWOperandMap(const std::string& nhwc_name, + const std::string& nchw_name); + common::Status SetNCHWToNHWCOperandMap(const std::string& nchw_name, + const std::string& nhwc_name); private: const NnApi* nnapi_{nullptr}; @@ -134,8 +143,8 @@ class ModelBuilder { std::unordered_set skipped_initializers_; - // All activation nodes (Relu, Relu1, Relu6) as a map - std::unordered_map activation_nodes_; + // All activation nodes (Relu, Relu1, Relu6) as a map + std::unordered_map activation_node_units_; std::unordered_map> op_support_checkers_; @@ -149,9 +158,14 @@ class ModelBuilder { std::vector input_index_vec_; std::vector output_index_vec_; - // Contains all quantized operators' input and the node(s) using the input - // In the form of {input_name, [node(s) using the input]} - std::unordered_map> all_quantized_op_inputs_; + // Contains all quantized operators' input and the NodeUnit(s) using the input + // In the form of {input_name, [NodeUnit(s) using the input]} + std::unordered_map> all_quantized_op_inputs_; + + // Holder for the NodeUnits in the graph, this will guarantee the NodeUnits is + // valid throughout the lifetime of the ModelBuilder + std::vector> node_unit_holder_; + std::unordered_map node_unit_map_; std::unordered_set unique_names_; @@ -164,32 +178,38 @@ class ModelBuilder { uint32_t next_index_ = 0; // Convert the onnx model to ANeuralNetworksModel - Status Prepare() ORT_MUST_USE_RESULT; + common::Status Prepare(); - Status GetTargetDevices() ORT_MUST_USE_RESULT; + common::Status GetTargetDevices(); // If a NNAPI operation will use initializers directly, we will add the initializers to the skip list void PreprocessInitializers(); // Preprocess all the activation nodes (Relu/Relu1/Relu6) for easy query later void PreprocessActivations(); // Copy and process all the initializers to NNAPI model - Status RegisterInitializers() ORT_MUST_USE_RESULT; - Status RegisterModelInputs() ORT_MUST_USE_RESULT; - Status AddOperations() ORT_MUST_USE_RESULT; - Status RegisterModelOutputs() ORT_MUST_USE_RESULT; + common::Status RegisterInitializers(); + common::Status RegisterModelInputs(); + common::Status AddOperations(); + common::Status RegisterModelOutputs(); // After constructing the NNAPI model, will set the shape inferencing record to the Model void RegisterModelShaper(); - Status SetOperandValue(uint32_t index, Model::NNMemory* memory, - size_t size, size_t offset) ORT_MUST_USE_RESULT; + // Get all quantized inputs in the underlying graph_viewer + void GetAllQuantizedOpInputs(); - Status AddNewNNAPIOperand(const android::nn::wrapper::OperandType& type, uint32_t& index) ORT_MUST_USE_RESULT; - Status AddNewOperand(const std::string& name, - const android::nn::wrapper::OperandType& operand_type, - bool is_nhwc, - uint32_t& index) ORT_MUST_USE_RESULT; + // Go through the underlying graph_viewer, and generate NodeUnits, Many initializing functions are + // using the result of PreprocessNodeUnits, this need to run early in the Prepare() + void PreprocessNodeUnits(); - static const IOpBuilder* GetOpBuilder(const Node& node); + common::Status SetOperandValue(uint32_t index, Model::NNMemory* memory, size_t size, size_t offset); + + common::Status AddNewNNAPIOperand(const android::nn::wrapper::OperandType& type, uint32_t& index); + common::Status AddNewOperand(const std::string& name, + const android::nn::wrapper::OperandType& operand_type, + bool is_nhwc, + uint32_t& index); + + static const IOpBuilder* GetOpBuilder(const NodeUnit& node_unit); }; } // namespace nnapi diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 66f870df15..bff260cb67 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -3,12 +3,13 @@ #include "op_builder.h" -#include -#include -#include -#include #include +#include "core/common/logging/logging.h" +#include "core/common/safeint.h" +#include "core/framework/tensorprotoutils.h" +#include "core/graph/graph_viewer.h" +#include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/cpu/tensor/slice_helper.h" @@ -16,9 +17,7 @@ #include "model_builder.h" #include "op_support_checker.h" -using onnxruntime::NodeUnit; using namespace android::nn::wrapper; -using std::vector; namespace onnxruntime { namespace nnapi { @@ -40,13 +39,7 @@ struct OpBuilderRegistrations { Status AddTransposeOperator(ModelBuilder& model_builder, const std::string& input, const std::string& perm_name, - vector perm, - const std::string& output, - bool output_is_nhwc) ORT_MUST_USE_RESULT; -Status AddTransposeOperator(ModelBuilder& model_builder, - const std::string& input, - const std::string& perm_name, - vector perm, + std::vector perm, const std::string& output, bool output_is_nhwc) { auto& shaper(model_builder.GetShaper()); @@ -69,10 +62,6 @@ Status AddTransposeOperator(ModelBuilder& model_builder, {output_operand_type}, {output_is_nhwc}); } -Status TransposeBetweenNCHWAndNHWC(ModelBuilder& model_builder, - const std::string& input, - const std::string& output, - bool nchw_to_nhwc) ORT_MUST_USE_RESULT; Status TransposeBetweenNCHWAndNHWC(ModelBuilder& model_builder, const std::string& input, const std::string& output, @@ -83,7 +72,7 @@ Status TransposeBetweenNCHWAndNHWC(ModelBuilder& model_builder, "TransposeBetweenNCHWAndNHWC input has to be a 4d tensor, actual dimensions: ", shaper[input].size()); std::string perm_name; - vector perm; + std::vector perm; if (nchw_to_nhwc) { perm_name = model_builder.GetUniqueName(input + "nchw_to_nhwc_perm"); perm = {0, 2, 3, 1}; @@ -110,18 +99,12 @@ Status TransposeBetweenNCHWAndNHWC(ModelBuilder& model_builder, return Status::OK(); } -Status TransposeNHWCToNCHW(ModelBuilder& model_builder, - const std::string& input, - const std::string& output) ORT_MUST_USE_RESULT; Status TransposeNHWCToNCHW(ModelBuilder& model_builder, const std::string& input, const std::string& output) { return TransposeBetweenNCHWAndNHWC(model_builder, input, output, false /* nchw_to_nhwc */); } -Status TransposeNCHWToNHWC(ModelBuilder& model_builder, - const std::string& input, - const std::string& output) ORT_MUST_USE_RESULT; Status TransposeNCHWToNHWC(ModelBuilder& model_builder, const std::string& input, const std::string& output) { @@ -130,22 +113,22 @@ Status TransposeNCHWToNHWC(ModelBuilder& model_builder, // Convert the input from nchw to nhwc // Caller should ensure input is currently in nchw format using ModelBuilder::IsOperandNHWC -Status GetNHWCInput(ModelBuilder& model_builder, const Node& node, size_t input_index, std::string& input) { - const auto& nchw_input = node.InputDefs()[input_index]->Name(); - if (!model_builder.GetNHWCOperand(nchw_input, input)) { - input = model_builder.GetUniqueName(nchw_input + "_nchw_to_nhwc"); - ORT_RETURN_IF_ERROR(TransposeNCHWToNHWC(model_builder, nchw_input, input)); +Status GetNHWCInput(ModelBuilder& model_builder, const NodeUnit& node_unit, size_t input_index, std::string& nhwc_input) { + const auto& nchw_input = node_unit.Inputs()[input_index].node_arg.Name(); + if (!model_builder.GetNHWCOperand(nchw_input, nhwc_input)) { + nhwc_input = model_builder.GetUniqueName(nchw_input + "_nchw_to_nhwc"); + ORT_RETURN_IF_ERROR(TransposeNCHWToNHWC(model_builder, nchw_input, nhwc_input)); } return Status::OK(); } // Convert the input from nhwc to nchw // Caller should ensure input is currently in nhwc format using ModelBuilder::IsOperandNHWC -Status GetNCHWInput(ModelBuilder& model_builder, const Node& node, size_t input_index, std::string& input) { - const auto& nhwc_input = node.InputDefs()[input_index]->Name(); - if (!model_builder.GetNCHWOperand(nhwc_input, input)) { - input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); - ORT_RETURN_IF_ERROR(TransposeNHWCToNCHW(model_builder, nhwc_input, input)); +Status GetNCHWInput(ModelBuilder& model_builder, const NodeUnit& node_unit, size_t input_index, std::string& nchw_input) { + const auto& nhwc_input = node_unit.Inputs()[input_index].node_arg.Name(); + if (!model_builder.GetNCHWOperand(nhwc_input, nchw_input)) { + nchw_input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); + ORT_RETURN_IF_ERROR(TransposeNHWCToNCHW(model_builder, nhwc_input, nchw_input)); } return Status::OK(); } @@ -154,12 +137,7 @@ Status GetNCHWInput(ModelBuilder& model_builder, const Node& node, size_t input_ // and return the layout type of output tensor // If both inputs have same layout, the output will have the same layout // Otherwise we will need transpose the nhwc input back to nchw, and output will be nchw -Status TransposeBinaryOpInputLayout(ModelBuilder& model_builder, const Node& node, - size_t input1_idx, size_t input2_idx, - std::string& input1, std::string& input2, - bool& output_is_nhwc) ORT_MUST_USE_RESULT; -Status TransposeBinaryOpInputLayout(ModelBuilder& model_builder, const Node& node, - size_t input1_idx, size_t input2_idx, +Status TransposeBinaryOpInputLayout(ModelBuilder& model_builder, const NodeUnit& node_unit, std::string& input1, std::string& input2, bool& output_is_nhwc) { bool input1_is_nhwc = model_builder.IsOperandNHWC(input1); @@ -170,10 +148,10 @@ Status TransposeBinaryOpInputLayout(ModelBuilder& model_builder, const Node& nod output_is_nhwc = input1_is_nhwc; } else if (input1_is_nhwc) { // need transpose input1 back to nchw - ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, input1_idx, input1)); + ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node_unit, 0, input1)); } else { // input2_is_nhwc // need transpose input2 back to nchw - ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, input2_idx, input2)); + ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node_unit, 1, input2)); } return Status::OK(); @@ -188,17 +166,7 @@ static Status AddBinaryOperator(int32_t op_type, const std::string& output, bool output_is_nhwc, float output_scale = 0.0f, - int32_t output_zero_point = 0) ORT_MUST_USE_RESULT; -static Status AddBinaryOperator(int32_t op_type, - ModelBuilder& model_builder, - const std::string& input1, - const std::string& input2, - bool add_activation, - int32_t fuse_code, - const std::string& output, - bool output_is_nhwc, - float output_scale, - int32_t output_zero_point) { + int32_t output_zero_point = 0) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); @@ -222,11 +190,7 @@ static Status AddBinaryOperator(int32_t op_type, static Status AddSqueezeOp(ModelBuilder& model_builder, const std::string& node_name, const std::string& input, const std::string& output, - vector axes) ORT_MUST_USE_RESULT; -static Status AddSqueezeOp(ModelBuilder& model_builder, - const std::string& node_name, - const std::string& input, const std::string& output, - vector axes) { + std::vector axes) { if (model_builder.GetNNAPIFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_2) { return ORT_MAKE_STATUS( ONNXRUNTIME, FAIL, "Squeeze is not supported on API level ", model_builder.GetNNAPIFeatureLevel()); @@ -283,11 +247,6 @@ enum DataLayout { // since NNAPI requires X and W to be same type for per-tensor quantization, // the initializer tensor W will be converted from int8 to uint8 by flip each byte by XOR 0x80 // byte ^ 0x80 == byte + 128 -static Status AddInitializerInNewLayout(ModelBuilder& model_builder, - const std::string& name, - const OperandType& source_operand_type, - DataLayout new_layout, - bool is_per_tensor_u8s8) ORT_MUST_USE_RESULT; static Status AddInitializerInNewLayout(ModelBuilder& model_builder, const std::string& name, const OperandType& source_operand_type, @@ -373,10 +332,6 @@ static Status AddInitializerInNewLayout(ModelBuilder& model_builder, // and input B is signed int8), in this case, since NNAPI requires A and B to be same type, // the initializer tensor B will be converted from int8 to uint8 by flip each byte by XOR 0x80 // byte ^ 0x80 == byte + 128 -static Status AddInitializerTransposed(ModelBuilder& model_builder, - const OperandType& source_operand_type, - const std::string& name, - bool is_per_tensor_u8s8) ORT_MUST_USE_RESULT; static Status AddInitializerTransposed(ModelBuilder& model_builder, const OperandType& source_operand_type, const std::string& name, @@ -430,13 +385,7 @@ static Status ComputeConvPads( const uint32_t weight_size_y, const uint32_t weight_size_x, const std::vector& onnx_pads, const std::vector& onnx_strides, const std::vector& onnx_dilations, AutoPadType auto_pad_type, bool nchw, - vector& pads_out) ORT_MUST_USE_RESULT; -static Status ComputeConvPads( - const Shape& input_dimen, - const uint32_t weight_size_y, const uint32_t weight_size_x, - const std::vector& onnx_pads, const std::vector& onnx_strides, const std::vector& onnx_dilations, - AutoPadType auto_pad_type, bool nchw, - vector& pads_out) { + std::vector& pads_out) { const int32_t input_size_y = nchw ? input_dimen[2] : input_dimen[1]; const int32_t input_size_x = nchw ? input_dimen[3] : input_dimen[2]; const int32_t stride_y = onnx_strides[0]; @@ -467,21 +416,11 @@ static Status ComputeConvPads( static Status HandleAutoPad(const Shape& input_shape, const uint32_t weight_size_y, const uint32_t weight_size_x, - const vector& onnx_strides, - const vector& onnx_dilations, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, AutoPadType auto_pad_type, bool use_nchw, - vector& onnx_pads, - int32_t& nnapi_padding_code, - bool& use_auto_pad) ORT_MUST_USE_RESULT; -static Status HandleAutoPad(const Shape& input_shape, - const uint32_t weight_size_y, - const uint32_t weight_size_x, - const vector& onnx_strides, - const vector& onnx_dilations, - AutoPadType auto_pad_type, - bool use_nchw, - vector& onnx_pads, + std::vector& onnx_pads, int32_t& nnapi_padding_code, bool& use_auto_pad) { use_auto_pad = false; @@ -498,7 +437,7 @@ static Status HandleAutoPad(const Shape& input_shape, } } else if (onnx_dilations == std::vector{1, 1}) { // Since NNAPI runs more efficiently using auto_pad, we try to map the NOTSET padding to auto_pad - vector same_upper_pads; + std::vector same_upper_pads; ORT_RETURN_IF_ERROR(ComputeConvPads(input_shape, weight_size_y, weight_size_x, onnx_pads, onnx_strides, onnx_dilations, AutoPadType::SAME_UPPER, use_nchw, @@ -516,20 +455,15 @@ static Status HandleAutoPad(const Shape& input_shape, // QLinearConv, QLinearMatmul, QLinearAdd // a, b are inputs, and y is output static Status GetBinaryOpQuantizationScaleAndZeroPoint( - const ModelBuilder& model_builder, const Node& node, - float& a_scale, float& b_scale, float& y_scale, - int32_t& a_zero_point, int32_t& b_zero_point, int32_t& y_zero_point) ORT_MUST_USE_RESULT; -static Status GetBinaryOpQuantizationScaleAndZeroPoint( - const ModelBuilder& model_builder, const Node& node, + const InitializedTensorSet& initializers, const NodeUnit& node_unit, float& a_scale, float& b_scale, float& y_scale, int32_t& a_zero_point, int32_t& b_zero_point, int32_t& y_zero_point) { - const auto& initializers = model_builder.GetInitializerTensors(); - ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 1, a_scale)); - ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 4, b_scale)); - ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 6, y_scale)); - ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 2, a_zero_point)); - ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 5, b_zero_point)); - ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 7, y_zero_point)); + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + initializers, node_unit.Inputs()[0], node_unit.ModelPath(), a_scale, a_zero_point)); + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + initializers, node_unit.Inputs()[1], node_unit.ModelPath(), b_scale, b_zero_point)); + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + initializers, node_unit.Outputs()[0], node_unit.ModelPath(), y_scale, y_zero_point)); return Status::OK(); } @@ -544,26 +478,21 @@ static Status GetBinaryOpQuantizationScaleAndZeroPoint( // will be convert to uint8 later, will return the same scale and 128 as zero point // Also will set is_per_tensor_u8s8 to true to be used later static Status GetConvMatMulOpQuantizationScaleAndZeroPoint( - const ModelBuilder& model_builder, const Node& node, + const ModelBuilder& model_builder, const NodeUnit& node_unit, float& a_scale, float& w_scale, float& y_scale, int32_t& a_zero_point, int32_t& w_zero_point, int32_t& y_zero_point, - optional>& w_scales, bool& is_per_tensor_u8s8) ORT_MUST_USE_RESULT; -static Status GetConvMatMulOpQuantizationScaleAndZeroPoint( - const ModelBuilder& model_builder, const Node& node, - float& a_scale, float& w_scale, float& y_scale, - int32_t& a_zero_point, int32_t& w_zero_point, int32_t& y_zero_point, - optional>& w_scales, bool& is_per_tensor_u8s8) { + optional>& w_scales, bool& is_per_tensor_u8s8) { is_per_tensor_u8s8 = false; + const auto& initializers(model_builder.GetInitializerTensors()); // Get scale and zero points // We will handle per-channel weight scale and zero point later ORT_RETURN_IF_ERROR( - GetBinaryOpQuantizationScaleAndZeroPoint(model_builder, node, + GetBinaryOpQuantizationScaleAndZeroPoint(initializers, node_unit, a_scale, w_scale, y_scale, a_zero_point, w_zero_point, y_zero_point)); - const auto input_defs = node.InputDefs(); - const auto& initializers(model_builder.GetInitializerTensors()); - const auto& weight_tensor = *initializers.at(input_defs[3]->Name()); + const auto& inputs = node_unit.Inputs(); + const auto& weight_tensor = *initializers.at(inputs[1].node_arg.Name()); // We are done here is this is u8u8 QLinearConv if (weight_tensor.data_type() == ONNX_NAMESPACE::TensorProto_DataType_UINT8) @@ -574,7 +503,7 @@ static Status GetConvMatMulOpQuantizationScaleAndZeroPoint( // For this case we will need to convert the int8 weight tensor to uint8 // And have same scale and 128 as zero point // The conversion of the weight tensor itself will be done in the OpBuilder - const auto& scale_tensor = *initializers.at(input_defs[4]->Name()); + const auto& scale_tensor = *initializers.at(inputs[1].quant_param->scale.Name()); int64_t scale_dim = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0]; if (scale_dim == 1) { w_zero_point = 128; @@ -593,7 +522,7 @@ static Status GetConvMatMulOpQuantizationScaleAndZeroPoint( ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_tensor)); const float* scales = reinterpret_cast(unpacked_tensor.data()); const size_t scales_size = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0]; - vector scales_vec(scales, scales + scales_size); + std::vector scales_vec(scales, scales + scales_size); w_scales = onnxruntime::make_optional(std::move(scales_vec)); return Status::OK(); } @@ -601,10 +530,6 @@ static Status GetConvMatMulOpQuantizationScaleAndZeroPoint( // NNAPI has the quantization scale and zero point embedded in the ANeuralNetworksOperandType // ONNX has the quantization scale and zero point as the inputs of the qlinear operators // We want to verify the scale and zeropoint of the ONNX inputs matches the values embedded in the NNAPI inputs -static Status IsValidInputQuantizedType(const ModelBuilder& model_builder, - const std::string& input_name, - float scale, - int32_t zero_point) ORT_MUST_USE_RESULT; static Status IsValidInputQuantizedType(const ModelBuilder& model_builder, const std::string& input_name, float scale, @@ -631,12 +556,7 @@ static Status IsValidConvWeightQuantizedType(const ModelBuilder& model_builder, const std::string& input_name, float scale, int32_t zero_point, - const optional>& scales) ORT_MUST_USE_RESULT; -static Status IsValidConvWeightQuantizedType(const ModelBuilder& model_builder, - const std::string& input_name, - float scale, - int32_t zero_point, - const optional>& scales) { + const optional>& scales) { // first verify as the weight has no per-channel quantization ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input_name, scale, zero_point)); @@ -656,57 +576,23 @@ static Status IsValidConvWeightQuantizedType(const ModelBuilder& model_builder, return Status::OK(); } -static void AddBinaryOpQuantizationScaleAndZeroPointToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) { - const auto& node = node_unit.GetNode(); - const auto input_defs(node.InputDefs()); - model_builder.AddInitializerToSkip(input_defs[1]->Name()); // a_scale - model_builder.AddInitializerToSkip(input_defs[2]->Name()); // a_zero_point - model_builder.AddInitializerToSkip(input_defs[4]->Name()); // b_scale - model_builder.AddInitializerToSkip(input_defs[5]->Name()); // b_zero_point - model_builder.AddInitializerToSkip(input_defs[6]->Name()); // y_scale - model_builder.AddInitializerToSkip(input_defs[7]->Name()); // y_zero_point +static void AddQuantizationScaleAndZeroPointToSkip(ModelBuilder& model_builder, + const NodeUnitIODef::QuantParam& quant_param) { + // If we reach here, we assume the io_def has quant_param + model_builder.AddInitializerToSkip(quant_param.scale.Name()); // scale + LOGS_DEFAULT(VERBOSE) << quant_param.scale.Name() << "is skipped"; + if (quant_param.zero_point) { + model_builder.AddInitializerToSkip(quant_param.zero_point->Name()); // zero_point + LOGS_DEFAULT(VERBOSE) << quant_param.zero_point->Name() << "is skipped"; + } } -Status GetQuantizedInputScaleAndZeroPoint(const InitializedTensorSet& initializers, - const NodeUnit& node_unit, - const std::string& input_name, - float& scale, - int32_t& zero_point) { - const auto& node = node_unit.GetNode(); - const auto& op_type = node.OpType(); - auto qlinear_op_type = GetQLinearOpType(node); - assert(qlinear_op_type != QLinearOpType::Unknown && - qlinear_op_type != QLinearOpType::QuantizeLinear); - - size_t scale_idx, zero_point_idx; - if (qlinear_op_type == QLinearOpType::DequantizeLinear || - qlinear_op_type == QLinearOpType::QLinearSigmoid || - qlinear_op_type == QLinearOpType::QLinearAveragePool) { - scale_idx = 1; - zero_point_idx = 2; - } else if (IsQLinearBinaryOp(qlinear_op_type)) { - const auto input_defs(node.InputDefs()); - if (input_name == input_defs[0]->Name()) { - scale_idx = 1; - zero_point_idx = 2; - } else if (input_name == input_defs[3]->Name()) { - scale_idx = 4; - zero_point_idx = 5; - } else { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "Unknown input: ", input_name, ", for op: ", op_type); - } - } else { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Unsupported op: ", op_type); - } - - ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, scale_idx, scale)); - zero_point = 0; - if (node.InputDefs().size() > zero_point_idx) { - ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, zero_point_idx, zero_point)); - } - - return Status::OK(); +// Ignore the input (with quantization scale and ZP if available) +// The input (usually weight) is already embedded in the NNAPI model +static void AddInputToSkip(ModelBuilder& model_builder, const NodeUnitIODef& io_def) { + model_builder.AddInitializerToSkip(io_def.node_arg.Name()); // main input + if (io_def.quant_param) + AddQuantizationScaleAndZeroPointToSkip(model_builder, *io_def.quant_param); } template @@ -731,20 +617,24 @@ class BaseOpBuilder : public IOpBuilder { public: virtual ~BaseOpBuilder() = default; virtual void AddInitializersToSkip(ModelBuilder& /* model_builder */, const NodeUnit& /* node_unit */) const override {} - Status AddToModelBuilder(ModelBuilder& model_builder, const NodeUnit& node_unit) const override final ORT_MUST_USE_RESULT; + Status AddToModelBuilder(ModelBuilder& model_builder, const NodeUnit& node_unit) const override final; protected: - virtual Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const ORT_MUST_USE_RESULT = 0; + virtual Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const = 0; + static bool IsOpSupported(const ModelBuilder& model_builder, const NodeUnit& node_unit) ORT_MUST_USE_RESULT; }; -Status BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, const NodeUnit& node_unit) const { +/* static */ bool BaseOpBuilder::IsOpSupported(const ModelBuilder& model_builder, const NodeUnit& node_unit) { OpSupportCheckParams params{ model_builder.GetNNAPIFeatureLevel(), model_builder.UseNCHW(), }; - ORT_RETURN_IF_NOT(IsNodeSupported(node_unit, model_builder.GetGraphViewer(), params), - "Unsupported operator ", node_unit.OpType()); + return IsNodeSupported(node_unit, model_builder.GetGraphViewer(), params); +} + +Status BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, const NodeUnit& node_unit) const { + ORT_RETURN_IF_NOT(IsOpSupported(model_builder, node_unit), "Unsupported operator ", node_unit.OpType()); ORT_RETURN_IF_ERROR(AddToModelBuilderImpl(model_builder, node_unit)); LOGS_DEFAULT(VERBOSE) << "Operator name: [" << node_unit.Name() << "] type: [" << node_unit.OpType() << "] was added"; @@ -761,14 +651,23 @@ class BinaryOpBuilder : public BaseOpBuilder { static void CreateSharedOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations); private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + static bool IsQuantizedOp(const NodeUnit& node_unit) ORT_MUST_USE_RESULT; // TODO, see if we want to move this to BaseOpBuilder + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; +/* static */ bool BinaryOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) { + // TODO, add support for QDQ NodeUnit + return node_unit.OpType() == "QLinearAdd"; +} + void BinaryOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& op = node_unit.OpType(); - if (op == "QLinearAdd") { - AddBinaryOpQuantizationScaleAndZeroPointToSkip(model_builder, node_unit); - } + if (!IsQuantizedOp(node_unit)) + return; + + const auto& inputs = node_unit.Inputs(); + AddQuantizationScaleAndZeroPointToSkip(model_builder, *inputs[0].quant_param); // a_scale, a_zp + AddQuantizationScaleAndZeroPointToSkip(model_builder, *inputs[1].quant_param); // b_scale, b_zp + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Outputs()[0].quant_param); // y_scale, y_zp } /* static */ void BinaryOpBuilder::CreateSharedOpBuilder( @@ -786,9 +685,8 @@ void BinaryOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const N } Status BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - const auto& op_type(node.OpType()); - const auto input_defs(node.InputDefs()); + const auto& op_type(node_unit.OpType()); + const auto& inputs = node_unit.Inputs(); int32_t op_code; bool add_activation = true; @@ -808,18 +706,13 @@ Status BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "UnaryOpBuilder, unknown op: ", op_type); } - size_t a_idx = 0, b_idx = 1; - if (op_is_qlinear) { - b_idx = 3; - } - - std::string input1 = input_defs[a_idx]->Name(); - std::string input2 = input_defs[b_idx]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + std::string input1 = inputs[0].node_arg.Name(); + std::string input2 = inputs[1].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = false; ORT_RETURN_IF_ERROR( - TransposeBinaryOpInputLayout(model_builder, node, a_idx, b_idx, input1, input2, output_is_nhwc)); + TransposeBinaryOpInputLayout(model_builder, node_unit, input1, input2, output_is_nhwc)); float a_scale = 0.0f, b_scale = 0.0f, @@ -829,9 +722,10 @@ Status BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const y_zero_point = 0; if (op_is_qlinear) { - ORT_RETURN_IF_ERROR(GetBinaryOpQuantizationScaleAndZeroPoint(model_builder, node, - a_scale, b_scale, y_scale, - a_zero_point, b_zero_point, y_zero_point)); + ORT_RETURN_IF_ERROR(GetBinaryOpQuantizationScaleAndZeroPoint( + model_builder.GetInitializerTensors(), node_unit, + a_scale, b_scale, y_scale, + a_zero_point, b_zero_point, y_zero_point)); } // Verify if the scale and zero point matchs from onnx input and nnapi input match @@ -842,7 +736,7 @@ Status BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE; if (add_activation) { - fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); + fuse_code = model_builder.FindActivation(node_unit); } return AddBinaryOperator(op_code, model_builder, @@ -857,24 +751,23 @@ Status BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const class ReluOpBuilder : public BaseOpBuilder { private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; Status ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& input = node.InputDefs()[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input = node_unit.Inputs()[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); // skip this relu if it is some op's fuse output if (Contains(model_builder.GetFusedActivations(), input)) { - LOGS_DEFAULT(VERBOSE) << "Relu Node [" << node.Name() << "] fused"; + LOGS_DEFAULT(VERBOSE) << "Relu Node [" << node_unit.Name() << "] fused"; model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, output_is_nhwc); } else { std::vector input_indices; @@ -892,17 +785,16 @@ Status ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N class TransposeOpBuilder : public BaseOpBuilder { private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); - auto input = node.InputDefs()[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); - NodeAttrHelper helper(node); - vector perm = helper.Get("perm", vector()); + const auto& input = node_unit.Inputs()[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); + NodeAttrHelper helper(node_unit); + std::vector perm = helper.Get("perm", std::vector()); auto input_dims = shaper[input].size(); if (perm.empty()) { for (int32_t i = input_dims - 1; i >= 0; i--) @@ -920,7 +812,7 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co perm[i] = axis_nchw_to_nhwc[perm[i]]; } - std::string perm_name = model_builder.GetUniqueName(node.Name() + input + "perm"); + std::string perm_name = model_builder.GetUniqueName(node_unit.Name() + input + "perm"); // It is possible this onnx transpose operator can be nchw->nhwc, but so far I don't see // any scenario will do this since onnx is nchw only, assume the output is always not nhwc @@ -938,17 +830,17 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co class ReshapeOpBuilder : public BaseOpBuilder { public: void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; - static Status AddReshapeOperator(ModelBuilder& model_builder, const Node& node, - const std::string& input, const std::vector& shape) ORT_MUST_USE_RESULT; + static Status AddReshapeOperator(ModelBuilder& model_builder, const NodeUnit& node_unit, + const std::string& input, const std::vector& shape); private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; - static bool CanSkipReshape(const ModelBuilder& model_builder, const Node& node, size_t input_rank, size_t output_rank); + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; + static bool CanSkipReshape(const ModelBuilder& model_builder, const NodeUnit& node_unit, + size_t input_rank, size_t output_rank); }; void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); + model_builder.AddInitializerToSkip(node_unit.Inputs()[1].node_arg.Name()); } // We can skip the Reshape if all the output edges satisfies both the following conditions @@ -963,25 +855,34 @@ void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const // between NNAPI CPU impl and Hardware Accelerator impl and will speed up the execution // If we are going to skip the reshape, we will still add correct shape and operand type for the output in // onnxruntime::nnapi::Model. -/* static */ bool ReshapeOpBuilder::CanSkipReshape(const ModelBuilder& model_builder, const Node& node, +/* static */ bool ReshapeOpBuilder::CanSkipReshape(const ModelBuilder& model_builder, const NodeUnit& node_unit, size_t input_rank, size_t output_rank) { - const auto& output = node.OutputDefs()[0]->Name(); + const auto& output_node_arg = node_unit.Outputs()[0].node_arg; + const auto& output_name = output_node_arg.Name(); + const auto& output_node = *node_unit.GetOutputNodes()[0]; + // We will go through all the output edges - for (auto it = node.OutputEdgesBegin(), end = node.OutputEdgesEnd(); it != end; ++it) { - const auto& op_type = it->GetNode().OpType(); + for (auto it = output_node.OutputEdgesBegin(), end = output_node.OutputEdgesEnd(); it != end; ++it) { + const auto& dest_node_unit = model_builder.GetNodeUnit(&it->GetNode()); + const auto& op_type = dest_node_unit.OpType(); // TODO add quantized matmul when reshape support quantized input if (op_type != "Gemm" && op_type != "MatMul") { LOGS_DEFAULT(VERBOSE) << "Reshape/Flatten can only be skipped when the output is Gemm/Matmul" << " or no op is using the output (output is graph output)" - << ", output name, " << output + << ", output name, " << output_name << " is used by " << op_type; return false; } + // Now the dest node is Gemm/Matmul, we want to make sure it is supported + if (!BaseOpBuilder::IsOpSupported(model_builder, node_unit)) { + return false; + } + // NNAPI ANEURALNETWORKS_FULLY_CONNECTED will only flatten the input 0 - if (it->GetDstArgIndex() != 0) { + if (&output_node_arg != &dest_node_unit.Inputs()[0].node_arg) { LOGS_DEFAULT(VERBOSE) << "Reshape/Flatten can only be skipped when the output is input 0 of Gemm/Matmul" - << ", output name, " << output; + << ", output name, " << output_name; return false; } @@ -989,7 +890,7 @@ void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const // And NNAPI ANEURALNETWORKS_FULLY_CONNECTED will only flatten input rank >= 2 if (input_rank < 2 || output_rank != 2) { LOGS_DEFAULT(VERBOSE) << "Reshape/Flatten can only be skipped when input_rank >= 2 and output_rank == 2" - << ", output name, " << output + << ", output name, " << output_name << ", the actual input_rank, " << input_rank << ", the actual output_rank, " << output_rank; return false; @@ -1000,26 +901,26 @@ void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const // Check if the Reshape output is a graph output, if so we cannot skip the Reshape // We do not care the case where the Reshape output is a dead end for (const auto* node_arg : model_builder.GetGraphViewer().GetOutputs()) { - if (node_arg->Name() == output) { + if (node_arg == &output_node_arg) { LOGS_DEFAULT(VERBOSE) << "Reshape/Flatten can not be skipped when the output is a graph output" - << ", output name, " << output; + << ", output name, " << output_name; return false; } } LOGS_DEFAULT(VERBOSE) << "Skipping Reshape/Flatten node [" - << node.Name() << "] with output, " << output; + << node_unit.Name() << "] with output, " << output_name; return true; } /* static */ Status ReshapeOpBuilder::AddReshapeOperator(ModelBuilder& model_builder, - const Node& node, + const NodeUnit& node_unit, const std::string& input, const std::vector& shape) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); ORT_RETURN_IF_ERROR(shaper.Reshape(input, shape, output)); auto input_rank = shaper[input].size(); auto output_rank = shaper[output].size(); @@ -1027,7 +928,7 @@ void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const // Since Reshape is not running using hardware in NNAPI for some CPU (e.g. Qualcomm SD for now) // We will try to see if we the skip the Reshape to prevent context switching between // NNAPI CPU impl and NNAPI hardware accelerator impl - if (CanSkipReshape(model_builder, node, input_rank, output_rank)) { + if (CanSkipReshape(model_builder, node_unit, input_rank, output_rank)) { // Since reshape can be skipped, only register the dimension and type, with same index and new name const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, false); @@ -1038,7 +939,7 @@ void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const input_indices.push_back(operand_indices.at(input)); // Add new shape Shape shape_dimen = {static_cast(shape.size())}; - std::string shape_name = model_builder.GetUniqueName(node.Name() + input + "newshape"); + std::string shape_name = model_builder.GetUniqueName(node_unit.Name() + input + "newshape"); OperandType shape_operand_type(Type::TENSOR_INT32, shape_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(shape_name, shape.data(), shape_operand_type)); input_indices.push_back(operand_indices.at(shape_name)); @@ -1051,17 +952,16 @@ void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const } Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& initializers(model_builder.GetInitializerTensors()); - auto input = node.InputDefs()[0]->Name(); + auto input = node_unit.Inputs()[0].node_arg.Name(); if (model_builder.IsOperandNHWC(input)) { // We want to transpose nhwc operand back to nchw before reshape - ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, 0, input)); + ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node_unit, 0, input)); } - const auto& shape_tensor = *initializers.at(node.InputDefs()[1]->Name()); + const auto& shape_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name()); std::vector unpacked_tensor; ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(shape_tensor, unpacked_tensor)); const int64_t* raw_shape = reinterpret_cast(unpacked_tensor.data()); @@ -1075,7 +975,7 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons shape[i] = dim == 0 ? input_shape[i] : dim; } - return AddReshapeOperator(model_builder, node, input, shape); + return AddReshapeOperator(model_builder, node_unit, input, shape); } #pragma endregion op_reshape @@ -1087,38 +987,37 @@ class BatchNormalizationOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); // skip everything except input0 for BatchNormalization - model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); // scale - model_builder.AddInitializerToSkip(node.InputDefs()[2]->Name()); // B - model_builder.AddInitializerToSkip(node.InputDefs()[3]->Name()); // mean - model_builder.AddInitializerToSkip(node.InputDefs()[4]->Name()); //var + model_builder.AddInitializerToSkip(node_unit.Inputs()[1].node_arg.Name()); // scale + model_builder.AddInitializerToSkip(node_unit.Inputs()[2].node_arg.Name()); // B + model_builder.AddInitializerToSkip(node_unit.Inputs()[3].node_arg.Name()); // mean + model_builder.AddInitializerToSkip(node_unit.Inputs()[4].node_arg.Name()); //var } Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); + const auto& inputs = node_unit.Inputs(); // For reshape we are not really doing anything but // register a new operand with new shape - const auto& input = node.InputDefs()[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input = inputs[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); - const auto& scale_tensor = *initializers.at(node.InputDefs()[1]->Name()); - const auto& bias_tensor = *initializers.at(node.InputDefs()[2]->Name()); - const auto& mean_tensor = *initializers.at(node.InputDefs()[3]->Name()); - const auto& var_tensor = *initializers.at(node.InputDefs()[4]->Name()); + const auto& scale_tensor = *initializers.at(inputs[1].node_arg.Name()); + const auto& bias_tensor = *initializers.at(inputs[2].node_arg.Name()); + const auto& mean_tensor = *initializers.at(inputs[3].node_arg.Name()); + const auto& var_tensor = *initializers.at(inputs[4].node_arg.Name()); const auto eps = helper.Get("epsilon", 1e-5f); const auto size = SafeInt(scale_tensor.dims()[0]); - vector a, b; + std::vector a, b; a.reserve(size); b.reserve(size); @@ -1144,9 +1043,9 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu bias_data[i]); } - const auto tensor_a_name = model_builder.GetUniqueName(node.Name() + input + "_imm_a"); - const auto tensor_b_name = model_builder.GetUniqueName(node.Name() + input + "_imm_b"); - const auto tensor_imm_product_name = model_builder.GetUniqueName(node.Name() + input + "_imm_mul"); + const auto tensor_a_name = model_builder.GetUniqueName(node_unit.Name() + input + "_imm_a"); + const auto tensor_b_name = model_builder.GetUniqueName(node_unit.Name() + input + "_imm_b"); + const auto tensor_imm_product_name = model_builder.GetUniqueName(node_unit.Name() + input + "_imm_mul"); Shape tensor_a_dimen = {size}; bool input_is_nhwc = model_builder.IsOperandNHWC(input); @@ -1180,7 +1079,7 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu output_is_nhwc)); // Add - int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); + int32_t fuse_code = model_builder.FindActivation(node_unit); ORT_RETURN_IF_ERROR(AddBinaryOperator(ANEURALNETWORKS_ADD, model_builder, tensor_imm_product_name, tensor_b_name, @@ -1201,24 +1100,22 @@ class PoolOpBuilder : public BaseOpBuilder { static void CreateSharedOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations); private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + static bool IsQuantizedOp(const NodeUnit& node_unit) ORT_MUST_USE_RESULT; // TODO, see if we want to move this to BaseOpBuilder + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; +/* static */ bool PoolOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) { + // TODO, add support for QDQ NodeUnit + return node_unit.OpType() == "QLinearAveragePool"; +} + void PoolOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - const auto& op = node.OpType(); - if (op != "QLinearAveragePool") + if (!IsQuantizedOp(node_unit)) return; - const auto input_defs = node.InputDefs(); - // skip input/output scales and zeropoints - model_builder.AddInitializerToSkip(input_defs[1]->Name()); // X_scale - model_builder.AddInitializerToSkip(input_defs[2]->Name()); // X_zero_point - model_builder.AddInitializerToSkip(input_defs[3]->Name()); // Y_scale - - if (input_defs.size() == 5) // has Y_zero_point input - model_builder.AddInitializerToSkip(input_defs[4]->Name()); // Y_zero_point + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Inputs()[0].quant_param); // x_scale, x_zp + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Outputs()[0].quant_param); // y_scale, y_zp } /* static */ void PoolOpBuilder::CreateSharedOpBuilder( @@ -1235,15 +1132,13 @@ void PoolOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Nod } Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); - auto input = node.InputDefs()[0]->Name(); + auto input = node_unit.Inputs()[0].node_arg.Name(); bool use_nchw = model_builder.UseNCHW(); bool input_is_nhwc = model_builder.IsOperandNHWC(input); bool output_is_nhwc = false; @@ -1252,12 +1147,12 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N } else { output_is_nhwc = true; if (!input_is_nhwc) { - ORT_RETURN_IF_ERROR(GetNHWCInput(model_builder, node, 0, input)); + ORT_RETURN_IF_ERROR(GetNHWCInput(model_builder, node_unit, 0, input)); } } - const auto& output = node.OutputDefs()[0]->Name(); - const auto& op_type = node.OpType(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); + const auto& op_type = node_unit.OpType(); int32_t op_code; bool is_qlinear_average_pool = op_type == "QLinearAveragePool"; @@ -1267,15 +1162,15 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N else // (op_type == "MaxPool" || op_type == "GlobalMaxPool") op_code = ANEURALNETWORKS_MAX_POOL_2D; - vector onnx_pads, onnx_strides, kernel_shape; + std::vector onnx_pads, onnx_strides, kernel_shape; bool use_auto_pad = false; int32_t nnapi_padding_code = ANEURALNETWORKS_PADDING_VALID; const auto& input_shape = shaper[input]; if (is_average_pool || op_type == "MaxPool") { const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET")); - kernel_shape = helper.Get("kernel_shape", vector{0, 0}); - onnx_strides = helper.Get("strides", vector{1, 1}); - onnx_pads = helper.Get("pads", vector{0, 0, 0, 0}); + kernel_shape = helper.Get("kernel_shape", std::vector{0, 0}); + onnx_strides = helper.Get("strides", std::vector{1, 1}); + onnx_pads = helper.Get("pads", std::vector{0, 0, 0, 0}); const auto weight_size_y = static_cast(kernel_shape[0]); const auto weight_size_x = static_cast(kernel_shape[1]); ORT_RETURN_IF_ERROR( @@ -1286,18 +1181,18 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N } else { // (op_type == "GlobalAveragePool" || op_type == "GlobalMaxPool") use_auto_pad = true; nnapi_padding_code = ANEURALNETWORKS_PADDING_VALID; - onnx_strides = vector{1, 1}; - onnx_pads = vector{0, 0, 0, 0}; + onnx_strides = std::vector{1, 1}; + onnx_pads = std::vector{0, 0, 0, 0}; if (use_nchw) { - kernel_shape = vector{static_cast(input_shape[2]), - static_cast(input_shape[3])}; + kernel_shape = std::vector{static_cast(input_shape[2]), + static_cast(input_shape[3])}; } else { - kernel_shape = vector{static_cast(input_shape[1]), - static_cast(input_shape[2])}; + kernel_shape = std::vector{static_cast(input_shape[1]), + static_cast(input_shape[2])}; } } - int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); + int32_t fuse_code = model_builder.FindActivation(node_unit); // Get output scale and zero point if this is QLinearAveragePool // Otherwise we will use the scale and zero point of the input @@ -1307,16 +1202,14 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N if (is_qlinear_average_pool) { const auto& initializers = model_builder.GetInitializerTensors(); float x_scale = 0.0f; - ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 1 /* idx */, x_scale)); int32_t x_zero_point = 0; - ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 2 /* idx */, x_zero_point)); + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + initializers, node_unit.Inputs()[0], node_unit.ModelPath(), x_scale, x_zero_point)); // Verify if the scale and zero point values from onnx input and nnapi input match ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input, x_scale, x_zero_point)); - - ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 3 /* idx */, y_scale)); - if (node.InputDefs().size() > 4) - ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 4 /* idx */, y_zero_point)); + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + initializers, node_unit.Outputs()[0], node_unit.ModelPath(), y_scale, y_zero_point)); } std::vector input_indices; @@ -1361,10 +1254,17 @@ class ConvOpBuilder : public BaseOpBuilder { static void CreateSharedOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations); private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + static bool IsQuantizedOp(const NodeUnit& node_unit) ORT_MUST_USE_RESULT; // TODO, see if we want to move this to BaseOpBuilder + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; -/* static */ void ConvOpBuilder::CreateSharedOpBuilder( +/* static */ bool ConvOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) { + // TODO, add support for QDQ NodeUnit + return node_unit.OpType() == "QLinearConv"; +} + +/* static */ void +ConvOpBuilder::CreateSharedOpBuilder( const std::string& op_type, OpBuilderRegistrations& op_registrations) { CreateSharedOpBuilderImpl( op_type, op_registrations, @@ -1375,50 +1275,42 @@ class ConvOpBuilder : public BaseOpBuilder { } void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - const auto& op = node.OpType(); - const auto input_defs = node.InputDefs(); - + const auto& inputs = node_unit.Inputs(); // skip the weight for conv as we need to transpose - if (op == "QLinearConv") { - AddBinaryOpQuantizationScaleAndZeroPointToSkip(model_builder, node_unit); - model_builder.AddInitializerToSkip(input_defs[3]->Name()); // w - if (input_defs.size() > 8) - model_builder.AddInitializerToSkip(input_defs[8]->Name()); // B + if (IsQuantizedOp(node_unit)) { + AddQuantizationScaleAndZeroPointToSkip(model_builder, *inputs[0].quant_param); // x_scale, x_zp + AddInputToSkip(model_builder, inputs[1]); // w, w_scale, w_zp + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Outputs()[0].quant_param); // y_scale, y_zp + if (inputs.size() > 2) + AddInputToSkip(model_builder, inputs[2]); // B, B_scale, B_zp } else { - model_builder.AddInitializerToSkip(input_defs[1]->Name()); // w + model_builder.AddInitializerToSkip(inputs[1].node_arg.Name()); // w } } Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); - NodeAttrHelper helper(node); - const auto input_defs = node.InputDefs(); - const auto& op_type = node.OpType(); - bool is_qlinear_conv = (op_type == "QLinearConv"); + NodeAttrHelper helper(node_unit); + const auto inputs = node_unit.Inputs(); + bool is_qlinear_conv = IsQuantizedOp(node_unit); // onnx strides are in the order height, width // while nnapi strides are in the order width, height - const auto onnx_strides = helper.Get("strides", vector{1, 1}); + const auto onnx_strides = helper.Get("strides", std::vector{1, 1}); // onnx pads are in the order top, left, bottom, right // while nnapi pads is in the order left, right, top, bottom - auto onnx_pads = helper.Get("pads", vector{0, 0, 0, 0}); + auto onnx_pads = helper.Get("pads", std::vector{0, 0, 0, 0}); // onnx dilations is in the order height, width // while nnapi dilations are in the order width, height - const auto onnx_dilations = helper.Get("dilations", vector{1, 1}); + const auto onnx_dilations = helper.Get("dilations", std::vector{1, 1}); const auto group = helper.Get("group", 1); - size_t x_idx = 0, - w_idx = is_qlinear_conv ? 3 : 1, - b_idx = is_qlinear_conv ? 8 : 2; - - auto input = input_defs[x_idx]->Name(); + auto input = inputs[0].node_arg.Name(); bool use_nchw = model_builder.UseNCHW(); bool input_is_nhwc = model_builder.IsOperandNHWC(input); bool output_is_nhwc = false; @@ -1427,13 +1319,13 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N } else { output_is_nhwc = true; if (!input_is_nhwc) { - ORT_RETURN_IF_ERROR(GetNHWCInput(model_builder, node, x_idx, input)); + ORT_RETURN_IF_ERROR(GetNHWCInput(model_builder, node_unit, 0, input)); } } - const auto& weight = input_defs[w_idx]->Name(); + const auto& weight = inputs[1].node_arg.Name(); const auto& weight_tensor = *initializers.at(weight); - auto conv_type = GetConvType(node, model_builder.GetGraphViewer().GetAllInitializedTensors()); + auto conv_type = GetConvType(node_unit, model_builder.GetInitializerTensors()); bool conv_2d = (conv_type == ConvType::Regular), depthwise_conv_2d = (conv_type == ConvType::Depthwise), grouped_conv_2d = (conv_type == ConvType::Grouped); @@ -1446,10 +1338,10 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N y_zero_point = 0; // this is for per-channel quantization weights - optional> w_scales; + optional> w_scales; bool is_per_tensor_u8s8 = false; if (is_qlinear_conv) { - ORT_RETURN_IF_ERROR(GetConvMatMulOpQuantizationScaleAndZeroPoint(model_builder, node, + ORT_RETURN_IF_ERROR(GetConvMatMulOpQuantizationScaleAndZeroPoint(model_builder, node_unit, x_scale, w_scale, y_scale, x_zero_point, w_zero_point, y_zero_point, w_scales, is_per_tensor_u8s8)); @@ -1505,8 +1397,8 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N ORT_RETURN_IF_ERROR(IsValidConvWeightQuantizedType(model_builder, weight, w_scale, w_zero_point, w_scales)); } - bool hasBias = (input_defs.size() > b_idx); - std::string bias = hasBias ? input_defs[b_idx]->Name() : weight + "_bias"; + bool hasBias = (inputs.size() > 2); + std::string bias = hasBias ? inputs[2].node_arg.Name() : weight + "_bias"; if (!hasBias) { const auto weight_dimen = shaper[weight]; Shape bias_dimen; @@ -1517,11 +1409,11 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N const auto& weight_type = operand_types.at(weight).type; if (weight_type == Type::TENSOR_FLOAT32) { - vector buffer(bias_dimen[0], 0.0f); + std::vector buffer(bias_dimen[0], 0.0f); OperandType bias_operand_type(Type::TENSOR_FLOAT32, bias_dimen, x_scale * w_scale); ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(bias, buffer.data(), bias_operand_type)); } else if (weight_type == Type::TENSOR_QUANT8_ASYMM || weight_type == Type::TENSOR_QUANT8_SYMM_PER_CHANNEL) { - vector buffer(bias_dimen[0], 0); + std::vector buffer(bias_dimen[0], 0); OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, x_scale * w_scale); ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(bias, buffer.data(), bias_operand_type)); } else { @@ -1582,7 +1474,7 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N } } - int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); + int32_t fuse_code = model_builder.FindActivation(node_unit); ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code); if (model_builder.GetNNAPIFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) { @@ -1600,7 +1492,7 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N } int32_t operationCode; - const auto& output = node.OutputDefs()[0]->Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); if (conv_2d || grouped_conv_2d) { operationCode = conv_2d ? ANEURALNETWORKS_CONV_2D : ANEURALNETWORKS_GROUPED_CONV_2D; @@ -1628,17 +1520,16 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N class CastOpBuilder : public BaseOpBuilder { private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; Status CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); - const auto& input = node.InputDefs()[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input = node_unit.Inputs()[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); auto to = helper.Get("to", 0); @@ -1669,25 +1560,24 @@ Status CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N class SoftMaxOpBuilder : public BaseOpBuilder { private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); const auto android_feature_level = model_builder.GetNNAPIFeatureLevel(); - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); - auto input = node.InputDefs()[0]->Name(); + auto input = node_unit.Inputs()[0].node_arg.Name(); bool input_is_nhwc = model_builder.IsOperandNHWC(input); bool output_is_nhwc = input_is_nhwc; if (android_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) { if (model_builder.IsOperandNHWC(input)) { output_is_nhwc = false; // We want to transpose nhwc operand back to nchw before softmax - ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, 0, input)); + ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node_unit, 0, input)); } } @@ -1697,7 +1587,7 @@ Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons axis = axis_nchw_to_nhwc[axis]; } - const auto& output = node.OutputDefs()[0]->Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); float beta = 1.f; std::vector input_indices; input_indices.push_back(operand_indices.at(input)); @@ -1721,20 +1611,18 @@ Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons class IdentityOpBuilder : public BaseOpBuilder { private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; Status IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - // Identity is not really going to do anything // Just register the dimension and type, with same index and new name auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& input = node.InputDefs()[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input = node_unit.Inputs()[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); std::vector input_indices; @@ -1756,9 +1644,15 @@ class GemmOpBuilder : public BaseOpBuilder { static void CreateSharedOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations); private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + static bool IsQuantizedOp(const NodeUnit& node_unit) ORT_MUST_USE_RESULT; // TODO, see if we want to move this to BaseOpBuilder + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; +/* static */ bool GemmOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) { + // TODO, add support for QDQ NodeUnit + return node_unit.OpType() == "QLinearMatMul"; +} + /* static */ void GemmOpBuilder::CreateSharedOpBuilder( const std::string& op_type, OpBuilderRegistrations& op_registrations) { CreateSharedOpBuilderImpl( @@ -1771,43 +1665,38 @@ class GemmOpBuilder : public BaseOpBuilder { } void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - - const auto& op = node.OpType(); - const auto input_defs(node.InputDefs()); - if (op == "MatMul") { - model_builder.AddInitializerToSkip(input_defs[1]->Name()); - } else if (op == "Gemm") { - NodeAttrHelper helper(node); - const auto transB = helper.Get("transB", 0); - if (transB == 0) - model_builder.AddInitializerToSkip(input_defs[1]->Name()); - } else if (op == "QLinearMatMul") { - AddBinaryOpQuantizationScaleAndZeroPointToSkip(model_builder, node_unit); - model_builder.AddInitializerToSkip(input_defs[3]->Name()); // b + const auto& inputs = node_unit.Inputs(); + if (IsQuantizedOp(node_unit)) { + AddQuantizationScaleAndZeroPointToSkip(model_builder, *inputs[0].quant_param); // b_scale, b_zp + AddInputToSkip(model_builder, inputs[1]); // b, b_scale, b_zp + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Outputs()[0].quant_param); // y_scale, y_zp + } else { + const auto& op = node_unit.OpType(); + if (op == "MatMul") { + model_builder.AddInitializerToSkip(inputs[1].node_arg.Name()); + } else if (op == "Gemm") { + NodeAttrHelper helper(node_unit); + const auto transB = helper.Get("transB", 0); + if (transB == 0) + model_builder.AddInitializerToSkip(inputs[1].node_arg.Name()); + } } } Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); - const auto& op = node.OpType(); - const auto input_defs(node.InputDefs()); - NodeAttrHelper helper(node); + const auto& op = node_unit.OpType(); + const auto& inputs = node_unit.Inputs(); + NodeAttrHelper helper(node_unit); bool is_qlinear_matmul = op == "QLinearMatMul"; - size_t a_idx = 0, - b_idx = is_qlinear_matmul ? 3 : 1, - c_idx = 2; // QLinearMatMul has no bias - - const auto& input1 = input_defs[a_idx]->Name(); - const auto& input2 = input_defs[b_idx]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input1 = inputs[0].node_arg.Name(); + const auto& input2 = inputs[1].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); const auto transB = helper.Get("transB", 0); float a_scale = 0.0f, @@ -1819,9 +1708,9 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N bool is_per_tensor_u8s8 = false; if (is_qlinear_matmul) { - optional> w_scales; + optional> w_scales; ORT_RETURN_IF_ERROR( - GetConvMatMulOpQuantizationScaleAndZeroPoint(model_builder, node, + GetConvMatMulOpQuantizationScaleAndZeroPoint(model_builder, node_unit, a_scale, b_scale, y_scale, a_zero_point, b_zero_point, y_zero_point, w_scales, is_per_tensor_u8s8)); @@ -1853,14 +1742,14 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N } uint32_t bias_idx; - bool has_bias = (op == "Gemm") && (input_defs.size() > 2); + bool has_bias = inputs.size() > 2; if (has_bias) { - const auto& bias = input_defs[c_idx]->Name(); + const auto& bias = inputs[2].node_arg.Name(); // We need squeeze the input tensor to 1d if necessary if (shaper[bias].size() > 1) { - std::string bias_squeezed = model_builder.GetUniqueName(node.Name() + op + "_bias_squeezed"); + std::string bias_squeezed = model_builder.GetUniqueName(node_unit.Name() + op + "_bias_squeezed"); // We will use squeeze all here - ORT_RETURN_IF_ERROR(AddSqueezeOp(model_builder, node.Name(), + ORT_RETURN_IF_ERROR(AddSqueezeOp(model_builder, node_unit.Name(), bias, bias_squeezed, {} /* axes */)); bias_idx = operand_indices.at(bias_squeezed); @@ -1873,7 +1762,7 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N } } else { // No C supplied, we need a vector of 0 - std::string bias = model_builder.GetUniqueName(node.Name() + op + "_bias"); + std::string bias = model_builder.GetUniqueName(node_unit.Name() + op + "_bias"); const auto& bias_type = operand_types.at(input2).type; const Shape& bias_dimen = {shaper[input2][0]}; if (bias_type == Type::TENSOR_FLOAT32) { @@ -1895,7 +1784,7 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N input_indices.push_back(operand_indices.at(input1)); // A input_indices.push_back(input_2_idx); // B input_indices.push_back(bias_idx); // C - int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); + int32_t fuse_code = model_builder.FindActivation(node_unit); ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code); ORT_RETURN_IF_ERROR(shaper.FC(input1, input2, output)); @@ -1915,24 +1804,21 @@ class UnaryOpBuilder : public BaseOpBuilder { static void CreateSharedOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations); private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + static bool IsQuantizedOp(const NodeUnit& node_unit) ORT_MUST_USE_RESULT; // TODO, see if we want to move this to BaseOpBuilder + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; +/* static */ bool UnaryOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) { + // TODO, add support for QDQ NodeUnit + return node_unit.OpType() == "QLinearSigmoid"; +} + void UnaryOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - const auto& op = node.OpType(); - if (op != "QLinearSigmoid") + if (!IsQuantizedOp(node_unit)) return; - const auto input_defs = node.InputDefs(); - - // skip input/output scales and zeropoints - model_builder.AddInitializerToSkip(input_defs[1]->Name()); // X_scale - model_builder.AddInitializerToSkip(input_defs[2]->Name()); // X_zero_point - model_builder.AddInitializerToSkip(input_defs[3]->Name()); // Y_scale - - if (input_defs.size() == 5) // has Y_zero_point input - model_builder.AddInitializerToSkip(input_defs[4]->Name()); // Y_zero_point + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Inputs()[0].quant_param); // x_scale, x_zp + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Outputs()[0].quant_param); // y_scale, y_zp } /* static */ void UnaryOpBuilder::CreateSharedOpBuilder( @@ -1954,14 +1840,13 @@ void UnaryOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const No } Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& op_type(node.OpType()); + const auto& op_type(node_unit.OpType()); - const auto& input = node.InputDefs()[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input = node_unit.Inputs()[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); @@ -1995,9 +1880,9 @@ Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const if (is_qlinear_sigmoid) { const auto& initializers = model_builder.GetInitializerTensors(); float x_scale = 0.0f; - ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 1, x_scale)); int32_t x_zero_point = 0; - ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 2, x_zero_point)); + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + initializers, node_unit.Inputs()[0], node_unit.ModelPath(), x_scale, x_zero_point)); // Verify if the scale and zero point values from onnx input and nnapi input match ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input, x_scale, x_zero_point)); @@ -2021,21 +1906,21 @@ Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const class ConcatOpBuilder : public BaseOpBuilder { private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); + const auto& inputs = node_unit.Inputs(); std::vector input_indices; - const auto& input0 = node.InputDefs()[0]->Name(); + const auto& input0 = inputs[0].node_arg.Name(); bool all_input_have_same_layout = true; bool output_is_nhwc = false; - const auto node_input_size = node.InputDefs().size(); + const auto node_input_size = inputs.size(); // First if the inputs are uint8, we need verify all the inputs have same scale and zero points if (operand_types.at(input0).type == android::nn::wrapper::Type::TENSOR_QUANT8_ASYMM) { @@ -2044,7 +1929,7 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const // Compare scale and zp of input0 to input1~n for (size_t i = 1; i < node_input_size; i++) { - const auto& type = operand_types.at(node.InputDefs()[i]->Name()); + const auto& type = operand_types.at(inputs[i].node_arg.Name()); ORT_RETURN_IF_NOT(scale == type.operandType.scale, "Input[", i, "]'s scale: ", type.operandType.scale, " is different than input[0]'s scale: ", scale); @@ -2059,31 +1944,31 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const for (size_t i = 0; i < node_input_size - 1; i++) { all_input_have_same_layout = all_input_have_same_layout && - model_builder.IsOperandNHWC(node.InputDefs()[i]->Name()) == - model_builder.IsOperandNHWC(node.InputDefs()[i + 1]->Name()); + model_builder.IsOperandNHWC(inputs[i].node_arg.Name()) == + model_builder.IsOperandNHWC(inputs[i + 1].node_arg.Name()); } - std::vector inputs; - inputs.reserve(node_input_size); + std::vector input_names; + input_names.reserve(node_input_size); if (all_input_have_same_layout) { // if all the inputs are of same layout, output will be the same layout output_is_nhwc = model_builder.IsOperandNHWC(input0); for (size_t i = 0; i < node_input_size; i++) { - auto input = node.InputDefs()[i]->Name(); + const auto& input = inputs[i].node_arg.Name(); input_indices.push_back(operand_indices.at(input)); - inputs.push_back(input); + input_names.push_back(input); } } else { // if all the inputs are not same layout, // will need transpos those nhwc tensors back to nchw for (size_t i = 0; i < node_input_size; i++) { - auto input = node.InputDefs()[i]->Name(); + auto input = inputs[i].node_arg.Name(); if (model_builder.IsOperandNHWC(input)) { - ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, i, input)); + ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node_unit, i, input)); } input_indices.push_back(operand_indices.at(input)); - inputs.push_back(input); + input_names.push_back(input); } } @@ -2099,8 +1984,8 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const } ADD_SCALAR_OPERAND(model_builder, input_indices, axis); - const auto& output = node.OutputDefs()[0]->Name(); - ORT_RETURN_IF_ERROR(shaper.Concat(inputs, axis, output)); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); + ORT_RETURN_IF_ERROR(shaper.Concat(input_names, axis, output)); OperandType output_operand_type = operand_types.at(input0); output_operand_type.SetDimensions(shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_CONCATENATION, input_indices, @@ -2117,25 +2002,24 @@ class SqueezeOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; - static Status GetAxes(ModelBuilder& model_builder, const Node& node, vector& axes); + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; + static Status GetAxes(ModelBuilder& model_builder, const NodeUnit& node_unit, std::vector& axes); }; void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - if (node.SinceVersion() > 12 && node.InputDefs().size() > 1) { - model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); + if (node_unit.SinceVersion() > 12 && node_unit.Inputs().size() > 1) { + model_builder.AddInitializerToSkip(node_unit.Inputs()[1].node_arg.Name()); } } /* static */ Status SqueezeOpBuilder::GetAxes(ModelBuilder& model_builder, - const Node& node, vector& axes) { + const NodeUnit& node_unit, std::vector& axes) { // Squeeze opset 13 use input as axes - if (node.SinceVersion() > 12) { + if (node_unit.SinceVersion() > 12) { // If axes is not supplied, return an empty axes as default to squeeze all - if (node.InputDefs().size() > 1) { + if (node_unit.Inputs().size() > 1) { const auto& initializers(model_builder.GetInitializerTensors()); - const auto& axes_tensor = *initializers.at(node.InputDefs()[1]->Name()); + const auto& axes_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name()); std::vector unpacked_tensor; ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor)); const int64_t* raw_axes = reinterpret_cast(unpacked_tensor.data()); @@ -2147,25 +2031,23 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const } } } else { - NodeAttrHelper helper(node); - axes = helper.Get("axes", vector()); + NodeAttrHelper helper(node_unit); + axes = helper.Get("axes", std::vector()); } return Status::OK(); } Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - - auto input = node.InputDefs()[0]->Name(); + auto input = node_unit.Inputs()[0].node_arg.Name(); if (model_builder.IsOperandNHWC(input)) { // We want to transpose nhwc operand back to nchw before squeeze - ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, 0, input)); + ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node_unit, 0, input)); } - vector axes; - ORT_RETURN_IF_ERROR(GetAxes(model_builder, node, axes)); - return AddSqueezeOp(model_builder, node.Name(), input, node.OutputDefs()[0]->Name(), axes); + std::vector axes; + ORT_RETURN_IF_ERROR(GetAxes(model_builder, node_unit, axes)); + return AddSqueezeOp(model_builder, node_unit.Name(), input, node_unit.Outputs()[0].node_arg.Name(), axes); } #pragma endregion @@ -2177,38 +2059,27 @@ class QuantizeLinearOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; void QuantizeLinearOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - const auto input_defs(node.InputDefs()); - - model_builder.AddInitializerToSkip(input_defs[1]->Name()); - - if (input_defs.size() == 3) // has zero_point input - model_builder.AddInitializerToSkip(input_defs[2]->Name()); + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Outputs()[0].quant_param); // y_scale, y_zp } Status QuantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); - const auto input_defs(node.InputDefs()); - const auto& input = input_defs[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input = node_unit.Inputs()[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); float scale = 0.0f; - ORT_RETURN_IF_ERROR(GetQuantizationScale(model_builder.GetInitializerTensors(), node, 1, scale)); int32_t zero_point = 0; + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + model_builder.GetInitializerTensors(), node_unit.Outputs()[0], node_unit.ModelPath(), scale, zero_point)); + Type output_type = Type::TENSOR_QUANT8_ASYMM; - - if (input_defs.size() == 3) { // Get zero point - ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(model_builder.GetInitializerTensors(), node, 2, zero_point)); - } - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(output_type, shaper[output], scale, zero_point); std::vector input_indices; @@ -2227,36 +2098,26 @@ class DequantizeLinearOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; void DequantizeLinearOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - const auto input_defs(node.InputDefs()); - - model_builder.AddInitializerToSkip(input_defs[1]->Name()); - - if (input_defs.size() == 3) // has zero_point input - model_builder.AddInitializerToSkip(input_defs[2]->Name()); + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Inputs()[0].quant_param); // x_scale, x_zp } Status DequantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); - const auto input_defs(node.InputDefs()); + const auto& inputs = node_unit.Inputs(); - const auto& input = input_defs[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input = inputs[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); float scale = 0.0; - ORT_RETURN_IF_ERROR(GetQuantizationScale(model_builder.GetInitializerTensors(), node, 1, scale)); int32_t zero_point = 0; - if (input_defs.size() == 3) { // Get zero point - ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(model_builder.GetInitializerTensors(), node, 2, zero_point)); - } + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + model_builder.GetInitializerTensors(), node_unit.Inputs()[0], node_unit.ModelPath(), scale, zero_point)); ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input, scale, zero_point)); @@ -2276,25 +2137,24 @@ Status DequantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil class LRNOpBuilder : public BaseOpBuilder { private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; Status LRNOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); const auto android_feature_level = model_builder.GetNNAPIFeatureLevel(); - auto input = node.InputDefs()[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + auto input = node_unit.Inputs()[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); if (android_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) { // on android api level 28, we need to transpose the nchw input to nhwc output_is_nhwc = true; if (!model_builder.IsOperandNHWC(input)) { - ORT_RETURN_IF_ERROR(GetNHWCInput(model_builder, node, 0, input)); + ORT_RETURN_IF_ERROR(GetNHWCInput(model_builder, node_unit, 0, input)); } } @@ -2338,40 +2198,39 @@ class ClipOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; void ClipOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - if (node.InputDefs().size() > 1) - model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); // min + const auto& inputs = node_unit.Inputs(); + if (inputs.size() > 1) + model_builder.AddInitializerToSkip(inputs[1].node_arg.Name()); // min - if (node.InputDefs().size() > 2) - model_builder.AddInitializerToSkip(node.InputDefs()[2]->Name()); // max + if (inputs.size() > 2) + model_builder.AddInitializerToSkip(inputs[2].node_arg.Name()); // max } Status ClipOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& input = node.InputDefs()[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input = node_unit.Inputs()[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); if (Contains(model_builder.GetFusedActivations(), input)) { - LOGS_DEFAULT(VERBOSE) << "Clip Node [" << node.Name() << "] fused"; + LOGS_DEFAULT(VERBOSE) << "Clip Node [" << node_unit.Name() << "] fused"; model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, output_is_nhwc); return Status::OK(); } float min, max; - GetClipMinMax(model_builder.GetInitializerTensors(), node, min, max, logging::LoggingManager::DefaultLogger()); + GetClipMinMax(model_builder.GetInitializerTensors(), node_unit.GetNode(), min, max, + logging::LoggingManager::DefaultLogger()); int32_t op_code; if (min == 0.0f && max == 6.0f) @@ -2398,34 +2257,33 @@ class ResizeOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; void ResizeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); + const auto& inputs = node_unit.Inputs(); // We don't really use ROI here, so add them to skipped list - model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); // ROI + model_builder.AddInitializerToSkip(inputs[1].node_arg.Name()); // ROI // We will still add scales to the skipped list even sizes are present // since there is no use of it, we will not process it later - model_builder.AddInitializerToSkip(node.InputDefs()[2]->Name()); // scales + model_builder.AddInitializerToSkip(inputs[2].node_arg.Name()); // scales - if (node.InputDefs().size() > 3) - model_builder.AddInitializerToSkip(node.InputDefs()[3]->Name()); // sizes + if (inputs.size() > 3) + model_builder.AddInitializerToSkip(inputs[3].node_arg.Name()); // sizes } Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); - NodeAttrHelper helper(node); - const auto input_defs = node.InputDefs(); + NodeAttrHelper helper(node_unit); + const auto& inputs = node_unit.Inputs(); const auto android_feature_level = model_builder.GetNNAPIFeatureLevel(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); - auto input = input_defs[0]->Name(); + auto input = inputs[0].node_arg.Name(); bool use_nchw = model_builder.UseNCHW(); bool input_is_nhwc = model_builder.IsOperandNHWC(input); bool output_is_nhwc = false; @@ -2434,7 +2292,7 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const } else { output_is_nhwc = true; if (!input_is_nhwc) { - ORT_RETURN_IF_ERROR(GetNHWCInput(model_builder, node, 0, input)); + ORT_RETURN_IF_ERROR(GetNHWCInput(model_builder, node_unit, 0, input)); } } @@ -2447,8 +2305,8 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const bool using_half_pixel = coord_trans_mode == "half_pixel"; bool using_align_corners = coord_trans_mode == "align_corners"; - if (input_defs.size() == 3) { // we are using scales - const auto& scales_name = input_defs[2]->Name(); + if (inputs.size() == 3) { // we are using scales + const auto& scales_name = inputs[2].node_arg.Name(); const auto& scales_tensor = *initializers.at(scales_name); std::vector unpacked_tensor; ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor)); @@ -2458,7 +2316,7 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ORT_RETURN_IF_ERROR( shaper.ResizeUsingScales(input, scale_h, scale_w, use_nchw, output)); } else { // we are using sizes - const auto& sizes_name = input_defs[3]->Name(); + const auto& sizes_name = inputs[3].node_arg.Name(); const auto& sizes_tensor = *initializers.at(sizes_name); std::vector unpacked_tensor; ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor)); @@ -2505,31 +2363,29 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const class FlattenOpBuilder : public BaseOpBuilder { private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; Status FlattenOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - - auto input = node.InputDefs()[0]->Name(); + auto input = node_unit.Inputs()[0].node_arg.Name(); if (model_builder.IsOperandNHWC(input)) { // We want to transpose nhwc operand back to nchw before reshape - ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, 0, input)); + ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node_unit, 0, input)); } // Flatten is basically a reshape to 2d tensor // Get the shape for Reshape here Shape input_shape; - GetShape(*node.InputDefs()[0], input_shape); + GetShape(node_unit.Inputs()[0].node_arg, input_shape); int32_t dim_1 = 1; int32_t dim_2 = 1; - GetFlattenOutputShape(node, input_shape, dim_1, dim_2); + GetFlattenOutputShape(node_unit, input_shape, dim_1, dim_2); // If the input is of dynamic shape, replace 0 (dynamic) dimension with -1 // We cannot have dim_1 and dim_2 both be 0 here, it was checked in IsOpSupportedImpl dim_1 = dim_1 == 0 ? -1 : dim_1; dim_2 = dim_2 == 0 ? -1 : dim_2; std::vector shape{dim_1, dim_2}; - return ReshapeOpBuilder::AddReshapeOperator(model_builder, node, input, shape); + return ReshapeOpBuilder::AddReshapeOperator(model_builder, node_unit, input, shape); } #pragma endregion @@ -2539,12 +2395,12 @@ Status FlattenOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons class MinMaxOpBuilder : public BaseOpBuilder { public: static void CreateSharedOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations); - static Status AddMinMaxOperator(ModelBuilder& model_builder, const Node& node, - const std::string& input1, const std::string& input2, - bool output_is_nhwc) ORT_MUST_USE_RESULT; private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; + static Status AddMinMaxOperator(ModelBuilder& model_builder, const NodeUnit& node_unit, + const std::string& input1, const std::string& input2, + bool output_is_nhwc); }; /* static */ void MinMaxOpBuilder::CreateSharedOpBuilder( @@ -2557,16 +2413,16 @@ class MinMaxOpBuilder : public BaseOpBuilder { }); } -/* static */ Status MinMaxOpBuilder::AddMinMaxOperator(ModelBuilder& model_builder, const Node& node, +/* static */ Status MinMaxOpBuilder::AddMinMaxOperator(ModelBuilder& model_builder, const NodeUnit& node_unit, const std::string& input1, const std::string& input2, bool output_is_nhwc) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); - const auto& op_type(node.OpType()); + const auto& op_type(node_unit.OpType()); int32_t op_code; if (op_type == "Min") op_code = ANEURALNETWORKS_MINIMUM; @@ -2588,17 +2444,14 @@ class MinMaxOpBuilder : public BaseOpBuilder { } Status MinMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - const auto input_defs(node.InputDefs()); - std::string input1 = input_defs[0]->Name(); - std::string input2 = input_defs[1]->Name(); + const auto& inputs = node_unit.Inputs(); + std::string input1 = inputs[0].node_arg.Name(); + std::string input2 = inputs[1].node_arg.Name(); bool output_is_nhwc = false; - ORT_RETURN_IF_ERROR(TransposeBinaryOpInputLayout(model_builder, node, - 0 /* input1_idx */, - 1 /* input2_idx */, + ORT_RETURN_IF_ERROR(TransposeBinaryOpInputLayout(model_builder, node_unit, input1, input2, output_is_nhwc)); - return AddMinMaxOperator(model_builder, node, input1, input2, output_is_nhwc); + return AddMinMaxOperator(model_builder, node_unit, input1, input2, output_is_nhwc); } #pragma endregion @@ -2607,21 +2460,19 @@ Status MinMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const class EluOpBuilder : public BaseOpBuilder { private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; Status EluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& input = node.InputDefs()[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input = node_unit.Inputs()[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); const auto alpha = helper.Get("alpha", 1.0f); std::vector input_indices; input_indices.push_back(operand_indices.at(input)); @@ -2639,32 +2490,28 @@ class SliceOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; private: - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override ORT_MUST_USE_RESULT; + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; }; void SliceOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - // Skip everything except input0 for Slice - const auto input_defs = node.InputDefs(); - model_builder.AddInitializerToSkip(input_defs[1]->Name()); // starts - model_builder.AddInitializerToSkip(input_defs[2]->Name()); // ends - if (input_defs.size() > 3) { - model_builder.AddInitializerToSkip(input_defs[3]->Name()); // axes - if (input_defs.size() > 4) { - model_builder.AddInitializerToSkip(input_defs[4]->Name()); // steps + const auto& inputs = node_unit.Inputs(); + model_builder.AddInitializerToSkip(inputs[1].node_arg.Name()); // starts + model_builder.AddInitializerToSkip(inputs[2].node_arg.Name()); // ends + if (inputs.size() > 3) { + model_builder.AddInitializerToSkip(inputs[3].node_arg.Name()); // axes + if (inputs.size() > 4) { + model_builder.AddInitializerToSkip(inputs[4].node_arg.Name()); // steps } } } Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& node = node_unit.GetNode(); - auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto input_defs = node.InputDefs(); - const auto& input_shape = shaper[input_defs[0]->Name()]; + const auto& inputs = node_unit.Inputs(); + const auto& input_shape = shaper[inputs[0].node_arg.Name()]; std::vector input_shape_64(input_shape.cbegin(), input_shape.cend()); SliceOp::PrepareForComputeMetadata compute_metadata(input_shape_64); @@ -2678,15 +2525,14 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const std::vector input_axes; std::vector input_steps; - const auto CopyInputData = [&node, &model_builder](size_t input_idx, std::vector& data) { + const auto CopyInputData = [&inputs, &model_builder](size_t input_idx, std::vector& data) { data.clear(); - const auto input_defs = node.InputDefs(); // This is an optional input, return empty vector - if (input_defs.size() <= input_idx) + if (inputs.size() <= input_idx) return Status::OK(); - const auto& input_name = input_defs[input_idx]->Name(); + const auto& input_name = inputs[input_idx].node_arg.Name(); const auto& initializers(model_builder.GetInitializerTensors()); const auto& tensor = *initializers.at(input_name); @@ -2728,8 +2574,8 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const std::back_inserter(nnapi_output_shape), [](int64_t i) { return SafeInt(i); }); - const auto& input = node.InputDefs()[0]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + const auto& input = inputs[0].node_arg.Name(); + const auto& output = node_unit.Outputs()[0].node_arg.Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); // No shape inference for Slice, everything is calculated here, we only need to add the output shape @@ -2744,14 +2590,14 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Shape param_dimen = {static_cast(input_shape.size())}; // helper function to add begin/end/strides of ANEURALNETWORKS_STRIDED_SLICE - const auto AddOperand = [&model_builder, &node, &input_indices, &operand_indices]( + const auto AddOperand = [&model_builder, &node_unit, &input_indices, &operand_indices]( const char* name, const Shape& shape, const std::vector& param_raw_data) { std::vector param_data; param_data.reserve(param_raw_data.size()); std::transform(param_raw_data.cbegin(), param_raw_data.cend(), std::back_inserter(param_data), [](int64_t i) { return SafeInt(i); }); - std::string param_name = model_builder.GetUniqueName(node.Name() + name); + std::string param_name = model_builder.GetUniqueName(node_unit.Name() + name); OperandType param_operand_type(Type::TENSOR_INT32, shape); ORT_RETURN_IF_ERROR( model_builder.AddOperandFromPersistMemoryBuffer(param_name, param_data.data(), param_operand_type)); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h index 46acbc4eff..6483c432f1 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h @@ -7,7 +7,6 @@ #include #include "core/graph/basic_types.h" -#include "core/session/onnxruntime_c_api.h" namespace onnxruntime { @@ -31,7 +30,7 @@ class IOpBuilder { virtual void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const = 0; // Add the operator to NNAPI model - virtual common::Status AddToModelBuilder(ModelBuilder& model_builder, const NodeUnit& node_unit) const ORT_MUST_USE_RESULT = 0; + virtual common::Status AddToModelBuilder(ModelBuilder& model_builder, const NodeUnit& node_unit) const = 0; }; // Get the lookup table with IOpBuilder delegates for different onnx operators @@ -40,13 +39,7 @@ class IOpBuilder { const std::unordered_map& GetOpBuilders(); // Transpose the NHWC input to NCHW output -common::Status TransposeNHWCToNCHW(ModelBuilder& model_builder, const std::string& input, const std::string& output) - ORT_MUST_USE_RESULT; - -// Get the quantized input's scale and zero point for the given input -common::Status GetQuantizedInputScaleAndZeroPoint(const InitializedTensorSet& initializers, - const NodeUnit& node_unit, const std::string& input_name, - float& scale, int32_t& zero_point) ORT_MUST_USE_RESULT; +common::Status TransposeNHWCToNCHW(ModelBuilder& model_builder, const std::string& input, const std::string& output); } // namespace nnapi } // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc index ca2ba5e90f..75eab4c837 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc @@ -1,19 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include -#include -#include -#include +#include "op_support_checker.h" +#include "core/common/logging/logging.h" +#include "core/common/safeint.h" +#include "core/framework/tensorprotoutils.h" +#include "core/graph/graph.h" #include "core/providers/common.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/shared/utils/utils.h" #include "helper.h" -#include "op_support_checker.h" - -using onnxruntime::NodeUnit; -using std::vector; namespace onnxruntime { namespace nnapi { @@ -25,19 +22,37 @@ struct OpSupportCheckerRegistrations { std::unordered_map op_support_checker_map; }; -bool HasExternalInitializer(const InitializedTensorSet& initializers, const Node& node) { - for (const auto* node_arg : node.InputDefs()) { - const auto& input_name(node_arg->Name()); - if (!Contains(initializers, input_name)) +bool HasExternalInitializer(const InitializedTensorSet& initializers, const NodeUnit& node_unit) { + const auto is_ext_initializer = + [&](const NodeArg& node_arg) { + const auto& input_name(node_arg.Name()); + if (!Contains(initializers, input_name)) + return false; + + const auto& tensor = *initializers.at(input_name); + if (tensor.has_data_location() && + tensor.data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) { + LOGS_DEFAULT(VERBOSE) << "Initializer [" << input_name + << "] with external data location are not currently supported"; + return true; + } + + return false; + }; + + const auto& inputs = node_unit.Inputs(); + for (const auto& input : inputs) { + if (is_ext_initializer(input.node_arg)) + return true; + + if (!input.quant_param) continue; - const auto& tensor = *initializers.at(input_name); - if (tensor.has_data_location() && - tensor.data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) { - LOGS_DEFAULT(VERBOSE) << "Initializer [" << input_name - << "] with external data location are not currently supported"; + if (is_ext_initializer(input.quant_param->scale)) + return true; + + if (input.quant_param->zero_point && is_ext_initializer(*input.quant_param->zero_point)) return true; - } } return false; @@ -118,10 +133,8 @@ bool BaseOpSupportChecker::IsOpSupported(const InitializedTensorSet& initializer if (!HasSupportedInputs(node_unit)) return false; - const auto& node = node_unit.GetNode(); - // We do not support external initializers for now - if (HasExternalInitializer(initializers, node)) + if (HasExternalInitializer(initializers, node_unit)) return false; if (!HasSupportedOpSet(node_unit)) @@ -247,30 +260,25 @@ int BinaryOpSupportChecker::GetMinSupportedOpSet(const NodeUnit& node_unit) cons } bool BinaryOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) const { - // TODO, change to use node unit and quant_param of IODef - const auto& node = node_unit.GetNode(); - bool is_qlinear_add = node.OpType() == "QLinearAdd"; - bool is_pow = node.OpType() == "Pow"; + bool is_qlinear_add = node_unit.OpType() == "QLinearAdd"; + bool is_pow = node_unit.OpType() == "Pow"; if (!is_qlinear_add && !is_pow) return BaseOpSupportChecker::HasSupportedInputsImpl(node_unit); if (is_qlinear_add) { // QLinearAdd - if (!HasValidBinaryOpQuantizedInputs(node)) + if (!HasValidBinaryOpQuantizedInputs(node_unit)) return false; } // Pow we only support both input as fp32 now if (is_pow) { - const auto& input1 = *node.InputDefs()[0]; - const auto& input2 = *node.InputDefs()[1]; - int32_t input_type_1; - if (!GetType(input1, input_type_1)) + if (!GetType(node_unit.Inputs()[0].node_arg, input_type_1)) return false; int32_t input_type_2; - if (!GetType(input2, input_type_2)) + if (!GetType(node_unit.Inputs()[1].node_arg, input_type_2)) return false; if (input_type_1 != ONNX_NAMESPACE::TensorProto_DataType_FLOAT || input_type_1 != input_type_2) { @@ -286,24 +294,18 @@ bool BinaryOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) c bool BinaryOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) const { - const auto& node = node_unit.GetNode(); - - const auto& op_type(node.OpType()); - const auto input_defs(node.InputDefs()); + const auto& op_type(node_unit.OpType()); + const auto& inputs = node_unit.Inputs(); bool op_is_qlinear = op_type == "QLinearAdd"; - size_t a_idx = 0, b_idx = 1; - if (op_is_qlinear) { - b_idx = 3; - } Shape input1_shape, input2_shape; - if (!GetShape(*input_defs[a_idx], input1_shape) || - !GetShape(*input_defs[b_idx], input2_shape)) + if (!GetShape(inputs[0].node_arg, input1_shape) || + !GetShape(inputs[1].node_arg, input2_shape)) return false; const auto input1_size = input1_shape.size(); const auto input2_size = input2_shape.size(); if (input1_size > 4 || input2_size > 4) { - LOGS_DEFAULT(VERBOSE) << node.OpType() << " only support up to 4d shape, input1 is " + LOGS_DEFAULT(VERBOSE) << op_type << " only support up to 4d shape, input1 is " << input1_size << "d shape, input 2 is " << input2_size << "d shape"; return false; @@ -312,7 +314,7 @@ bool BinaryOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi if (op_is_qlinear) { // For QLinearAdd, we only support uint8 output now int32_t output_type; - if (!GetType(*node.OutputDefs()[0], output_type)) + if (!GetType(node_unit.Outputs()[0].node_arg, output_type)) return false; if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { @@ -322,13 +324,16 @@ bool BinaryOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi return false; } - // All scale/zero points are initializer scalars - // a/b/y_scale - if (!HasValidQuantizationScales(initializers, node, {1, 4, 6}, params)) + // Check input scales and ZPs + if (!HasValidQuantizationScales(initializers, node_unit, {0, 1}, params, true /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0, 1}, true /* is_input */)) return false; - // a/b/y_zero_point - if (!HasValidQuantizationZeroPoints(initializers, node, {2, 5, 7})) + // Check output scale and ZP + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) return false; } @@ -354,9 +359,8 @@ class TransposeOpSupportChecker : public BaseOpSupportChecker { bool TransposeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); Shape input_shape; - if (!GetShape(*node.InputDefs()[0], input_shape)) + if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape)) return false; const auto input_size = input_shape.size(); @@ -400,15 +404,15 @@ class ReshapeOpSupportChecker : public BaseOpSupportChecker { bool ReshapeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); - const auto& perm_name = node.InputDefs()[1]->Name(); + const auto& inputs = node_unit.Inputs(); + const auto& perm_name = inputs[1].node_arg.Name(); if (!Contains(initializers, perm_name)) { LOGS_DEFAULT(VERBOSE) << "New shape of reshape must be known"; return false; } Shape input_shape; - if (!GetShape(*node.InputDefs()[0], input_shape)) + if (!GetShape(inputs[0].node_arg, input_shape)) return false; if (input_shape.size() > 4 || input_shape.empty()) { @@ -427,7 +431,7 @@ bool ReshapeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& init const int64_t* raw_perm = reinterpret_cast(unpacked_tensor.data()); const auto perm_size = SafeInt(perm_tensor.dims()[0]); - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); const bool allow_zero = helper.Get("allowzero ", 0) == 1; for (uint32_t i = 0; i < perm_size; i++) { // NNAPI reshape does not support 0 as dimension @@ -462,16 +466,15 @@ class BatchNormalizationOpSupportChecker : public BaseOpSupportChecker { bool BatchNormalizationOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); - if (node.OutputDefs().size() != 1) { + if (node_unit.Outputs().size() != 1) { LOGS_DEFAULT(VERBOSE) << "Your onnx model may be in training mode, please export " "it in test mode."; return false; } - const auto& input_defs = node.InputDefs(); + const auto& inputs = node_unit.Inputs(); Shape input_shape; - if (!GetShape(*input_defs[0], input_shape)) + if (!GetShape(inputs[0].node_arg, input_shape)) return false; const auto input_size = input_shape.size(); @@ -481,17 +484,17 @@ bool BatchNormalizationOpSupportChecker::IsOpSupportedImpl(const InitializedTens return false; } - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); const auto spatial = helper.Get("spatial", 1); if (spatial != 1) { LOGS_DEFAULT(VERBOSE) << "Non-spatial BN is not supported"; return false; } - const auto& scale_name = input_defs[1]->Name(); - const auto& b_name = input_defs[2]->Name(); - const auto& mean_name = input_defs[3]->Name(); - const auto& var_name = input_defs[4]->Name(); + const auto& scale_name = inputs[1].node_arg.Name(); + const auto& b_name = inputs[2].node_arg.Name(); + const auto& mean_name = inputs[3].node_arg.Name(); + const auto& var_name = inputs[4].node_arg.Name(); if (!Contains(initializers, scale_name)) { LOGS_DEFAULT(VERBOSE) << "Scale of BN must be known"; return false; @@ -548,25 +551,24 @@ class PoolOpSupportChecker : public BaseOpSupportChecker { bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) const { - const auto& node = node_unit.GetNode(); - const auto& op_name = node.Name(); - const auto& op_type = node.OpType(); - const auto& input_defs = node.InputDefs(); + const auto& op_name = node_unit.Name(); + const auto& op_type = node_unit.OpType(); + const auto& inputs = node_unit.Inputs(); Shape input_shape; - if (!GetShape(*input_defs[0], input_shape)) + if (!GetShape(inputs[0].node_arg, input_shape)) return false; const auto input_size = input_shape.size(); if (input_size != 4) { LOGS_DEFAULT(VERBOSE) << op_type << " only supports rank-4 tensor, input [" - << input_defs[0]->Name() << "] has actual dim count " << input_size; + << inputs[0].node_arg.Name() << "] has actual dim count " << input_size; return false; } bool is_qlinear_average_pool = op_type == "QLinearAveragePool"; if (op_type == "AveragePool" || op_type == "MaxPool" || is_qlinear_average_pool) { - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); const auto count_include_pad = helper.Get("count_include_pad", 0); if (count_include_pad == 1) { @@ -596,7 +598,7 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial return false; } - if (node.OutputDefs().size() != 1) { + if (node_unit.Outputs().size() != 1) { LOGS_DEFAULT(VERBOSE) << "Argmax in maxpooling is not supported"; return false; } @@ -607,37 +609,38 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial // We need to check if we have valid scales and zero points for QLinearAveragePool if (is_qlinear_average_pool) { - if (input_defs.size() < 4) + // Check input scales and ZPs + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */)) return false; - // the output zero point can be optional - bool has_output_zp = input_defs.size() == 5; + // Check output scale and ZP - if (!HasValidQuantizationScales(initializers, node, {1, 3}, params)) + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) return false; - - if (!HasValidQuantizationZeroPoints(initializers, node, - has_output_zp - ? std::vector{2} - : std::vector{2, 4})) { + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) return false; - } // NNAPI requires Quantized Average Pool has same scale and zero point for both input and output float input_scale = 0.0f; - auto status = GetQuantizationScale(initializers, node, 1, input_scale); + int32_t input_zp = 0; + auto status = GetQuantizationScaleAndZeroPoint( + initializers, node_unit.Inputs()[0], node_unit.ModelPath(), input_scale, input_zp); if (!status.IsOK()) { LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name - << "] GetQuantizationScale for input_scale failed, message: " + << "] GetQuantizationScaleAndZeroPoint for input_scale/zp failed, message: " << status.ErrorMessage(); return false; } float output_scale = 0.0f; - status = GetQuantizationScale(initializers, node, 3, output_scale); + int32_t output_zp = 0; + status = GetQuantizationScaleAndZeroPoint( + initializers, node_unit.Outputs()[0], node_unit.ModelPath(), output_scale, output_zp); if (!status.IsOK()) { LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name - << "] GetQuantizationScale for output_scale failed, message: " + << "] GetQuantizationScaleAndZeroPoint for output_scale/zp failed, message: " << status.ErrorMessage(); return false; } @@ -649,26 +652,6 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial return false; } - int32_t input_zp = 0; - int32_t output_zp = 0; - status = GetQuantizationZeroPoint(initializers, node, 2, input_zp); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name - << "] GetQuantizationZeroPoint for input_zp failed, message: " - << status.ErrorMessage(); - return false; - } - - if (has_output_zp) { - status = GetQuantizationZeroPoint(initializers, node, 4, output_zp); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name - << "] GetQuantizationZeroPoint for output_zp failed, message: " - << status.ErrorMessage(); - return false; - } - } - if (input_zp != output_zp) { LOGS_DEFAULT(VERBOSE) << "Op [" << op_type << "] name [" << op_name << "] has different input_zp: " << input_zp @@ -681,26 +664,24 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial } bool PoolOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) const { - // TODO, change to use node unit and quant_param of IODef - const auto& node = node_unit.GetNode(); - bool is_max_pool = node.OpType() == "MaxPool"; - bool is_qlinear_average_pool = node.OpType() == "QLinearAveragePool"; + bool is_max_pool = node_unit.OpType() == "MaxPool"; + bool is_qlinear_average_pool = node_unit.OpType() == "QLinearAveragePool"; if (!is_max_pool && !is_qlinear_average_pool) return BaseOpSupportChecker::HasSupportedInputsImpl(node_unit); if (is_qlinear_average_pool) { - return HasValidUnaryOpQuantizedInputs(node); + return HasValidUnaryOpQuantizedInputs(node_unit); } // is_max_pool // For max pool, we can support both float and uint8 input int32_t input_type; - if (!GetType(*node.InputDefs()[0], input_type)) + if (!GetType(node_unit.Inputs()[0].node_arg, input_type)) return false; if (input_type != ONNX_NAMESPACE::TensorProto_DataType_FLOAT && input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { - LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() + LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() << "] Input type: [" << input_type << "] is not supported for now"; return false; @@ -741,13 +722,11 @@ class ConvOpSupportChecker : public BaseOpSupportChecker { } bool ConvOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) const { - // TODO, change to use node unit and quant_param of IODef - const auto& node = node_unit.GetNode(); - if (node.OpType() != "QLinearConv") + if (node_unit.OpType() != "QLinearConv") return BaseOpSupportChecker::HasSupportedInputsImpl(node_unit); // QLinearConv only supports input of uint8 for now - if (!HasValidBinaryOpQuantizedInputs(node)) + if (!HasValidBinaryOpQuantizedInputs(node_unit)) return false; return true; @@ -755,21 +734,19 @@ bool ConvOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) con bool ConvOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) const { - const auto& node = node_unit.GetNode(); - const auto& op_type = node.OpType(); + const auto& op_type = node_unit.OpType(); const bool is_qlinear_conv = (op_type == "QLinearConv"); // We don't support nhwc com.microsoft.QLinearConv for now - if (is_qlinear_conv && node.Domain() == kMSDomain) { + if (is_qlinear_conv && node_unit.Domain() == kMSDomain) { LOGS_DEFAULT(VERBOSE) << "com.microsoft.QLinearConv is not supported"; return false; } - const auto input_defs = node.InputDefs(); - NodeAttrHelper helper(node); - size_t w_idx = is_qlinear_conv ? 3 : 1; + const auto& inputs = node_unit.Inputs(); + NodeAttrHelper helper(node_unit); const auto group = helper.Get("group", 1); - const auto weight_name = input_defs[w_idx]->Name(); + const auto weight_name = inputs[1].node_arg.Name(); if (Contains(initializers, weight_name)) { const auto& tensor = *initializers.at(weight_name); if (tensor.dims().size() != 4) { @@ -777,8 +754,8 @@ bool ConvOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial return false; } - const auto onnx_dilations = helper.Get("dilations", vector{1, 1}); - if (onnx_dilations != vector{1, 1}) { + const auto onnx_dilations = helper.Get("dilations", std::vector{1, 1}); + if (onnx_dilations != std::vector{1, 1}) { if (group != 1 && tensor.dims()[1] != 1) { LOGS_DEFAULT(VERBOSE) << "dilation is not supported on grouped conv"; return false; @@ -798,7 +775,7 @@ bool ConvOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial if (is_qlinear_conv) { // For QLinearConv, we only support uint8 output now int32_t output_type; - if (!GetType(*node.OutputDefs()[0], output_type)) + if (!GetType(node_unit.Outputs()[0].node_arg, output_type)) return false; if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { @@ -808,17 +785,21 @@ bool ConvOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial return false; } - if (input_defs.size() > 8 && !Contains(initializers, input_defs[8]->Name())) { + if (inputs.size() > 2 && !Contains(initializers, inputs[2].node_arg.Name())) { LOGS_DEFAULT(VERBOSE) << "Bias of QLinearConv must be known"; return false; } - // a/b/y_scale - if (!HasValidQuantizationScales(initializers, node, {1, 4, 6}, params)) + // Check input scales and ZPs + if (!HasValidQuantizationScales(initializers, node_unit, {0, 1}, params, true /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0, 1}, true /* is_input */)) return false; - // a/b/y_zero_point - if (!HasValidQuantizationZeroPoints(initializers, node, {2, 5, 7})) + // Check output scale and ZP + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) return false; } @@ -845,8 +826,7 @@ class CastOpSupportChecker : public BaseOpSupportChecker { bool CastOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); const auto to = helper.Get("to", 0); if (to != ONNX_NAMESPACE::TensorProto::FLOAT && to != ONNX_NAMESPACE::TensorProto::INT32) { @@ -874,9 +854,8 @@ class SoftMaxOpSupportChecker : public BaseOpSupportChecker { bool SoftMaxOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, const OpSupportCheckParams& params) const { - const auto& node = node_unit.GetNode(); Shape input_shape; - if (!GetShape(*node.InputDefs()[0], input_shape)) + if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape)) return false; const auto input_size = input_shape.size(); @@ -887,7 +866,7 @@ bool SoftMaxOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* i } if (params.android_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) { - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); int32_t axis = helper.Get("axis", 1); if (axis != 1) { LOGS_DEFAULT(VERBOSE) @@ -917,13 +896,11 @@ class GemmOpSupportChecker : public BaseOpSupportChecker { }; bool GemmOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) const { - // TODO, change to use node unit and quant_param of IODef - const auto& node = node_unit.GetNode(); - if (node.OpType() != "QLinearMatMul") + if (node_unit.OpType() != "QLinearMatMul") return BaseOpSupportChecker::HasSupportedInputsImpl(node_unit); // QLinearMatMul - if (!HasValidBinaryOpQuantizedInputs(node)) + if (!HasValidBinaryOpQuantizedInputs(node_unit)) return false; return true; @@ -985,19 +962,13 @@ int GemmOpSupportChecker::GetMinSupportedOpSet(const NodeUnit& node_unit) const bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) const { - const auto& node = node_unit.GetNode(); - const auto& op_type = node.OpType(); - const auto input_defs(node.InputDefs()); - size_t a_idx = 0, b_idx = 1, c_idx = 2; // A*B+C + const auto& op_type = node_unit.OpType(); + const auto& inputs = node_unit.Inputs(); bool is_qlinear_matmul = op_type == "QLinearMatMul"; - if (is_qlinear_matmul) { - a_idx = 0; - b_idx = 3; - } Shape a_shape; { - if (!GetShape(*input_defs[a_idx], a_shape)) + if (!GetShape(inputs[0].node_arg, a_shape)) return false; if (a_shape.size() != 2) { @@ -1008,7 +979,7 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial Shape b_shape; { - if (!GetShape(*input_defs[b_idx], b_shape)) + if (!GetShape(inputs[1].node_arg, b_shape)) return false; if (b_shape.size() != 2) { @@ -1021,7 +992,7 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial // Only support // 1. A*B'+C // 2. A*B+C and B is an initializer - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); const auto transA = helper.Get("transA", 0); const auto transB = helper.Get("transB", 0); const auto alpha = helper.Get("alpha", 1.0f); @@ -1037,14 +1008,14 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial return false; } - if (transB == 0 && !Contains(initializers, input_defs[b_idx]->Name())) { + if (transB == 0 && !Contains(initializers, inputs[1].node_arg.Name())) { LOGS_DEFAULT(VERBOSE) << "B of Gemm must be known if transB != 1"; return false; } - if (input_defs.size() == 3) { + if (inputs.size() == 3) { Shape c_shape; - if (!GetShape(*input_defs[c_idx], c_shape)) + if (!GetShape(inputs[2].node_arg, c_shape)) return false; uint32_t c_size; @@ -1062,7 +1033,7 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial } } else if (op_type == "MatMul" || is_qlinear_matmul) { // Only support A*B B is an initializer - if (!Contains(initializers, input_defs[b_idx]->Name())) { + if (!Contains(initializers, inputs[1].node_arg.Name())) { LOGS_DEFAULT(VERBOSE) << "B of MatMul must be known"; return false; } @@ -1070,7 +1041,7 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial if (is_qlinear_matmul) { // For QLinearMatMul, we only support uint8 output now int32_t output_type; - if (!GetType(*node.OutputDefs()[0], output_type)) + if (!GetType(node_unit.Outputs()[0].node_arg, output_type)) return false; if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { @@ -1081,12 +1052,16 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial } // All scale/zero points are initializer scalars - // a/b/y_scale - if (!HasValidQuantizationScales(initializers, node, {1, 4, 6}, params)) + // Check input scales and ZPs + if (!HasValidQuantizationScales(initializers, node_unit, {0, 1}, params, true /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0, 1}, true /* is_input */)) return false; - // a/b/y_zero_point - if (!HasValidQuantizationZeroPoints(initializers, node, {2, 5, 7})) + // Check output scale and ZP + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) return false; } } else { @@ -1116,7 +1091,7 @@ class UnaryOpSupportChecker : public BaseOpSupportChecker { int GetMinSupportedOpSet(const NodeUnit& node_unit) const override; - static bool IsQuantizedOpSupported(const InitializedTensorSet& initializers, const Node& node, + static bool IsQuantizedOpSupported(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params); }; @@ -1140,9 +1115,8 @@ class UnaryOpSupportChecker : public BaseOpSupportChecker { bool UnaryOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) const { - const auto& node = node_unit.GetNode(); - if (node.OpType() == "QLinearSigmoid") - return IsQuantizedOpSupported(initializers, node, params); + if (node_unit.OpType() == "QLinearSigmoid") + return IsQuantizedOpSupported(initializers, node_unit, params); else // Everything except "QLinearSigmoid" are by default supported return true; } @@ -1163,13 +1137,11 @@ int32_t UnaryOpSupportChecker::GetMinSupportedNNAPIFeatureLevel(const NodeUnit& } bool UnaryOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) const { - // TODO, change to use node unit and quant_param of IODef - const auto& node = node_unit.GetNode(); // We only need to override input check for QLinearSigmoid - if (node.OpType() != "QLinearSigmoid") + if (node_unit.OpType() != "QLinearSigmoid") return BaseOpSupportChecker::HasSupportedInputsImpl(node_unit); - return HasValidUnaryOpQuantizedInputs(node); + return HasValidUnaryOpQuantizedInputs(node_unit); } // All ops except "Sin" opset 5- uses consumed_inputs attribute which is not supported for now @@ -1183,35 +1155,35 @@ int UnaryOpSupportChecker::GetMinSupportedOpSet(const NodeUnit& node_unit) const } /* static */ bool UnaryOpSupportChecker::IsQuantizedOpSupported( - const InitializedTensorSet& initializers, const Node& node, const OpSupportCheckParams& params) { - const auto& op_type = node.OpType(); + const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) { + const auto& op_type = node_unit.OpType(); ORT_ENFORCE(op_type == "QLinearSigmoid"); - const auto& op_name = node.Name(); - const auto input_defs(node.InputDefs()); - // const auto output_defs(node.OutputDefs()); + const auto& op_name = node_unit.Name(); - if (input_defs.size() < 4) + // Check input scales and ZPs + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */)) return false; - bool has_output_zp = input_defs.size() == 5; - - if (!HasValidQuantizationScales(initializers, node, {1, 3}, params)) + // Check output scale and ZP + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) return false; - if (!HasValidQuantizationZeroPoints(initializers, node, - has_output_zp - ? std::vector{2} - : std::vector{2, 4})) - return false; + return false; // NNAPI requires the scale be 1.f/256 and zero point to be 0 // See https://android.googlesource.com/platform/frameworks/ml/+/refs/heads/android10-c2f2-release/nn/common/operations/Activation.cpp#180 float output_scale = 0.0f; - auto status = GetQuantizationScale(initializers, node, 3, output_scale); + int32_t output_zp = 0; + auto status = GetQuantizationScaleAndZeroPoint(initializers, node_unit.Outputs()[0], node_unit.ModelPath(), + output_scale, output_zp); if (!status.IsOK()) { LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name - << "] GetQuantizationScale failed, message: " << status.ErrorMessage(); + << "] GetQuantizationScaleAndZeroPoint failed, message: " << status.ErrorMessage(); return false; } @@ -1221,20 +1193,10 @@ int UnaryOpSupportChecker::GetMinSupportedOpSet(const NodeUnit& node_unit) const return false; } - int32_t output_zp; - if (has_output_zp) { - status = GetQuantizationZeroPoint(initializers, node, 4, output_zp); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name - << "] GetQuantizationZeroPoint failed, message: " << status.ErrorMessage(); - return false; - } - - if (output_zp != 0) { - LOGS_DEFAULT(VERBOSE) << "Op [" << op_type << "] name [" << op_name - << "] output zero point can only be 0, actual zero point: " << output_scale; - return false; - } + if (output_zp != 0) { + LOGS_DEFAULT(VERBOSE) << "Op [" << op_type << "] name [" << op_name + << "] output zero point can only be 0, actual zero point: " << output_scale; + return false; } return true; @@ -1254,9 +1216,8 @@ class ConcatOpSupportChecker : public BaseOpSupportChecker { bool ConcatOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); Shape input_shape; - if (!GetShape(*node.InputDefs()[0], input_shape)) + if (GetShape(node_unit.Inputs()[0].node_arg, input_shape)) return false; const auto input_size = input_shape.size(); @@ -1302,21 +1263,21 @@ class SqueezeOpSupportChecker : public BaseOpSupportChecker { bool SqueezeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); + const auto& inputs = node_unit.Inputs(); Shape input_shape; - if (!GetShape(*node.InputDefs()[0], input_shape)) + if (!GetShape(inputs[0].node_arg, input_shape)) return false; - const auto input_size = input_shape.size(); - if (input_size > 4 || input_size == 0) { + const auto input_rank = input_shape.size(); + if (input_rank > 4 || input_rank == 0) { LOGS_DEFAULT(VERBOSE) << "Squeeze only supports 1-4d shape, input is " - << input_size << "d shape"; + << input_rank << "d shape"; return false; } // Squeeze opset 13 use input 1 as axes, if we have input 1 then it need to be an initializer - if (node.SinceVersion() > 12 && node.InputDefs().size() > 1) { - const auto& axes_name = node.InputDefs()[1]->Name(); + if (node_unit.SinceVersion() > 12 && inputs.size() > 1) { + const auto& axes_name = inputs[1].node_arg.Name(); if (!Contains(initializers, axes_name)) { LOGS_DEFAULT(VERBOSE) << "Input axes of Squeeze must be known"; return false; @@ -1343,28 +1304,23 @@ class QuantizeLinearOpSupportChecker : public BaseOpSupportChecker { bool QuantizeLinearOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) const { - const auto& node = node_unit.GetNode(); - const auto input_defs(node.InputDefs()); - const auto output_defs(node.OutputDefs()); - int32_t output_type; - if (!GetType(*output_defs[0], output_type)) + if (!GetType(node_unit.Outputs()[0].node_arg, output_type)) return false; if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { - LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() + LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() << "] output type: [" << output_type << "] is not supported for now"; return false; } - if (!HasValidQuantizationScales(initializers, node, {1}, params)) + // For QuantizeLinear only output is quantized + // Check output scale and ZP + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) return false; - - if (input_defs.size() == 3) { // has zero_point input - if (!HasValidQuantizationZeroPoints(initializers, node, {2})) - return false; - } return true; } @@ -1387,15 +1343,12 @@ class DequantizeLinearOpSupportChecker : public BaseOpSupportChecker { bool DequantizeLinearOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) const { - const auto& node = node_unit.GetNode(); - const auto input_defs(node.InputDefs()); - if (!HasValidQuantizationScales(initializers, node, {1}, params)) + // For DequantizeLinear only input is quantized + // Check input scale and ZP + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */)) return false; - - if (input_defs.size() == 3) { // has zero_point input - if (!HasValidQuantizationZeroPoints(initializers, node, {2})) - return false; - } return true; } @@ -1432,9 +1385,8 @@ class LRNOpSupportChecker : public BaseOpSupportChecker { bool LRNOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); Shape input_shape; - if (!GetShape(*node.InputDefs()[0], input_shape)) + if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape)) return false; const auto input_size = input_shape.size(); @@ -1459,9 +1411,8 @@ class ClipOpSupportChecker : public BaseOpSupportChecker { bool ClipOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); float min, max; - if (!GetClipMinMax(initializers, node, min, max, logging::LoggingManager::DefaultLogger())) + if (!GetClipMinMax(initializers, node_unit.GetNode(), min, max, logging::LoggingManager::DefaultLogger())) return false; // We only supoort relu6 or relu1 @@ -1496,9 +1447,8 @@ class ResizeOpSupportChecker : public BaseOpSupportChecker { bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) const { - const auto& node = node_unit.GetNode(); Shape input_shape; - if (!GetShape(*node.InputDefs()[0], input_shape)) + if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape)) return false; const auto input_size = input_shape.size(); @@ -1509,7 +1459,7 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi } { // check attributes - NodeAttrHelper helper(node); + NodeAttrHelper helper(node_unit); const auto mode = helper.Get("mode", "nearest"); bool is_linear_resize = mode == "linear"; bool is_nearest_resize = mode == "nearest"; @@ -1556,27 +1506,27 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi } { // scales and sizes (if present) must be initializers - const auto input_defs = node.InputDefs(); - if (input_defs.size() < 3) { + const auto inputs = node_unit.Inputs(); + if (inputs.size() < 3) { LOGS_DEFAULT(VERBOSE) << "Input scales or sizes of Resize must be known"; return false; } // scales - if (input_defs.size() == 3 && !Contains(initializers, input_defs[2]->Name())) { + if (inputs.size() == 3 && !Contains(initializers, inputs[2].node_arg.Name())) { LOGS_DEFAULT(VERBOSE) << "Input scales of Resize must be known"; return false; } // sizes - if (input_defs.size() > 3 && !Contains(initializers, input_defs[3]->Name())) { + if (inputs.size() > 3 && !Contains(initializers, inputs[3].node_arg.Name())) { LOGS_DEFAULT(VERBOSE) << "Input sizes of Resize must be known"; return false; } // We want to check if the scales or sizes are not trying to resize on N/C channels here - if (input_defs.size() == 3) { // we are using scales - const auto& scales_tensor = *initializers.at(input_defs[2]->Name()); + if (inputs.size() == 3) { // we are using scales + const auto& scales_tensor = *initializers.at(inputs[2].node_arg.Name()); std::vector unpacked_tensor; auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor); if (!status.IsOK()) { @@ -1594,7 +1544,7 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi } } else { // we are using sizes - const auto& sizes_name = input_defs[3]->Name(); + const auto& sizes_name = inputs[3].node_arg.Name(); const auto& sizes_tensor = *initializers.at(sizes_name); std::vector unpacked_tensor; auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor); @@ -1659,9 +1609,8 @@ class FlattenOpSupportChecker : public BaseOpSupportChecker { bool FlattenOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); Shape input_shape; - if (!GetShape(*node.InputDefs()[0], input_shape)) + if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape)) return false; if (input_shape.size() > 4 || input_shape.empty()) { @@ -1672,7 +1621,7 @@ bool FlattenOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* i int32_t dim_1 = 1; int32_t dim_2 = 1; - GetFlattenOutputShape(node, input_shape, dim_1, dim_2); + GetFlattenOutputShape(node_unit, input_shape, dim_1, dim_2); if (dim_1 == 0 && dim_2 == 0) { LOGS_DEFAULT(VERBOSE) << "The dynamical input shape " << Shape2String(input_shape) @@ -1717,11 +1666,10 @@ class MinMaxOpSupportChecker : public BaseOpSupportChecker { bool MinMaxOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); // TODO support 2+ inputs for Min/Max op - if (node.InputDefs().size() != 2) { - LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() << "] only supports 2 inputs, " - << "actual input number, " << node.InputDefs().size(); + if (node_unit.Inputs().size() != 2) { + LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() << "] only supports 2 inputs, " + << "actual input number, " << node_unit.Inputs().size(); return false; } @@ -1763,9 +1711,8 @@ class SliceOpSupportChecker : public BaseOpSupportChecker { bool SliceOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { - const auto& node = node_unit.GetNode(); Shape input_shape; - if (!GetShape(*node.InputDefs()[0], input_shape)) + if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape)) return false; if (input_shape.size() > 4) { @@ -1780,19 +1727,19 @@ bool SliceOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initia return false; } - if (!CheckIsInitializer(initializers, node, 1, "starts")) { + if (!CheckIsInitializer(initializers, node_unit, node_unit.Inputs()[1].node_arg.Name(), "starts")) { return false; } - if (!CheckIsInitializer(initializers, node, 2, "ends")) { + if (!CheckIsInitializer(initializers, node_unit, node_unit.Inputs()[2].node_arg.Name(), "ends")) { return false; } - const auto& input_defs = node.InputDefs(); - if (input_defs.size() > 3) { - if (!CheckIsInitializer(initializers, node, 3, "axes")) { + const auto& inputs = node_unit.Inputs(); + if (inputs.size() > 3) { + if (!CheckIsInitializer(initializers, node_unit, node_unit.Inputs()[3].node_arg.Name(), "axes")) { return false; } - if (input_defs.size() > 4) { - if (!CheckIsInitializer(initializers, node, 4, "steps")) { + if (inputs.size() > 4) { + if (!CheckIsInitializer(initializers, node_unit, node_unit.Inputs()[4].node_arg.Name(), "steps")) { return false; } } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc index 9d675ecfa8..1652237622 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc @@ -3,20 +3,17 @@ #include "core/providers/common.h" -#include "helper.h" #include "shaper.h" +#include "helper.h" namespace onnxruntime { namespace nnapi { -using std::string; -using std::vector; - std::pair ComputeConvOutputShape(const uint32_t input_size_y, const uint32_t input_size_x, const uint32_t weight_size_y, const uint32_t weight_size_x, - const vector& onnx_pads, - const vector& onnx_strides, - const vector& onnx_dilations) { + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations) { int32_t padding_top = onnx_pads[0]; int32_t padding_bottom = onnx_pads[2]; int32_t padding_left = onnx_pads[1]; @@ -53,9 +50,9 @@ std::pair ComputeConvOutputShape(const uint32_t input_size_y Status Shaper::Conv(const std::string& input_name, const std::string& weight_name, - const vector& onnx_pads, - const vector& onnx_strides, - const vector& onnx_dilations, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, bool nchw, const std::string& output_name) { SHAPER_FUNC(Conv, @@ -150,9 +147,9 @@ Status Shaper::ResizeUsingOutputSizes(const std::string& input_name, Status Shaper::ConvImpl(const std::string& input_name, const std::string& weight_name, - const vector& onnx_pads, - const vector& onnx_strides, - const vector& onnx_dilations, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, bool nchw, const std::string& output_name) { const Shape& input_dimen = shape_map_.at(input_name); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h index b9299454dc..8656328804 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h @@ -6,7 +6,8 @@ #include #include #include -#include + +#include "core/common/status.h" namespace onnxruntime { namespace nnapi { @@ -20,115 +21,103 @@ class Shaper { return shape_map_.at(key); } - Status Conv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) ORT_MUST_USE_RESULT; + common::Status Conv(const std::string& input_name, + const std::string& weight_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, + bool nchw, + const std::string& output_name); - Status DepthwiseConv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) ORT_MUST_USE_RESULT; + common::Status DepthwiseConv(const std::string& input_name, + const std::string& weight_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, + bool nchw, + const std::string& output_name); - Status Pool(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name) ORT_MUST_USE_RESULT; + common::Status Pool(const std::string& input_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& kernel_shape, + bool nchw, + const std::string& output_name); - Status Reshape(const std::string& input_name, const std::vector& shape, const std::string& output_name) - ORT_MUST_USE_RESULT; + common::Status Reshape(const std::string& input_name, const std::vector& shape, const std::string& output_name); - Status Transpose(const std::string& input_name, const std::vector& perm, const std::string& output_name) - ORT_MUST_USE_RESULT; + common::Status Transpose(const std::string& input_name, const std::vector& perm, const std::string& output_name); - Status Eltwise(const std::string& input1_name, const std::string& input2_name, const std::string& output_name) - ORT_MUST_USE_RESULT; + common::Status Eltwise(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); - Status Identity(const std::string& input_name, const std::string& output_name) ORT_MUST_USE_RESULT; + common::Status Identity(const std::string& input_name, const std::string& output_name); - Status FC(const std::string& input1_name, const std::string& input2_name, const std::string& output_name) - ORT_MUST_USE_RESULT; + common::Status FC(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); - Status Concat(const std::vector& input_names, const int32_t axis, const std::string& output_name) - ORT_MUST_USE_RESULT; + common::Status Concat(const std::vector& input_names, const int32_t axis, const std::string& output_name); - Status Squeeze(const std::string& input_name, const std::vector& axes, const std::string& output_name) - ORT_MUST_USE_RESULT; + common::Status Squeeze(const std::string& input_name, const std::vector& axes, const std::string& output_name); - Status ResizeUsingScales(const std::string& input_name, - const float scale_h, const float scale_w, - bool nchw, - const std::string& output_name) ORT_MUST_USE_RESULT; - Status ResizeUsingOutputSizes(const std::string& input_name, - const uint32_t output_h, const uint32_t output_w, - bool nchw, - const std::string& output_name) ORT_MUST_USE_RESULT; + common::Status ResizeUsingScales(const std::string& input_name, + const float scale_h, const float scale_w, + bool nchw, + const std::string& output_name); + common::Status ResizeUsingOutputSizes(const std::string& input_name, + const uint32_t output_h, const uint32_t output_w, + bool nchw, + const std::string& output_name); // If the shape of certain input is dynamic // Use the following 2 functions to update the particular shape // and calculate the new output shape // Only perform this when the NNAPI model is finalized! - Status UpdateShape(const std::string& name, const Shape& new_shape) ORT_MUST_USE_RESULT; - Status UpdateDynamicDimensions() ORT_MUST_USE_RESULT; + common::Status UpdateShape(const std::string& name, const Shape& new_shape); + common::Status UpdateDynamicDimensions(); void Clear(); private: - Status ConvImpl(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) ORT_MUST_USE_RESULT; + common::Status ConvImpl(const std::string& input_name, + const std::string& weight_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, + bool nchw, + const std::string& output_name); - Status DepthwiseConvImpl(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) ORT_MUST_USE_RESULT; + common::Status DepthwiseConvImpl(const std::string& input_name, + const std::string& weight_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, + bool nchw, + const std::string& output_name); - Status PoolImpl(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name) ORT_MUST_USE_RESULT; + common::Status PoolImpl(const std::string& input_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& kernel_shape, + bool nchw, + const std::string& output_name); - Status ReshapeImpl(const std::string& input_name, const std::vector& shape, const std::string& output_name) - ORT_MUST_USE_RESULT; - Status TransposeImpl(const std::string& input_name, const std::vector& perm, const std::string& output_name) - ORT_MUST_USE_RESULT; - Status EltwiseImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name) - ORT_MUST_USE_RESULT; - Status IdentityImpl(const std::string& input_name, const std::string& output_name) ORT_MUST_USE_RESULT; - Status FCImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name) - ORT_MUST_USE_RESULT; - Status ConcatImpl(const std::vector& input_names, const int32_t axis, const std::string& output_name) - ORT_MUST_USE_RESULT; - Status SqueezeImpl(const std::string& input_names, const std::vector& axes, const std::string& output_name) - ORT_MUST_USE_RESULT; - Status ResizeUsingScalesImpl(const std::string& input_name, - const float scale_h, const float scale_w, - bool nchw, - const std::string& output_name) ORT_MUST_USE_RESULT; - Status ResizeUsingOutputSizesImpl(const std::string& input_name, - const uint32_t output_h, const uint32_t output_w, - bool nchw, - const std::string& output_name) ORT_MUST_USE_RESULT; + common::Status ReshapeImpl(const std::string& input_name, const std::vector& shape, const std::string& output_name); + common::Status TransposeImpl(const std::string& input_name, const std::vector& perm, const std::string& output_name); + common::Status EltwiseImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); + common::Status IdentityImpl(const std::string& input_name, const std::string& output_name); + common::Status FCImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); + common::Status ConcatImpl(const std::vector& input_names, const int32_t axis, const std::string& output_name); + common::Status SqueezeImpl(const std::string& input_names, const std::vector& axes, const std::string& output_name); + common::Status ResizeUsingScalesImpl(const std::string& input_name, + const float scale_h, const float scale_w, + bool nchw, + const std::string& output_name); + common::Status ResizeUsingOutputSizesImpl(const std::string& input_name, + const uint32_t output_h, const uint32_t output_w, + bool nchw, + const std::string& output_name); std::unordered_map shape_map_; - std::vector> shape_ops_; + std::vector> shape_ops_; }; } // namespace nnapi diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc index 7a2036252a..887384e6bd 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include - #include "model.h" + +#include "core/common/logging/logging.h" #include "core/providers/common.h" #include "core/providers/nnapi/nnapi_builtin/builders/helper.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h index 8ce72538af..6326e60cf9 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h @@ -103,7 +103,7 @@ class Model { // this output may need special handling bool IsScalarOutput(const std::string& output_name) const; - Status PrepareForExecution(std::unique_ptr& execution) ORT_MUST_USE_RESULT; + common::Status PrepareForExecution(std::unique_ptr& execution); private: const NnApi* nnapi_{nullptr}; @@ -143,7 +143,7 @@ class Model { void AddScalarOutput(const std::string& output_name); - void SetShaper(const Shaper shaper) { shaper_ = shaper; } + void SetShaper(const Shaper& shaper) { shaper_ = shaper; } int32_t GetNNAPIFeatureLevel() const; }; @@ -172,17 +172,16 @@ class Execution { // Set the input/output data buffers // These need to be called before calling Predict() - Status SetInputBuffers(const std::vector& inputs) ORT_MUST_USE_RESULT; - Status SetOutputBuffers(const std::vector& outputs) ORT_MUST_USE_RESULT; + common::Status SetInputBuffers(const std::vector& inputs); + common::Status SetOutputBuffers(const std::vector& outputs); // Execute the NNAPI model // if there is dynamic output shape, will output the actual output shapes - Status Predict(const std::vector& dynamic_outputs, std::vector& dynamic_output_shapes) - ORT_MUST_USE_RESULT; + common::Status Predict(const std::vector& dynamic_outputs, std::vector& dynamic_output_shapes); private: - Status SetInputBuffer(const int32_t index, const InputBuffer& input) ORT_MUST_USE_RESULT; - Status SetOutputBuffer(const int32_t index, const OutputBuffer& output) ORT_MUST_USE_RESULT; + common::Status SetInputBuffer(const int32_t index, const InputBuffer& input); + common::Status SetOutputBuffer(const int32_t index, const OutputBuffer& output); const NnApi* nnapi_{nullptr}; ANeuralNetworksExecution* execution_; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index fa876a7ef6..85a0cf3ad1 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -21,8 +21,6 @@ #include "core/providers/nnapi/nnapi_builtin/model.h" #endif -using onnxruntime::NodeUnit; - namespace onnxruntime { namespace { @@ -189,14 +187,6 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view } #ifdef __ANDROID__ -static Status GetOutputBuffer(Ort::CustomOpApi& ort, - OrtKernelContext* context, - const nnapi::Model& model, - const std::string& output_name, - const std::vector& output_shape, - const android::nn::wrapper::Type output_type, - void** output_buffer) ORT_MUST_USE_RESULT; - static Status GetOutputBuffer(Ort::CustomOpApi& ort, OrtKernelContext* context, const nnapi::Model& model, diff --git a/onnxruntime/core/providers/shared/node_unit/node_unit.cc b/onnxruntime/core/providers/shared/node_unit/node_unit.cc index 80492ce701..d443fe858f 100644 --- a/onnxruntime/core/providers/shared/node_unit/node_unit.cc +++ b/onnxruntime/core/providers/shared/node_unit/node_unit.cc @@ -6,40 +6,170 @@ namespace onnxruntime { +namespace { + +// The QLinearOpType GetQLinearOpType, is very similar to the one in NNAPI +// However, the NNAPI ones are only the subset of the ones here, +// TODO, make these shared +enum class QLinearOpType : uint8_t { + Unknown, // Unknown or not a linear quantized op + DequantizeLinear, + QuantizeLinear, + QLinearConv, + QLinearMatMul, + QLinearAdd, + QLinearSigmoid, + QLinearAveragePool, + QLinearMul, + QLinearReduceMean, + QLinearConcat, + QLinearGlobalAveragePool, + QLinearLeakyRelu, +}; + +QLinearOpType GetQLinearOpType(const onnxruntime::Node& node) { + const auto& op_type = node.OpType(); + if (op_type == "DequantizeLinear") + return QLinearOpType::DequantizeLinear; + else if (op_type == "QuantizeLinear") + return QLinearOpType::QuantizeLinear; + else if (op_type == "QLinearConv") + return QLinearOpType::QLinearConv; + else if (op_type == "QLinearMatMul") + return QLinearOpType::QLinearMatMul; + else if (op_type == "QLinearAdd") + return QLinearOpType::QLinearAdd; + else if (op_type == "QLinearSigmoid") + return QLinearOpType::QLinearSigmoid; + else if (op_type == "QLinearAveragePool") + return QLinearOpType::QLinearAveragePool; + else if (op_type == "QLinearMul") + return QLinearOpType::QLinearMul; + else if (op_type == "QLinearReduceMean") + return QLinearOpType::QLinearReduceMean; + else if (op_type == "QLinearConcat") + return QLinearOpType::QLinearConcat; + else if (op_type == "QLinearGlobalAveragePool") + return QLinearOpType::QLinearGlobalAveragePool; + else if (op_type == "QLinearLeakyRelu") + return QLinearOpType::QLinearLeakyRelu; + + return QLinearOpType::Unknown; +} + +// Ops have 1 input +bool IsUnaryQLinearOp(QLinearOpType type) { + return type == QLinearOpType::QLinearSigmoid || + type == QLinearOpType::QLinearAveragePool || + type == QLinearOpType::QLinearGlobalAveragePool || + type == QLinearOpType::QLinearLeakyRelu || + type == QLinearOpType::QLinearReduceMean; +} + +// Ops have 2 inputs +bool IsBinaryQLinearOp(QLinearOpType type) { + return type == QLinearOpType::QLinearConv || + type == QLinearOpType::QLinearMatMul || + type == QLinearOpType::QLinearAdd || + type == QLinearOpType::QLinearMul; +} + +// Ops have 1 or more inputs +bool IsVariadicQLinearOp(QLinearOpType type) { + return type == QLinearOpType::QLinearConcat; +} + +} // namespace + NodeUnit::NodeUnit(const Node& node) - : nodes_{&node}, - node_(node), + : output_nodes_{&node}, + target_node_(node), type_(Type::SingleNode) { InitForNode(); } -const std::string& NodeUnit::Domain() const noexcept { return node_.Domain(); } -const std::string& NodeUnit::OpType() const noexcept { return node_.OpType(); } -const std::string& NodeUnit::Name() const noexcept { return node_.Name(); } -int NodeUnit::SinceVersion() const noexcept { return node_.SinceVersion(); } -NodeIndex NodeUnit::Index() const noexcept { return node_.Index(); } -const Path& NodeUnit::ModelPath() const noexcept { return node_.ModelPath(); } -ProviderType NodeUnit::GetExecutionProviderType() const noexcept { return node_.GetExecutionProviderType(); } +const std::string& NodeUnit::Domain() const noexcept { return target_node_.Domain(); } +const std::string& NodeUnit::OpType() const noexcept { return target_node_.OpType(); } +const std::string& NodeUnit::Name() const noexcept { return target_node_.Name(); } +int NodeUnit::SinceVersion() const noexcept { return target_node_.SinceVersion(); } +NodeIndex NodeUnit::Index() const noexcept { return target_node_.Index(); } +const Path& NodeUnit::ModelPath() const noexcept { return target_node_.ModelPath(); } +ProviderType NodeUnit::GetExecutionProviderType() const noexcept { return target_node_.GetExecutionProviderType(); } void NodeUnit::InitForNode() { - const auto& input_defs = node_.InputDefs(); - const auto& output_defs = node_.OutputDefs(); - // The 1st step is to hookup the NodeUnit with the NNAPI builder interface - // So we are not handling quantization here now - // TODO, enable quantization - // auto qlinear_type = GetQLinearOpType(node_); - // if (qlinear_type == QLinearOpType::Unknown) { - // Not a Qlinear op, add all inputs/outputs - auto add_all_io = [](std::vector& defs, - const ConstPointerContainer>& node_defs) { - defs.reserve(node_defs.size()); + const auto& input_defs = target_node_.InputDefs(); + const auto& output_defs = target_node_.OutputDefs(); + auto qlinear_type = GetQLinearOpType(target_node_); + if (qlinear_type == QLinearOpType::Unknown || + IsVariadicQLinearOp(qlinear_type)) { // TODO, add variadic support + // Not a Qlinear op, add all inputs / outputs + auto add_all_io = [](std::vector& defs, + const ConstPointerContainer>& node_defs) { + defs.reserve(node_defs.size()); - for (const auto def : node_defs) { - defs.push_back(NodeUnit::IODef{*def, std::nullopt}); + for (const auto def : node_defs) { + defs.push_back(NodeUnitIODef{*def, std::nullopt}); + } + }; + add_all_io(inputs_, input_defs); + add_all_io(outputs_, output_defs); + } else if (IsUnaryQLinearOp(qlinear_type)) { + // Unary QLinear Op has 5 inputs + // x, x_scale, x_zp, y_scale, y_zp (optional) + inputs_.push_back(NodeUnitIODef{ + *input_defs[0], + NodeUnitIODef::QuantParam{*input_defs[1], input_defs[2]}}); + + outputs_.push_back(NodeUnitIODef{ + *output_defs[0], + NodeUnitIODef::QuantParam{*input_defs[3], + input_defs.size() > 4 + ? input_defs[4] + : nullptr}}); + } else if (IsBinaryQLinearOp(qlinear_type)) { + // Binary QLinear Op has 9 inputs + // x1, x1_scale, x1_zp, x2/w, x2_scale, x2_zp, y_scale , y_zp, B + inputs_.push_back(NodeUnitIODef{ + *input_defs[0], + NodeUnitIODef::QuantParam{*input_defs[1], input_defs[2]}}); + inputs_.push_back(NodeUnitIODef{ + *input_defs[3], + NodeUnitIODef::QuantParam{*input_defs[4], input_defs[5]}}); + + if (input_defs.size() == 9) { // has Bias + inputs_.push_back(NodeUnitIODef{ + *input_defs[8], + std::nullopt}); // for Bias the scale and zp are optional } - }; - add_all_io(input_defs_, input_defs); - add_all_io(output_defs_, output_defs); + + outputs_.push_back(NodeUnitIODef{ + *output_defs[0], + NodeUnitIODef::QuantParam{*input_defs[6], input_defs[7]}}); + } else if (qlinear_type == QLinearOpType::DequantizeLinear) { + // DequantizeLinear has 3 inputs + // x, x_scale, x_zp + // output is not quantized + inputs_.push_back(NodeUnitIODef{ + *input_defs[0], + NodeUnitIODef::QuantParam{*input_defs[1], + input_defs.size() == 3 + ? input_defs[2] + : nullptr}}); + outputs_.push_back(NodeUnitIODef{*output_defs[0], std::nullopt}); + } else if (qlinear_type == QLinearOpType::QuantizeLinear) { + // QuantizeLinear the input is not quantized and has 3 inputs + // x, y_scale, y_zp (optional) + // The output is quantized + inputs_.push_back(NodeUnitIODef{*input_defs[0], std::nullopt}); + outputs_.push_back(NodeUnitIODef{ + *output_defs[0], + NodeUnitIODef::QuantParam{*input_defs[1], + input_defs.size() == 3 + ? input_defs[2] + : nullptr}}); + } else { + ORT_THROW("The QLinear op [", static_cast(qlinear_type), "] is not supported"); + } } } // namespace onnxruntime diff --git a/onnxruntime/core/providers/shared/node_unit/node_unit.h b/onnxruntime/core/providers/shared/node_unit/node_unit.h index d94b0e2fc5..e109703c93 100644 --- a/onnxruntime/core/providers/shared/node_unit/node_unit.h +++ b/onnxruntime/core/providers/shared/node_unit/node_unit.h @@ -21,6 +21,20 @@ namespace QDQ { struct NodeGroup; } +// Definition of one input or output +// If the optional quant_param is present, then this is a quantized input, +// otherwise this is a regular input +struct NodeUnitIODef { + // The quantization parameter, scale is manadatory, and zero_point is optional + struct QuantParam { + const NodeArg& scale; + const NodeArg* zero_point{nullptr}; + }; + + const NodeArg& node_arg; + const std::optional quant_param; +}; + /** @class NodeUnit Class to represent a single node or a QDQ group of nodes, which will be used as a single unit. @@ -33,27 +47,13 @@ class NodeUnit { QDQGroup, // The NodeUnit contain a QDQ group of nodes, such as "DQ->Sigmoid->Q" }; - // Definition of one input or output - // If the optional quant_param is present, then this is a quantized input, - // otherwise this is a regular input - struct IODef { - // The quantization parmeter, scale is manadatory, and zero_point is optional - struct QuantParam { - const NodeArg& scale; - const NodeArg* zero_point{nullptr}; - }; - - const NodeArg& node_arg; - const std::optional quant_param; - }; - public: explicit NodeUnit(const Node& node); Type UnitType() const noexcept { return type_; } - const std::vector& Inputs() const noexcept { return input_defs_; } - const std::vector& Outputs() const noexcept { return output_defs_; } + const std::vector& Inputs() const noexcept { return inputs_; } + const std::vector& Outputs() const noexcept { return outputs_; } const std::string& Domain() const noexcept; const std::string& OpType() const noexcept; @@ -63,16 +63,15 @@ class NodeUnit { const Path& ModelPath() const noexcept; ProviderType GetExecutionProviderType() const noexcept; - const Node& GetNode() const noexcept { return node_; } - - const std::vector GetAllNodes() const noexcept { return nodes_; } + const Node& GetNode() const noexcept { return target_node_; } + const std::vector GetOutputNodes() const noexcept { return output_nodes_; } private: - std::vector input_defs_; - std::vector output_defs_; + std::vector inputs_; + std::vector outputs_; - const std::vector nodes_; // all nodes in this NodeUnit - const Node& node_; // target Node + const std::vector output_nodes_; // all the nodes producing outputs for this NodeUnit + const Node& target_node_; Type type_; void InitForNode(); // Initializing for single Node diff --git a/onnxruntime/core/providers/shared/utils/utils.cc b/onnxruntime/core/providers/shared/utils/utils.cc index 6f38a8e368..d0e33062d7 100644 --- a/onnxruntime/core/providers/shared/utils/utils.cc +++ b/onnxruntime/core/providers/shared/utils/utils.cc @@ -8,6 +8,7 @@ #include #include #include +#include "core/providers/shared/node_unit/node_unit.h" namespace onnxruntime { @@ -81,6 +82,9 @@ bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node, NodeAttrHelper::NodeAttrHelper(const onnxruntime::Node& node) : node_attributes_(node.GetAttributes()) {} +NodeAttrHelper::NodeAttrHelper(const NodeUnit& node_unit) + : node_attributes_(node_unit.GetNode().GetAttributes()) {} + float NodeAttrHelper::Get(const std::string& key, float def_val) const { if (!HasAttr(key)) return def_val; diff --git a/onnxruntime/core/providers/shared/utils/utils.h b/onnxruntime/core/providers/shared/utils/utils.h index 925df731fc..26898aa95e 100644 --- a/onnxruntime/core/providers/shared/utils/utils.h +++ b/onnxruntime/core/providers/shared/utils/utils.h @@ -17,6 +17,7 @@ class Logger; class Node; class NodeArg; +class NodeUnit; // Get the min/max of a Clip operator. // If min/max are not known initializer tensors, will return false @@ -34,7 +35,10 @@ bool GetType(const NodeArg& node_arg, int32_t& type, const logging::Logger& logg */ class NodeAttrHelper { public: - NodeAttrHelper(const onnxruntime::Node& node); + explicit NodeAttrHelper(const Node& node); + + // Get the attributes from the target node of the node_unit + explicit NodeAttrHelper(const NodeUnit& node_unit); float Get(const std::string& key, float def_val) const; @@ -52,7 +56,7 @@ class NodeAttrHelper { bool HasAttr(const std::string& key) const; private: - const onnxruntime::NodeAttributes& node_attributes_; + const NodeAttributes& node_attributes_; }; } // namespace onnxruntime From c1c9fa18bfe257929dae7dc7447cfd672e232c73 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 19 Jan 2022 07:43:44 +1000 Subject: [PATCH 03/59] C#: Avoid inefficient DenseTensor ctor in ToTensor extensions (#10240) * Update extension helpers to avoid inefficient construction of DenseTensor. Add tests for extension helpers. --- .../Tensors/ArrayTensorExtensions.shared.cs | 78 ++++++++++++- .../Tensors/DenseTensor.shared.cs | 58 +++++++--- .../Tensors/Tensor.shared.cs | 22 +++- ...crosoft.ML.OnnxRuntime.Tests.Common.csproj | 1 + .../Tensors/ArrayTensorExtensionsTests.cs | 105 ++++++++++++++++++ ...oft.ML.OnnxRuntime.Tests.NetCoreApp.csproj | 3 + 6 files changed, 241 insertions(+), 26 deletions(-) create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Tensors/ArrayTensorExtensionsTests.cs diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/ArrayTensorExtensions.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/ArrayTensorExtensions.shared.cs index b26d215840..f6f57bd0b9 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/ArrayTensorExtensions.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/ArrayTensorExtensions.shared.cs @@ -27,7 +27,12 @@ namespace Microsoft.ML.OnnxRuntime.Tensors /// A 1-dimensional DenseTensor<T> with the same length and content as . public static DenseTensor ToTensor(this T[] array) { - return new DenseTensor(array); + // DenseTensor(Array, ...) is not efficient so do the copy here. + var dimensions = new int[] { array.Length }; + T[] copy = new T[array.Length]; + array.CopyTo(copy, 0); + + return new DenseTensor(new Memory(copy), dimensions); } /// @@ -39,7 +44,25 @@ namespace Microsoft.ML.OnnxRuntime.Tensors /// A 2-dimensional DenseTensor<T> with the same dimensions and content as . public static DenseTensor ToTensor(this T[,] array, bool reverseStride = false) { - return new DenseTensor(array, reverseStride); + if (reverseStride) + { + // we need logic from the DenseTensor ctor to be applied during copying + return new DenseTensor(array, reverseStride); + } + else + { + // it's more efficient to copy and flatten to 1D T[] and construct DenseTensor with Memory + T[] copy = new T[array.Length]; + var dimensions = new int[] { array.GetLength(0), array.GetLength(1) }; + + long idx = 0; + foreach (var item in array) + { + copy[idx++] = item; + } + + return new DenseTensor(new Memory(copy), dimensions); + } } /// @@ -51,7 +74,56 @@ namespace Microsoft.ML.OnnxRuntime.Tensors /// A 3-dimensional DenseTensor<T> with the same dimensions and content as . public static DenseTensor ToTensor(this T[,,] array, bool reverseStride = false) { - return new DenseTensor(array, reverseStride); + if (reverseStride) + { + // we need logic from the DenseTensor ctor to be applied during copying + return new DenseTensor(array, reverseStride); + } + else + { + // it's more efficient to copy and flatten to 1D T[] and construct DenseTensor with Memory + T[] copy = new T[array.Length]; + var dimensions = new int[] { array.GetLength(0), array.GetLength(1), array.GetLength(2) }; + + long idx = 0; + foreach (var item in array) + { + copy[idx++] = item; + } + + return new DenseTensor(new Memory(copy), dimensions); + } + } + + /// + /// Creates a copy of this four-dimensional array as a DenseTensor<T> + /// + /// Type contained in the array to copy to the DenseTensor<T>. + /// The array to create a DenseTensor<T> from. + /// False (default) to indicate that the first dimension is most major (farthest apart) and the last dimension is most minor (closest together): akin to row-major in a rank-2 tensor. True to indicate that the last dimension is most major (farthest apart) and the first dimension is most minor (closest together): akin to column-major in a rank-2 tensor. + /// A 4-dimensional DenseTensor<T> with the same dimensions and content as . + public static DenseTensor ToTensor(this T[,,,] array, bool reverseStride = false) + { + if (reverseStride) + { + // we need logic from the DenseTensor ctor to be applied during copying + return new DenseTensor(array, reverseStride); + } + else + { + // it's more efficient to copy and flatten to 1D T[] and construct DenseTensor with Memory + T[] copy = new T[array.Length]; + var dimensions = new int[] { + array.GetLength(0), array.GetLength(1), array.GetLength(2), array.GetLength(3) }; + + long idx = 0; + foreach (var item in array) + { + copy[idx++] = item; + } + + return new DenseTensor(new Memory(copy), dimensions); + } } /// diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/DenseTensor.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/DenseTensor.shared.cs index 7dc3d6bd15..997b5eeb24 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/DenseTensor.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/DenseTensor.shared.cs @@ -51,6 +51,7 @@ namespace Microsoft.ML.OnnxRuntime.Tensors backingArray[index++] = (T)item; } } + memory = backingArray; } @@ -66,26 +67,42 @@ namespace Microsoft.ML.OnnxRuntime.Tensors /// /// Initializes a rank-n Tensor using the dimensions specified in . /// - /// An span of integers that represent the size of each dimension of the DenseTensor to create. - /// False (default) to indicate that the first dimension is most major (farthest apart) and the last dimension is most minor (closest together): akin to row-major in a rank-2 tensor. True to indicate that the last dimension is most major (farthest apart) and the first dimension is most minor (closest together): akin to column-major in a rank-2 tensor. + /// + /// An span of integers that represent the size of each dimension of the DenseTensor to create. + /// + /// + /// False (default) to indicate that the first dimension is most major (farthest apart) and the last dimension + /// is most minor (closest together): akin to row-major in a rank-2 tensor. + /// True to indicate that the last dimension is most major (farthest apart) and the first dimension is most + /// minor (closest together): akin to column-major in a rank-2 tensor. + /// public DenseTensor(ReadOnlySpan dimensions, bool reverseStride = false) : base(dimensions, reverseStride) { memory = new T[Length]; } /// - /// Constructs a new DenseTensor of the specifed dimensions, wrapping existing backing memory for the contents. + /// Constructs a new DenseTensor of the specified dimensions, wrapping existing backing memory for the contents. /// /// - /// An span of integers that represent the size of each dimension of the DenseTensor to create. - /// False (default) to indicate that the first dimension is most major (farthest apart) and the last dimension is most minor (closest together): akin to row-major in a rank-2 tensor. True to indicate that the last dimension is most major (farthest apart) and the first dimension is most minor (closest together): akin to column-major in a rank-2 tensor. - public DenseTensor(Memory memory, ReadOnlySpan dimensions, bool reverseStride = false) : base(dimensions, reverseStride) + /// + /// An span of integers that represent the size of each dimension of the DenseTensor to create. + /// + /// False (default) to indicate that the first dimension is most major (farthest apart) and the last dimension + /// is most minor (closest together): akin to row-major in a rank-2 tensor. + /// True to indicate that the last dimension is most major (farthest apart) and the first dimension is most + /// minor (closest together): akin to column-major in a rank-2 tensor. + /// + public DenseTensor(Memory memory, ReadOnlySpan dimensions, bool reverseStride = false) + : base(dimensions, reverseStride) { this.memory = memory; if (Length != memory.Length) { - throw new ArgumentException($"Length of {nameof(memory)} ({memory.Length}) must match product of {nameof(dimensions)} ({Length})."); + throw new ArgumentException( + $"Length of {nameof(memory)} ({memory.Length}) must match product of " + + $"{nameof(dimensions)} ({Length})."); } } @@ -95,8 +112,8 @@ namespace Microsoft.ML.OnnxRuntime.Tensors public Memory Buffer => memory; /// - /// Gets the value at the specied index, where index is a linearized version of n-dimension indices using strides. - /// For a scalar, use index = 0 + /// Gets the value at the specified index, where index is a linearized version of n-dimension indices + /// using strides. For a scalar, use index = 0 /// /// An integer index computed as a dot-product of indices. /// The value at the specified position in this Tensor. @@ -106,8 +123,8 @@ namespace Microsoft.ML.OnnxRuntime.Tensors } /// - /// Sets the value at the specied index, where index is a linearized version of n-dimension indices using strides. - /// For a scalar, use index = 0 + /// Sets the value at the specified index, where index is a linearized version of n-dimension indices + /// using strides. For a scalar, use index = 0 /// /// An integer index computed as a dot-product of indices. /// The new value to set at the specified position in this Tensor. @@ -130,7 +147,9 @@ namespace Microsoft.ML.OnnxRuntime.Tensors } if (array.Length < arrayIndex + Length) { - throw new ArgumentException("The number of elements in the Tensor is greater than the available space from index to the end of the destination array.", nameof(array)); + throw new ArgumentException( + "The number of elements in the Tensor is greater than the available space from index to " + + "the end of the destination array.", nameof(array)); } Buffer.Span.CopyTo(array.AsSpan(arrayIndex)); @@ -165,14 +184,17 @@ namespace Microsoft.ML.OnnxRuntime.Tensors /// A shallow copy of this tensor. public override Tensor Clone() { - return new DenseTensor(Buffer.ToArray(), dimensions, IsReversedStride); + // create copy + return new DenseTensor(new Memory(memory.ToArray()), dimensions, IsReversedStride); } /// - /// Creates a new Tensor of a different type with the specified dimensions and the same layout as this tensor with elements initialized to their default value. + /// Creates a new Tensor of a different type with the specified dimensions and the same layout as this tensor + /// with elements initialized to their default value. /// /// Type contained in the returned Tensor. - /// An span of integers that represent the size of each dimension of the DenseTensor to create. + /// + /// An span of integers that represent the size of each dimension of the DenseTensor to create. /// A new tensor with the same layout as this tensor but different type and dimensions. public override Tensor CloneEmpty(ReadOnlySpan dimensions) { @@ -182,7 +204,8 @@ namespace Microsoft.ML.OnnxRuntime.Tensors /// /// Reshapes the current tensor to new dimensions, using the same backing storage. /// - /// An span of integers that represent the size of each dimension of the DenseTensor to create. + /// + /// An span of integers that represent the size of each dimension of the DenseTensor to create. /// A new tensor that reinterprets backing Buffer of this tensor with different dimensions. public override Tensor Reshape(ReadOnlySpan dimensions) { @@ -191,7 +214,8 @@ namespace Microsoft.ML.OnnxRuntime.Tensors if (newSize != Length) { - throw new ArgumentException($"Cannot reshape array due to mismatch in lengths, currently {Length} would become {newSize}.", nameof(dimensions)); + throw new ArgumentException($"Cannot reshape array due to mismatch in lengths, " + + "currently {Length} would become {newSize}.", nameof(dimensions)); } return new DenseTensor(Buffer, dimensions, IsReversedStride); diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/Tensor.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/Tensor.shared.cs index 94b48d10c8..bb7eea2ad1 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/Tensor.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Tensors/Tensor.shared.cs @@ -429,12 +429,16 @@ namespace Microsoft.ML.OnnxRuntime.Tensors } /// - /// Creates a n+1-dimension tensor using the specified n-dimension diagonal at the specified offset from the center. Values not on the diagonal will be filled with zeros. + /// Creates a n+1-dimension tensor using the specified n-dimension diagonal at the specified offset + /// from the center. Values not on the diagonal will be filled with zeros. /// - /// type contained within the Tensor. Typically a value type such as int, double, float, etc. + /// + /// type contained within the Tensor. Typically a value type such as int, double, float, etc. /// Tensor representing the diagonal to build the new tensor from. - /// Offset of diagonal to set in returned tensor. 0 for the main diagonal, less than zero for diagonals below, greater than zero from diagonals above. - /// A new tensor of the same layout and order as of one higher rank, with the values of along the specified diagonal and zeros elsewhere. + /// Offset of diagonal to set in returned tensor. 0 for the main diagonal, + /// less than zero for diagonals below, greater than zero from diagonals above. + /// A new tensor of the same layout and order as of one higher rank, + /// with the values of along the specified diagonal and zeros elsewhere. public static Tensor CreateFromDiagonal(Tensor diagonal, int offset) { if (diagonal.Rank < 1) @@ -678,10 +682,16 @@ namespace Microsoft.ML.OnnxRuntime.Tensors } /// - /// Initializes tensor with same dimensions as array, content of array is ignored. ReverseStride=true gives a stride of 1-element width to the first dimension (0). ReverseStride=false gives a stride of 1-element width to the last dimension (n-1). + /// Initializes tensor with same dimensions as array, content of array is ignored. + /// ReverseStride=true gives a stride of 1-element width to the first dimension (0). + /// ReverseStride=false gives a stride of 1-element width to the last dimension (n-1). /// /// Array from which to derive dimensions. - /// False (default) to indicate that the first dimension is most major (farthest apart) and the last dimension is most minor (closest together): akin to row-major in a rank-2 tensor. True to indicate that the last dimension is most major (farthest apart) and the first dimension is most minor (closest together): akin to column-major in a rank-2 tensor. + /// + /// False (default) to indicate that the first dimension is most major (farthest apart) and the + /// last dimension is most minor (closest together): akin to row-major in a rank-2 tensor. + /// True to indicate that the last dimension is most major (farthest apart) and the first dimension + /// is most minor (closest together): akin to column-major in a rank-2 tensor. protected Tensor(Array fromArray, bool reverseStride) : base(typeof(T)) { if (fromArray == null) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Microsoft.ML.OnnxRuntime.Tests.Common.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Microsoft.ML.OnnxRuntime.Tests.Common.csproj index 706920746b..853b3818ee 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Microsoft.ML.OnnxRuntime.Tests.Common.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Microsoft.ML.OnnxRuntime.Tests.Common.csproj @@ -77,6 +77,7 @@ + diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Tensors/ArrayTensorExtensionsTests.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Tensors/ArrayTensorExtensionsTests.cs new file mode 100644 index 0000000000..5e1e82f809 --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Tensors/ArrayTensorExtensionsTests.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Xunit; +using Microsoft.ML.OnnxRuntime.Tensors; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.ML.OnnxRuntime.Tests.ArrayTensorExtensions +{ + public class ArrayTensorExtensionsTests + { + static void CheckValues(IEnumerable expected, DenseTensor tensor) + { + foreach (var pair in expected.Zip(tensor.Buffer.ToArray(), Tuple.Create)) + { + Assert.Equal(pair.Item1, pair.Item2); + } + } + + [Fact] + public void ConstructFrom1D() + { + var array = new int[] { 1, 2, 3, 4 }; + var tensor = array.ToTensor(); + + var expectedDims = new int[] { 4 }; + Assert.Equal(tensor.Length, array.Length); + Assert.Equal(expectedDims, tensor.Dimensions.ToArray()); + CheckValues(array.Cast(), tensor); + } + + [Fact] + public void ConstructFrom2D() + { + var array = new int[,] { { 1, 2 } , { 3, 4 } }; + var tensor = array.ToTensor(); + + var expectedDims = new int[] { 2, 2 }; + Assert.Equal(tensor.Length, array.Length); + Assert.Equal(expectedDims, tensor.Dimensions.ToArray()); + CheckValues(array.Cast(), tensor); + } + + [Fact] + public void ConstructFrom3D() + { + var array = new int[,,] { { { 1, 2 }, { 3, 4 } }, + { { 5, 6 }, { 7, 8 } } }; + var tensor = array.ToTensor(); + + var expectedDims = new int[] { 2, 2, 2 }; + Assert.Equal(tensor.Length, array.Length); + Assert.Equal(expectedDims, tensor.Dimensions.ToArray()); + CheckValues(array.Cast(), tensor); + } + + [Fact] + public void ConstructFrom3DWithDim1() + { + var array = new int[,,] { { { 1, 2 } }, + { { 3, 4 } } }; + var tensor = array.ToTensor(); + + var expectedDims = new int[] { 2, 1, 2 }; + Assert.Equal(tensor.Length, array.Length); + Assert.Equal(expectedDims, tensor.Dimensions.ToArray()); + CheckValues(array.Cast(), tensor); + } + + [Fact] + public void ConstructFrom4D() + { + var array = new int[,,,] { + { { { 1, 2 }, { 3, 4 } }, + { { 5, 6 }, { 7, 8 } } } + }; + var tensor = array.ToTensor(); + + var expectedDims = new int[] { 1, 2, 2, 2 }; + Assert.Equal(tensor.Length, array.Length); + Assert.Equal(expectedDims, tensor.Dimensions.ToArray()); + CheckValues(array.Cast(), tensor); + } + + [Fact] + public void ConstructFrom5D() + { + var array = new int[,,,,] { + { { { { 1, 2 }, { 3, 4 } }, + { { 5, 6 }, { 7, 8 } } } } + }; + + // 5D requires cast to Array + Array a = (Array)array; + var tensor = a.ToTensor(); + + var expectedDims = new int[] { 1, 1, 2, 2, 2 }; + Assert.Equal(tensor.Length, array.Length); + Assert.Equal(expectedDims, tensor.Dimensions.ToArray()); + CheckValues(array.Cast(), tensor); + } + } +} diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj index 6a3a7ed4c0..49ec3af9e9 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj @@ -89,6 +89,9 @@ TensorTests.cs + + ArrayTensorExtensionsTests.cs + From 712f4e403d9fa962e0104ad6c785bfdab6cc0007 Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Tue, 18 Jan 2022 14:00:10 -0800 Subject: [PATCH 04/59] [js/common] upgrade marked@4.0.10 (Dependbot warning) (#10313) --- js/common/package-lock.json | 2428 +++++++++++++++++------------------ js/common/package.json | 42 +- 2 files changed, 1225 insertions(+), 1245 deletions(-) diff --git a/js/common/package-lock.json b/js/common/package-lock.json index 80876b946a..74ec8d829d 100644 --- a/js/common/package-lock.json +++ b/js/common/package-lock.json @@ -1,1292 +1,1272 @@ { + "name": "onnxruntime-common", + "version": "1.11.0", + "lockfileVersion": 1, + "requires": true, "dependencies": { - "browserslist": { - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.16.6.tgz", - "requires": { - "escalade": "^3.1.1", - "electron-to-chromium": "^1.3.723", - "colorette": "^1.2.2", - "caniuse-lite": "^1.0.30001219", - "node-releases": "^1.1.71" - }, - "version": "4.16.6", - "integrity": "sha512-Wspk/PqO+4W9qp5iUTJsa1B/QrYn1keNCcEP5OvP7WBwT4KaDly0uONYmC6Xa3Z5IqnUgS0KcgLYu1l74x0ZXQ==", - "dev": true - }, - "npm-run-path": { - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "requires": { - "path-key": "^3.0.0" - }, - "version": "4.0.1", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dev": true - }, "@discoveryjs/json-ext": { - "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.2.tgz", - "version": "0.5.2", - "integrity": "sha512-HyYEUDeIj5rRQU2Hk5HTB2uHsbRQpF70nvMhVzi+VJR0X+xNEhjPui4/kBf3VeH/wqD28PT4sVOm8qqLjBrSZg==", + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.2.tgz", + "integrity": "sha512-HyYEUDeIj5rRQU2Hk5HTB2uHsbRQpF70nvMhVzi+VJR0X+xNEhjPui4/kBf3VeH/wqD28PT4sVOm8qqLjBrSZg==", "dev": true - }, - "graceful-fs": { - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz", - "version": "4.2.6", - "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==", - "dev": true - }, - "json-parse-better-errors": { - "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "version": "1.0.2", - "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", - "dev": true - }, - "shebang-regex": { - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "version": "3.0.0", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true - }, - "locate-path": { - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "requires": { - "p-locate": "^4.1.0" - }, - "version": "5.0.0", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true - }, - "chrome-trace-event": { - "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", - "version": "1.0.3", - "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==", - "dev": true - }, - "@webassemblyjs/helper-wasm-bytecode": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.0.tgz", - "version": "1.11.0", - "integrity": "sha512-MbmhvxXExm542tWREgSFnOVo07fDpsBJg3sIl6fSp9xuu75eGz5lz31q7wTLffwL3Za7XNRCMZy210+tnsUSEA==", - "dev": true - }, - "punycode": { - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "version": "2.1.1", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true - }, - "to-regex-range": { - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "requires": { - "is-number": "^7.0.0" - }, - "version": "5.0.1", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true - }, - "mimic-fn": { - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "version": "2.1.0", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true - }, - "color-convert": { - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "requires": { - "color-name": "~1.1.4" - }, - "version": "2.0.1", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true - }, - "@types/eslint-scope": { - "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.0.tgz", - "requires": { - "@types/eslint": "*", - "@types/estree": "*" - }, - "version": "3.7.0", - "integrity": "sha512-O/ql2+rrCUe2W2rs7wMR+GqPRcgB6UiqN5RhrR5xruFlY7l9YLMn0ZkDzjoHLeiFkR8MCQZVudUuuvQ2BLC9Qw==", - "dev": true - }, - "fs.realpath": { - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "version": "1.0.0", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true - }, - "has": { - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "requires": { - "function-bind": "^1.1.1" - }, - "version": "1.0.3", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true - }, - "json-schema-traverse": { - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "version": "0.4.1", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "resolve": { - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", - "requires": { - "path-parse": "^1.0.6", - "is-core-module": "^2.2.0" - }, - "version": "1.20.0", - "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", - "dev": true - }, - "resolve-from": { - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "version": "5.0.0", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true - }, - "p-try": { - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "version": "2.2.0", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true - }, - "randombytes": { - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "requires": { - "safe-buffer": "^5.1.0" - }, - "version": "2.1.0", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dev": true - }, - "get-stream": { - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "version": "6.0.1", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true - }, - "marked": { - "resolved": "https://registry.npmjs.org/marked/-/marked-3.0.4.tgz", - "version": "3.0.4", - "integrity": "sha512-jBo8AOayNaEcvBhNobg6/BLhdsK3NvnKWJg33MAAPbvTWiG4QBn9gpW1+7RssrKu4K1dKlN+0goVQwV41xEfOA==", - "dev": true - }, - "webpack-merge": { - "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.7.3.tgz", - "requires": { - "clone-deep": "^4.0.1", - "wildcard": "^2.0.0" - }, - "version": "5.7.3", - "integrity": "sha512-6/JUQv0ELQ1igjGDzHkXbVDRxkfA57Zw7PfiupdLFJYrgFqY5ZP8xxbpp2lU3EPwYx89ht5Z/aDkD40hFCm5AA==", - "dev": true - }, - "@webassemblyjs/wasm-parser": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.0.tgz", - "requires": { - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/utf8": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0" - }, - "version": "1.11.0", - "integrity": "sha512-6L285Sgu9gphrcpDXINvm0M9BskznnzJTE7gYkjDbxET28shDqp27wpruyx3C2S/dvEwiigBwLA1cz7lNUi0kw==", - "dev": true - }, - "is-core-module": { - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.3.0.tgz", - "requires": { - "has": "^1.0.3" - }, - "version": "2.3.0", - "integrity": "sha512-xSphU2KG9867tsYdLD4RWQ1VqdFl4HTO9Thf3I/3dLEfr0dbPTWKsuCKrgqMljg4nPE+Gq0VCnzT3gr0CyBmsw==", - "dev": true - }, - "resolve-cwd": { - "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", - "requires": { - "resolve-from": "^5.0.0" - }, - "version": "3.0.0", - "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", - "dev": true - }, - "@webassemblyjs/wast-printer": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.0.tgz", - "requires": { - "@xtuc/long": "4.2.2", - "@webassemblyjs/ast": "1.11.0" - }, - "version": "1.11.0", - "integrity": "sha512-Fg5OX46pRdTgB7rKIUojkh9vXaVN6sGYCnEiJN1GYkb0RPwShZXp6KTDqmoMdQPKhcroOXh3fEzmkWmCYaKYhQ==", - "dev": true - }, - "execa": { - "resolved": "https://registry.npmjs.org/execa/-/execa-5.0.0.tgz", - "requires": { - "onetime": "^5.1.2", - "npm-run-path": "^4.0.1", - "human-signals": "^2.1.0", - "merge-stream": "^2.0.0", - "strip-final-newline": "^2.0.0", - "signal-exit": "^3.0.3", - "get-stream": "^6.0.0", - "is-stream": "^2.0.0", - "cross-spawn": "^7.0.3" - }, - "version": "5.0.0", - "integrity": "sha512-ov6w/2LCiuyO4RLYGdpFGjkcs0wMTgGE8PrkTHikeUy5iJekXyPIKUjifk5CsE0pt7sMCrMZ3YNqoCj6idQOnQ==", - "dev": true - }, - "shallow-clone": { - "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", - "requires": { - "kind-of": "^6.0.2" - }, - "version": "3.0.1", - "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", - "dev": true - }, - "is-number": { - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "version": "7.0.0", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true - }, - "@webpack-cli/serve": { - "resolved": "https://registry.npmjs.org/@webpack-cli/serve/-/serve-1.4.0.tgz", - "version": "1.4.0", - "integrity": "sha512-xgT/HqJ+uLWGX+Mzufusl3cgjAcnqYYskaB7o0vRcwOEfuu6hMzSILQpnIzFMGsTaeaX4Nnekl+6fadLbl1/Vg==", - "dev": true - }, - "isobject": { - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "version": "3.0.1", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", - "dev": true - }, - "esrecurse": { - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "version": "4.3.0", - "dependencies": { - "estraverse": { - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", - "version": "5.2.0", - "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", - "dev": true - } - }, - "requires": { - "estraverse": "^5.2.0" - } - }, - "clone-deep": { - "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", - "requires": { - "shallow-clone": "^3.0.0", - "is-plain-object": "^2.0.4", - "kind-of": "^6.0.2" - }, - "version": "4.0.1", - "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", - "dev": true - }, - "enhanced-resolve": { - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.8.0.tgz", - "requires": { - "tapable": "^2.2.0", - "graceful-fs": "^4.2.4" - }, - "version": "5.8.0", - "integrity": "sha512-Sl3KRpJA8OpprrtaIswVki3cWPiPKxXuFxJXBp+zNb6s6VwNWwFRUdtmzd2ReUut8n+sCPx7QCtQ7w5wfJhSgQ==", - "dev": true - }, - "semver": { - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "requires": { - "lru-cache": "^6.0.0" - }, - "version": "7.3.5", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", - "dev": true - }, - "has-flag": { - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "version": "4.0.0", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true - }, + }, "@types/eslint": { - "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.2.10.tgz", + "version": "7.2.10", + "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.2.10.tgz", + "integrity": "sha512-kUEPnMKrqbtpCq/KTaGFFKAcz6Ethm2EjCoKIDaCmfRBWLbFuTcOJfTlorwbnboXBzahqWLgUp1BQeKHiJzPUQ==", + "dev": true, "requires": { - "@types/json-schema": "*", - "@types/estree": "*" - }, - "version": "7.2.10", - "integrity": "sha512-kUEPnMKrqbtpCq/KTaGFFKAcz6Ethm2EjCoKIDaCmfRBWLbFuTcOJfTlorwbnboXBzahqWLgUp1BQeKHiJzPUQ==", - "dev": true - }, - "merge-stream": { - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "version": "2.0.0", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", - "dev": true - }, - "@webassemblyjs/leb128": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.0.tgz", - "requires": { - "@xtuc/long": "4.2.2" - }, - "version": "1.11.0", - "integrity": "sha512-aqbsHa1mSQAbeeNcl38un6qVY++hh8OpCOzxhixSYgbRfNWcxJNJQwe2rezK9XEcssJbbWIkblaJRwGMS9zp+g==", - "dev": true - }, - "color-name": { - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "version": "1.1.4", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - }, - "mime-types": { - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz", - "requires": { - "mime-db": "1.47.0" - }, - "version": "2.1.30", - "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==", - "dev": true - }, - "es-module-lexer": { - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.4.1.tgz", - "version": "0.4.1", - "integrity": "sha512-ooYciCUtfw6/d2w56UVeqHPcoCFAiJdz5XOkYpv/Txl1HMUozpXjz/2RIQgqwKdXNDPSF1W7mJCFse3G+HDyAA==", - "dev": true - }, - "tapable": { - "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.0.tgz", - "version": "2.2.0", - "integrity": "sha512-FBk4IesMV1rBxX2tfiK8RAmogtWn53puLOQlvO8XuwlgxcYbP4mVPS9Ph4aeamSyyVjOl24aYWAuc8U5kCVwMw==", - "dev": true - }, - "minimatch": { - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "requires": { - "brace-expansion": "^1.1.7" - }, - "version": "3.0.4", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true - }, - "find-up": { - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "version": "4.1.0", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true - }, - "source-map-support": { - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", - "requires": { - "source-map": "^0.6.0", - "buffer-from": "^1.0.0" - }, - "version": "0.5.19", - "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", - "dev": true - }, - "@webassemblyjs/helper-wasm-section": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.0.tgz", - "requires": { - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0" - }, - "version": "1.11.0", - "integrity": "sha512-3Eb88hcbfY/FCukrg6i3EH8H2UsD7x8Vy47iVJrP967A9JGqgBVL9aH71SETPx1JrGsOUVLo0c7vMCN22ytJew==", - "dev": true - }, - "buffer-from": { - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "version": "1.1.1", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "dev": true - }, - "ajv": { - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "requires": { - "uri-js": "^4.2.2", - "json-schema-traverse": "^0.4.1", - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0" - }, - "version": "6.12.6", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true - }, - "shebang-command": { - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "requires": { - "shebang-regex": "^3.0.0" - }, - "version": "2.0.0", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true - }, - "webpack-cli": { - "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-4.7.0.tgz", - "integrity": "sha512-7bKr9182/sGfjFm+xdZSwgQuFjgEcy0iCTIBxRUeteJ2Kr8/Wz0qNJX+jw60LU36jApt4nmMkep6+W5AKhok6g==", - "dev": true, - "version": "4.7.0", - "dependencies": { - "commander": { - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "version": "7.2.0", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", - "dev": true - } - }, - "requires": { - "import-local": "^3.0.2", - "@discoveryjs/json-ext": "^0.5.0", - "colorette": "^1.2.1", - "v8-compile-cache": "^2.2.0", - "execa": "^5.0.0", - "@webpack-cli/info": "^1.2.4", - "commander": "^7.0.0", - "fastest-levenshtein": "^1.0.12", - "@webpack-cli/configtest": "^1.0.3", - "@webpack-cli/serve": "^1.4.0", - "webpack-merge": "^5.7.3", - "rechoir": "^0.7.0", - "interpret": "^2.2.0" + "@types/estree": "*", + "@types/json-schema": "*" } - }, - "fast-deep-equal": { - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "version": "3.1.3", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + }, + "@types/eslint-scope": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.0.tgz", + "integrity": "sha512-O/ql2+rrCUe2W2rs7wMR+GqPRcgB6UiqN5RhrR5xruFlY7l9YLMn0ZkDzjoHLeiFkR8MCQZVudUuuvQ2BLC9Qw==", + "dev": true, + "requires": { + "@types/eslint": "*", + "@types/estree": "*" + } + }, + "@types/estree": { + "version": "0.0.47", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.47.tgz", + "integrity": "sha512-c5ciR06jK8u9BstrmJyO97m+klJrrhCf9u3rLu3DEAJBirxRqSCvDQoYKmxuYwQI5SZChAWu+tq9oVlGRuzPAg==", "dev": true - }, + }, + "@types/json-schema": { + "version": "7.0.7", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz", + "integrity": "sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==", + "dev": true + }, + "@types/node": { + "version": "15.0.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-15.0.2.tgz", + "integrity": "sha512-p68+a+KoxpoB47015IeYZYRrdqMUcpbK8re/zpFB8Ld46LHC1lPEbp3EXgkEhAYEcPvjJF6ZO+869SQ0aH1dcA==", + "dev": true + }, + "@webassemblyjs/ast": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.0.tgz", + "integrity": "sha512-kX2W49LWsbthrmIRMbQZuQDhGtjyqXfEmmHyEi4XWnSZtPmxY0+3anPIzsnRb45VH/J55zlOfWvZuY47aJZTJg==", + "dev": true, + "requires": { + "@webassemblyjs/helper-numbers": "1.11.0", + "@webassemblyjs/helper-wasm-bytecode": "1.11.0" + } + }, + "@webassemblyjs/floating-point-hex-parser": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.0.tgz", + "integrity": "sha512-Q/aVYs/VnPDVYvsCBL/gSgwmfjeCb4LW8+TMrO3cSzJImgv8lxxEPM2JA5jMrivE7LSz3V+PFqtMbls3m1exDA==", + "dev": true + }, + "@webassemblyjs/helper-api-error": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.0.tgz", + "integrity": "sha512-baT/va95eXiXb2QflSx95QGT5ClzWpGaa8L7JnJbgzoYeaA27FCvuBXU758l+KXWRndEmUXjP0Q5fibhavIn8w==", + "dev": true + }, + "@webassemblyjs/helper-buffer": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.0.tgz", + "integrity": "sha512-u9HPBEl4DS+vA8qLQdEQ6N/eJQ7gT7aNvMIo8AAWvAl/xMrcOSiI2M0MAnMCy3jIFke7bEee/JwdX1nUpCtdyA==", + "dev": true + }, + "@webassemblyjs/helper-numbers": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.0.tgz", + "integrity": "sha512-DhRQKelIj01s5IgdsOJMKLppI+4zpmcMQ3XboFPLwCpSNH6Hqo1ritgHgD0nqHeSYqofA6aBN/NmXuGjM1jEfQ==", + "dev": true, + "requires": { + "@webassemblyjs/floating-point-hex-parser": "1.11.0", + "@webassemblyjs/helper-api-error": "1.11.0", + "@xtuc/long": "4.2.2" + } + }, + "@webassemblyjs/helper-wasm-bytecode": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.0.tgz", + "integrity": "sha512-MbmhvxXExm542tWREgSFnOVo07fDpsBJg3sIl6fSp9xuu75eGz5lz31q7wTLffwL3Za7XNRCMZy210+tnsUSEA==", + "dev": true + }, + "@webassemblyjs/helper-wasm-section": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.0.tgz", + "integrity": "sha512-3Eb88hcbfY/FCukrg6i3EH8H2UsD7x8Vy47iVJrP967A9JGqgBVL9aH71SETPx1JrGsOUVLo0c7vMCN22ytJew==", + "dev": true, + "requires": { + "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/helper-buffer": "1.11.0", + "@webassemblyjs/helper-wasm-bytecode": "1.11.0", + "@webassemblyjs/wasm-gen": "1.11.0" + } + }, "@webassemblyjs/ieee754": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.0.tgz", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.0.tgz", + "integrity": "sha512-KXzOqpcYQwAfeQ6WbF6HXo+0udBNmw0iXDmEK5sFlmQdmND+tr773Ti8/5T/M6Tl/413ArSJErATd8In3B+WBA==", + "dev": true, "requires": { "@xtuc/ieee754": "^1.2.0" - }, - "version": "1.11.0", - "integrity": "sha512-KXzOqpcYQwAfeQ6WbF6HXo+0udBNmw0iXDmEK5sFlmQdmND+tr773Ti8/5T/M6Tl/413ArSJErATd8In3B+WBA==", - "dev": true - }, - "is-plain-object": { - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + } + }, + "@webassemblyjs/leb128": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.0.tgz", + "integrity": "sha512-aqbsHa1mSQAbeeNcl38un6qVY++hh8OpCOzxhixSYgbRfNWcxJNJQwe2rezK9XEcssJbbWIkblaJRwGMS9zp+g==", + "dev": true, "requires": { - "isobject": "^3.0.1" - }, - "version": "2.0.4", - "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "@xtuc/long": "4.2.2" + } + }, + "@webassemblyjs/utf8": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.0.tgz", + "integrity": "sha512-A/lclGxH6SpSLSyFowMzO/+aDEPU4hvEiooCMXQPcQFPPJaYcPQNKGOCLUySJsYJ4trbpr+Fs08n4jelkVTGVw==", "dev": true - }, - "estraverse": { - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "version": "4.3.0", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", - "dev": true - }, - "isexe": { - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "version": "2.0.0", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", - "dev": true - }, + }, + "@webassemblyjs/wasm-edit": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.0.tgz", + "integrity": "sha512-JHQ0damXy0G6J9ucyKVXO2j08JVJ2ntkdJlq1UTiUrIgfGMmA7Ik5VdC/L8hBK46kVJgujkBIoMtT8yVr+yVOQ==", + "dev": true, + "requires": { + "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/helper-buffer": "1.11.0", + "@webassemblyjs/helper-wasm-bytecode": "1.11.0", + "@webassemblyjs/helper-wasm-section": "1.11.0", + "@webassemblyjs/wasm-gen": "1.11.0", + "@webassemblyjs/wasm-opt": "1.11.0", + "@webassemblyjs/wasm-parser": "1.11.0", + "@webassemblyjs/wast-printer": "1.11.0" + } + }, + "@webassemblyjs/wasm-gen": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.0.tgz", + "integrity": "sha512-BEUv1aj0WptCZ9kIS30th5ILASUnAPEvE3tVMTrItnZRT9tXCLW2LEXT8ezLw59rqPP9klh9LPmpU+WmRQmCPQ==", + "dev": true, + "requires": { + "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/helper-wasm-bytecode": "1.11.0", + "@webassemblyjs/ieee754": "1.11.0", + "@webassemblyjs/leb128": "1.11.0", + "@webassemblyjs/utf8": "1.11.0" + } + }, + "@webassemblyjs/wasm-opt": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.0.tgz", + "integrity": "sha512-tHUSP5F4ywyh3hZ0+fDQuWxKx3mJiPeFufg+9gwTpYp324mPCQgnuVKwzLTZVqj0duRDovnPaZqDwoyhIO8kYg==", + "dev": true, + "requires": { + "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/helper-buffer": "1.11.0", + "@webassemblyjs/wasm-gen": "1.11.0", + "@webassemblyjs/wasm-parser": "1.11.0" + } + }, + "@webassemblyjs/wasm-parser": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.0.tgz", + "integrity": "sha512-6L285Sgu9gphrcpDXINvm0M9BskznnzJTE7gYkjDbxET28shDqp27wpruyx3C2S/dvEwiigBwLA1cz7lNUi0kw==", + "dev": true, + "requires": { + "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/helper-api-error": "1.11.0", + "@webassemblyjs/helper-wasm-bytecode": "1.11.0", + "@webassemblyjs/ieee754": "1.11.0", + "@webassemblyjs/leb128": "1.11.0", + "@webassemblyjs/utf8": "1.11.0" + } + }, + "@webassemblyjs/wast-printer": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.0.tgz", + "integrity": "sha512-Fg5OX46pRdTgB7rKIUojkh9vXaVN6sGYCnEiJN1GYkb0RPwShZXp6KTDqmoMdQPKhcroOXh3fEzmkWmCYaKYhQ==", + "dev": true, + "requires": { + "@webassemblyjs/ast": "1.11.0", + "@xtuc/long": "4.2.2" + } + }, "@webpack-cli/configtest": { - "resolved": "https://registry.npmjs.org/@webpack-cli/configtest/-/configtest-1.0.3.tgz", - "version": "1.0.3", - "integrity": "sha512-WQs0ep98FXX2XBAfQpRbY0Ma6ADw8JR6xoIkaIiJIzClGOMqVRvPCWqndTxf28DgFopWan0EKtHtg/5W1h0Zkw==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@webpack-cli/configtest/-/configtest-1.0.3.tgz", + "integrity": "sha512-WQs0ep98FXX2XBAfQpRbY0Ma6ADw8JR6xoIkaIiJIzClGOMqVRvPCWqndTxf28DgFopWan0EKtHtg/5W1h0Zkw==", "dev": true - }, - "onetime": { - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "requires": { - "mimic-fn": "^2.1.0" - }, - "version": "5.1.2", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true - }, - "ansi-styles": { - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "requires": { - "color-convert": "^2.0.1" - }, - "version": "4.3.0", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true - }, - "acorn": { - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.2.4.tgz", - "version": "8.2.4", - "integrity": "sha512-Ibt84YwBDDA890eDiDCEqcbwvHlBvzzDkU2cGBBDDI1QWT12jTiXIOn2CIw5KK4i6N5Z2HUxwYjzriDyqaqqZg==", - "dev": true - }, - "@webassemblyjs/helper-buffer": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.0.tgz", - "version": "1.11.0", - "integrity": "sha512-u9HPBEl4DS+vA8qLQdEQ6N/eJQ7gT7aNvMIo8AAWvAl/xMrcOSiI2M0MAnMCy3jIFke7bEee/JwdX1nUpCtdyA==", - "dev": true - }, - "interpret": { - "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", - "version": "2.2.0", - "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==", - "dev": true - }, - "@types/estree": { - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.47.tgz", - "version": "0.0.47", - "integrity": "sha512-c5ciR06jK8u9BstrmJyO97m+klJrrhCf9u3rLu3DEAJBirxRqSCvDQoYKmxuYwQI5SZChAWu+tq9oVlGRuzPAg==", - "dev": true - }, - "uri-js": { - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "requires": { - "punycode": "^2.1.0" - }, - "version": "4.4.1", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true - }, - "@xtuc/long": { - "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", - "version": "4.2.2", - "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", - "dev": true - }, - "node-releases": { - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.71.tgz", - "version": "1.1.71", - "integrity": "sha512-zR6HoT6LrLCRBwukmrVbHv0EpEQjksO6GmFcZQQuCAy139BEsoVKPYnf3jongYW83fAa1torLGYwxxky/p28sg==", - "dev": true - }, - "lunr": { - "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", - "version": "2.3.9", - "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==", - "dev": true - }, - "cross-spawn": { - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "requires": { - "shebang-command": "^2.0.0", - "which": "^2.0.1", - "path-key": "^3.1.0" - }, - "version": "7.0.3", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true - }, - "schema-utils": { - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.0.0.tgz", - "requires": { - "ajv-keywords": "^3.5.2", - "ajv": "^6.12.5", - "@types/json-schema": "^7.0.6" - }, - "version": "3.0.0", - "integrity": "sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==", - "dev": true - }, + }, "@webpack-cli/info": { - "resolved": "https://registry.npmjs.org/@webpack-cli/info/-/info-1.2.4.tgz", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@webpack-cli/info/-/info-1.2.4.tgz", + "integrity": "sha512-ogE2T4+pLhTTPS/8MM3IjHn0IYplKM4HbVNMCWA9N4NrdPzunwenpCsqKEXyejMfRu6K8mhauIPYf8ZxWG5O6g==", + "dev": true, "requires": { "envinfo": "^7.7.3" - }, - "version": "1.2.4", - "integrity": "sha512-ogE2T4+pLhTTPS/8MM3IjHn0IYplKM4HbVNMCWA9N4NrdPzunwenpCsqKEXyejMfRu6K8mhauIPYf8ZxWG5O6g==", + } + }, + "@webpack-cli/serve": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@webpack-cli/serve/-/serve-1.4.0.tgz", + "integrity": "sha512-xgT/HqJ+uLWGX+Mzufusl3cgjAcnqYYskaB7o0vRcwOEfuu6hMzSILQpnIzFMGsTaeaX4Nnekl+6fadLbl1/Vg==", "dev": true - }, - "wrappy": { - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "version": "1.0.2", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + }, + "@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", "dev": true - }, - "escalade": { - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "version": "3.1.1", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + }, + "@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "dev": true - }, - "picomatch": { - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.3.tgz", - "version": "2.2.3", - "integrity": "sha512-KpELjfwcCDUb9PeigTs2mBJzXUPzAuP2oPcA989He8Rte0+YUAjw1JVedDhuTKPkHjSYzMN3npC9luThGYEKdg==", + }, + "acorn": { + "version": "8.2.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.2.4.tgz", + "integrity": "sha512-Ibt84YwBDDA890eDiDCEqcbwvHlBvzzDkU2cGBBDDI1QWT12jTiXIOn2CIw5KK4i6N5Z2HUxwYjzriDyqaqqZg==", "dev": true - }, - "@webassemblyjs/wasm-gen": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.0.tgz", + }, + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, "requires": { - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/utf8": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0" - }, - "version": "1.11.0", - "integrity": "sha512-BEUv1aj0WptCZ9kIS30th5ILASUnAPEvE3tVMTrItnZRT9tXCLW2LEXT8ezLw59rqPP9klh9LPmpU+WmRQmCPQ==", + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", "dev": true - }, + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "braces": { - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, "requires": { "fill-range": "^7.0.1" - }, - "version": "3.0.2", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true - }, - "@webassemblyjs/helper-api-error": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.0.tgz", - "version": "1.11.0", - "integrity": "sha512-baT/va95eXiXb2QflSx95QGT5ClzWpGaa8L7JnJbgzoYeaA27FCvuBXU758l+KXWRndEmUXjP0Q5fibhavIn8w==", - "dev": true - }, - "terser": { - "resolved": "https://registry.npmjs.org/terser/-/terser-5.7.0.tgz", - "integrity": "sha512-HP5/9hp2UaZt5fYkuhNBR8YyRcT8juw8+uFbAme53iN9hblvKnLUTKkmwJG6ocWpIKf8UK4DoeWG4ty0J6S6/g==", - "dev": true, - "version": "5.7.0", - "dependencies": { - "source-map": { - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "version": "0.7.3", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", - "dev": true - } - }, - "requires": { - "source-map": "~0.7.2", - "source-map-support": "~0.5.19", - "commander": "^2.20.0" } - }, - "import-local": { - "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.2.tgz", + }, + "browserslist": { + "version": "4.16.6", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.16.6.tgz", + "integrity": "sha512-Wspk/PqO+4W9qp5iUTJsa1B/QrYn1keNCcEP5OvP7WBwT4KaDly0uONYmC6Xa3Z5IqnUgS0KcgLYu1l74x0ZXQ==", + "dev": true, "requires": { - "resolve-cwd": "^3.0.0", - "pkg-dir": "^4.2.0" - }, - "version": "3.0.2", - "integrity": "sha512-vjL3+w0oulAVZ0hBHnxa/Nm5TAurf9YLQJDhqRZyqb+VKGOB6LU8t9H1Nr5CIo16vh9XfJTOoHwU0B71S557gA==", - "dev": true - }, - "webpack-sources": { - "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-2.2.0.tgz", - "requires": { - "source-map": "^0.6.1", - "source-list-map": "^2.0.1" - }, - "version": "2.2.0", - "integrity": "sha512-bQsA24JLwcnWGArOKUxYKhX3Mz/nK1Xf6hxullKERyktjNMC4x8koOeaDNTA2fEJ09BdWLbM/iTW0ithREUP0w==", - "dev": true - }, - "terser-webpack-plugin": { - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.1.1.tgz", - "requires": { - "source-map": "^0.6.1", - "p-limit": "^3.1.0", - "jest-worker": "^26.6.2", - "schema-utils": "^3.0.0", - "terser": "^5.5.1", - "serialize-javascript": "^5.0.1" - }, - "version": "5.1.1", - "integrity": "sha512-5XNNXZiR8YO6X6KhSGXfY0QrGrCRlSwAEjIIrlRQR4W8nP69TaJUlh3bkuac6zzgspiGPfKEHcY295MMVExl5Q==", - "dev": true - }, - "webpack": { - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.36.2.tgz", - "requires": { - "enhanced-resolve": "^5.8.0", - "browserslist": "^4.14.5", - "schema-utils": "^3.0.0", - "graceful-fs": "^4.2.4", - "json-parse-better-errors": "^1.0.2", - "chrome-trace-event": "^1.0.2", - "mime-types": "^2.1.27", - "es-module-lexer": "^0.4.0", - "tapable": "^2.1.1", - "webpack-sources": "^2.1.1", - "terser-webpack-plugin": "^5.1.1", - "@types/eslint-scope": "^3.7.0", - "eslint-scope": "^5.1.1", - "@webassemblyjs/wasm-edit": "1.11.0", - "events": "^3.2.0", - "neo-async": "^2.6.2", - "@webassemblyjs/ast": "1.11.0", - "acorn": "^8.2.1", - "glob-to-regexp": "^0.4.1", - "@types/estree": "^0.0.47", - "@webassemblyjs/wasm-parser": "1.11.0", - "watchpack": "^2.0.0", - "loader-runner": "^4.2.0" - }, - "version": "5.36.2", - "integrity": "sha512-XJumVnnGoH2dV+Pk1VwgY4YT6AiMKpVoudUFCNOXMIVrEKPUgEwdIfWPjIuGLESAiS8EdIHX5+TiJz/5JccmRg==", - "dev": true - }, - "inflight": { - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "requires": { - "wrappy": "1", - "once": "^1.3.0" - }, - "version": "1.0.6", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true - }, - "source-list-map": { - "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", - "version": "2.0.1", - "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==", - "dev": true - }, - "function-bind": { - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "version": "1.1.1", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true - }, - "kind-of": { - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "version": "6.0.3", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", - "dev": true - }, - "mime-db": { - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", - "version": "1.47.0", - "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==", - "dev": true - }, - "vscode-textmate": { - "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-5.2.0.tgz", - "version": "5.2.0", - "integrity": "sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ==", - "dev": true - }, - "source-map": { - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "version": "0.6.1", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true - }, - "p-limit": { - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "requires": { - "yocto-queue": "^0.1.0" - }, - "version": "3.1.0", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true - }, - "jest-worker": { - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-26.6.2.tgz", - "requires": { - "supports-color": "^7.0.0", - "@types/node": "*", - "merge-stream": "^2.0.0" - }, - "version": "26.6.2", - "integrity": "sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ==", - "dev": true - }, - "pkg-dir": { - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", - "requires": { - "find-up": "^4.0.0" - }, - "version": "4.2.0", - "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "dev": true - }, - "safe-buffer": { - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "version": "5.2.1", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true - }, - "onigasm": { - "resolved": "https://registry.npmjs.org/onigasm/-/onigasm-2.2.5.tgz", - "integrity": "sha512-F+th54mPc0l1lp1ZcFMyL/jTs2Tlq4SqIHKIXGZOR/VkHkF9A7Fr5rRr5+ZG/lWeRsyrClLYRq7s/yFQ/XhWCA==", - "dev": true, - "version": "2.2.5", - "dependencies": { - "yallist": { - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "version": "3.1.1", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true - }, - "lru-cache": { - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "requires": { - "yallist": "^3.0.2" - }, - "version": "5.1.1", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true - } - }, - "requires": { - "lru-cache": "^5.1.1" + "caniuse-lite": "^1.0.30001219", + "colorette": "^1.2.2", + "electron-to-chromium": "^1.3.723", + "escalade": "^3.1.1", + "node-releases": "^1.1.71" } - }, - "concat-map": { - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "version": "0.0.1", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", "dev": true - }, - "@webassemblyjs/ast": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.0.tgz", - "requires": { - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/helper-numbers": "1.11.0" - }, - "version": "1.11.0", - "integrity": "sha512-kX2W49LWsbthrmIRMbQZuQDhGtjyqXfEmmHyEi4XWnSZtPmxY0+3anPIzsnRb45VH/J55zlOfWvZuY47aJZTJg==", - "dev": true - }, - "path-exists": { - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "version": "4.0.0", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true - }, - "path-key": { - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "version": "3.1.1", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true - }, - "balanced-match": { - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "version": "1.0.2", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true - }, - "ts-loader": { - "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.1.2.tgz", - "requires": { - "chalk": "^4.1.0", - "enhanced-resolve": "^5.0.0", - "semver": "^7.3.4", - "micromatch": "^4.0.0" - }, - "version": "9.1.2", - "integrity": "sha512-ryMgATvLLl+z8zQvdlm6Pep0slmwxFWIEnq/5VdiLVjqQXnFJgO+qNLGIIP+d2N2jsFZ9MibZCVDb2bSp7OmEA==", - "dev": true - }, - "@webassemblyjs/helper-numbers": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.0.tgz", - "requires": { - "@webassemblyjs/helper-api-error": "1.11.0", - "@xtuc/long": "4.2.2", - "@webassemblyjs/floating-point-hex-parser": "1.11.0" - }, - "version": "1.11.0", - "integrity": "sha512-DhRQKelIj01s5IgdsOJMKLppI+4zpmcMQ3XboFPLwCpSNH6Hqo1ritgHgD0nqHeSYqofA6aBN/NmXuGjM1jEfQ==", - "dev": true - }, - "supports-color": { - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "requires": { - "has-flag": "^4.0.0" - }, - "version": "7.2.0", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true - }, - "path-is-absolute": { - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "version": "1.0.1", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true - }, - "micromatch": { - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", - "requires": { - "picomatch": "^2.2.3", - "braces": "^3.0.1" - }, - "version": "4.0.4", - "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", - "dev": true - }, - "envinfo": { - "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.8.1.tgz", - "version": "7.8.1", - "integrity": "sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw==", - "dev": true - }, - "chalk": { - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz", - "requires": { - "supports-color": "^7.1.0", - "ansi-styles": "^4.1.0" - }, - "version": "4.1.1", - "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==", - "dev": true - }, - "p-locate": { - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "version": "4.1.0", - "dependencies": { - "p-limit": { - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "requires": { - "p-try": "^2.0.0" - }, - "version": "2.3.0", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true - } - }, - "requires": { - "p-limit": "^2.2.0" - } - }, - "brace-expansion": { - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - }, - "version": "1.1.11", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true - }, - "loader-runner": { - "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.2.0.tgz", - "version": "4.2.0", - "integrity": "sha512-92+huvxMvYlMzMt0iIOukcwYBFpkYJdpl2xsZ7LrlayO7E8SOv+JJUEK17B/dJIHAOLMfh2dZZ/Y18WgmGtYNw==", - "dev": true - }, - "yocto-queue": { - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "version": "0.1.0", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true - }, - "lru-cache": { - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "requires": { - "yallist": "^4.0.0" - }, - "version": "6.0.0", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true - }, - "inherits": { - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "version": "2.0.4", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "yallist": { - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "version": "4.0.0", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, + }, "caniuse-lite": { - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001223.tgz", - "version": "1.0.30001223", - "integrity": "sha512-k/RYs6zc/fjbxTjaWZemeSmOjO0JJV+KguOBA3NwPup8uzxM1cMhR2BD9XmO86GuqaqTCO8CgkgH9Rz//vdDiA==", + "version": "1.0.30001223", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001223.tgz", + "integrity": "sha512-k/RYs6zc/fjbxTjaWZemeSmOjO0JJV+KguOBA3NwPup8uzxM1cMhR2BD9XmO86GuqaqTCO8CgkgH9Rz//vdDiA==", "dev": true - }, - "human-signals": { - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "version": "2.1.0", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + }, + "chalk": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz", + "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "chrome-trace-event": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", + "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==", "dev": true - }, - "path-parse": { - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "version": "1.0.7", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + }, + "clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "requires": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true - }, + }, "colorette": { - "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.2.tgz", - "version": "1.2.2", - "integrity": "sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.2.tgz", + "integrity": "sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==", "dev": true - }, + }, "commander": { - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "version": "2.20.3", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true - }, - "signal-exit": { - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", - "version": "3.0.3", - "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true - }, - "typescript": { - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.4.tgz", - "version": "4.2.4", - "integrity": "sha512-V+evlYHZnQkaz8TRBuxTA92yZBPotr5H+WhQ7bD3hZUndx5tGOa1fuCgeSjxAzM1RiN5IzvadIXTVefuuwZCRg==", - "dev": true - }, - "@types/json-schema": { - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz", - "version": "7.0.7", - "integrity": "sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==", - "dev": true - }, - "is-stream": { - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "version": "2.0.0", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", - "dev": true - }, - "shiki": { - "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.9.11.tgz", + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, "requires": { - "jsonc-parser": "^3.0.0", - "vscode-textmate": "5.2.0", - "onigasm": "^2.2.5" - }, - "version": "0.9.11", - "integrity": "sha512-tjruNTLFhU0hruCPoJP0y+B9LKOmcqUhTpxn7pcJB3fa+04gFChuEmxmrUfOJ7ZO6Jd+HwMnDHgY3lv3Tqonuw==", - "dev": true - }, - "@webassemblyjs/utf8": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.0.tgz", - "version": "1.11.0", - "integrity": "sha512-A/lclGxH6SpSLSyFowMzO/+aDEPU4hvEiooCMXQPcQFPPJaYcPQNKGOCLUySJsYJ4trbpr+Fs08n4jelkVTGVw==", - "dev": true - }, - "@webassemblyjs/wasm-opt": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.0.tgz", - "requires": { - "@webassemblyjs/wasm-parser": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0" - }, - "version": "1.11.0", - "integrity": "sha512-tHUSP5F4ywyh3hZ0+fDQuWxKx3mJiPeFufg+9gwTpYp324mPCQgnuVKwzLTZVqj0duRDovnPaZqDwoyhIO8kYg==", - "dev": true - }, - "rechoir": { - "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.7.0.tgz", - "requires": { - "resolve": "^1.9.0" - }, - "version": "0.7.0", - "integrity": "sha512-ADsDEH2bvbjltXEP+hTIAmeFekTFK0V2BTxMkok6qILyAJEXV0AFfoWcAq4yfll5VdIMd/RVXq0lR+wQi5ZU3Q==", - "dev": true - }, - "strip-final-newline": { - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "version": "2.0.0", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "dev": true - }, - "jsonc-parser": { - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz", - "version": "3.0.0", - "integrity": "sha512-fQzRfAbIBnR0IQvftw9FJveWiHp72Fg20giDrHz6TdfB12UH/uue0D3hm57UB5KgAVuniLMCaS8P1IMj9NR7cA==", - "dev": true - }, - "eslint-scope": { - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", - "requires": { - "estraverse": "^4.1.1", - "esrecurse": "^4.3.0" - }, - "version": "5.1.1", - "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", - "dev": true - }, - "which": { - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "requires": { - "isexe": "^2.0.0" - }, - "version": "2.0.2", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true - }, - "@webassemblyjs/wasm-edit": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.0.tgz", - "requires": { - "@webassemblyjs/wasm-parser": "1.11.0", - "@webassemblyjs/helper-wasm-section": "1.11.0", - "@webassemblyjs/wast-printer": "1.11.0", - "@webassemblyjs/wasm-opt": "1.11.0", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0" - }, - "version": "1.11.0", - "integrity": "sha512-JHQ0damXy0G6J9ucyKVXO2j08JVJ2ntkdJlq1UTiUrIgfGMmA7Ik5VdC/L8hBK46kVJgujkBIoMtT8yVr+yVOQ==", - "dev": true - }, - "events": { - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "version": "3.3.0", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "dev": true - }, - "neo-async": { - "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", - "version": "2.6.2", - "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", - "dev": true - }, - "v8-compile-cache": { - "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", - "version": "2.3.0", - "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", - "dev": true - }, + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, "electron-to-chromium": { - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.727.tgz", - "version": "1.3.727", - "integrity": "sha512-Mfz4FIB4FSvEwBpDfdipRIrwd6uo8gUDoRDF4QEYb4h4tSuI3ov594OrjU6on042UlFHouIJpClDODGkPcBSbg==", + "version": "1.3.727", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.727.tgz", + "integrity": "sha512-Mfz4FIB4FSvEwBpDfdipRIrwd6uo8gUDoRDF4QEYb4h4tSuI3ov594OrjU6on042UlFHouIJpClDODGkPcBSbg==", "dev": true - }, - "glob": { - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + }, + "enhanced-resolve": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.8.0.tgz", + "integrity": "sha512-Sl3KRpJA8OpprrtaIswVki3cWPiPKxXuFxJXBp+zNb6s6VwNWwFRUdtmzd2ReUut8n+sCPx7QCtQ7w5wfJhSgQ==", + "dev": true, "requires": { - "inherits": "2", - "minimatch": "^3.0.4", - "inflight": "^1.0.4", - "fs.realpath": "^1.0.0", - "path-is-absolute": "^1.0.0", - "once": "^1.3.0" - }, - "version": "7.1.7", - "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + } + }, + "envinfo": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.8.1.tgz", + "integrity": "sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw==", "dev": true - }, - "fastest-levenshtein": { - "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.12.tgz", - "version": "1.0.12", - "integrity": "sha512-On2N+BpYJ15xIC974QNVuYGMOlEVt4s0EOI3wwMqOmK1fdDY+FN/zltPV8vosq4ad4c/gJ1KHScUn/6AWIgiow==", + }, + "es-module-lexer": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.4.1.tgz", + "integrity": "sha512-ooYciCUtfw6/d2w56UVeqHPcoCFAiJdz5XOkYpv/Txl1HMUozpXjz/2RIQgqwKdXNDPSF1W7mJCFse3G+HDyAA==", "dev": true - }, - "ajv-keywords": { - "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", - "version": "3.5.2", - "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", "dev": true - }, - "serialize-javascript": { - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", + }, + "eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, "requires": { - "randombytes": "^2.1.0" - }, - "version": "5.0.1", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + } + }, + "esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "requires": { + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true + } + } + }, + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", "dev": true - }, - "@types/node": { - "resolved": "https://registry.npmjs.org/@types/node/-/node-15.0.2.tgz", - "version": "15.0.2", - "integrity": "sha512-p68+a+KoxpoB47015IeYZYRrdqMUcpbK8re/zpFB8Ld46LHC1lPEbp3EXgkEhAYEcPvjJF6ZO+869SQ0aH1dcA==", + }, + "events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", "dev": true - }, - "@xtuc/ieee754": { - "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", - "version": "1.2.0", - "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + }, + "execa": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.0.0.tgz", + "integrity": "sha512-ov6w/2LCiuyO4RLYGdpFGjkcs0wMTgGE8PrkTHikeUy5iJekXyPIKUjifk5CsE0pt7sMCrMZ3YNqoCj6idQOnQ==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + } + }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true - }, + }, "fast-json-stable-stringify": { - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "version": "2.1.0", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true - }, - "typedoc": { - "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.22.3.tgz", - "requires": { - "lunr": "^2.3.9", - "glob": "^7.1.7", - "shiki": "^0.9.10", - "marked": "^3.0.3", - "minimatch": "^3.0.4" - }, - "version": "0.22.3", - "integrity": "sha512-EOWf9Vf3Vfb/jzBzr87uoLybQw9fx3iyXLUcpQn9F2Ks1/ZJN9iGeBbYRU+VNqrWvV4T+aS7Ife7GFEJUf0ohQ==", + }, + "fastest-levenshtein": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.12.tgz", + "integrity": "sha512-On2N+BpYJ15xIC974QNVuYGMOlEVt4s0EOI3wwMqOmK1fdDY+FN/zltPV8vosq4ad4c/gJ1KHScUn/6AWIgiow==", "dev": true - }, + }, "fill-range": { - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, "requires": { "to-regex-range": "^5.0.1" - }, - "version": "7.0.1", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true - }, - "watchpack": { - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.1.1.tgz", + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, "requires": { - "graceful-fs": "^4.1.2", - "glob-to-regexp": "^0.4.1" - }, - "version": "2.1.1", - "integrity": "sha512-Oo7LXCmc1eE1AjyuSBmtC3+Wy4HcV8PxWh2kP6fOl8yTlNS7r0K9l1ao2lrrUza7V39Y3D/BbJgY8VeSlc5JKw==", + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true - }, - "@webassemblyjs/floating-point-hex-parser": { - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.0.tgz", - "version": "1.11.0", - "integrity": "sha512-Q/aVYs/VnPDVYvsCBL/gSgwmfjeCb4LW8+TMrO3cSzJImgv8lxxEPM2JA5jMrivE7LSz3V+PFqtMbls3m1exDA==", + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", "dev": true - }, - "wildcard": { - "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", - "version": "2.0.0", - "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==", + }, + "get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", "dev": true - }, + }, + "glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "glob-to-regexp": { - "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", - "version": "0.4.1", - "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", "dev": true - }, + }, + "graceful-fs": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz", + "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==", + "dev": true + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true + }, + "import-local": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.2.tgz", + "integrity": "sha512-vjL3+w0oulAVZ0hBHnxa/Nm5TAurf9YLQJDhqRZyqb+VKGOB6LU8t9H1Nr5CIo16vh9XfJTOoHwU0B71S557gA==", + "dev": true, + "requires": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "interpret": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", + "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==", + "dev": true + }, + "is-core-module": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.3.0.tgz", + "integrity": "sha512-xSphU2KG9867tsYdLD4RWQ1VqdFl4HTO9Thf3I/3dLEfr0dbPTWKsuCKrgqMljg4nPE+Gq0VCnzT3gr0CyBmsw==", + "dev": true, + "requires": { + "has": "^1.0.3" + } + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, + "is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "requires": { + "isobject": "^3.0.1" + } + }, + "is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", + "dev": true + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", + "dev": true + }, + "jest-worker": { + "version": "26.6.2", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-26.6.2.tgz", + "integrity": "sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ==", + "dev": true, + "requires": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^7.0.0" + } + }, + "json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "jsonc-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz", + "integrity": "sha512-fQzRfAbIBnR0IQvftw9FJveWiHp72Fg20giDrHz6TdfB12UH/uue0D3hm57UB5KgAVuniLMCaS8P1IMj9NR7cA==", + "dev": true + }, + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + }, + "loader-runner": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.2.0.tgz", + "integrity": "sha512-92+huvxMvYlMzMt0iIOukcwYBFpkYJdpl2xsZ7LrlayO7E8SOv+JJUEK17B/dJIHAOLMfh2dZZ/Y18WgmGtYNw==", + "dev": true + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "lunr": { + "version": "2.3.9", + "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", + "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==", + "dev": true + }, + "marked": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.0.10.tgz", + "integrity": "sha512-+QvuFj0nGgO970fySghXGmuw+Fd0gD2x3+MqCWLIPf5oxdv1Ka6b2q+z9RP01P/IaKPMEramy+7cNy/Lw8c3hw==", + "dev": true + }, + "merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true + }, + "micromatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", + "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", + "dev": true, + "requires": { + "braces": "^3.0.1", + "picomatch": "^2.2.3" + } + }, + "mime-db": { + "version": "1.47.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz", + "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==", + "dev": true + }, + "mime-types": { + "version": "2.1.30", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz", + "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==", + "dev": true, + "requires": { + "mime-db": "1.47.0" + } + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true + }, + "node-releases": { + "version": "1.1.71", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.71.tgz", + "integrity": "sha512-zR6HoT6LrLCRBwukmrVbHv0EpEQjksO6GmFcZQQuCAy139BEsoVKPYnf3jongYW83fAa1torLGYwxxky/p28sg==", + "dev": true + }, + "npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "requires": { + "path-key": "^3.0.0" + } + }, "once": { - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, "requires": { "wrappy": "1" - }, - "version": "1.4.0", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + } + }, + "onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + }, + "dependencies": { + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + } + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "picomatch": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.3.tgz", + "integrity": "sha512-KpELjfwcCDUb9PeigTs2mBJzXUPzAuP2oPcA989He8Rte0+YUAjw1JVedDhuTKPkHjSYzMN3npC9luThGYEKdg==", + "dev": true + }, + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "requires": { + "find-up": "^4.0.0" + } + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true + }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "requires": { + "safe-buffer": "^5.1.0" + } + }, + "rechoir": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.7.0.tgz", + "integrity": "sha512-ADsDEH2bvbjltXEP+hTIAmeFekTFK0V2BTxMkok6qILyAJEXV0AFfoWcAq4yfll5VdIMd/RVXq0lR+wQi5ZU3Q==", + "dev": true, + "requires": { + "resolve": "^1.9.0" + } + }, + "resolve": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", + "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", + "dev": true, + "requires": { + "is-core-module": "^2.2.0", + "path-parse": "^1.0.6" + } + }, + "resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "requires": { + "resolve-from": "^5.0.0" + } + }, + "resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + }, + "schema-utils": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.0.0.tgz", + "integrity": "sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.6", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + } + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "serialize-javascript": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", + "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "dev": true, + "requires": { + "randombytes": "^2.1.0" + } + }, + "shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "requires": { + "kind-of": "^6.0.2" + } + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "shiki": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.10.0.tgz", + "integrity": "sha512-iczxaIYeBFHTFrQPb9DVy2SKgYxC4Wo7Iucm7C17cCh2Ge/refnvHscUOxM85u57MfLoNOtjoEFUWt9gBexblA==", + "dev": true, + "requires": { + "jsonc-parser": "^3.0.0", + "vscode-oniguruma": "^1.6.1", + "vscode-textmate": "5.2.0" + } + }, + "signal-exit": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", + "dev": true + }, + "source-list-map": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", + "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==", + "dev": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + }, + "tapable": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.0.tgz", + "integrity": "sha512-FBk4IesMV1rBxX2tfiK8RAmogtWn53puLOQlvO8XuwlgxcYbP4mVPS9Ph4aeamSyyVjOl24aYWAuc8U5kCVwMw==", + "dev": true + }, + "terser": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.7.0.tgz", + "integrity": "sha512-HP5/9hp2UaZt5fYkuhNBR8YyRcT8juw8+uFbAme53iN9hblvKnLUTKkmwJG6ocWpIKf8UK4DoeWG4ty0J6S6/g==", + "dev": true, + "requires": { + "commander": "^2.20.0", + "source-map": "~0.7.2", + "source-map-support": "~0.5.19" + }, + "dependencies": { + "source-map": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", + "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", + "dev": true + } + } + }, + "terser-webpack-plugin": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.1.1.tgz", + "integrity": "sha512-5XNNXZiR8YO6X6KhSGXfY0QrGrCRlSwAEjIIrlRQR4W8nP69TaJUlh3bkuac6zzgspiGPfKEHcY295MMVExl5Q==", + "dev": true, + "requires": { + "jest-worker": "^26.6.2", + "p-limit": "^3.1.0", + "schema-utils": "^3.0.0", + "serialize-javascript": "^5.0.1", + "source-map": "^0.6.1", + "terser": "^5.5.1" + } + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + }, + "ts-loader": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.1.2.tgz", + "integrity": "sha512-ryMgATvLLl+z8zQvdlm6Pep0slmwxFWIEnq/5VdiLVjqQXnFJgO+qNLGIIP+d2N2jsFZ9MibZCVDb2bSp7OmEA==", + "dev": true, + "requires": { + "chalk": "^4.1.0", + "enhanced-resolve": "^5.0.0", + "micromatch": "^4.0.0", + "semver": "^7.3.4" + } + }, + "typedoc": { + "version": "0.22.11", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.22.11.tgz", + "integrity": "sha512-pVr3hh6dkS3lPPaZz1fNpvcrqLdtEvXmXayN55czlamSgvEjh+57GUqfhAI1Xsuu/hNHUT1KNSx8LH2wBP/7SA==", + "dev": true, + "requires": { + "glob": "^7.2.0", + "lunr": "^2.3.9", + "marked": "^4.0.10", + "minimatch": "^3.0.4", + "shiki": "^0.10.0" + } + }, + "typescript": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.4.tgz", + "integrity": "sha512-V+evlYHZnQkaz8TRBuxTA92yZBPotr5H+WhQ7bD3hZUndx5tGOa1fuCgeSjxAzM1RiN5IzvadIXTVefuuwZCRg==", + "dev": true + }, + "uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "requires": { + "punycode": "^2.1.0" + } + }, + "v8-compile-cache": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", + "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", + "dev": true + }, + "vscode-oniguruma": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.6.1.tgz", + "integrity": "sha512-vc4WhSIaVpgJ0jJIejjYxPvURJavX6QG41vu0mGhqywMkQqulezEqEQ3cO3gc8GvcOpX6ycmKGqRoROEMBNXTQ==", + "dev": true + }, + "vscode-textmate": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-5.2.0.tgz", + "integrity": "sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ==", + "dev": true + }, + "watchpack": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.1.1.tgz", + "integrity": "sha512-Oo7LXCmc1eE1AjyuSBmtC3+Wy4HcV8PxWh2kP6fOl8yTlNS7r0K9l1ao2lrrUza7V39Y3D/BbJgY8VeSlc5JKw==", + "dev": true, + "requires": { + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.1.2" + } + }, + "webpack": { + "version": "5.36.2", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.36.2.tgz", + "integrity": "sha512-XJumVnnGoH2dV+Pk1VwgY4YT6AiMKpVoudUFCNOXMIVrEKPUgEwdIfWPjIuGLESAiS8EdIHX5+TiJz/5JccmRg==", + "dev": true, + "requires": { + "@types/eslint-scope": "^3.7.0", + "@types/estree": "^0.0.47", + "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/wasm-edit": "1.11.0", + "@webassemblyjs/wasm-parser": "1.11.0", + "acorn": "^8.2.1", + "browserslist": "^4.14.5", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^5.8.0", + "es-module-lexer": "^0.4.0", + "eslint-scope": "^5.1.1", + "events": "^3.2.0", + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.2.4", + "json-parse-better-errors": "^1.0.2", + "loader-runner": "^4.2.0", + "mime-types": "^2.1.27", + "neo-async": "^2.6.2", + "schema-utils": "^3.0.0", + "tapable": "^2.1.1", + "terser-webpack-plugin": "^5.1.1", + "watchpack": "^2.0.0", + "webpack-sources": "^2.1.1" + } + }, + "webpack-cli": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-4.7.0.tgz", + "integrity": "sha512-7bKr9182/sGfjFm+xdZSwgQuFjgEcy0iCTIBxRUeteJ2Kr8/Wz0qNJX+jw60LU36jApt4nmMkep6+W5AKhok6g==", + "dev": true, + "requires": { + "@discoveryjs/json-ext": "^0.5.0", + "@webpack-cli/configtest": "^1.0.3", + "@webpack-cli/info": "^1.2.4", + "@webpack-cli/serve": "^1.4.0", + "colorette": "^1.2.1", + "commander": "^7.0.0", + "execa": "^5.0.0", + "fastest-levenshtein": "^1.0.12", + "import-local": "^3.0.2", + "interpret": "^2.2.0", + "rechoir": "^0.7.0", + "v8-compile-cache": "^2.2.0", + "webpack-merge": "^5.7.3" + }, + "dependencies": { + "commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "dev": true + } + } + }, + "webpack-merge": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.7.3.tgz", + "integrity": "sha512-6/JUQv0ELQ1igjGDzHkXbVDRxkfA57Zw7PfiupdLFJYrgFqY5ZP8xxbpp2lU3EPwYx89ht5Z/aDkD40hFCm5AA==", + "dev": true, + "requires": { + "clone-deep": "^4.0.1", + "wildcard": "^2.0.0" + } + }, + "webpack-sources": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-2.2.0.tgz", + "integrity": "sha512-bQsA24JLwcnWGArOKUxYKhX3Mz/nK1Xf6hxullKERyktjNMC4x8koOeaDNTA2fEJ09BdWLbM/iTW0ithREUP0w==", + "dev": true, + "requires": { + "source-list-map": "^2.0.1", + "source-map": "^0.6.1" + } + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "wildcard": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", + "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==", + "dev": true + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true } - }, - "version": "1.11.0", - "requires": true, - "name": "onnxruntime-common", - "lockfileVersion": 1 -} \ No newline at end of file + } +} diff --git a/js/common/package.json b/js/common/package.json index 0b5ad1857a..00ff56f8f1 100644 --- a/js/common/package.json +++ b/js/common/package.json @@ -1,31 +1,31 @@ { - "license": "MIT", - "unpkg": "dist/ort-common.min.js", - "name": "onnxruntime-common", + "license": "MIT", + "unpkg": "dist/ort-common.min.js", + "name": "onnxruntime-common", "repository": { - "url": "https://github.com/Microsoft/onnxruntime.git", + "url": "https://github.com/Microsoft/onnxruntime.git", "type": "git" - }, - "author": "fs-eire", - "module": "dist/lib/index.js", - "version": "1.11.0", - "jsdelivr": "dist/ort-common.min.js", + }, + "author": "fs-eire", + "module": "dist/lib/index.js", + "version": "1.11.0", + "jsdelivr": "dist/ort-common.min.js", "scripts": { "prepare": "tsc && webpack" - }, + }, "keywords": [ - "ONNX", - "ONNXRuntime", + "ONNX", + "ONNXRuntime", "ONNX Runtime" - ], + ], "devDependencies": { - "ts-loader": "^9.1.2", - "webpack": "^5.36.2", - "typescript": "^4.2.4", - "typedoc": "^0.22.3", + "ts-loader": "^9.1.2", + "typedoc": "^0.22.11", + "typescript": "^4.2.4", + "webpack": "^5.36.2", "webpack-cli": "^4.7.0" - }, - "main": "dist/ort-common.node.js", - "types": "dist/lib/index.d.ts", + }, + "main": "dist/ort-common.node.js", + "types": "dist/lib/index.d.ts", "description": "ONNXRuntime JavaScript API library" -} \ No newline at end of file +} From e27f2dc932f7c875c7f300cb5ebd4f12c62e6681 Mon Sep 17 00:00:00 2001 From: Yi-Hong Lyu Date: Wed, 19 Jan 2022 06:37:34 +0800 Subject: [PATCH 05/59] int8/uint8 support for Argmax for opset 1, 11, 12 (#10296) --- docs/OperatorKernels.md | 4 ++-- .../core/providers/cpu/cpu_execution_provider.cc | 12 ++++++++++++ .../providers/cpu/reduction/reduction_ops.cc | 4 ++++ .../cpu/reduction/reduction_ops_test.cc | 4 ++-- .../testdata/kernel_def_hashes/onnx.cpu.json | 16 ++++++++++++++++ 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/docs/OperatorKernels.md b/docs/OperatorKernels.md index 7e311e15c0..4d885fdb15 100644 --- a/docs/OperatorKernels.md +++ b/docs/OperatorKernels.md @@ -26,8 +26,8 @@ Do not modify directly.* |Affine|*in* X:**T**
*out* Y:**T**|1+|**T** = tensor(float)| |And|*in* A:**T**
*in* B:**T**
*out* C:**T1**|7+|**T** = tensor(bool)
**T1** = tensor(bool)| |ArgMax|*in* data:**T**
*out* reduced:**tensor(int64)**|13+|**T** = tensor(double), tensor(float), tensor(int32), tensor(int8), tensor(uint8)| -|||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32)| -|||[1, 10]|**T** = tensor(float), tensor(int32)| +|||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int8), tensor(uint8)| +|||[1, 10]|**T** = tensor(float), tensor(int32), tensor(int8), tensor(uint8)| |ArgMin|*in* data:**T**
*out* reduced:**tensor(int64)**|13+|**T** = tensor(double), tensor(float), tensor(int32)| |||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32)| |||[1, 10]|**T** = tensor(float), tensor(int32)| diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index eb12b7c4fd..ef5e821319 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -183,6 +183,8 @@ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOn class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 10, double, ReduceSumSquare); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 10, float, ArgMax); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 10, int8_t, ArgMax); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 10, uint8_t, ArgMax); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 10, int32_t, ArgMax); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 10, float, ArgMin); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, 10, int32_t, ArgMin); @@ -340,6 +342,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, uint8_t, DynamicQuantizeLinear); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float, ArgMax); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, ArgMax); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, int8_t, ArgMax); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, uint8_t, ArgMax); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, int32_t, ArgMax); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float, ArgMin); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, ArgMin); @@ -998,6 +1002,10 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { double, ReduceSumSquare)>, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo(1)); test.AddAttribute("keepdims", static_cast(1)); test.AddInput("data", {3, 2, 2}, @@ -2219,7 +2219,7 @@ TEST(ReductionOpTest, ArgMax_int8) { } TEST(ReductionOpTest, ArgMax_uint8) { - OpTester test("ArgMax", 13); + OpTester test("ArgMax"); test.AddAttribute("axis", static_cast(1)); test.AddAttribute("keepdims", static_cast(1)); test.AddInput("data", {3, 2, 2}, diff --git a/onnxruntime/test/testdata/kernel_def_hashes/onnx.cpu.json b/onnxruntime/test/testdata/kernel_def_hashes/onnx.cpu.json index 88974c2b26..0aac7d8962 100644 --- a/onnxruntime/test/testdata/kernel_def_hashes/onnx.cpu.json +++ b/onnxruntime/test/testdata/kernel_def_hashes/onnx.cpu.json @@ -139,6 +139,10 @@ "And ai.onnx CPUExecutionProvider", 7931711152704979424 ], + [ + "ArgMax ai.onnx CPUExecutionProvider", + 725939091209285112 + ], [ "ArgMax ai.onnx CPUExecutionProvider", 1506815612154586624 @@ -163,6 +167,14 @@ "ArgMax ai.onnx CPUExecutionProvider", 3776111440658888968 ], + [ + "ArgMax ai.onnx CPUExecutionProvider", + 4620340115180881848 + ], + [ + "ArgMax ai.onnx CPUExecutionProvider", + 10486830423259333928 + ], [ "ArgMax ai.onnx CPUExecutionProvider", 12157492629288967928 @@ -179,6 +191,10 @@ "ArgMax ai.onnx CPUExecutionProvider", 14910778597193697496 ], + [ + "ArgMax ai.onnx CPUExecutionProvider", + 18394842867298045376 + ], [ "ArgMin ai.onnx CPUExecutionProvider", 3259714541817453752 From 62eab67f796bf6ae693542cacb3f9fa5ac69829e Mon Sep 17 00:00:00 2001 From: Yi-Hong Lyu Date: Wed, 19 Jan 2022 06:47:33 +0800 Subject: [PATCH 06/59] Fuse DQ -> ArgMax into ArgMax (#10274) --- .../optimizer/qdq_transformer/qdq_util.cc | 26 +++++++++++++ .../core/optimizer/qdq_transformer/qdq_util.h | 7 ++++ .../qdq_selector_action_transformer.cc | 24 ++++++++++++ .../selectors_actions/qdq_selectors.cc | 24 ++++++++++++ .../selectors_actions/qdq_selectors.h | 15 +++++++ .../test/optimizer/qdq_transformer_test.cc | 39 +++++++++++++++++++ 6 files changed, 135 insertions(+) diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc index 14ed791cb0..0c5710b6fd 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc @@ -58,5 +58,31 @@ bool IsQDQPairSupported( *q_scale.data() == *dq_scale.data(); } +bool IsDQSupported( + const Node& dq_node, + const std::function& get_const_initializer) { + ConstPointerContainer> dq_input_defs = dq_node.InputDefs(); + + // DQ contains optional input is not supported + // non-scalar DQ scale and zero point needs are not supported + if (dq_input_defs.size() != InputIndex::TOTAL_COUNT || + !optimizer_utils::IsScalar(*dq_input_defs[InputIndex::SCALE_ID]) || + !optimizer_utils::IsScalar(*dq_input_defs[InputIndex::ZERO_POINT_ID])) { + return false; + } + + // if DQ scale and zero point are not constant, return false + const ONNX_NAMESPACE::TensorProto* dq_scale_tensor_proto = + get_const_initializer(dq_input_defs[InputIndex::SCALE_ID]->Name()); + const ONNX_NAMESPACE::TensorProto* dq_zp_tensor_proto = + get_const_initializer(dq_input_defs[InputIndex::ZERO_POINT_ID]->Name()); + if (nullptr == dq_zp_tensor_proto || + nullptr == dq_scale_tensor_proto) { + return false; + } + + return true; +} + } // namespace QDQ } // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_util.h b/onnxruntime/core/optimizer/qdq_transformer/qdq_util.h index a6690cd782..22a06e7506 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/qdq_util.h +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_util.h @@ -36,5 +36,12 @@ bool IsQDQPairSupported( const std::function& get_const_initializer, const Path& model_path); +// Check if DQ is supported in the QDQ transformer. It requires: +// 1. DQ doesn't have optional input. +// 2. scale and zero point is constant scalar +bool IsDQSupported( + const Node& dq_node, + const std::function& get_const_initializer); + } // namespace QDQ } // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc index e39a11a1e8..50bc405378 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc @@ -45,6 +45,29 @@ void DropQDQNodesRules(SelectorActionRegistry& qdq_selector_action_registry) { #endif } +// create rules for ops that don't change the data +void DropDQNodesRules(SelectorActionRegistry& qdq_selector_action_registry) { + // 2 nodes. DQ, target. Merge into target and remove DQ. + const std::string action_name{"dropDQ"}; + NTO::NodeLocation dq{NTO::NodeType::kInput, 0}; + + // Move DQ input 0 to target input 0. + std::vector moves{ + MoveToSlot(dq, ArgType::kInput, 0, ArgType::kInput, 0)}; + + std::unique_ptr action = std::make_unique(std::move(moves)); + +#if !defined(ORT_MINIMAL_BUILD) + std::unique_ptr selector = std::make_unique(); + qdq_selector_action_registry.RegisterSelectorAndAction(action_name, + {{"ArgMax", {}}}, + std::move(selector), + std::move(action)); +#else + qdq_selector_action_registry.RegisterAction(action_name, std::move(action)); +#endif +} + void UnaryOpQDQRules(SelectorActionRegistry& qdq_selector_action_registry) { // 3 nodes. DQ, target, Q // Replace with internal QLinear version of operator. Delete all original nodes. @@ -148,6 +171,7 @@ SelectorActionRegistry CreateSelectorActionRegistry(bool is_int8_allowed) { SelectorActionRegistry qdq_selector_action_registry; DropQDQNodesRules(qdq_selector_action_registry); + DropDQNodesRules(qdq_selector_action_registry); UnaryOpQDQRules(qdq_selector_action_registry); BinaryOpQDQRules(qdq_selector_action_registry); VariadicOpQDQRules(qdq_selector_action_registry); diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc index 582d215cfc..56f3015903 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc @@ -104,6 +104,30 @@ bool DropQDQNodeGroupSelector::Check(const GraphViewer& graph_viewer, return IsQDQPairSupported(q_node, dq_node, get_const_initializer, graph_viewer.ModelPath()); } +bool DropDQNodeGroupSelector::CheckDQNodes(const Node& node, const std::vector& dq_nodes) const { + int num_dq_inputs = NumActualValues(node, true); + + return num_dq_inputs == gsl::narrow_cast(dq_nodes.size()); +} + +bool DropDQNodeGroupSelector::Check(const GraphViewer& graph_viewer, + const Node& node, + const std::vector& dq_nodes, + const std::vector& q_nodes) const { + if (!CheckDQNodes(node, dq_nodes)) { + return false; + } + + (void)q_nodes; + const Node& dq_node = *dq_nodes.front(); + + auto get_const_initializer = [&graph_viewer](const std::string& initializer_name) { + return graph_viewer.GetConstantInitializer(initializer_name, true); + }; + + return IsDQSupported(dq_node, get_const_initializer); +} + bool UnaryNodeGroupSelector::Check(const GraphViewer& graph_viewer, const Node& node, const std::vector& dq_nodes, const std::vector& q_nodes) const { diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h index 75ec972c9a..398cfc1cce 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h @@ -56,6 +56,16 @@ class DropQDQNodeGroupSelector : public NodeGroupSelector { const std::vector& q_nodes) const override; }; +// Single DQ -> node. +class DropDQNodeGroupSelector : public NodeGroupSelector { + // base check that we have the expected number of DQ inputs. + bool CheckDQNodes(const Node& node, const std::vector& dq_nodes) const; + + bool Check(const GraphViewer& graph_viewer, const Node& node, + const std::vector& dq_nodes, + const std::vector& q_nodes) const override; +}; + // single input. default is to only support uint8. class UnaryNodeGroupSelector : public NodeGroupSelector { bool Check(const GraphViewer& graph_viewer, const Node& node, @@ -142,6 +152,11 @@ class DropQDQNodesSelector : public BaseSelector { DropQDQNodesSelector() : BaseSelector(std::make_unique()) {} }; +class DropDQNodesSelector : public BaseSelector { + public: + DropDQNodesSelector() : BaseSelector(std::make_unique()) {} +}; + class UnarySelector : public BaseSelector { public: UnarySelector() : BaseSelector(std::make_unique()) {} diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index 4eea3be246..cbb716bfdd 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -819,6 +819,45 @@ TEST(QDQTransformerTests, ResizeReshape) { test_case({1, 2, 26, 42}, {4}); } +TEST(QDQTransformerTests, ArgMax) { + auto test_case = [&](const std::vector& input_shape, + int axis, + int keepdims, + int select_last_index) { + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput(input_shape, + std::numeric_limits::min(), + std::numeric_limits::max()); + auto* output_arg = builder.MakeOutput(); + + // add DQ + auto* dq_output = builder.MakeIntermediate(); + builder.AddDequantizeLinearNode(input_arg, .003f, 1, dq_output); + + // add ArgMax + Node& argmax_node = builder.AddNode("ArgMax", {dq_output}, {output_arg}); + argmax_node.AddAttribute("axis", static_cast(axis)); + argmax_node.AddAttribute("keepdims", static_cast(keepdims)); + argmax_node.AddAttribute("select_last_index", static_cast(select_last_index)); + }; + + auto check_argmax_graph = [&](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["ArgMax"], 1); + EXPECT_EQ(op_to_count["DequantizeLinear"], 0); + }; + + TransformerTester(build_test_case, check_argmax_graph, + TransformerLevel::Level1, + TransformerLevel::Level2, + /* opset_version */ 13); + }; + + test_case({2, 13, 12, 37}, 1, 0, 0); + test_case({2, 13, 12, 37}, 0, 1, 0); + test_case({2, 13, 12, 37}, 0, 0, 1); +} + TEST(QDQTransformerTests, QLinearMatMul) { auto test_case = [&](const std::vector& input1_shape, const std::vector& input2_shape) { auto build_test_case = [&](ModelTestBuilder& builder) { From b038f4e56f90a810b47cf8176b84306a4c8bed57 Mon Sep 17 00:00:00 2001 From: Sunghoon <35605090+hanbitmyths@users.noreply.github.com> Date: Tue, 18 Jan 2022 18:05:04 -0800 Subject: [PATCH 07/59] Add a build option to create a WebAssembly static library (#10184) * add p50 in test * Add a build option to create a WebAssembly static library Co-authored-by: Yulong Wang --- cmake/CMakeLists.txt | 1 + cmake/onnxruntime_webassembly.cmake | 251 ++++++++++++++++++------ js/README.md | 4 + onnxruntime/test/wasm/test_inference.cc | 42 ++++ onnxruntime/test/wasm/test_main.cc | 9 + tools/ci_build/build.py | 5 + 6 files changed, 251 insertions(+), 61 deletions(-) create mode 100644 onnxruntime/test/wasm/test_inference.cc create mode 100644 onnxruntime/test/wasm/test_main.cc diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index e1b4c053c1..07ce345345 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -147,6 +147,7 @@ option(onnxruntime_USE_MPI "Build with MPI support" OFF) # build WebAssembly option(onnxruntime_BUILD_WEBASSEMBLY "Enable this option to create WebAssembly byte codes" OFF) +option(onnxruntime_BUILD_WEBASSEMBLY_STATIC_LIB "Enable this option to create WebAssembly static library" OFF) option(onnxruntime_ENABLE_WEBASSEMBLY_THREADS "Enable this option to create WebAssembly byte codes with multi-threads support" OFF) option(onnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_CATCHING "Enable this option to turn on exception catching" OFF) option(onnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_THROWING "Enable this option to turn on exception throwing even if the build disabled exceptions support" OFF) diff --git a/cmake/onnxruntime_webassembly.cmake b/cmake/onnxruntime_webassembly.cmake index 99db7cb7c6..cb9b3c7029 100644 --- a/cmake/onnxruntime_webassembly.cmake +++ b/cmake/onnxruntime_webassembly.cmake @@ -1,15 +1,85 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -file(GLOB_RECURSE onnxruntime_webassembly_src CONFIGURE_DEPENDS - "${ONNXRUNTIME_ROOT}/wasm/api.cc" -) +function(bundle_static_library bundled_target_name) + function(recursively_collect_dependencies input_target) + set(input_link_libraries LINK_LIBRARIES) + get_target_property(input_type ${input_target} TYPE) + if (${input_type} STREQUAL "INTERFACE_LIBRARY") + set(input_link_libraries INTERFACE_LINK_LIBRARIES) + endif() + get_target_property(public_dependencies ${input_target} ${input_link_libraries}) + foreach(dependency IN LISTS public_dependencies) + if(TARGET ${dependency}) + get_target_property(alias ${dependency} ALIASED_TARGET) + if (TARGET ${alias}) + set(dependency ${alias}) + endif() + get_target_property(type ${dependency} TYPE) + if (${type} STREQUAL "STATIC_LIBRARY") + list(APPEND static_libs ${dependency}) + endif() -source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_webassembly_src}) + get_property(library_already_added GLOBAL PROPERTY ${target_name}_static_bundle_${dependency}) + if (NOT library_already_added) + set_property(GLOBAL PROPERTY ${target_name}_static_bundle_${dependency} ON) + recursively_collect_dependencies(${dependency}) + endif() + endif() + endforeach() + set(static_libs ${static_libs} PARENT_SCOPE) + endfunction() -add_executable(onnxruntime_webassembly - ${onnxruntime_webassembly_src} -) + foreach(target_name IN ITEMS ${ARGN}) + list(APPEND static_libs ${target_name}) + recursively_collect_dependencies(${target_name}) + endforeach() + + list(REMOVE_DUPLICATES static_libs) + + set(bundled_target_full_name + ${CMAKE_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${bundled_target_name}${CMAKE_STATIC_LIBRARY_SUFFIX}) + + file(WRITE ${CMAKE_BINARY_DIR}/${bundled_target_name}.ar.in + "CREATE ${bundled_target_full_name}\n" ) + + foreach(target IN LISTS static_libs) + file(APPEND ${CMAKE_BINARY_DIR}/${bundled_target_name}.ar.in + "ADDLIB $\n") + endforeach() + + file(APPEND ${CMAKE_BINARY_DIR}/${bundled_target_name}.ar.in "SAVE\n") + file(APPEND ${CMAKE_BINARY_DIR}/${bundled_target_name}.ar.in "END\n") + + file(GENERATE + OUTPUT ${CMAKE_BINARY_DIR}/${bundled_target_name}.ar + INPUT ${CMAKE_BINARY_DIR}/${bundled_target_name}.ar.in) + + set(ar_tool ${CMAKE_AR}) + if (CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(ar_tool ${CMAKE_CXX_COMPILER_AR}) + endif() + + add_custom_command( + COMMAND ${ar_tool} -M < ${CMAKE_BINARY_DIR}/${bundled_target_name}.ar + OUTPUT ${bundled_target_full_name} + COMMENT "Bundling ${bundled_target_name}" + VERBATIM) + + add_custom_target(bundling_target ALL DEPENDS ${bundled_target_full_name}) + foreach(target_name IN ITEMS ${ARGN}) + add_dependencies(bundling_target ${target_name}) + endforeach() + + add_library(${bundled_target_name} STATIC IMPORTED) + foreach(target_name IN ITEMS ${ARGN}) + set_target_properties(${bundled_target_name} + PROPERTIES + IMPORTED_LOCATION ${bundled_target_full_name} + INTERFACE_INCLUDE_DIRECTORIES $) + endforeach() + add_dependencies(${bundled_target_name} bundling_target) +endfunction() if (NOT onnxruntime_ENABLE_WEBASSEMBLY_THREADS) add_compile_definitions( @@ -22,68 +92,127 @@ endif() target_compile_options(onnx PRIVATE -Wno-unused-parameter -Wno-unused-variable) -target_link_libraries(onnxruntime_webassembly PRIVATE - nsync_cpp - ${PROTOBUF_LIB} - onnx - onnx_proto - onnxruntime_common - onnxruntime_flatbuffers - onnxruntime_framework - onnxruntime_graph - onnxruntime_mlas - onnxruntime_optimizer - onnxruntime_providers - onnxruntime_session - onnxruntime_util - re2::re2 -) +if (onnxruntime_BUILD_WEBASSEMBLY_STATIC_LIB) + bundle_static_library(onnxruntime_webassembly + nsync_cpp + ${PROTOBUF_LIB} + onnx + onnx_proto + onnxruntime_common + onnxruntime_flatbuffers + onnxruntime_framework + onnxruntime_graph + onnxruntime_mlas + onnxruntime_optimizer + onnxruntime_providers + onnxruntime_session + onnxruntime_util + re2::re2 + ) -set(EXPORTED_RUNTIME_METHODS "['stackAlloc','stackRestore','stackSave','UTF8ToString','stringToUTF8','lengthBytesUTF8']") + file(GLOB_RECURSE onnxruntime_webassembly_test_src CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/test/wasm/test_main.cc" + "${ONNXRUNTIME_ROOT}/test/wasm/test_inference.cc" + ) -set_target_properties(onnxruntime_webassembly PROPERTIES LINK_FLAGS " \ - -s \"EXPORTED_RUNTIME_METHODS=${EXPORTED_RUNTIME_METHODS}\" \ - -s WASM=1 \ - -s NO_EXIT_RUNTIME=0 \ - -s ALLOW_MEMORY_GROWTH=1 \ - -s MODULARIZE=1 \ - -s EXPORT_ALL=0 \ - -s LLD_REPORT_UNDEFINED \ - -s VERBOSE=0 \ - -s NO_FILESYSTEM=1 \ - -s MALLOC=${onnxruntime_WEBASSEMBLY_MALLOC} \ - --closure 1 \ - --no-entry") + source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_webassembly_test_src}) -if (CMAKE_BUILD_TYPE STREQUAL "Debug") - set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=2 -s SAFE_HEAP=1 -s STACK_OVERFLOW_CHECK=1 -s DEMANGLE_SUPPORT=1") + add_executable(onnxruntime_webassembly_test + ${onnxruntime_webassembly_test_src} + ) + + set_target_properties(onnxruntime_webassembly_test PROPERTIES LINK_FLAGS + "-s ALLOW_MEMORY_GROWTH=1 -s \"EXPORTED_RUNTIME_METHODS=['FS']\" --preload-file ${CMAKE_CURRENT_BINARY_DIR}/testdata@/testdata -s EXIT_RUNTIME=1" + ) + + target_link_libraries(onnxruntime_webassembly_test PUBLIC + onnxruntime_webassembly + GTest::gtest + ) + + find_program(NODE_EXECUTABLE node required) + if (NOT NODE_EXECUTABLE) + message(FATAL_ERROR "Node is required for a test") + endif() + + add_test(NAME onnxruntime_webassembly_test + COMMAND ${NODE_EXECUTABLE} onnxruntime_webassembly_test.js + WORKING_DIRECTORY $ + ) else() - set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=0 -s SAFE_HEAP=0 -s STACK_OVERFLOW_CHECK=0 -s DEMANGLE_SUPPORT=0") -endif() + file(GLOB_RECURSE onnxruntime_webassembly_src CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/wasm/api.cc" + ) -# Set link flag to enable exceptions support, this will override default disabling exception throwing behavior when disable exceptions. -if (onnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_THROWING) - set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s DISABLE_EXCEPTION_THROWING=0") -endif() + source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_webassembly_src}) -if (onnxruntime_ENABLE_WEBASSEMBLY_PROFILING) - set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " --profiling --profiling-funcs") -endif() + add_executable(onnxruntime_webassembly + ${onnxruntime_webassembly_src} + ) -if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) - if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD) - set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasmSimdThreaded -s USE_PTHREADS=1") - set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-simd-threaded") + target_link_libraries(onnxruntime_webassembly PRIVATE + nsync_cpp + ${PROTOBUF_LIB} + onnx + onnx_proto + onnxruntime_common + onnxruntime_flatbuffers + onnxruntime_framework + onnxruntime_graph + onnxruntime_mlas + onnxruntime_optimizer + onnxruntime_providers + onnxruntime_session + onnxruntime_util + re2::re2 + ) + + set(EXPORTED_RUNTIME_METHODS "['stackAlloc','stackRestore','stackSave','UTF8ToString','stringToUTF8','lengthBytesUTF8']") + + set_target_properties(onnxruntime_webassembly PROPERTIES LINK_FLAGS " \ + -s \"EXPORTED_RUNTIME_METHODS=${EXPORTED_RUNTIME_METHODS}\" \ + -s WASM=1 \ + -s NO_EXIT_RUNTIME=0 \ + -s ALLOW_MEMORY_GROWTH=1 \ + -s MODULARIZE=1 \ + -s EXPORT_ALL=0 \ + -s LLD_REPORT_UNDEFINED \ + -s VERBOSE=0 \ + -s NO_FILESYSTEM=1 \ + -s MALLOC=${onnxruntime_WEBASSEMBLY_MALLOC} \ + --closure 1 \ + --no-entry") + + if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=2 -s SAFE_HEAP=1 -s STACK_OVERFLOW_CHECK=1 -s DEMANGLE_SUPPORT=1") else() - set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasmThreaded -s USE_PTHREADS=1") - set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-threaded") + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=0 -s SAFE_HEAP=0 -s STACK_OVERFLOW_CHECK=0 -s DEMANGLE_SUPPORT=0") endif() -else() - if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD) - set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasmSimd") - set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-simd") + + # Set link flag to enable exceptions support, this will override default disabling exception throwing behavior when disable exceptions. + if (onnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_THROWING) + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s DISABLE_EXCEPTION_THROWING=0") + endif() + + if (onnxruntime_ENABLE_WEBASSEMBLY_PROFILING) + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " --profiling --profiling-funcs") + endif() + + if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD) + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasmSimdThreaded -s USE_PTHREADS=1") + set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-simd-threaded") + else() + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasmThreaded -s USE_PTHREADS=1") + set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-threaded") + endif() else() - set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasm") - set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm") + if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD) + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasmSimd") + set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-simd") + else() + set_property(TARGET onnxruntime_webassembly APPEND_STRING PROPERTY LINK_FLAGS " -s EXPORT_NAME=ortWasm") + set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm") + endif() endif() -endif() +endif() \ No newline at end of file diff --git a/js/README.md b/js/README.md index 69b66dcc6d..88d707bb42 100644 --- a/js/README.md +++ b/js/README.md @@ -301,6 +301,7 @@ It should be able to consumed by both from projects that uses NPM packages (thro #### Reduced WebAssembly artifacts By default, the WebAssembly artifacts from onnxruntime-web package allows use of both standard ONNX models (.onnx) and ORT format models (.ort). There is an option to use a minimal build of ONNX Runtime to reduce the binary size, which only supports ORT format models. See also [ORT format model](https://onnxruntime.ai/docs/tutorials/mobile/overview.html) for more information. + #### Reduced JavaScript bundle file fize By default, the main bundle file `ort.min.js` of ONNX Runtime Web contains all features. However, its size is over 500kB and for some scenarios we want a smaller sized bundle file, if we don't use all the features. The following table lists all available bundles with their support status of features. @@ -313,6 +314,9 @@ By default, the main bundle file `ort.min.js` of ONNX Runtime Web contains all f |ort.wasm.min.js|148.56|44KB|X|O|O|O|X| |ort.wasm-core.min.js|40.56KB|12.74KB|X|O|X|X|X| +#### Build ONNX Runtime as a WebAssembly static library + +When `--build_wasm_static_lib` is given instead of `--build_wasm`, it builds a WebAssembly static library of ONNX Runtime and creates a `libonnxruntime_webassembly.a` file at a build output directory. Developers who have their own C/C++ project and build it as WebAssembly with ONNX Runtime, this build option would be useful. This static library is not published by a pipeline, so a manual build is required if necessary. ## onnxruntime-react-native diff --git a/onnxruntime/test/wasm/test_inference.cc b/onnxruntime/test/wasm/test_inference.cc new file mode 100644 index 0000000000..644c99bad6 --- /dev/null +++ b/onnxruntime/test/wasm/test_inference.cc @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "core/session/onnxruntime_cxx_api.h" + +TEST(WebAssemblyTest, test) { + Ort::Env ort_env; + Ort::Session session{ort_env, "testdata/mul_1.onnx", Ort::SessionOptions{nullptr}}; + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + + std::array input_data{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + std::array input_shape{3, 2}; + Ort::Value input_tensor = Ort::Value::CreateTensor(memory_info, + input_data.data(), input_data.size(), + input_shape.data(), input_shape.size()); + + std::array output_data{}; + std::array output_shape{3, 2}; + Ort::Value output_tensor = Ort::Value::CreateTensor(memory_info, + output_data.data(), output_data.size(), + output_shape.data(), output_shape.size()); + + const char* input_names[] = {"X"}; + const char* output_names[] = {"Y"}; + + session.Run(Ort::RunOptions{nullptr}, input_names, &input_tensor, 1, output_names, &output_tensor, 1); + + std::array expected_data{1.0f, 4.0f, 9.0f, 16.0f, 25.0f, 36.0f}; + std::vector expected_shape{3, 2}; + + auto type_info = output_tensor.GetTensorTypeAndShapeInfo(); + ASSERT_EQ(type_info.GetShape(), expected_shape); + auto total_len = type_info.GetElementCount(); + ASSERT_EQ(total_len, expected_data.size()); + + float* result = output_tensor.GetTensorMutableData(); + for (size_t i = 0; i != total_len; ++i) { + ASSERT_EQ(expected_data[i], result[i]); + } +} \ No newline at end of file diff --git a/onnxruntime/test/wasm/test_main.cc b/onnxruntime/test/wasm/test_main.cc new file mode 100644 index 0000000000..5f64e61f5c --- /dev/null +++ b/onnxruntime/test/wasm/test_main.cc @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index edb2433d7e..56272d448a 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -355,6 +355,7 @@ def parse_arguments(): # WebAssembly build parser.add_argument("--build_wasm", action='store_true', help="Build for WebAssembly") + parser.add_argument("--build_wasm_static_lib", action='store_true', help="Build for WebAssembly static library") parser.add_argument("--enable_wasm_simd", action='store_true', help="Enable WebAssembly SIMD") parser.add_argument( "--disable_wasm_exception_catching", action='store_true', @@ -822,6 +823,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_ENABLE_MEMORY_PROFILE=" + ("ON" if args.enable_memory_profile else "OFF"), "-Donnxruntime_ENABLE_CUDA_LINE_NUMBER_INFO=" + ("ON" if args.enable_cuda_line_info else "OFF"), "-Donnxruntime_BUILD_WEBASSEMBLY=" + ("ON" if args.build_wasm else "OFF"), + "-Donnxruntime_BUILD_WEBASSEMBLY_STATIC_LIB=" + ("ON" if args.build_wasm_static_lib else "OFF"), "-Donnxruntime_ENABLE_WEBASSEMBLY_SIMD=" + ("ON" if args.enable_wasm_simd else "OFF"), "-Donnxruntime_ENABLE_WEBASSEMBLY_EXCEPTION_CATCHING=" + ("OFF" if args.disable_wasm_exception_catching else "ON"), @@ -2056,6 +2058,9 @@ def main(): if args.nnapi_min_api < 27: raise BuildError("--nnapi_min_api should be 27+") + if args.build_wasm_static_lib: + args.build_wasm = True + if args.build_wasm: if not args.disable_wasm_exception_catching and args.disable_exceptions: # When '--disable_exceptions' is set, we set '--disable_wasm_exception_catching' as well From 7b14c70cfe68b3a51b6e98a74b3ca89ed0493dae Mon Sep 17 00:00:00 2001 From: David Fan <30608893+jiafatom@users.noreply.github.com> Date: Tue, 18 Jan 2022 22:06:37 -0800 Subject: [PATCH 08/59] [ortmodule] Ensure contiguous tensor into forward pass (#10315) --- .../orttraining/python/training/ortmodule/_training_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/orttraining/orttraining/python/training/ortmodule/_training_manager.py b/orttraining/orttraining/python/training/ortmodule/_training_manager.py index 1f1a607f51..29ffb0dc70 100644 --- a/orttraining/orttraining/python/training/ortmodule/_training_manager.py +++ b/orttraining/orttraining/python/training/ortmodule/_training_manager.py @@ -52,6 +52,9 @@ class TrainingManager(GraphExecutionManager): forward_inputs = C.OrtValueVector() forward_inputs.reserve(len(inputs)) for input in inputs: + # TODO: Non-contiguous tensor input in execution_session_run_forward, need tensor copy. + if not input.is_contiguous(): + input = input.contiguous() if input.device.type == 'ort': forward_inputs.push_back(C.aten_ort_tensor_to_ort_value(input)) else: From a656c55a754f3758872b8864aa5640a7e2be6a48 Mon Sep 17 00:00:00 2001 From: jingyanwangms <47403504+jingyanwangms@users.noreply.github.com> Date: Wed, 19 Jan 2022 10:26:27 -0800 Subject: [PATCH 09/59] Add _force_exportable_set and pass debug_options (#10282) * Add _force_exportable_set and pass debug_options * Update orttraining/orttraining/python/training/ortmodule/experimental/hierarchical_ortmodule/_hierarchical_ortmodule.py Co-authored-by: Wei-Sheng Chin * nit fix * Update orttraining/orttraining/python/training/ortmodule/experimental/hierarchical_ortmodule/_hierarchical_ortmodule.py Co-authored-by: Wei-Sheng Chin Co-authored-by: Wei-Sheng Chin --- .../_hierarchical_ortmodule.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/orttraining/orttraining/python/training/ortmodule/experimental/hierarchical_ortmodule/_hierarchical_ortmodule.py b/orttraining/orttraining/python/training/ortmodule/experimental/hierarchical_ortmodule/_hierarchical_ortmodule.py index 5c79c68f37..ad2af5df6b 100644 --- a/orttraining/orttraining/python/training/ortmodule/experimental/hierarchical_ortmodule/_hierarchical_ortmodule.py +++ b/orttraining/orttraining/python/training/ortmodule/experimental/hierarchical_ortmodule/_hierarchical_ortmodule.py @@ -7,6 +7,10 @@ from ... import ORTModule from .... import ortmodule from ...debug_options import DebugOptions +# nn.Module's in this set are considered exportable to ONNX. +# For other nn.Module's, torch.onnx.export is called to check if +# they are exportable. +_force_exportable_set = set([torch.nn.Linear, torch.nn.Identity, torch.nn.modules.linear.NonDynamicallyQuantizableLinear]) class HierarchicalORTModule(torch.nn.Module): ''' @@ -48,8 +52,7 @@ class HierarchicalORTModule(torch.nn.Module): self._initialized = False super(HierarchicalORTModule, self).__init__() self._original_module = module - if not debug_options: - self._debug_options = DebugOptions() + self._debug_options = debug_options if debug_options else DebugOptions() def _initialize(self, *args, **kwargs): handle_pool = [] @@ -81,6 +84,11 @@ class HierarchicalORTModule(torch.nn.Module): # "module" can be wrapped as ORTModule. Otherwise, "module" is # not exportable to ONNX. def check_exportable(module): + # forward functions of classes in _force_exportable_set may not be called + # thus not in module_arg_pool + if type(module) in _force_exportable_set: + exportable_list[module] = True + return True sub_dict = module._modules if not sub_dict: # No sub-module exists, so this module is a leaf From 90e2a4b936d0ab79806d5e1ecb87cc45b11537df Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Thu, 20 Jan 2022 07:46:35 +1000 Subject: [PATCH 10/59] Fix GH Issue 10305 by adding implicit inputs to consumer nodes map. (#10319) --- onnxruntime/core/graph/graph.cc | 19 +- .../optimizer/transpose_optimizer_test.cc | 364 +++++++++--------- .../test/testdata/ort_github_issue_10305.onnx | Bin 0 -> 729 bytes .../test/testdata/ort_github_issue_10305.py | 58 +++ 4 files changed, 258 insertions(+), 183 deletions(-) create mode 100644 onnxruntime/test/testdata/ort_github_issue_10305.onnx create mode 100644 onnxruntime/test/testdata/ort_github_issue_10305.py diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 54dfca8217..2c19452a88 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -1066,7 +1066,7 @@ void Node::ForEachDef(std::function loaded from model file, construct // a object and Resolve() it. -//Status Graph::LoadGraph(const GraphProto& graph_proto, +// Status Graph::LoadGraph(const GraphProto& graph_proto, // const std::unordered_map& domain_to_version, // Version ir_version, // std::unique_ptr& new_graph) { @@ -1607,16 +1607,19 @@ Status Graph::BuildConnections(std::unordered_set& outer_scope_node // now build connections within this Graph instance node_arg_to_producer_node_.clear(); node_arg_to_consumer_nodes_.clear(); + for (auto& node : Nodes()) { // Need mutable input defs to be able to set any outer scope NodeArg implicit inputs auto& input_args = node.MutableInputDefs(); auto& output_args = node.MutableOutputDefs(); - if (!output_args.empty()) { - for (const auto* output_arg : output_args) { - if (output_arg->Exists()) { - node_arg_to_producer_node_.insert({output_arg->Name(), node.Index()}); - } + for (const auto* implicit_input : node.ImplicitInputDefs()) { + node_arg_to_consumer_nodes_[implicit_input->Name()].insert(node.Index()); + } + + for (const auto* output_arg : output_args) { + if (output_arg->Exists()) { + node_arg_to_producer_node_.insert({output_arg->Name(), node.Index()}); } } @@ -2650,7 +2653,7 @@ void Graph::InitFunctionBodyForNode(Node& node) { << node.Name() << "' optype " << node.OpType() #ifndef ORT_NO_EXCEPTIONS << ". Error message " << e.what() -#endif //ORT_NO_EXCEPTIONS +#endif // ORT_NO_EXCEPTIONS << ". Execution will fail if ORT does not have a specialized kernel for this op"; // Return without using this function op's expansion. No need to fail just yet. // If ORT has a specialized kernel for this op then execution will proceed @@ -3476,7 +3479,7 @@ void Graph::CleanUnusedInitializersAndNodeArgs(const std::unordered_set used_args; used_args.reserve(node_args_.size()); - //Node Args we want to preserved even not being used + // Node Args we want to preserved even not being used std::unordered_set node_args_to_preserve; if (initializer_names_to_preserve) { node_args_to_preserve.reserve(initializer_names_to_preserve->size()); diff --git a/onnxruntime/test/optimizer/transpose_optimizer_test.cc b/onnxruntime/test/optimizer/transpose_optimizer_test.cc index a76843413e..855baaf575 100644 --- a/onnxruntime/test/optimizer/transpose_optimizer_test.cc +++ b/onnxruntime/test/optimizer/transpose_optimizer_test.cc @@ -294,209 +294,209 @@ TEST(TransposeOptimizerTests, TestPadNonconst) { // Todo: renable tests on resize transformer after adding NHWC support in upsample op on cpu // https://github.com/microsoft/onnxruntime/issues/9857 -//TEST(TransposeOptimizerTests, TestResize) { -// auto build_test_case_1 = [&](ModelTestBuilder& builder) { -// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); -// auto* const_1 = builder.MakeInitializer({4}, {0.3f, 2.5f, 1.0f, 0.7f}); -// auto* transpose_1_out_0 = builder.MakeIntermediate(); -// auto* resize_1_out_0 = builder.MakeIntermediate(); -// auto* transpose_2_out_0 = builder.MakeOutput(); +// TEST(TransposeOptimizerTests, TestResize) { +// auto build_test_case_1 = [&](ModelTestBuilder& builder) { +// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); +// auto* const_1 = builder.MakeInitializer({4}, {0.3f, 2.5f, 1.0f, 0.7f}); +// auto* transpose_1_out_0 = builder.MakeIntermediate(); +// auto* resize_1_out_0 = builder.MakeIntermediate(); +// auto* transpose_2_out_0 = builder.MakeOutput(); // -// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); -// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); -// builder.AddNode("Resize", {transpose_1_out_0, const_1}, {resize_1_out_0}); -// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); -// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); -// }; +// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); +// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); +// builder.AddNode("Resize", {transpose_1_out_0, const_1}, {resize_1_out_0}); +// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); +// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); +// }; // -// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { -// int transpose_cost = EstimateTransposeCost(session.GetGraph()); -// EXPECT_EQ(transpose_cost, 0); -// }; +// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { +// int transpose_cost = EstimateTransposeCost(session.GetGraph()); +// EXPECT_EQ(transpose_cost, 0); +// }; // -// TransformerTester(build_test_case_1, -// check_optimized_graph_1, -// TransformerLevel::Default, -// TransformerLevel::Level1, -// /*opset_version*/ 10); -//} +// TransformerTester(build_test_case_1, +// check_optimized_graph_1, +// TransformerLevel::Default, +// TransformerLevel::Level1, +// /*opset_version*/ 10); +// } // -//TEST(TransposeOptimizerTests, TestResizeOpset11) { -// auto build_test_case_1 = [&](ModelTestBuilder& builder) { -// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); -// auto* const_1 = builder.MakeInitializer({8}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}); -// auto* const_2 = builder.MakeInitializer({4}, {0.3f, 2.5f, 1.0f, 0.7f}); -// auto* transpose_1_out_0 = builder.MakeIntermediate(); -// auto* resize_1_out_0 = builder.MakeIntermediate(); -// auto* transpose_2_out_0 = builder.MakeOutput(); +// TEST(TransposeOptimizerTests, TestResizeOpset11) { +// auto build_test_case_1 = [&](ModelTestBuilder& builder) { +// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); +// auto* const_1 = builder.MakeInitializer({8}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}); +// auto* const_2 = builder.MakeInitializer({4}, {0.3f, 2.5f, 1.0f, 0.7f}); +// auto* transpose_1_out_0 = builder.MakeIntermediate(); +// auto* resize_1_out_0 = builder.MakeIntermediate(); +// auto* transpose_2_out_0 = builder.MakeOutput(); // -// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); -// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); -// builder.AddNode("Resize", {transpose_1_out_0, const_1, const_2}, {resize_1_out_0}); -// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); -// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); -// }; +// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); +// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); +// builder.AddNode("Resize", {transpose_1_out_0, const_1, const_2}, {resize_1_out_0}); +// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); +// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); +// }; // -// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { -// int transpose_cost = EstimateTransposeCost(session.GetGraph()); -// EXPECT_EQ(transpose_cost, 0); -// }; +// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { +// int transpose_cost = EstimateTransposeCost(session.GetGraph()); +// EXPECT_EQ(transpose_cost, 0); +// }; // -// TransformerTester(build_test_case_1, -// check_optimized_graph_1, -// TransformerLevel::Default, -// TransformerLevel::Level1, -// /*opset_version*/ 11); -//} +// TransformerTester(build_test_case_1, +// check_optimized_graph_1, +// TransformerLevel::Default, +// TransformerLevel::Level1, +// /*opset_version*/ 11); +// } // -//TEST(TransposeOptimizerTests, TestResizeOpset15) { -// auto build_test_case_1 = [&](ModelTestBuilder& builder) { -// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); -// auto* const_1 = builder.MakeInitializer({4}, {0.3f, 2.5f, 1.0f, 0.7f}); -// auto* transpose_1_out_0 = builder.MakeIntermediate(); -// auto* resize_1_out_0 = builder.MakeIntermediate(); -// auto* transpose_2_out_0 = builder.MakeOutput(); -// auto empty_arg = NodeArg("", nullptr); +// TEST(TransposeOptimizerTests, TestResizeOpset15) { +// auto build_test_case_1 = [&](ModelTestBuilder& builder) { +// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); +// auto* const_1 = builder.MakeInitializer({4}, {0.3f, 2.5f, 1.0f, 0.7f}); +// auto* transpose_1_out_0 = builder.MakeIntermediate(); +// auto* resize_1_out_0 = builder.MakeIntermediate(); +// auto* transpose_2_out_0 = builder.MakeOutput(); +// auto empty_arg = NodeArg("", nullptr); // -// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); -// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); -// builder.AddNode("Resize", {transpose_1_out_0, &empty_arg, const_1}, {resize_1_out_0}); -// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); -// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); -// }; +// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); +// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); +// builder.AddNode("Resize", {transpose_1_out_0, &empty_arg, const_1}, {resize_1_out_0}); +// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); +// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); +// }; // -// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { -// int transpose_cost = EstimateTransposeCost(session.GetGraph()); -// EXPECT_EQ(transpose_cost, 0); -// }; +// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { +// int transpose_cost = EstimateTransposeCost(session.GetGraph()); +// EXPECT_EQ(transpose_cost, 0); +// }; // -// TransformerTester(build_test_case_1, -// check_optimized_graph_1, -// TransformerLevel::Default, -// TransformerLevel::Level1, -// /*opset_version*/ 15); -//} +// TransformerTester(build_test_case_1, +// check_optimized_graph_1, +// TransformerLevel::Default, +// TransformerLevel::Level1, +// /*opset_version*/ 15); +// } // -//TEST(TransposeOptimizerTests, TestResizeSizeRoi) { -// auto build_test_case_1 = [&](ModelTestBuilder& builder) { -// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); -// auto* const_1 = builder.MakeInitializer({8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f}); -// auto* const_2 = builder.MakeInitializer({4}, {10, 9, 8, 7}); -// auto* transpose_1_out_0 = builder.MakeIntermediate(); -// auto* resize_1_out_0 = builder.MakeIntermediate(); -// auto* transpose_2_out_0 = builder.MakeOutput(); -// auto empty_arg = NodeArg("", nullptr); +// TEST(TransposeOptimizerTests, TestResizeSizeRoi) { +// auto build_test_case_1 = [&](ModelTestBuilder& builder) { +// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); +// auto* const_1 = builder.MakeInitializer({8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f}); +// auto* const_2 = builder.MakeInitializer({4}, {10, 9, 8, 7}); +// auto* transpose_1_out_0 = builder.MakeIntermediate(); +// auto* resize_1_out_0 = builder.MakeIntermediate(); +// auto* transpose_2_out_0 = builder.MakeOutput(); +// auto empty_arg = NodeArg("", nullptr); // -// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); -// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); -// auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, const_1, &empty_arg, const_2}, {resize_1_out_0}); -// resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize"); -// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); -// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); -// }; +// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); +// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); +// auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, const_1, &empty_arg, const_2}, {resize_1_out_0}); +// resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize"); +// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); +// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); +// }; // -// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { -// int transpose_cost = EstimateTransposeCost(session.GetGraph()); -// EXPECT_EQ(transpose_cost, 0); -// }; +// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { +// int transpose_cost = EstimateTransposeCost(session.GetGraph()); +// EXPECT_EQ(transpose_cost, 0); +// }; // -// TransformerTester(build_test_case_1, -// check_optimized_graph_1, -// TransformerLevel::Default, -// TransformerLevel::Level1, -// /*opset_version*/ 15); -//} +// TransformerTester(build_test_case_1, +// check_optimized_graph_1, +// TransformerLevel::Default, +// TransformerLevel::Level1, +// /*opset_version*/ 15); +// } // -//TEST(TransposeOptimizerTests, TestResizeRoiScalesZeroRank0) { -// auto build_test_case_1 = [&](ModelTestBuilder& builder) { -// auto* input = builder.MakeInput({1, 512, 512, 3}, -// std::numeric_limits::min(), -// std::numeric_limits::max()); -// auto* resize_in_roi = builder.MakeInitializer({0}, {}); -// auto* resize_in_scales = builder.MakeInitializer({0}, {}); -// auto* resize_in_sizes = builder.MakeInitializer({4}, {1, 256, 32, 32}); +// TEST(TransposeOptimizerTests, TestResizeRoiScalesZeroRank0) { +// auto build_test_case_1 = [&](ModelTestBuilder& builder) { +// auto* input = builder.MakeInput({1, 512, 512, 3}, +// std::numeric_limits::min(), +// std::numeric_limits::max()); +// auto* resize_in_roi = builder.MakeInitializer({0}, {}); +// auto* resize_in_scales = builder.MakeInitializer({0}, {}); +// auto* resize_in_sizes = builder.MakeInitializer({4}, {1, 256, 32, 32}); // -// auto* transpose1_out_transposed = builder.MakeIntermediate(); -// auto* resize_out_Y = builder.MakeIntermediate(); -// auto* output = builder.MakeOutput(); +// auto* transpose1_out_transposed = builder.MakeIntermediate(); +// auto* resize_out_Y = builder.MakeIntermediate(); +// auto* output = builder.MakeOutput(); // -// auto& transpose_1 = builder.AddNode("Transpose", {input}, {transpose1_out_transposed}); -// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); -// builder.AddNode("Resize", -// {transpose1_out_transposed, resize_in_roi, resize_in_scales, resize_in_sizes}, -// {resize_out_Y}); -// auto& transpose_2 = builder.AddNode("Transpose", {resize_out_Y}, {output}); -// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); -// }; +// auto& transpose_1 = builder.AddNode("Transpose", {input}, {transpose1_out_transposed}); +// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); +// builder.AddNode("Resize", +// {transpose1_out_transposed, resize_in_roi, resize_in_scales, resize_in_sizes}, +// {resize_out_Y}); +// auto& transpose_2 = builder.AddNode("Transpose", {resize_out_Y}, {output}); +// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); +// }; // -// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { -// int transpose_cost = EstimateTransposeCost(session.GetGraph()); -// EXPECT_EQ(transpose_cost, 0); -// }; +// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { +// int transpose_cost = EstimateTransposeCost(session.GetGraph()); +// EXPECT_EQ(transpose_cost, 0); +// }; // -// TransformerTester(build_test_case_1, -// check_optimized_graph_1, -// TransformerLevel::Default, -// TransformerLevel::Level1); -//} +// TransformerTester(build_test_case_1, +// check_optimized_graph_1, +// TransformerLevel::Default, +// TransformerLevel::Level1); +// } // -//TEST(TransposeOptimizerTests, TestResizeNonconst) { -// auto build_test_case_1 = [&](ModelTestBuilder& builder) { -// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); -// auto* input1_arg = MakeInput(builder, {{8}}, {8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f}); -// auto* input2_arg = MakeInput(builder, {{4}}, {4}, {0.3f, 2.5f, 1.0f, 0.7f}); -// auto* transpose_1_out_0 = builder.MakeIntermediate(); -// auto* resize_1_out_0 = builder.MakeIntermediate(); -// auto* transpose_2_out_0 = builder.MakeOutput(); +// TEST(TransposeOptimizerTests, TestResizeNonconst) { +// auto build_test_case_1 = [&](ModelTestBuilder& builder) { +// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); +// auto* input1_arg = MakeInput(builder, {{8}}, {8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f}); +// auto* input2_arg = MakeInput(builder, {{4}}, {4}, {0.3f, 2.5f, 1.0f, 0.7f}); +// auto* transpose_1_out_0 = builder.MakeIntermediate(); +// auto* resize_1_out_0 = builder.MakeIntermediate(); +// auto* transpose_2_out_0 = builder.MakeOutput(); // -// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); -// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); -// auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, input1_arg, input2_arg}, {resize_1_out_0}); -// resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize"); -// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); -// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); -// }; +// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); +// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); +// auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, input1_arg, input2_arg}, {resize_1_out_0}); +// resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize"); +// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); +// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); +// }; // -// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { -// int transpose_cost = EstimateTransposeCost(session.GetGraph()); -// EXPECT_EQ(transpose_cost, 0); -// }; +// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { +// int transpose_cost = EstimateTransposeCost(session.GetGraph()); +// EXPECT_EQ(transpose_cost, 0); +// }; // -// TransformerTester(build_test_case_1, -// check_optimized_graph_1, -// TransformerLevel::Default, -// TransformerLevel::Level1, -// /*opset_version*/ 11); -//} +// TransformerTester(build_test_case_1, +// check_optimized_graph_1, +// TransformerLevel::Default, +// TransformerLevel::Level1, +// /*opset_version*/ 11); +// } // -//TEST(TransposeOptimizerTests, TestResizeNonconstOpset13) { -// auto build_test_case_1 = [&](ModelTestBuilder& builder) { -// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); -// auto* input1_arg = MakeInput(builder, {{8}}, {8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f}); -// auto* input2_arg = MakeInput(builder, {{4}}, {4}, {0.3f, 2.5f, 1.0f, 0.7f}); -// auto* transpose_1_out_0 = builder.MakeIntermediate(); -// auto* resize_1_out_0 = builder.MakeIntermediate(); -// auto* transpose_2_out_0 = builder.MakeOutput(); +// TEST(TransposeOptimizerTests, TestResizeNonconstOpset13) { +// auto build_test_case_1 = [&](ModelTestBuilder& builder) { +// auto* input0_arg = MakeInput(builder, {{4, -1, 2, -1}}, {4, 6, 2, 10}, 0.0, 1.0); +// auto* input1_arg = MakeInput(builder, {{8}}, {8}, {0.1f, 0.2f, 0.3f, 0.4f, 0.9f, 0.8f, 0.7f, 0.6f}); +// auto* input2_arg = MakeInput(builder, {{4}}, {4}, {0.3f, 2.5f, 1.0f, 0.7f}); +// auto* transpose_1_out_0 = builder.MakeIntermediate(); +// auto* resize_1_out_0 = builder.MakeIntermediate(); +// auto* transpose_2_out_0 = builder.MakeOutput(); // -// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); -// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); -// auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, input1_arg, input2_arg}, {resize_1_out_0}); -// resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize"); -// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); -// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); -// }; +// auto& transpose_1 = builder.AddNode("Transpose", {input0_arg}, {transpose_1_out_0}); +// transpose_1.AddAttribute("perm", std::vector{0, 3, 1, 2}); +// auto& resize_1 = builder.AddNode("Resize", {transpose_1_out_0, input1_arg, input2_arg}, {resize_1_out_0}); +// resize_1.AddAttribute("coordinate_transformation_mode", "tf_crop_and_resize"); +// auto& transpose_2 = builder.AddNode("Transpose", {resize_1_out_0}, {transpose_2_out_0}); +// transpose_2.AddAttribute("perm", std::vector{0, 2, 3, 1}); +// }; // -// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { -// int transpose_cost = EstimateTransposeCost(session.GetGraph()); -// EXPECT_EQ(transpose_cost, 0); -// }; +// auto check_optimized_graph_1 = [&](InferenceSessionWrapper& session) { +// int transpose_cost = EstimateTransposeCost(session.GetGraph()); +// EXPECT_EQ(transpose_cost, 0); +// }; // -// TransformerTester(build_test_case_1, -// check_optimized_graph_1, -// TransformerLevel::Default, -// TransformerLevel::Level1, -// /*opset_version*/ 13); -//} +// TransformerTester(build_test_case_1, +// check_optimized_graph_1, +// TransformerLevel::Default, +// TransformerLevel::Level1, +// /*opset_version*/ 13); +// } TEST(TransposeOptimizerTests, TestAdd) { auto build_test_case_1 = [&](ModelTestBuilder& builder) { @@ -3869,5 +3869,19 @@ TEST(TransposeOptimizerTests, RegressionTest_GitHubIssue9671) { ASSERT_STATUS_OK(session_object.Initialize()); // optimizers run during initialization } +// regression test for a model where the transpose optimizations incorrectly removed a node providing an implicit +// input to a subgraph. fixed by updating Graph::BuildConnections to add implicit inputs to node_arg_to_consumer_nodes_ +// see https://github.com/microsoft/onnxruntime/issues/10305 for more details. +TEST(TransposeOptimizerTests, RegressionTest_GitHubIssue10305) { + Status status; + auto model_uri = ORT_TSTR("testdata/ort_github_issue_10305.onnx"); + + SessionOptions so; + so.session_logid = "TransposeOptimizerTests.RegressionTest_GitHubIssue10305"; + InferenceSession session_object{so, GetEnvironment()}; + ASSERT_STATUS_OK(session_object.Load(model_uri)); + ASSERT_STATUS_OK(session_object.Initialize()); // optimizers run during initialization +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/testdata/ort_github_issue_10305.onnx b/onnxruntime/test/testdata/ort_github_issue_10305.onnx new file mode 100644 index 0000000000000000000000000000000000000000..f4b0d1f839d47f48574c386699935435bdefec6e GIT binary patch literal 729 zcma))!A^rf5Qbd}0%J>?J=lcCm=I5#pcf+tQ#tFYp15qFY$~x`5Ei38O<%{GkKzM( za4AKRw222`hxzB*e|A>WyblGgK}}dza?imhJP%osr3LjI6Mh26((jvuJp?(U`Fe(C zM>A=IRJek687Ku0a}GVqVmvH(p2RdJ#VT1;%;W9Cv6OlB1E$bb?8>o{R)$Qicq%D< zqmHG{>lM69V64LzBFGt8q$T5UT9nZ;53>jINNGlvX~LEyVbB*=I3>?vE}|Ogn#ygO zX09byANvb|9HsH=^b^75-#!K_As{tDR2!{Ocr)djICia`cw@ankqQ$`V32T{hdfCc zVdWYFXqt*;kTeur95T99={uD`w6+pER|r%mqkkcUs^b=87zsvS*pQlMgA@#RVZzAv z=xjqnBh^Dkemy|DU&qL=CTWkGsB`xAxk%_XLjHjv1ZTh3YmbY->`bZM?HKB7fDI6C Ru+~_QCUwHy*OxDihToTmzS{r* literal 0 HcmV?d00001 diff --git a/onnxruntime/test/testdata/ort_github_issue_10305.py b/onnxruntime/test/testdata/ort_github_issue_10305.py new file mode 100644 index 0000000000..e8d4829448 --- /dev/null +++ b/onnxruntime/test/testdata/ort_github_issue_10305.py @@ -0,0 +1,58 @@ +import onnx +from onnx import helper +from onnx import TensorProto + +# Loop is so the Tranpose output is used in a subgraph +loop_body = helper.make_graph( + [ + helper.make_node("Add", ["transpose:0", "loop_state_in"], ["loop_state_out"], "Add1"), + ], + "Loop_body", + [ + helper.make_tensor_value_info('iteration_num', TensorProto.INT64, [1]), + helper.make_tensor_value_info('subgraph_keep_going_in', TensorProto.BOOL, [1]), + helper.make_tensor_value_info('loop_state_in', TensorProto.FLOAT, [1]) + ], + [ + helper.make_tensor_value_info('subgraph_keep_going_in', TensorProto.BOOL, [1]), + helper.make_tensor_value_info('loop_state_out', TensorProto.FLOAT, [2, 2, 2]), + ], + [ + ] +) + +# Create the main graph +graph_proto = helper.make_graph( + [ + # add a Transpose that can be moved past the Slice + helper.make_node('Transpose', inputs=['input:0'], outputs=['transpose:0'], name='transpose0', perm=[1, 0, 2]), + helper.make_node('Slice', + inputs=['transpose:0', 'start', 'end'], + outputs=['strided_slice:0'], name='slice0'), + helper.make_node('Squeeze', + inputs=['strided_slice:0', 'start'], + outputs=['out:0'], + name='squeeze0'), + helper.make_node("Loop", ["max_trip_count", "subgraph_keep_going_in", "state_var_in"], ["out:1"], "Loop1", + body=loop_body) + ], + "Main_graph", + [ + helper.make_tensor_value_info('input:0', TensorProto.FLOAT, [2, 2, 2]), + helper.make_tensor_value_info('state_var_in', TensorProto.FLOAT, [1]), + ], + [ + helper.make_tensor_value_info('out:0', TensorProto.FLOAT, [2, 2]), + helper.make_tensor_value_info('out:1', TensorProto.FLOAT, [2, 2, 2]), + ], + [ + helper.make_tensor('start', TensorProto.INT64, [1], [0]), + helper.make_tensor('end', TensorProto.INT64, [1], [1]), + helper.make_tensor('max_trip_count', TensorProto.INT64, [1], [1]), + helper.make_tensor('subgraph_keep_going_in', TensorProto.BOOL, [1], [1]), + ] +) + +model = helper.make_model(graph_proto) +onnx.checker.check_model(model, True) +onnx.save(model, 'ort_github_issue_10305.onnx') \ No newline at end of file From 4aa7cee0d8baf7485932b64370b4fc7dc1611874 Mon Sep 17 00:00:00 2001 From: Abhishek Jindal Date: Wed, 19 Jan 2022 14:20:09 -0800 Subject: [PATCH 11/59] Abjindal/clean eager backend (#10055) * clearing map for eager mode backends * clearing map for eager mode backends manager * making OrtBackendsManager an extern variable and trying to delete it * cleaning backends manager when the python interpret exits * adding ifdef for eager mode code * disabling warning for pybind state file * disabling warning for python module file * running clang auto format and reducing redundancy * remove new line * moving declaration to a new header file * adding the header file for eager mode for python module * removing source files for eager mode * add source file for python module in eager mode * Update orttraining/orttraining/python/orttraining_python_module_eager.h Co-authored-by: Thiago Crepaldi Co-authored-by: Thiago Crepaldi --- cmake/onnxruntime_python.cmake | 1 + .../orttraining/eager/ort_backends.cpp | 8 ++--- orttraining/orttraining/eager/ort_backends.h | 2 -- orttraining/orttraining/eager/ort_eager.cpp | 9 ++--- .../python/orttraining_python_module.cc | 33 +++++++++++++++++++ .../python/orttraining_python_module_eager.h | 9 +++++ 6 files changed, 50 insertions(+), 12 deletions(-) create mode 100644 orttraining/orttraining/python/orttraining_python_module_eager.h diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 4a1a6a360b..824d14e302 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -126,6 +126,7 @@ if (onnxruntime_ENABLE_EAGER_MODE) set_source_files_properties("${ORTTRAINING_ROOT}/orttraining/eager/ort_hooks.cpp" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) set_source_files_properties("${ORTTRAINING_ROOT}/orttraining/eager/ort_ops.cpp" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) set_source_files_properties("${ORTTRAINING_ROOT}/orttraining/eager/ort_util.cpp" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) + set_source_files_properties("${ORTTRAINING_ROOT}/orttraining/python/orttraining_python_module.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter) endif() if (MSVC) target_compile_options(onnxruntime_pybind11_state PRIVATE "/wd4100" "/wd4324" "/wd4458" "/wd4127" "/wd4193" "/wd4624" "/wd4702") diff --git a/orttraining/orttraining/eager/ort_backends.cpp b/orttraining/orttraining/eager/ort_backends.cpp index e14ba9d2a1..6e3c2080a6 100644 --- a/orttraining/orttraining/eager/ort_backends.cpp +++ b/orttraining/orttraining/eager/ort_backends.cpp @@ -8,6 +8,7 @@ #include "ort_backends.h" #include "ort_log.h" #include "core/platform/env.h" +#include "orttraining/python/orttraining_python_module_eager.h" //use the environment from python module @@ -24,12 +25,7 @@ namespace torch_ort { namespace eager { using namespace onnxruntime; - -ORTBackendsManager& GetORTBackendsManager() { - auto& env = onnxruntime::python::GetTrainingORTEnv(); - static ORTBackendsManager instance {env.GetLoggingManager()->DefaultLogger()}; - return instance; -} +using namespace onnxruntime::python; onnxruntime::ORTInvoker& GetORTInvoker(const at::Device device) { return GetORTBackendsManager().GetInvoker(device); diff --git a/orttraining/orttraining/eager/ort_backends.h b/orttraining/orttraining/eager/ort_backends.h index 4f3d885a5e..484ab1b192 100644 --- a/orttraining/orttraining/eager/ort_backends.h +++ b/orttraining/orttraining/eager/ort_backends.h @@ -43,8 +43,6 @@ private: std::unordered_map device_ep_info_; }; -ORTBackendsManager& GetORTBackendsManager(); - onnxruntime::ORTInvoker& GetORTInvoker(const at::Device device); } // namespace eager diff --git a/orttraining/orttraining/eager/ort_eager.cpp b/orttraining/orttraining/eager/ort_eager.cpp index f679335258..f7d3d730c3 100644 --- a/orttraining/orttraining/eager/ort_eager.cpp +++ b/orttraining/orttraining/eager/ort_eager.cpp @@ -12,6 +12,7 @@ #include "ort_customops.h" #include "torch/csrc/autograd/python_variable.h" #include "core/framework/tensor.h" +#include "orttraining/python/orttraining_python_module_eager.h" namespace onnxruntime{ namespace python{ @@ -49,7 +50,7 @@ OrtValue ORTTensor_toORTValue(const at::Tensor& data) at::Tensor OrtValue_To_ATen_Tensor(OrtValue& ortvalue) { auto& ort_tensor = ortvalue.Get(); - size_t ort_device_idx = torch_ort::eager::GetORTBackendsManager().GetOrtDeviceIndex(ort_tensor.Location()); + size_t ort_device_idx = GetORTBackendsManager().GetOrtDeviceIndex(ort_tensor.Location()); return torch_ort::eager::aten_tensor_from_ort( std::move(ortvalue), at::TensorOptions() @@ -78,15 +79,15 @@ void addObjectMethodsForEager(py::module& m){ m.def("set_device", [](size_t device_index, const std::string& provider_type, const std::unordered_map& arguments){ - auto status = torch_ort::eager::GetORTBackendsManager().set_device(device_index, provider_type, arguments); + auto status = GetORTBackendsManager().set_device(device_index, provider_type, arguments); if (!status.IsOK()) throw std::runtime_error(status.ErrorMessage()); }); m.def("get_ort_device", [](size_t torch_device_index){ - return torch_ort::eager::GetORTBackendsManager().GetOrtDeviceInfo(torch_device_index); + return GetORTBackendsManager().GetOrtDeviceInfo(torch_device_index); }); m.def("get_ort_device_provider_info", [](size_t torch_device_index){ - return torch_ort::eager::GetORTBackendsManager().GetOrtDeviceProviderInfo(torch_device_index); + return GetORTBackendsManager().GetOrtDeviceProviderInfo(torch_device_index); }); auto customop_module = m.def_submodule("custom_ops"); diff --git a/orttraining/orttraining/python/orttraining_python_module.cc b/orttraining/orttraining/python/orttraining_python_module.cc index ec998bb5d6..3096d1c2cd 100644 --- a/orttraining/orttraining/python/orttraining_python_module.cc +++ b/orttraining/orttraining/python/orttraining_python_module.cc @@ -8,6 +8,10 @@ #include "core/providers/get_execution_providers.h" #include "core/session/provider_bridge_ort.h" +#ifdef ENABLE_EAGER_MODE +#include "orttraining/python/orttraining_python_module_eager.h" +#endif + namespace onnxruntime { namespace python { namespace py = pybind11; @@ -208,6 +212,32 @@ Environment& GetTrainingORTEnv() { return ort_training_env->GetORTEnv(); } +#ifdef ENABLE_EAGER_MODE +using namespace torch_ort::eager; +static std::unique_ptr ort_backends_manager_instance; + +void InitializeBackendsManager() { + auto initialize = [&]() { + static bool initialized = false; + if (initialized) { + return; + } + // Initialization of the module + auto& env = onnxruntime::python::GetTrainingORTEnv(); + ort_backends_manager_instance = std::make_unique(env.GetLoggingManager()->DefaultLogger()); + initialized = true; + }; + initialize(); +} + +ORTBackendsManager& GetORTBackendsManager() { + if (!ort_backends_manager_instance) { + InitializeBackendsManager(); + } + return *ort_backends_manager_instance; +} +#endif + void ResolveExtraProviderOptions(const std::vector& provider_types, const ProviderOptionsMap& original_provider_options_map, ProviderOptionsMap& merged_options){ @@ -325,6 +355,9 @@ PYBIND11_MODULE(onnxruntime_pybind11_state, m) { auto atexit = py::module_::import("atexit"); atexit.attr("register")(py::cpp_function([]() { ort_training_env = nullptr; +#ifdef ENABLE_EAGER_MODE + ort_backends_manager_instance = nullptr; +#endif })); } diff --git a/orttraining/orttraining/python/orttraining_python_module_eager.h b/orttraining/orttraining/python/orttraining_python_module_eager.h new file mode 100644 index 0000000000..ef8d511d84 --- /dev/null +++ b/orttraining/orttraining/python/orttraining_python_module_eager.h @@ -0,0 +1,9 @@ +#include "orttraining/eager/ort_backends.h" + +namespace onnxruntime { +namespace python { + +torch_ort::eager::ORTBackendsManager& GetORTBackendsManager(); + +} +} From 001cc539683d5e294d7b306d57e5d5bbb8422d73 Mon Sep 17 00:00:00 2001 From: Vincent Wang Date: Thu, 20 Jan 2022 18:17:28 +0800 Subject: [PATCH 12/59] Fix CUDA10.2 Build Break for BFloat16 Change (#10331) * fix build break on cuda 10.2 * fix linux build --- .../providers/cuda/shared_inc/fpgeneric.h | 111 +++++++----------- 1 file changed, 45 insertions(+), 66 deletions(-) diff --git a/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h b/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h index 9e1e0fe52d..1ec4440f00 100644 --- a/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h +++ b/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h @@ -112,32 +112,25 @@ inline cublasStatus_t cublasGemmHelper(cublasHandle_t handle, } } -inline cublasStatus_t cublasGemmHelper(cublasHandle_t handle, - cublasOperation_t transa, - cublasOperation_t transb, - int m, int n, int k, - const BFloat16* alpha, - const BFloat16* A, int lda, - const BFloat16* B, int ldb, - const BFloat16* beta, - BFloat16* C, int ldc, +#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 +inline cublasStatus_t cublasGemmHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, + int n, int k, const BFloat16* alpha, const BFloat16* A, int lda, + const BFloat16* B, int ldb, const BFloat16* beta, BFloat16* C, int ldc, const cudaDeviceProp& /*prop*/) { float h_a = alpha->ToFloat(); float h_b = beta->ToFloat(); // accumulating in FP32 - return cublasGemmEx(handle, - transa, - transb, - m, n, k, - &h_a, - A, CUDA_R_16BF, lda, - B, CUDA_R_16BF, ldb, - &h_b, - C, CUDA_R_16BF, ldc, - CUBLAS_COMPUTE_32F, - CUBLAS_GEMM_DEFAULT); + return cublasGemmEx(handle, transa, transb, m, n, k, &h_a, A, CUDA_R_16BF, lda, B, CUDA_R_16BF, ldb, &h_b, C, + CUDA_R_16BF, ldc, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); } +#else +inline cublasStatus_t cublasGemmHelper(cublasHandle_t, cublasOperation_t, cublasOperation_t, int, int, int, + const BFloat16*, const BFloat16*, int, const BFloat16*, int, const BFloat16*, + BFloat16*, int, const cudaDeviceProp&) { + return CUBLAS_STATUS_NOT_SUPPORTED; +} +#endif // batched gemm inline cublasStatus_t cublasGemmBatchedHelper(cublasHandle_t handle, @@ -236,34 +229,27 @@ inline cublasStatus_t cublasGemmBatchedHelper(cublasHandle_t handle, } } -inline cublasStatus_t cublasGemmBatchedHelper(cublasHandle_t handle, - cublasOperation_t transa, - cublasOperation_t transb, - int m, int n, int k, - const BFloat16* alpha, - const BFloat16* Aarray[], int lda, - const BFloat16* Barray[], int ldb, - const BFloat16* beta, - BFloat16* Carray[], int ldc, - int batch_count, +#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 +inline cublasStatus_t cublasGemmBatchedHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, + int m, int n, int k, const BFloat16* alpha, const BFloat16* Aarray[], + int lda, const BFloat16* Barray[], int ldb, const BFloat16* beta, + BFloat16* Carray[], int ldc, int batch_count, const cudaDeviceProp& /*prop*/) { float h_a = alpha->ToFloat(); float h_b = beta->ToFloat(); // accumulating in FP32 - return cublasGemmBatchedEx(handle, - transa, - transb, - m, n, k, - &h_a, - (const void**)Aarray, CUDA_R_16BF, lda, - (const void**)Barray, CUDA_R_16BF, ldb, - &h_b, - (void**)Carray, CUDA_R_16BF, ldc, - batch_count, - CUDA_R_32F, - CUBLAS_GEMM_DEFAULT); + return cublasGemmBatchedEx(handle, transa, transb, m, n, k, &h_a, (const void**)Aarray, CUDA_R_16BF, lda, + (const void**)Barray, CUDA_R_16BF, ldb, &h_b, (void**)Carray, CUDA_R_16BF, ldc, + batch_count, CUDA_R_32F, CUBLAS_GEMM_DEFAULT); } +#else +inline cublasStatus_t cublasGemmBatchedHelper(cublasHandle_t, cublasOperation_t, cublasOperation_t, int, int, int, + const BFloat16*, const BFloat16*[], int, const BFloat16*[], int, + const BFloat16*, BFloat16*[], int, int, const cudaDeviceProp&) { + return CUBLAS_STATUS_NOT_SUPPORTED; +} +#endif // strided batched gemm inline cublasStatus_t cublasGemmStridedBatchedHelper(cublasHandle_t handle, @@ -425,36 +411,29 @@ inline cublasStatus_t cublasGemmStridedBatchedHelper(cublasHandle_t handle, } } -inline cublasStatus_t cublasGemmStridedBatchedHelper(cublasHandle_t handle, - cublasOperation_t transa, - cublasOperation_t transb, - int m, int n, int k, - const BFloat16* alpha, - const BFloat16* A, int lda, - long long int strideA, - const BFloat16* B, int ldb, - long long int strideB, - const BFloat16* beta, - BFloat16* C, int ldc, - long long int strideC, - int batch_count, +#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 +inline cublasStatus_t cublasGemmStridedBatchedHelper(cublasHandle_t handle, cublasOperation_t transa, + cublasOperation_t transb, int m, int n, int k, + const BFloat16* alpha, const BFloat16* A, int lda, + long long int strideA, const BFloat16* B, int ldb, + long long int strideB, const BFloat16* beta, BFloat16* C, int ldc, + long long int strideC, int batch_count, const cudaDeviceProp& /*prop*/) { float h_a = alpha->ToFloat(); float h_b = beta->ToFloat(); // accumulating in FP32 - return cublasGemmStridedBatchedEx(handle, - transa, - transb, - m, n, k, - &h_a, - A, CUDA_R_16BF, lda, strideA, - B, CUDA_R_16BF, ldb, strideB, - &h_b, - C, CUDA_R_16BF, ldc, strideC, - batch_count, - CUDA_R_32F, + return cublasGemmStridedBatchedEx(handle, transa, transb, m, n, k, &h_a, A, CUDA_R_16BF, lda, strideA, B, CUDA_R_16BF, + ldb, strideB, &h_b, C, CUDA_R_16BF, ldc, strideC, batch_count, CUDA_R_32F, CUBLAS_GEMM_DEFAULT); } +#else +inline cublasStatus_t cublasGemmStridedBatchedHelper(cublasHandle_t, cublasOperation_t, cublasOperation_t, int, int, + int, const BFloat16*, const BFloat16*, int, long long int, + const BFloat16*, int, long long int, const BFloat16*, BFloat16*, + int, long long int, int, const cudaDeviceProp&) { + return CUBLAS_STATUS_NOT_SUPPORTED; +} +#endif // transpose using geam inline cublasStatus_t cublasTransposeHelper(cudaStream_t, cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, const float* alpha, const float* A, int lda, const float* beta, const float* B, int ldb, float* C, int ldc) { From c67594694c7d3cc8a27032318706e504e16c9c7b Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Thu, 20 Jan 2022 09:10:19 -0800 Subject: [PATCH 13/59] Add ability to set onnx opset version from json config (#10223) --- .../json_config/_load_config_from_json.py | 16 +++++++++++++++- ...ng_test_ortmodule_experimental_json_config.py | 12 +++++++++--- ...est_ortmodule_experimental_json_config_1.json | 3 ++- ...est_ortmodule_experimental_json_config_2.json | 3 ++- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/orttraining/orttraining/python/training/ortmodule/experimental/json_config/_load_config_from_json.py b/orttraining/orttraining/python/training/ortmodule/experimental/json_config/_load_config_from_json.py index 8c3d6508d5..371ffe3fee 100644 --- a/orttraining/orttraining/python/training/ortmodule/experimental/json_config/_load_config_from_json.py +++ b/orttraining/orttraining/python/training/ortmodule/experimental/json_config/_load_config_from_json.py @@ -13,6 +13,7 @@ from . import JSON_PATH_ENVIRONMENT_KEY from ..._fallback import _FallbackPolicy from ..._graph_execution_manager import _SkipCheck from ...debug_options import DebugOptions, LogLevel, _SaveOnnxOptions +from onnxruntime.training import ortmodule log = logging.getLogger(__name__) @@ -190,6 +191,16 @@ def _load_fallback_policy(ortmodule_config_accessor, data): ortmodule_config_accessor._fallback_manager.policy = fallback_policy +def _load_onnx_opset_version(ortmodule_config_accessor, data): + """Loads OnnxOpsetVersion from json file onto ORTModule.""" + + assert hasattr(data, _load_onnx_opset_version.loading_key) + log.info(f"Found keyword {_load_onnx_opset_version.loading_key} in json. Loading attributes from file.") + + assert isinstance(data.OnnxOpsetVersion, int), f"{_load_onnx_opset_version.loading_key} must be an int" + ortmodule.ONNX_OPSET_VERSION = data.OnnxOpsetVersion + + def _define_load_function_keys(): """Define static key variables for each loading function""" @@ -204,6 +215,7 @@ def _define_load_function_keys(): _load_debug_options.loading_key = "DebugOptions" _load_use_memory_efficient_gradient.loading_key = "UseMemoryEfficientGradient" _load_fallback_policy.loading_key = "FallbackPolicy" + _load_onnx_opset_version.loading_key = "OnnxOpsetVersion" def load_from_json(ortmodule, path=None): @@ -246,6 +258,7 @@ def load_from_json(ortmodule, path=None): "FALLBACK_UNSUPPORTED_ONNX_MODEL", "FALLBACK_BAD_INITIALIZATION", ], + "OnnxOpsetVersion": 14 # int defining the opset version to be used during export } Args: @@ -281,7 +294,8 @@ def load_from_json(ortmodule, path=None): _load_skip_check.loading_key: _load_skip_check, _load_debug_options.loading_key: _load_debug_options, _load_use_memory_efficient_gradient.loading_key: _load_use_memory_efficient_gradient, - _load_fallback_policy.loading_key: _load_fallback_policy + _load_fallback_policy.loading_key: _load_fallback_policy, + _load_onnx_opset_version.loading_key: _load_onnx_opset_version } for training_mode in [True, False]: diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config.py index 28f26da119..d2bcff7c0c 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config.py @@ -1,7 +1,7 @@ import os import torch -from onnxruntime.training.ortmodule import ORTModule +from onnxruntime.training import ortmodule from onnxruntime.capi import _pybind_state as C from onnxruntime.training.ortmodule.experimental.json_config import load_from_json @@ -22,7 +22,7 @@ class Net(torch.nn.Module): def test_load_config_from_json_1(): device = 'cuda' - model = ORTModule(Net().to(device)) + model = ortmodule.ORTModule(Net().to(device)) # load from json once. path_to_json = os.path.join(os.getcwd(), 'orttraining_test_ortmodule_experimental_json_config_2.json') @@ -72,9 +72,12 @@ def test_load_config_from_json_1(): # test fallback policy assert ort_model_attributes._fallback_manager.policy.value == 1 + # assert onnx opset version + assert ortmodule.ONNX_OPSET_VERSION == 13 + def test_load_config_from_json_2(): device = 'cuda' - model = ORTModule(Net().to(device)) + model = ortmodule.ORTModule(Net().to(device)) # load from json once. path_to_json = os.path.join(os.getcwd(), 'orttraining_test_ortmodule_experimental_json_config_1.json') @@ -123,3 +126,6 @@ def test_load_config_from_json_2(): # test fallback policy assert ort_model_attributes._fallback_manager.policy.value == 250 + + # assert onnx opset version + assert ortmodule.ONNX_OPSET_VERSION == 12 diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config_1.json b/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config_1.json index 47d1ee4425..bf51e5d710 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config_1.json +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config_1.json @@ -27,5 +27,6 @@ "FallbackPolicy": [ "FALLBACK_DISABLE" - ] + ], + "OnnxOpsetVersion": 13 } diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config_2.json b/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config_2.json index 6ebecc5ded..9c42a211a1 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config_2.json +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_experimental_json_config_2.json @@ -31,5 +31,6 @@ "FALLBACK_UNSUPPORTED_TORCH_MODEL", "FALLBACK_UNSUPPORTED_ONNX_MODEL", "FALLBACK_BAD_INITIALIZATION" - ] + ], + "OnnxOpsetVersion": 12 } From 2dcb69685e8b48a24d5a01e34e58bd0c0270dca9 Mon Sep 17 00:00:00 2001 From: "Tang, Cheng" Date: Thu, 20 Jan 2022 10:06:09 -0800 Subject: [PATCH 14/59] support type promotion in binary poerators in eager mode (#10285) Co-authored-by: Cheng Tang --- orttraining/orttraining/eager/opgen/opgen.py | 2 +- .../orttraining/eager/opgen/opgen/atenops.py | 7 ++ .../eager/opgen/opgen/custom_ops.py | 2 + .../eager/opgen/opgen/generator.py | 52 ++++++-- orttraining/orttraining/eager/ort_aten.cpp | 115 ++++++++++++++---- orttraining/orttraining/eager/ort_aten.h | 7 ++ orttraining/orttraining/eager/test/ort_ops.py | 10 ++ 7 files changed, 163 insertions(+), 32 deletions(-) diff --git a/orttraining/orttraining/eager/opgen/opgen.py b/orttraining/orttraining/eager/opgen/opgen.py index 71e0542894..7fe7f02370 100755 --- a/orttraining/orttraining/eager/opgen/opgen.py +++ b/orttraining/orttraining/eager/opgen/opgen.py @@ -22,7 +22,7 @@ parser.add_argument('--custom_ops', action='store_true', help='Whether we are ge args = parser.parse_args() ops_module = SourceFileLoader("opgen.customop", args.ops_module).load_module() -ortgen = ORTGen(ops_module.ops, custom_ops=args.custom_ops) +ortgen = ORTGen(ops_module.ops, type_promotion_ops=ops_module.type_promotion_ops, custom_ops=args.custom_ops) regdecs_path = args.header_file print(f"INFO: Using ATen RegistrationDeclations from: {regdecs_path}") diff --git a/orttraining/orttraining/eager/opgen/opgen/atenops.py b/orttraining/orttraining/eager/opgen/opgen/atenops.py index e64f4a7d87..b9994b9b96 100644 --- a/orttraining/orttraining/eager/opgen/opgen/atenops.py +++ b/orttraining/orttraining/eager/opgen/opgen/atenops.py @@ -26,6 +26,7 @@ class GeluGrad(ONNXOp): self.domain = kMSDomain ops = {} +type_promotion_ops = [] for binary_op, onnx_op in { 'add': Add('self', Mul('alpha', 'other')), @@ -37,6 +38,7 @@ for binary_op, onnx_op in { name = f'aten::{binary_op}{variant}.{dtype}' if name not in ops: ops[f'aten::{binary_op}{variant}.{dtype}'] = deepcopy(onnx_op) + type_promotion_ops.append(f'aten::{binary_op}{variant}.{dtype}') for unary_op in [ 'abs','acos','acosh', 'asinh', 'atanh', 'asin', 'atan', 'ceil', 'cos', @@ -92,3 +94,8 @@ hand_implemented = { } ops = {**ops, **hand_implemented} +# TODO: this is a temporary whitelist for ops need type promotion +# Need to enhance the support for onnx type constrains to automatically +# resolve whether the op need type promotion. +# Will remove this list in the future. +type_promotion_ops = (*type_promotion_ops, 'aten::gelu_backward') diff --git a/orttraining/orttraining/eager/opgen/opgen/custom_ops.py b/orttraining/orttraining/eager/opgen/opgen/custom_ops.py index 4fe53bbbf9..a49d4be751 100644 --- a/orttraining/orttraining/eager/opgen/opgen/custom_ops.py +++ b/orttraining/orttraining/eager/opgen/opgen/custom_ops.py @@ -13,3 +13,5 @@ from opgen.onnxops import * ops = { 'gemm': Gemm('A', 'B', 'C', 'alpha', 'beta', 'transA', 'transB') } + +type_promotion_ops = {} diff --git a/orttraining/orttraining/eager/opgen/opgen/generator.py b/orttraining/orttraining/eager/opgen/opgen/generator.py index d58eb5d813..7226c504bd 100644 --- a/orttraining/orttraining/eager/opgen/opgen/generator.py +++ b/orttraining/orttraining/eager/opgen/opgen/generator.py @@ -105,11 +105,13 @@ class ORTGen: def __init__( self, ops: Optional[Dict[str, ONNXOp]] = None, - custom_ops : bool = False): + custom_ops : bool = False, + type_promotion_ops : List = ()): self._mapped_ops = {} if ops: self.register_many(ops) - self._custom_ops = custom_ops + self._custom_ops = custom_ops + self.type_promotion_ops = type_promotion_ops def register(self, aten_name: str, onnx_op: ONNXOp): self._mapped_ops[aten_name] = onnx_op @@ -310,6 +312,28 @@ class ORTGen: # Perform kernel fission on the ATen op to yield a chain of ORT Invokes # e.g. aten::add(x, y, α) -> onnx::Add(x, onnx::Mul(α, y)) + + # whether need type promotion + need_type_promotion = False + if mapped_func.mapped_op_name in self.type_promotion_ops: + types_from_tensor = [] + types_from_scalar = [] + for onnx_op_index, onnx_op in enumerate(ctx.ops): + for op_input in onnx_op.inputs: + if isinstance(op_input, Outputs): + continue + cpp_param = cpp_func.get_parameter(op_input) + if cpp_param: + if cpp_param.parameter_type.desugar().identifier_tokens[0].value == 'Tensor': + types_from_tensor.append(f'{op_input}.scalar_type()') + elif cpp_param.parameter_type.desugar().identifier_tokens[0].value == 'Scalar': + types_from_scalar.append(f'{op_input}.type()') + if len(types_from_tensor) > 0 or len(types_from_scalar) > 0 : + need_type_promotion = True + writer.writeline('auto promoted_type = PromoteScalarTypesWithCategory({%s}, {%s});' + % (','.join(types_from_tensor), ','.join(types_from_scalar))) + writer.writeline() + for onnx_op_index, onnx_op in enumerate(ctx.ops): # Torch -> ORT inputs for op_input in onnx_op.inputs: @@ -324,6 +348,14 @@ class ORTGen: writer.write(f'auto ort_input_{op_input} = ') writer.writeline(f'create_ort_value(invoker, {op_input});') + if need_type_promotion: + type_func_str = 'type()' if cpp_param.parameter_type.desugar().identifier_tokens[0].value == 'Scalar' else 'scalar_type()' + writer.write(f'if ({op_input}.{type_func_str} != *promoted_type)') + writer.writeline('{') + writer.push_indent() + writer.writeline(f'ort_input_{op_input} = CastToType(invoker, ort_input_{op_input}, *promoted_type);') + writer.pop_indent() + writer.writeline('}') # Torch kwargs -> ORT attributes attrs = { k:v for k, v in onnx_op.attributes.items() if v and v.value } @@ -403,17 +435,23 @@ class ORTGen: # TODO: Assert return type if not return_alias_info: + # tensor options + writer.write(f'at::TensorOptions tensor_options = {first_param.identifier.value}') + if first_param.parameter_type.desugar().identifier_tokens[0].value == 'TensorList': + writer.write('[0]') + writer.write('.options()') + if need_type_promotion: + writer.write('.dtype(*promoted_type)') + writer.writeline(';') + writer.writeline('return aten_tensor_from_ort(') writer.push_indent() if isinstance(cpp_func.return_type, ast.TemplateType) and cpp_func.return_type.identifier_tokens[-1].value == 'std::vector': writer.writeline(f'{return_outputs},') - writer.writeline(f'{first_param.identifier.value}.options());') + writer.writeline('tensor_options);') else: writer.writeline(f'std::move({return_outputs}[0]),') - writer.write(first_param.identifier.value) - if first_param.parameter_type.desugar().identifier_tokens[0].value == 'TensorList': - writer.write('[0]') - writer.writeline('.options());') + writer.writeline('tensor_options);') writer.pop_indent() return diff --git a/orttraining/orttraining/eager/ort_aten.cpp b/orttraining/orttraining/eager/ort_aten.cpp index 4bbfc2c2db..b9d6630437 100644 --- a/orttraining/orttraining/eager/ort_aten.cpp +++ b/orttraining/orttraining/eager/ort_aten.cpp @@ -7,11 +7,15 @@ #include #include +#include +#include + + namespace torch_ort { namespace eager { //#pragma region Helpers - +using NodeAttributes = onnxruntime::NodeAttributes; namespace { inline bool is_device_supported(at::DeviceType type) { return type == at::kORT || type == at::kCPU; @@ -218,6 +222,92 @@ bool IsSupportedType(at::TensorList tensors, const std::vector& return IsSupportedType(tensors[0], valid_types); } +ONNX_NAMESPACE::TensorProto_DataType GetONNXTensorProtoDataType(at::ScalarType dtype){ + switch (dtype){ + case at::kFloat: + return ONNX_NAMESPACE::TensorProto_DataType_FLOAT; + case at::kDouble: + return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE; + case at::kHalf: + return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16; + case at::kBFloat16: + return ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16; + case at::kInt: + return ONNX_NAMESPACE::TensorProto_DataType_INT32; + case at::kShort: + return ONNX_NAMESPACE::TensorProto_DataType_INT16; + case at::kLong: + return ONNX_NAMESPACE::TensorProto_DataType_INT64; + case at::kBool: + return ONNX_NAMESPACE::TensorProto_DataType_BOOL; + default: + ORT_THROW("Unsupport aten scalar type: ", dtype); + } +} + +static c10::optional PromoteScalarTypes( + const std::vector& types) { + if (types.empty()) { + return at::nullopt; + } + auto st = types[0]; + for (const auto i : c10::irange(1, types.size())) { + st = c10::promoteTypes(st, types[i]); + } + return st; +} + + +c10::optional PromoteScalarTypesWithCategory( + const std::vector& typesFromTensors, + const std::vector& typesFromScalars) { + auto typeFromTensor = PromoteScalarTypes(typesFromTensors); + auto typeFromScalar = PromoteScalarTypes(typesFromScalars); + + auto getTypeCategory = [](c10::ScalarType t) { + if (c10::kBool == t) { + return 1; + } + if (c10::isIntegralType(t, /*includeBool=*/false)) { + return 2; + } + if (c10::isFloatingType(t)) { + return 3; + } + return 0; + }; + + if (c10::nullopt == typeFromScalar) { + return typeFromTensor; + } else if (c10::nullopt == typeFromTensor) { + return typeFromScalar; + } + + auto typeCategoryFromTensor = getTypeCategory(typeFromTensor.value()); + auto typeCategoryFromScalar = getTypeCategory(typeFromScalar.value()); + + if (typeCategoryFromScalar > typeCategoryFromTensor) { + return typeFromScalar; + } + return typeFromTensor; +} + +OrtValue CastToType(onnxruntime::ORTInvoker& invoker, const OrtValue& input, at::ScalarType type){ + std::vector output(1); + NodeAttributes attrs(2); + attrs["to"] = create_ort_attribute( + "to", GetONNXTensorProtoDataType(type), at::ScalarType::Long); + + auto status = invoker.Invoke("Cast", { + std::move(input), + }, output, &attrs); + + if (!status.IsOK()) + throw std::runtime_error( + "ORT return failure status:" + status.ErrorMessage()); + return output[0]; +} + //#pragma endregion //#pragma region Hand-Implemented ATen Ops @@ -316,29 +406,6 @@ at::Tensor view(const at::Tensor& self, at::IntArrayRef size) { self.options()); } -ONNX_NAMESPACE::TensorProto_DataType GetONNXTensorProtoDataType(at::ScalarType dtype){ - switch (dtype){ - case at::kFloat: - return ONNX_NAMESPACE::TensorProto_DataType_FLOAT; - case at::kDouble: - return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE; - case at::kHalf: - return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16; - case at::kBFloat16: - return ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16; - case at::kInt: - return ONNX_NAMESPACE::TensorProto_DataType_INT32; - case at::kShort: - return ONNX_NAMESPACE::TensorProto_DataType_INT16; - case at::kLong: - return ONNX_NAMESPACE::TensorProto_DataType_INT64; - case at::kBool: - return ONNX_NAMESPACE::TensorProto_DataType_BOOL; - default: - ORT_THROW("Unsupport aten scalar type: ", dtype); - } -} - at::Tensor& copy_( at::Tensor& self, const at::Tensor& src, diff --git a/orttraining/orttraining/eager/ort_aten.h b/orttraining/orttraining/eager/ort_aten.h index 1f3190cabe..b01e370a25 100644 --- a/orttraining/orttraining/eager/ort_aten.h +++ b/orttraining/orttraining/eager/ort_aten.h @@ -118,5 +118,12 @@ bool IsSupportedType(c10::optional val, const std::vector& valid_types); +c10::optional PromoteScalarTypesWithCategory( + const std::vector& typesFromTensors, + const std::vector& typesFromScalars); + +ONNX_NAMESPACE::TensorProto_DataType GetONNXTensorProtoDataType(at::ScalarType dtype); + +OrtValue CastToType(onnxruntime::ORTInvoker& invoker, const OrtValue& input, at::ScalarType type); } // namespace eager } // namespace torch_ort \ No newline at end of file diff --git a/orttraining/orttraining/eager/test/ort_ops.py b/orttraining/orttraining/eager/test/ort_ops.py index 84be88d73a..e12b71a2f1 100644 --- a/orttraining/orttraining/eager/test/ort_ops.py +++ b/orttraining/orttraining/eager/test/ort_ops.py @@ -17,6 +17,16 @@ class OrtOpTests(unittest.TestCase): cpu_twos = cpu_ones + cpu_ones ort_twos = ort_ones + ort_ones assert torch.allclose(cpu_twos, ort_twos.cpu()) + + def test_type_promotion_add(self): + device = self.get_device() + x = torch.ones(2, 5, dtype = torch.int64) + y = torch.ones(2, 5, dtype = torch.float32) + ort_x = x.to(device) + ort_y = y.to(device) + ort_z = ort_x + ort_y + assert ort_z.dtype == torch.float32 + assert torch.allclose(ort_z.cpu(), (x + y)) def test_add_alpha(self): device = self.get_device() From d2b14249683d3e25994c2209027dbf5df9d98c16 Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Thu, 20 Jan 2022 16:30:18 -0800 Subject: [PATCH 15/59] fix bugs in cpuid_info (#10334) * fix serveral bugs in cpuid_info --- onnxruntime/core/common/cpuid_info.cc | 15 +++------------ onnxruntime/core/common/cpuid_info.h | 10 ++++++++-- onnxruntime/core/mlas/lib/compute.cpp | 12 ++++++------ onnxruntime/core/mlas/lib/convsym.cpp | 2 +- onnxruntime/core/mlas/lib/dgemm.cpp | 6 +++--- onnxruntime/core/mlas/lib/erf.cpp | 2 +- onnxruntime/core/mlas/lib/logistic.cpp | 2 +- onnxruntime/core/mlas/lib/mlasi.h | 6 +++++- onnxruntime/core/mlas/lib/platform.cpp | 8 +------- onnxruntime/core/mlas/lib/qdwconv.cpp | 8 ++++---- onnxruntime/core/mlas/lib/qgemm.cpp | 4 ++-- onnxruntime/core/mlas/lib/qgemm.h | 8 ++++---- onnxruntime/core/mlas/lib/qgemm_kernel_avx2.cpp | 10 +++++----- onnxruntime/core/mlas/lib/qladd.cpp | 4 ++-- onnxruntime/core/mlas/lib/quantize.cpp | 6 +++--- onnxruntime/core/mlas/lib/sgemm.cpp | 16 ++++++++-------- onnxruntime/core/mlas/lib/snchwc.cpp | 12 ++++++------ onnxruntime/core/mlas/lib/tanh.cpp | 2 +- 18 files changed, 64 insertions(+), 69 deletions(-) diff --git a/onnxruntime/core/common/cpuid_info.cc b/onnxruntime/core/common/cpuid_info.cc index b2590ccc3d..f9b084b682 100644 --- a/onnxruntime/core/common/cpuid_info.cc +++ b/onnxruntime/core/common/cpuid_info.cc @@ -1,13 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if defined(_M_IX86) || (defined(_M_X64) && !defined(_M_ARM64EC)) || defined(__i386__) || defined(__x86_64__) -#define CPUIDINFO_ARCH_X86 -#endif - -#if defined(_M_ARM64) || defined(__aarch64__) || defined(_M_ARM) || defined(__arm__) -#define CPUIDINFO_ARCH_ARM -#endif +#include "core/common/cpuid_info.h" +#include "core/common/logging/logging.h" +#include "core/common/logging/severity.h" #if defined(CPUIDINFO_ARCH_X86) #include @@ -30,9 +26,6 @@ #endif #include -#include "core/common/cpuid_info.h" -#include "core/common/logging/logging.h" -#include "core/common/logging/severity.h" #if _WIN32 #define HAS_WINDOWS_DESKTOP WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) @@ -65,8 +58,6 @@ static inline int XGETBV() { } #endif // CPUIDINFO_ARCH_X86 -CPUIDInfo CPUIDInfo::instance_; - CPUIDInfo::CPUIDInfo() { #if (defined(CPUIDINFO_ARCH_X86) || defined(CPUIDINFO_ARCH_ARM)) && defined(CPUINFO_SUPPORTED) diff --git a/onnxruntime/core/common/cpuid_info.h b/onnxruntime/core/common/cpuid_info.h index 754cc2e023..66fc21ff55 100644 --- a/onnxruntime/core/common/cpuid_info.h +++ b/onnxruntime/core/common/cpuid_info.h @@ -5,6 +5,14 @@ #include "core/common/common.h" +#if defined(_M_IX86) || (defined(_M_X64) && !defined(_M_ARM64EC)) || defined(__i386__) || defined(__x86_64__) +#define CPUIDINFO_ARCH_X86 +#endif + +#if defined(_M_ARM64) || defined(__aarch64__) || defined(_M_ARM) || defined(__arm__) +#define CPUIDINFO_ARCH_ARM +#endif + namespace onnxruntime { class CPUIDInfo { @@ -41,8 +49,6 @@ class CPUIDInfo { bool pytorch_cpuinfo_init_{false}; #endif bool has_arm_neon_dot_{false}; - - static CPUIDInfo instance_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/mlas/lib/compute.cpp b/onnxruntime/core/mlas/lib/compute.cpp index af5ee95ac8..1183510551 100644 --- a/onnxruntime/core/mlas/lib/compute.cpp +++ b/onnxruntime/core/mlas/lib/compute.cpp @@ -271,7 +271,7 @@ Return Value: --*/ { #if defined(MLAS_TARGET_AMD64) - MlasPlatform.ComputeExpF32Kernel(Input, Output, N); + GetMlasPlatform().ComputeExpF32Kernel(Input, Output, N); #else MlasComputeExpF32Kernel(Input, Output, N); #endif @@ -850,7 +850,7 @@ Return Value: // #if defined(MLAS_TARGET_AMD64) - float Maximum = MlasPlatform.ReduceMaximumF32Kernel(Input, D); + float Maximum = GetMlasPlatform().ReduceMaximumF32Kernel(Input, D); #else float Maximum = MlasReduceMaximumF32Kernel(Input, D); #endif @@ -863,7 +863,7 @@ Return Value: // #if defined(MLAS_TARGET_AMD64) - float Accumulation = MlasPlatform.ComputeSumExpF32Kernel(Input, nullptr, D, &NegativeMaximum); + float Accumulation = GetMlasPlatform().ComputeSumExpF32Kernel(Input, nullptr, D, &NegativeMaximum); #else float Accumulation = MlasComputeSumExpF32Kernel(Input, nullptr, D, &NegativeMaximum); #endif @@ -875,7 +875,7 @@ Return Value: float Parameters[] = { NegativeMaximum, std::log(Accumulation)}; #if defined(MLAS_TARGET_AMD64) - MlasPlatform.ComputeLogSoftmaxOutputF32Kernel(Input, Output, D, Parameters); + GetMlasPlatform().ComputeLogSoftmaxOutputF32Kernel(Input, Output, D, Parameters); #else MlasComputeLogSoftmaxOutputF32Kernel(Input, Output, D, Parameters); #endif @@ -888,7 +888,7 @@ Return Value: // #if defined(MLAS_TARGET_AMD64) - float Accumulation = MlasPlatform.ComputeSumExpF32Kernel(Input, Output, D, &NegativeMaximum); + float Accumulation = GetMlasPlatform().ComputeSumExpF32Kernel(Input, Output, D, &NegativeMaximum); #else float Accumulation = MlasComputeSumExpF32Kernel(Input, Output, D, &NegativeMaximum); #endif @@ -900,7 +900,7 @@ Return Value: float Parameters[] = { 1.0f / Accumulation }; #if defined(MLAS_TARGET_AMD64) - MlasPlatform.ComputeSoftmaxOutputF32Kernel(Output, D, Parameters); + GetMlasPlatform().ComputeSoftmaxOutputF32Kernel(Output, D, Parameters); #else MlasComputeSoftmaxOutputF32Kernel(Output, D, Parameters); #endif diff --git a/onnxruntime/core/mlas/lib/convsym.cpp b/onnxruntime/core/mlas/lib/convsym.cpp index 4deecc3553..b5c82044ea 100644 --- a/onnxruntime/core/mlas/lib/convsym.cpp +++ b/onnxruntime/core/mlas/lib/convsym.cpp @@ -205,7 +205,7 @@ MLAS_FORCEINLINE const MLAS_CONV_SYM_DISPATCH* GetConvSymDispatch(bool InputIsSigned){ - return InputIsSigned ? MlasPlatform.ConvSymS8S8Dispatch : MlasPlatform.ConvSymU8S8Dispatch; + return InputIsSigned ? GetMlasPlatform().ConvSymS8S8Dispatch : GetMlasPlatform().ConvSymU8S8Dispatch; } size_t diff --git a/onnxruntime/core/mlas/lib/dgemm.cpp b/onnxruntime/core/mlas/lib/dgemm.cpp index b98d6f8bb2..1ef63d03c8 100644 --- a/onnxruntime/core/mlas/lib/dgemm.cpp +++ b/onnxruntime/core/mlas/lib/dgemm.cpp @@ -531,7 +531,7 @@ Return Value: size_t RowsHandled; #if defined(MLAS_TARGET_AMD64_IX86) || defined (MLAS_TARGET_POWER) - RowsHandled = MlasPlatform.GemmDoubleKernel(A, B, C, CountK, CountM, CountN, lda, ldc, alpha, ZeroMode); + RowsHandled = GetMlasPlatform().GemmDoubleKernel(A, B, C, CountK, CountM, CountN, lda, ldc, alpha, ZeroMode); #else if (ZeroMode) { RowsHandled = MlasDgemmKernelZero(A, B, C, CountK, CountM, CountN, lda, ldc, alpha); @@ -830,10 +830,10 @@ MlasGemmBatch( ptrdiff_t TargetThreadCount; - if (Complexity < double(MLAS_DGEMM_THREAD_COMPLEXITY * MlasPlatform.MaximumThreadCount)) { + if (Complexity < double(MLAS_DGEMM_THREAD_COMPLEXITY * GetMlasPlatform().MaximumThreadCount)) { TargetThreadCount = ptrdiff_t(Complexity / double(MLAS_DGEMM_THREAD_COMPLEXITY)) + 1; } else { - TargetThreadCount = MlasPlatform.MaximumThreadCount; + TargetThreadCount = GetMlasPlatform().MaximumThreadCount; } ptrdiff_t MaximumThreadCount = MlasGetMaximumThreadCount(ThreadPool); diff --git a/onnxruntime/core/mlas/lib/erf.cpp b/onnxruntime/core/mlas/lib/erf.cpp index ebd5a3dd54..b45bd5162a 100644 --- a/onnxruntime/core/mlas/lib/erf.cpp +++ b/onnxruntime/core/mlas/lib/erf.cpp @@ -262,7 +262,7 @@ Return Value: --*/ { #if defined(MLAS_TARGET_AMD64) - MlasPlatform.ErfKernelRoutine(Input, Output, N); + GetMlasPlatform().ErfKernelRoutine(Input, Output, N); #else MlasErfKernel(Input, Output, N); #endif diff --git a/onnxruntime/core/mlas/lib/logistic.cpp b/onnxruntime/core/mlas/lib/logistic.cpp index a650b60819..ecca39f974 100644 --- a/onnxruntime/core/mlas/lib/logistic.cpp +++ b/onnxruntime/core/mlas/lib/logistic.cpp @@ -179,7 +179,7 @@ Return Value: --*/ { #if defined(MLAS_TARGET_AMD64) - MlasPlatform.LogisticKernelRoutine(Input, Output, N); + GetMlasPlatform().LogisticKernelRoutine(Input, Output, N); #else MlasLogisticKernel(Input, Output, N); #endif diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index 50ad470710..77b5038786 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -841,7 +841,11 @@ struct MLAS_PLATFORM { }; -extern MLAS_PLATFORM MlasPlatform; +inline +MLAS_PLATFORM& GetMlasPlatform(){ + static MLAS_PLATFORM MlasPlatform; + return MlasPlatform; +} // // Threading support. diff --git a/onnxruntime/core/mlas/lib/platform.cpp b/onnxruntime/core/mlas/lib/platform.cpp index 9e747ce07a..67e1e1675e 100644 --- a/onnxruntime/core/mlas/lib/platform.cpp +++ b/onnxruntime/core/mlas/lib/platform.cpp @@ -43,12 +43,6 @@ MLASCPUIDInfo::MLASCPUIDInfo() { has_arm_neon_dot_ = ((getauxval(AT_HWCAP) & HWC #endif #endif // MLAS_TARGET_ARM64 -// -// Stores the platform information. -// - -MLAS_PLATFORM MlasPlatform; - #ifdef MLAS_TARGET_AMD64_IX86 // @@ -422,7 +416,7 @@ Return Value: --*/ { #if defined(MLAS_TARGET_AMD64) - return MlasPlatform.PreferredBufferAlignment; + return GetMlasPlatform().PreferredBufferAlignment; #else return MLAS_DEFAULT_PREFERRED_BUFFER_ALIGNMENT; #endif diff --git a/onnxruntime/core/mlas/lib/qdwconv.cpp b/onnxruntime/core/mlas/lib/qdwconv.cpp index cf98100bc5..921addab2c 100644 --- a/onnxruntime/core/mlas/lib/qdwconv.cpp +++ b/onnxruntime/core/mlas/lib/qdwconv.cpp @@ -288,14 +288,14 @@ Return Value: if (InputIsSigned) { if (FilterIsSigned) { - MlasPlatform.ConvDepthwiseS8S8Kernel( + GetMlasPlatform().ConvDepthwiseS8S8Kernel( reinterpret_cast(Input), static_cast(InputZeroPoint), reinterpret_cast(Filter), static_cast(FilterZeroPoint), Output, Channels, OutputCount, KernelSize ); } else { - MlasPlatform.ConvDepthwiseS8U8Kernel( + GetMlasPlatform().ConvDepthwiseS8U8Kernel( reinterpret_cast(Input), static_cast(InputZeroPoint), reinterpret_cast(Filter), static_cast(FilterZeroPoint), Output, Channels, OutputCount, KernelSize @@ -304,14 +304,14 @@ Return Value: } else { if (FilterIsSigned) { - MlasPlatform.ConvDepthwiseU8S8Kernel( + GetMlasPlatform().ConvDepthwiseU8S8Kernel( reinterpret_cast(Input), static_cast(InputZeroPoint), reinterpret_cast(Filter), static_cast(FilterZeroPoint), Output, Channels, OutputCount, KernelSize ); } else { - MlasPlatform.ConvDepthwiseU8U8Kernel( + GetMlasPlatform().ConvDepthwiseU8U8Kernel( reinterpret_cast(Input), static_cast(InputZeroPoint), reinterpret_cast(Filter), static_cast(FilterZeroPoint), Output, Channels, OutputCount, KernelSize diff --git a/onnxruntime/core/mlas/lib/qgemm.cpp b/onnxruntime/core/mlas/lib/qgemm.cpp index 1a4c1c3f62..dce27166fb 100644 --- a/onnxruntime/core/mlas/lib/qgemm.cpp +++ b/onnxruntime/core/mlas/lib/qgemm.cpp @@ -165,10 +165,10 @@ MlasGemmBatch( ptrdiff_t TargetThreadCount; - if (Complexity < double(MLAS_QGEMM_THREAD_COMPLEXITY * MlasPlatform.MaximumThreadCount)) { + if (Complexity < double(MLAS_QGEMM_THREAD_COMPLEXITY * GetMlasPlatform().MaximumThreadCount)) { TargetThreadCount = ptrdiff_t(Complexity / double(MLAS_QGEMM_THREAD_COMPLEXITY)) + 1; } else { - TargetThreadCount = MlasPlatform.MaximumThreadCount; + TargetThreadCount = GetMlasPlatform().MaximumThreadCount; } ptrdiff_t MaximumThreadCount = MlasGetMaximumThreadCount(ThreadPool); diff --git a/onnxruntime/core/mlas/lib/qgemm.h b/onnxruntime/core/mlas/lib/qgemm.h index c4d3532d2a..924cde75c6 100644 --- a/onnxruntime/core/mlas/lib/qgemm.h +++ b/onnxruntime/core/mlas/lib/qgemm.h @@ -710,21 +710,21 @@ MlasGemmQuantGetDispatch( #if defined(MLAS_TARGET_AMD64_IX86) if (!AIsSigned) { if (BIsSigned) { - GemmQuantDispatch = MlasPlatform.GemmU8S8Dispatch; + GemmQuantDispatch = GetMlasPlatform().GemmU8S8Dispatch; } else { - GemmQuantDispatch = MlasPlatform.GemmU8U8Dispatch; + GemmQuantDispatch = GetMlasPlatform().GemmU8U8Dispatch; } } #elif defined(MLAS_TARGET_ARM64) if(BIsSigned) { - if(MlasPlatform.GemmU8X8Dispatch == &MlasGemmU8X8DispatchNeon) { + if(GetMlasPlatform().GemmU8X8Dispatch == &MlasGemmU8X8DispatchNeon) { GemmQuantDispatch = &MlasGemmX8S8DispatchNeon; } else { GemmQuantDispatch = AIsSigned? &MlasGemmS8S8DispatchSdot : &MlasGemmU8X8DispatchUdot; } } else if(!AIsSigned) { - GemmQuantDispatch = MlasPlatform.GemmU8X8Dispatch; + GemmQuantDispatch = GetMlasPlatform().GemmU8X8Dispatch; } #elif defined(MLAS_TARGET_ARM64EC) || (defined(MLAS_TARGET_ARM) && !defined(_MSC_VER)) if(BIsSigned || !AIsSigned) { diff --git a/onnxruntime/core/mlas/lib/qgemm_kernel_avx2.cpp b/onnxruntime/core/mlas/lib/qgemm_kernel_avx2.cpp index d9451b878d..810a25bf8d 100644 --- a/onnxruntime/core/mlas/lib/qgemm_kernel_avx2.cpp +++ b/onnxruntime/core/mlas/lib/qgemm_kernel_avx2.cpp @@ -107,7 +107,7 @@ MlasGemmQuantTryGemvKernel( ) { if (!AIsSigned && BIsSigned) { - MlasPlatform.GemvU8S8Kernel(A, B, C, CountK, CountN, ldb); + GetMlasPlatform().GemvU8S8Kernel(A, B, C, CountK, CountN, ldb); return true; } @@ -179,8 +179,8 @@ MlasGemmQuantKernel( bool ZeroMode ) { - return MlasPlatform.GemmU8S8Kernel(A, B, C, PackedCountK, CountM, CountN, ldc, - RowSumBuffer, ColumnSumBuffer, ZeroPointB, ZeroMode); + return GetMlasPlatform().GemmU8S8Kernel(A, B, C, PackedCountK, CountM, CountN, ldc, + RowSumBuffer, ColumnSumBuffer, ZeroPointB, ZeroMode); } const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8S8DispatchAvx2 = { @@ -260,8 +260,8 @@ MlasGemmQuantKernel( bool ZeroMode ) { - return MlasPlatform.GemmU8U8Kernel(A, B, C, PackedCountK, CountM, CountN, ldc, - RowSumBuffer, ColumnSumBuffer, ZeroPointB, ZeroMode); + return GetMlasPlatform().GemmU8U8Kernel(A, B, C, PackedCountK, CountM, CountN, ldc, + RowSumBuffer, ColumnSumBuffer, ZeroPointB, ZeroMode); } const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8U8DispatchAvx2 = { diff --git a/onnxruntime/core/mlas/lib/qladd.cpp b/onnxruntime/core/mlas/lib/qladd.cpp index 313b025ea4..02676fe4ca 100644 --- a/onnxruntime/core/mlas/lib/qladd.cpp +++ b/onnxruntime/core/mlas/lib/qladd.cpp @@ -520,7 +520,7 @@ MlasQLinearAdd( ) { #if defined(MLAS_TARGET_AMD64) - MlasPlatform.QLinearAddS8Kernel( + GetMlasPlatform().QLinearAddS8Kernel( #else MlasQLinearAddKernel( #endif @@ -545,7 +545,7 @@ MlasQLinearAdd( ) { #if defined(MLAS_TARGET_AMD64) - MlasPlatform.QLinearAddU8Kernel( + GetMlasPlatform().QLinearAddU8Kernel( #else MlasQLinearAddKernel( #endif diff --git a/onnxruntime/core/mlas/lib/quantize.cpp b/onnxruntime/core/mlas/lib/quantize.cpp index 632800e8fd..558b6ec360 100644 --- a/onnxruntime/core/mlas/lib/quantize.cpp +++ b/onnxruntime/core/mlas/lib/quantize.cpp @@ -248,7 +248,7 @@ MlasQuantizeLinear( ) { #if defined(MLAS_TARGET_AMD64) - MlasPlatform.QuantizeLinearS8Kernel( + GetMlasPlatform().QuantizeLinearS8Kernel( #else MlasQuantizeLinearS8Kernel( #endif @@ -267,7 +267,7 @@ MlasQuantizeLinear( ) { #if defined(MLAS_TARGET_AMD64) - MlasPlatform.QuantizeLinearU8Kernel( + GetMlasPlatform().QuantizeLinearU8Kernel( #else MlasQuantizeLinearU8Kernel( #endif @@ -989,7 +989,7 @@ Return Value: --*/ { #if defined(MLAS_TARGET_AMD64) - MlasPlatform.ReduceMinimumMaximumF32Kernel(Input, Min, Max, N); + GetMlasPlatform().ReduceMinimumMaximumF32Kernel(Input, Min, Max, N); #else MlasReduceMinimumMaximumF32Kernel(Input, Min, Max, N); #endif diff --git a/onnxruntime/core/mlas/lib/sgemm.cpp b/onnxruntime/core/mlas/lib/sgemm.cpp index 62170a2573..1ce64712d6 100644 --- a/onnxruntime/core/mlas/lib/sgemm.cpp +++ b/onnxruntime/core/mlas/lib/sgemm.cpp @@ -475,7 +475,7 @@ Return Value: #if defined(MLAS_TARGET_AMD64) MLAS_SGEMM_TRANSPOSE_PACKB_BLOCK_ROUTINE* SgemmTransposePackB16x4Routine = - MlasPlatform.TransposePackB16x4Routine; + GetMlasPlatform().TransposePackB16x4Routine; while (x >= 4) { @@ -1062,7 +1062,7 @@ Return Value: size_t RowsHandled; #if defined(MLAS_TARGET_AMD64_IX86) || defined(MLAS_TARGET_POWER) - RowsHandled = MlasPlatform.GemmFloatKernel(A, B, C, CountK, CountM, CountN, lda, ldc, alpha, ZeroMode); + RowsHandled = GetMlasPlatform().GemmFloatKernel(A, B, C, CountK, CountM, CountN, lda, ldc, alpha, ZeroMode); #else if (ZeroMode) { RowsHandled = MlasSgemmKernelZero(A, B, C, CountK, CountM, CountN, lda, ldc, alpha); @@ -1163,9 +1163,9 @@ Return Value: MLAS_SGEMM_KERNEL_M1_ROUTINE* SgemmKernelM1Routine; if (TransB == CblasNoTrans) { - SgemmKernelM1Routine = MlasPlatform.KernelM1Routine; + SgemmKernelM1Routine = GetMlasPlatform().KernelM1Routine; } else { - SgemmKernelM1Routine = MlasPlatform.KernelM1TransposeBRoutine; + SgemmKernelM1Routine = GetMlasPlatform().KernelM1TransposeBRoutine; } if (SgemmKernelM1Routine != nullptr) { @@ -1198,9 +1198,9 @@ Return Value: MLAS_SGEMM_KERNEL_M1_ROUTINE* SgemmKernelM1Routine; if (TransA == CblasNoTrans) { - SgemmKernelM1Routine = MlasPlatform.KernelM1TransposeBRoutine; + SgemmKernelM1Routine = GetMlasPlatform().KernelM1TransposeBRoutine; } else { - SgemmKernelM1Routine = MlasPlatform.KernelM1Routine; + SgemmKernelM1Routine = GetMlasPlatform().KernelM1Routine; } if (SgemmKernelM1Routine != nullptr) { @@ -1580,10 +1580,10 @@ MlasGemmBatch( ptrdiff_t TargetThreadCount; - if (Complexity < double(MLAS_SGEMM_THREAD_COMPLEXITY * MlasPlatform.MaximumThreadCount)) { + if (Complexity < double(MLAS_SGEMM_THREAD_COMPLEXITY * GetMlasPlatform().MaximumThreadCount)) { TargetThreadCount = ptrdiff_t(Complexity / double(MLAS_SGEMM_THREAD_COMPLEXITY)) + 1; } else { - TargetThreadCount = MlasPlatform.MaximumThreadCount; + TargetThreadCount = GetMlasPlatform().MaximumThreadCount; } ptrdiff_t MaximumThreadCount = MlasGetMaximumThreadCount(ThreadPool); diff --git a/onnxruntime/core/mlas/lib/snchwc.cpp b/onnxruntime/core/mlas/lib/snchwc.cpp index 2bc0b5d661..74d65f934a 100644 --- a/onnxruntime/core/mlas/lib/snchwc.cpp +++ b/onnxruntime/core/mlas/lib/snchwc.cpp @@ -102,7 +102,7 @@ Return Value: --*/ { #if defined(MLAS_TARGET_AMD64) - return MlasPlatform.NchwcBlockSize; + return GetMlasPlatform().NchwcBlockSize; #else return 1; #endif @@ -675,7 +675,7 @@ struct MLAS_NCHWC_CONV_NCHWC_ALGORITHM : MLAS_NCHWC_GROUPED_CONV_ALGORITHM const size_t BlockedOutputWidth = BlockSize * OutputWidth; #if defined(MLAS_TARGET_AMD64) - MLAS_CONV_FLOAT_KERNEL* Kernel = MlasPlatform.ConvNchwcFloatKernel; + MLAS_CONV_FLOAT_KERNEL* Kernel = GetMlasPlatform().ConvNchwcFloatKernel; #else MLAS_CONV_FLOAT_KERNEL* Kernel = MlasConvNchwcFloatKernel; #endif @@ -785,7 +785,7 @@ struct MLAS_NCHWC_CONV_NCHW_ALGORITHM : MLAS_NCHWC_GROUPED_CONV_ALGORITHM const size_t BlockedOutputWidth = BlockSize * OutputWidth; #if defined(MLAS_TARGET_AMD64) - MLAS_CONV_FLOAT_KERNEL* Kernel = MlasPlatform.ConvNchwFloatKernel; + MLAS_CONV_FLOAT_KERNEL* Kernel = GetMlasPlatform().ConvNchwFloatKernel; #else MLAS_CONV_FLOAT_KERNEL* Kernel = MlasConvNchwFloatKernel; #endif @@ -880,7 +880,7 @@ struct MLAS_NCHWC_CONV_POINTWISE_ALGORITHM : MLAS_NCHWC_GROUPED_CONV_ALGORITHM const size_t OutputStrideBytes = BlockSize * OutputSize * sizeof(float); #if defined(MLAS_TARGET_AMD64) - MLAS_CONV_POINTWISE_FLOAT_KERNEL* Kernel = MlasPlatform.ConvPointwiseFloatKernel; + MLAS_CONV_POINTWISE_FLOAT_KERNEL* Kernel = GetMlasPlatform().ConvPointwiseFloatKernel; #else MLAS_CONV_POINTWISE_FLOAT_KERNEL* Kernel = MlasConvPointwiseFloatKernel; #endif @@ -1017,7 +1017,7 @@ struct MLAS_NCHWC_CONV_DEPTHWISE_ALGORITHM : MLAS_NCHWC_CONV_ALGORITHM const size_t BlockedOutputWidth = BlockSize * OutputWidth; #if defined(MLAS_TARGET_AMD64) - MLAS_CONV_DEPTHWISE_FLOAT_KERNEL* Kernel = MlasPlatform.ConvDepthwiseFloatKernel; + MLAS_CONV_DEPTHWISE_FLOAT_KERNEL* Kernel = GetMlasPlatform().ConvDepthwiseFloatKernel; #else MLAS_CONV_DEPTHWISE_FLOAT_KERNEL* Kernel = MlasConvDepthwiseFloatKernel; #endif @@ -1132,7 +1132,7 @@ struct MLAS_NCHWC_POOL_ALGORITHM : MLAS_NCHWC_NN_ALGORITHM const size_t InputStrideBytes = DilatedInputWidthBytes - KernelWidth * DilationWidthBytes; #if defined(MLAS_TARGET_AMD64) - MLAS_POOL_FLOAT_KERNEL* Kernel = MlasPlatform.PoolFloatKernel[WorkBlock->PoolingKind]; + MLAS_POOL_FLOAT_KERNEL* Kernel = GetMlasPlatform().PoolFloatKernel[WorkBlock->PoolingKind]; #else MLAS_POOL_FLOAT_KERNEL* Kernel = PoolKernels[WorkBlock->PoolingKind]; #endif diff --git a/onnxruntime/core/mlas/lib/tanh.cpp b/onnxruntime/core/mlas/lib/tanh.cpp index 846e12b9d6..9750337237 100644 --- a/onnxruntime/core/mlas/lib/tanh.cpp +++ b/onnxruntime/core/mlas/lib/tanh.cpp @@ -177,7 +177,7 @@ Return Value: --*/ { #if defined(MLAS_TARGET_AMD64) - MlasPlatform.TanhKernelRoutine(Input, Output, N); + GetMlasPlatform().TanhKernelRoutine(Input, Output, N); #else MlasTanhKernel(Input, Output, N); #endif From eee627fde9da46e279154a15af581f621f2873fc Mon Sep 17 00:00:00 2001 From: Olivia Jain Date: Fri, 21 Jan 2022 13:20:53 -0800 Subject: [PATCH 16/59] Track Session Creation Time (#10281) * add back previous changes lost in merge * post session to dashboard * post session creation time to dashboard * fix trt 8 functionality: * add component governance * Remove hardcoded values * Update linux-gpu-tensorrt-daily-perf-pipeline.yml for Azure Pipelines * cleanup errors * post results only once * checkout 8.0 GA * try build 8.0 without building shared lib * add back build_shared_lib, not the problem * add upload_time to table * use identifier to post * Shorten to TRT x.x * shorten commit hash using rev_parse * use shortened commit hash * use nvidia's default TRT_VERSION --- dockerfiles/Dockerfile.tensorrt | 7 +- .../python/tools/tensorrt/perf/benchmark.py | 85 ++++++++++++++++--- .../tools/tensorrt/perf/benchmark_wrapper.py | 11 +++ .../tools/tensorrt/perf/build/build_image.sh | 6 +- .../tools/tensorrt/perf/build/checkout_trt.sh | 3 +- .../python/tools/tensorrt/perf/perf_utils.py | 5 +- .../python/tools/tensorrt/perf/post.py | 28 ++++-- ...linux-gpu-tensorrt-daily-perf-pipeline.yml | 8 +- 8 files changed, 122 insertions(+), 31 deletions(-) diff --git a/dockerfiles/Dockerfile.tensorrt b/dockerfiles/Dockerfile.tensorrt index 9b4db6b5f5..9947c54a82 100644 --- a/dockerfiles/Dockerfile.tensorrt +++ b/dockerfiles/Dockerfile.tensorrt @@ -6,7 +6,6 @@ # nVidia TensorRT Base Image ARG TRT_CONTAINER_VERSION=21.12 -ARG TRT_VERSION=8.2.1.8 FROM nvcr.io/nvidia/tensorrt:${TRT_CONTAINER_VERSION}-py3 ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime @@ -26,8 +25,8 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR # Checkout appropriate TRT_VERSION and build RUN cd onnxruntime &&\ - trt_version=${TRT_VERSION%.*.*} &&\ + trt_version=${TRT_VERSION:0:3} &&\ ./onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh ${trt_version} &&\ - /bin/sh build.sh --parallel --build_shared_lib --skip_submodule_sync --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /usr/lib/x86_64-linux-gnu/ --config Release --build_wheel --skip_tests --skip_submodule_sync --cmake_extra_defines '"CMAKE_CUDA_ARCHITECTURES='${CMAKE_CUDA_ARCHITECTURES}'"' &&\ + /bin/sh build.sh --parallel --build_shared_lib --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /usr/lib/x86_64-linux-gnu/ --config Release --build_wheel --skip_tests --skip_submodule_sync --cmake_extra_defines '"CMAKE_CUDA_ARCHITECTURES='${CMAKE_CUDA_ARCHITECTURES}'"' &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\ - cd .. \ No newline at end of file + cd .. diff --git a/onnxruntime/python/tools/tensorrt/perf/benchmark.py b/onnxruntime/python/tools/tensorrt/perf/benchmark.py index dedaa5d960..9510125b9f 100644 --- a/onnxruntime/python/tools/tensorrt/perf/benchmark.py +++ b/onnxruntime/python/tools/tensorrt/perf/benchmark.py @@ -42,6 +42,7 @@ trt_native_fp16_gain = 'TRT_Standalone_fp16_gain(%)' FAIL_MODEL_FILE = ".fail_model_map" LATENCY_FILE = ".latency_map" METRICS_FILE = ".metrics_map" +SESSION_FILE = ".session_map" MEMORY_FILE = './temp_memory.csv' def split_and_sort_output(string_list): @@ -612,7 +613,6 @@ def update_metrics_map_ori(model_to_metrics, name, ep_to_operator): logger.info('TRT FP16 operator map:') pp.pprint(trt_fp16_op_map) - ################################################################################################### # # model: {ep1: {error_type: xxx, error_message: xxx}, ep2: {error_type: xx, error_message: xx}} @@ -910,13 +910,18 @@ def run_symbolic_shape_inference(model_path, new_model_path): logger.error(e) return False, "Symbolic shape inference error" +def time_and_create_session(model_path, providers, session_options): + start = datetime.now() + session = onnxruntime.InferenceSession(model_path, providers=providers, sess_options=session_options) + end = datetime.now() + creation_time = (end - start).total_seconds() + return session, creation_time + def create_session(model_path, providers, session_options): logger.info(model_path) try: - session = onnxruntime.InferenceSession(model_path, providers=providers, sess_options=session_options) - return session - + return time_and_create_session(model_path, providers, session_options) except Exception as e: # shape inference required on model if "shape inference" in str(e): @@ -926,9 +931,8 @@ def create_session(model_path, providers, session_options): status = run_symbolic_shape_inference(model_path, new_model_path) if not status[0]: # symbolic shape inference error e = status[1] - raise Exception(e) - session = onnxruntime.InferenceSession(new_model_path, providers=providers, sess_options=session_options) - return session + raise Exception(e) + return time_and_create_session(new_model_path, providers, session_options) else: raise Exception(e) @@ -938,7 +942,8 @@ def run_onnxruntime(args, models): model_to_latency = {} # model -> cuda and tensorrt latency model_to_metrics = {} # model -> metrics from profiling file model_to_fail_ep = {} # model -> failing ep - + model_to_session = {} # models -> session creation time + ep_list = [] if args.ep: ep_list.append(args.ep) @@ -1021,7 +1026,6 @@ def run_onnxruntime(args, models): if standalone_trt_fp16 == ep: fp16 = True - print(fp16) inputs, ref_outputs = get_test_data(fp16, test_data_dir, all_inputs_shape) # generate random input data if args.input_data == "random": @@ -1045,7 +1049,7 @@ def run_onnxruntime(args, models): # create onnxruntime inference session try: - sess = create_session(model_path, providers, options) + sess, _ = create_session(model_path, providers, options) except Exception as e: logger.error(e) @@ -1133,12 +1137,16 @@ def run_onnxruntime(args, models): # create onnxruntime inference session try: - sess = create_session(model_path, ep_to_provider_list[ep], options) + sess, creation_time = create_session(model_path, ep_to_provider_list[ep], options) + except Exception as e: logger.error(e) update_fail_model_map(model_to_fail_ep, name, ep, 'runtime error', e) continue + if creation_time: + model_to_session[name] = copy.deepcopy({ep: creation_time}) + sess.disable_fallback() logger.info("start to inference {} with {} ...".format(name, ep)) @@ -1199,7 +1207,7 @@ def run_onnxruntime(args, models): # end of model - return success_results, model_to_latency, model_to_fail_ep, model_to_metrics + return success_results, model_to_latency, model_to_fail_ep, model_to_metrics, model_to_session def calculate_gain(value, ep1, ep2): ep1_latency = float(value[ep1]['average_latency_ms']) @@ -1356,6 +1364,54 @@ def output_specs(info, csv_filename): 'Version': [cpu_version, gpu_version, tensorrt_version, cuda_version, cudnn_version]}) table.to_csv(csv_filename, index=False) +def output_session_creation(results, csv_filename): + need_write_header = True + if os.path.exists(csv_filename): + need_write_header = False + + with open(csv_filename, mode="a", newline='') as csv_file: + column_names = [model_title] + for provider in ort_provider_list: + column_names.append(provider + session_ending) + + csv_writer = csv.writer(csv_file) + + + csv_writer = csv.writer(csv_file) + + if need_write_header: + csv_writer.writerow(column_names) + + cpu_time = "" + cuda_fp32_time = "" + trt_fp32_time = "" + cuda_fp16_time = "" + trt_fp16_time = "" + + for model_name, ep_dict in results.items(): + for ep, time in ep_dict.items(): + if ep == cpu: + cpu_time = time + elif ep == cuda: + cuda_fp32_time = time + elif ep == trt: + trt_fp32_time = time + elif ep == cuda_fp16: + cuda_fp16_time = time + elif ep == trt_fp16: + trt_fp16_time = time + else: + continue + + row = [model_name, + cpu_time, + cuda_fp32_time, + trt_fp32_time, + cuda_fp16_time, + trt_fp16_time] + csv_writer.writerow(row) + + def output_latency(results, csv_filename): need_write_header = True if os.path.exists(csv_filename): @@ -1657,7 +1713,7 @@ def main(): parse_models_helper(args, models) perf_start_time = datetime.now() - success_results, model_to_latency, model_to_fail_ep, model_to_metrics = run_onnxruntime(args, models) + success_results, model_to_latency, model_to_fail_ep, model_to_metrics, model_to_session = run_onnxruntime(args, models) perf_end_time = datetime.now() logger.info("Done running the perf.") @@ -1720,5 +1776,8 @@ def main(): csv_filename = os.path.join(path, csv_filename) output_metrics(model_to_metrics, csv_filename) + if len(model_to_session) > 0: + write_map_to_file(model_to_session, SESSION_FILE) + if __name__ == "__main__": main() diff --git a/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py b/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py index d2542c645c..29d5a0d42e 100644 --- a/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py +++ b/onnxruntime/python/tools/tensorrt/perf/benchmark_wrapper.py @@ -52,6 +52,7 @@ def main(): benchmark_success_csv = 'success.csv' benchmark_latency_csv = 'latency.csv' benchmark_status_csv = 'status.csv' + benchmark_session_csv = 'session.csv' specs_csv = 'specs.csv' for model, model_info in models.items(): @@ -119,6 +120,16 @@ def main(): output_metrics(model_to_metrics, os.path.join(path, benchmark_metrics_csv)) logger.info("\nSaved model metrics results to {}".format(benchmark_metrics_csv)) + logger.info("\n=========================================") + logger.info("======= Models/EPs session creation =======") + logger.info("=========================================") + + if os.path.exists(SESSION_FILE): + model_to_session = read_map_from_file(SESSION_FILE) + pretty_print(pp, model_to_session) + output_session_creation(model_to_session, os.path.join(path, benchmark_session_csv)) + logger.info("\nSaved session creation results to {}".format(benchmark_session_csv)) + elif args.running_mode == "benchmark": logger.info("\n=========================================================") logger.info("========== Failing Models/EPs (accumulated) ==============") diff --git a/onnxruntime/python/tools/tensorrt/perf/build/build_image.sh b/onnxruntime/python/tools/tensorrt/perf/build/build_image.sh index 88a0119e07..8d4bfb49be 100755 --- a/onnxruntime/python/tools/tensorrt/perf/build/build_image.sh +++ b/onnxruntime/python/tools/tensorrt/perf/build/build_image.sh @@ -7,7 +7,7 @@ o) TRT_DOCKERFILE_PATH=${OPTARG};; p) PERF_DOCKERFILE_PATH=${OPTARG};; b) ORT_BRANCH=${OPTARG};; i) IMAGE_NAME=${OPTARG};; -t) TRT=${OPTARG};; +t) TRT_CONTAINER=${OPTARG};; v) TRT_VERSION=${OPTARG};; c) CMAKE_CUDA_ARCHITECTURES=${OPTARG};; esac @@ -15,5 +15,5 @@ done IMAGE=onnxruntime -docker build --no-cache -t $IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg TRT=$TRT --build-arg TRT_VERSION=$TRT_VERSION --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH -f $TRT_DOCKERFILE_PATH . -docker build --no-cache --build-arg IMAGE=$IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH --build-arg TRT_VERSION=$TRT_VERSION -t $IMAGE_NAME -f $PERF_DOCKERFILE_PATH . \ No newline at end of file +docker build --no-cache -t $IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg TRT_CONTAINER_VERSION=$TRT_CONTAINER --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH -f $TRT_DOCKERFILE_PATH . +docker build --no-cache --build-arg IMAGE=$IMAGE --build-arg CMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES --build-arg ONNXRUNTIME_BRANCH=$ORT_BRANCH --build-arg TRT_VERSION=$TRT_VERSION -t $IMAGE_NAME -f $PERF_DOCKERFILE_PATH . diff --git a/onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh b/onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh index 20e51fba2e..8634eb41f8 100755 --- a/onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh +++ b/onnxruntime/python/tools/tensorrt/perf/build/checkout_trt.sh @@ -1,5 +1,6 @@ #!/bin/bash +echo "checking out onnx-tensorrt for correct version" echo "$1" if [ ! "$1" = "8.2" ] then @@ -15,4 +16,4 @@ then git checkout "$1"'.1' fi cd $CUR_PWD -fi \ No newline at end of file +fi diff --git a/onnxruntime/python/tools/tensorrt/perf/perf_utils.py b/onnxruntime/python/tools/tensorrt/perf/perf_utils.py index 08528147fe..5182147fa3 100644 --- a/onnxruntime/python/tools/tensorrt/perf/perf_utils.py +++ b/onnxruntime/python/tools/tensorrt/perf/perf_utils.py @@ -33,7 +33,8 @@ group_title = 'Group' avg_ending = ' \nmean (ms)' percentile_ending = ' \n90th percentile (ms)' memory_ending = ' \npeak memory usage (MiB)' - +session_ending = ' \n session creation time (s)' +ort_provider_list = [cpu, cuda, trt, cuda_fp16, trt_fp16] provider_list = [cpu, cuda, trt, standalone_trt, cuda_fp16, trt_fp16, standalone_trt_fp16] table_headers = [model_title] + provider_list @@ -260,4 +261,4 @@ def get_profile_metrics(path, profile_already_parsed, logger=None): logger.info("No profile metrics got.") return None - return data[-1] \ No newline at end of file + return data[-1] diff --git a/onnxruntime/python/tools/tensorrt/perf/post.py b/onnxruntime/python/tools/tensorrt/perf/post.py index 6285a64772..037be08571 100644 --- a/onnxruntime/python/tools/tensorrt/perf/post.py +++ b/onnxruntime/python/tools/tensorrt/perf/post.py @@ -25,6 +25,7 @@ latency = 'latency' status = 'status' latency_over_time = 'latency_over_time' specs = 'specs' +session = 'session' time_string_format = '%Y-%m-%d %H:%M:%S' @@ -92,16 +93,23 @@ def get_status(status, model_group): status = adjust_columns(status, status_columns, status_db_columns, model_group) return status -def get_specs(specs, branch, commit_id): +def get_specs(specs, branch, commit_id, upload_time): specs = specs.append({'.': 6, 'Spec': 'Branch', 'Version' : branch}, ignore_index=True) specs = specs.append({'.': 7, 'Spec': 'CommitId', 'Version' : commit_id}, ignore_index=True) + specs = specs.append({'.': 8, 'Spec': 'UploadTime', 'Version' : upload_time}, ignore_index=True) return specs -def write_table(ingest_client, table, table_name, trt_version, upload_time): +def get_session(session, model_group): + session_columns = session.keys() + session_db_columns = [model_title] + ort_provider_list + session = adjust_columns(session, session_columns, session_db_columns, model_group) + return session + +def write_table(ingest_client, table, table_name, upload_time, identifier): if table.empty: return - table = table.assign(TrtVersion=trt_version) # add TrtVersion table = table.assign(UploadTime=upload_time) # add UploadTime + table = table.assign(Identifier=identifier) # add Identifier ingestion_props = IngestionProperties( database=database, table=table_name, @@ -115,6 +123,9 @@ def get_time(): date_time = time.strftime(time_string_format) return date_time +def get_identifier(commit_id, trt_version, branch): + return commit_id + '_' + trt_version + '_' + branch + def main(): args = parse_arguments() @@ -123,14 +134,15 @@ def main(): kcsb_ingest = KustoConnectionStringBuilder.with_az_cli_authentication(cluster_ingest) ingest_client = QueuedIngestClient(kcsb_ingest) date_time = get_time() - + identifier = get_identifier(args.commit_hash, args.trt_version, args.branch) + try: result_file = args.report_folder folders = os.listdir(result_file) os.chdir(result_file) - tables = [fail, memory, latency, status, latency_over_time, specs] + tables = [fail, memory, latency, status, latency_over_time, specs, session] table_results = {} for table_name in tables: table_results[table_name] = pd.DataFrame() @@ -140,8 +152,10 @@ def main(): csv_filenames = os.listdir() for csv in csv_filenames: table = parse_csv(csv) + if session in csv: + table_results[session] = table_results[session].append(get_session(table, model_group), ignore_index=True) if specs in csv: - table_results[specs] = table_results[specs].append(get_specs(table, args.branch, args.commit_hash), ignore_index=True) + table_results[specs] = table_results[specs].append(get_specs(table, args.branch, args.commit_hash, date_time), ignore_index=True) if fail in csv: table_results[fail] = table_results[fail].append(get_failures(table, model_group), ignore_index=True) if latency in csv: @@ -154,7 +168,7 @@ def main(): for table in tables: print('writing ' + table + ' to database') db_table_name = 'ep_model_' + table - write_table(ingest_client, table_results[table], db_table_name, args.trt_version, date_time) + write_table(ingest_client, table_results[table], db_table_name, date_time, identifier) except BaseException as e: print(str(e)) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml index 6b18a83e93..1282b6e669 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml @@ -171,6 +171,12 @@ jobs: scriptLocation: inlineScript scriptType: bash inlineScript: | - python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory)/Artifact/result -c $(Build.SourceVersion) -u "https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" -t ${{ parameters.TrtVersion }} -b $(branch) + short_hash=$(git rev-parse --short HEAD^) && + python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory)/Artifact/result -c $short_hash -u "https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" -t ${{ parameters.TrtVersion }} -b $(branch) + + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - template: templates/clean-agent-build-directory-step.yml \ No newline at end of file From 13e277525cff3b9e9d8a0b712bf191893ec9a114 Mon Sep 17 00:00:00 2001 From: Cheng Tang Date: Thu, 20 Jan 2022 21:36:29 +0000 Subject: [PATCH 17/59] fix whitelist --- orttraining/orttraining/eager/opgen/opgen/atenops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orttraining/orttraining/eager/opgen/opgen/atenops.py b/orttraining/orttraining/eager/opgen/opgen/atenops.py index b9994b9b96..36710af30c 100644 --- a/orttraining/orttraining/eager/opgen/opgen/atenops.py +++ b/orttraining/orttraining/eager/opgen/opgen/atenops.py @@ -94,7 +94,7 @@ hand_implemented = { } ops = {**ops, **hand_implemented} -# TODO: this is a temporary whitelist for ops need type promotion +# TODO: this is a temporary allowlist for ops need type promotion # Need to enhance the support for onnx type constrains to automatically # resolve whether the op need type promotion. # Will remove this list in the future. From 141606534c8e9f3089437b2300a7a715ec8b7d54 Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Fri, 21 Jan 2022 13:37:59 -0800 Subject: [PATCH 18/59] Add support for FusedAdam to be mathematically equivalent to pytorch/AdamW (#10106) --- .../python/training/optim/__init__.py | 2 +- .../python/training/optim/fused_adam.py | 34 +++++++----- .../cuda/fused_ops/multi_tensor_adam.cu | 20 +++++-- .../python/orttraining_test_ortmodule_api.py | 53 ++++++++++++++++++- 4 files changed, 89 insertions(+), 20 deletions(-) diff --git a/orttraining/orttraining/python/training/optim/__init__.py b/orttraining/orttraining/python/training/optim/__init__.py index 7d35a84b40..291268307d 100644 --- a/orttraining/orttraining/python/training/optim/__init__.py +++ b/orttraining/orttraining/python/training/optim/__init__.py @@ -2,5 +2,5 @@ from .config import _OptimizerConfig, AdamConfig, LambConfig, SGDConfig from .lr_scheduler import _LRScheduler, ConstantWarmupLRScheduler, CosineWarmupLRScheduler,\ LinearWarmupLRScheduler, PolyWarmupLRScheduler -from .fused_adam import FusedAdam +from .fused_adam import FusedAdam, AdamWMode from .fp16_optimizer import FP16_Optimizer diff --git a/orttraining/orttraining/python/training/optim/fused_adam.py b/orttraining/orttraining/python/training/optim/fused_adam.py index a6c36752d2..e655468e41 100644 --- a/orttraining/orttraining/python/training/optim/fused_adam.py +++ b/orttraining/orttraining/python/training/optim/fused_adam.py @@ -12,13 +12,22 @@ This file is adapted from fused adam in NVIDIA/apex, commit a109f85 import torch from ._multi_tensor_apply import MultiTensorApply +from enum import IntEnum + + +class AdamWMode(IntEnum): + ADAM_L2_REGULARIZATION = 0 # Adam with L2 regularization + ADAMW_TRANSFORMERS = 1 # Adam with weight decay implemented to be equivalent to Transformers/AdamW + ADAMW_TORCH = 2 # Adam with weight decay implemented to be equivalent to torch/AdamW class FusedAdam(torch.optim.Optimizer): """Implements Adam algorithm. - The algorithmic implementation is mathematically equivalent to Transformers/AdamW - as defined here: https://github.com/huggingface/transformers/blob/61f64262692ac7dc90e2e0bdeb7e79d9cd607a66/src/transformers/optimization.py#L349-L370 + The algorithmic implementation is mathematically equivalent to + `Transformers/AdamW `_ + when adam_w_mode = 1 and `torch/Adam `_ + when adam_w_mode = 2 Currently GPU-only. @@ -27,7 +36,7 @@ class FusedAdam(torch.optim.Optimizer): * Fusion of the Adam update's elementwise operations * A multi-tensor apply launch that batches the elementwise updates applied to all the model's parameters into one or a few kernel launches. - Adam was been proposed in `Adam: A Method for Stochastic Optimization`_. + Adam was proposed in `Adam: A Method for Stochastic Optimization`_. Arguments: params (iterable): iterable of parameters to optimize or dicts defining @@ -38,11 +47,11 @@ class FusedAdam(torch.optim.Optimizer): eps (float, optional): term added to the denominator to improve numerical stability. (default: 1e-8) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) - amsgrad (boolean, optional): whether to use the AMSGrad variant of this - algorithm from the paper `On the Convergence of Adam and Beyond`_ - (default: False) NOT SUPPORTED in FusedAdam! - adam_w_mode (boolean, optional): Apply L2 regularization or weight decay - True for decoupled weight decay(also known as AdamW) (default: True) + adam_w_mode (AdamWMode, optional): Apply L2 regularization or weight decay + (AdamWMode.ADAM_L2_REGULARIZATION), decoupled weight decay with + transformers/AdamW mathematical implementation (AdamWMode.ADAMW_TRANSFORMERS) + or decoupled weight decay with transformers/AdamW implementation + (AdamWMode.ADAMW_TORCH) (default: AdamWMode.ADAMW_TRANSFORMERS) set_grad_none (bool, optional): whether set grad to None when zero_grad() method is called. (default: True) @@ -58,23 +67,20 @@ class FusedAdam(torch.optim.Optimizer): betas=(0.9, 0.999), eps=1e-6, - adam_w_mode=True, + adam_w_mode=AdamWMode.ADAMW_TRANSFORMERS, weight_decay=0., - amsgrad=False, - set_grad_none=False): + set_grad_none=True): # The FusedAdam implementation is mathematically equivalent to # transformers AdamW. The input arguments also have the same defaults. - if amsgrad: - raise RuntimeError('FusedAdam does not support the AMSGrad variant.') defaults = dict(lr=lr, bias_correction=bias_correction, betas=betas, eps=eps, weight_decay=weight_decay) super(FusedAdam, self).__init__(params, defaults) - self._adam_w_mode = 1 if adam_w_mode else 0 + self._adam_w_mode = adam_w_mode self._set_grad_none = set_grad_none # Skip buffer diff --git a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/fused_ops/multi_tensor_adam.cu b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/fused_ops/multi_tensor_adam.cu index 65601b2e19..affb0afa91 100644 --- a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/fused_ops/multi_tensor_adam.cu +++ b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/fused_ops/multi_tensor_adam.cu @@ -24,8 +24,9 @@ #define ILP 4 typedef enum { - ADAM_MODE_0 = 0, // L2 regularization mode - ADAM_MODE_1 = 1 // Decoupled weight decay mode(AdamW) + ADAM_MODE_0 = 0, // L2 regularization mode + ADAM_MODE_1 = 1, // Decoupled weight decay mode (AdamW) as implemented in transformers/AdamW + ADAM_MODE_2 = 2 // Decoupled weight decay mode (AdamW) as implemented in pytorch/AdamW } adamMode_t; using MATH_T = float; @@ -40,6 +41,8 @@ struct AdamFunctor { const float epsilon, const float lr, const float lr_corrected, + const float bias_correction1, + const float bias_correction2, adamMode_t mode, const float decay) { @@ -91,13 +94,20 @@ struct AdamFunctor { r_v[ii] = beta2 * r_v[ii] + (1 - beta2) * r_g[ii] * r_g[ii]; MATH_T denom = sqrtf(r_v[ii]) + epsilon; r_p[ii] = r_p[ii] - (lr_corrected * r_m[ii] / denom); - } else { // weight decay + } else if (mode == ADAM_MODE_1) { // weight decay // Adapted to be mathematically equivalent to transformers AdamW r_m[ii] = beta1 * r_m[ii] + (1 - beta1) * r_g[ii]; r_v[ii] = beta2 * r_v[ii] + (1 - beta2) * r_g[ii] * r_g[ii]; MATH_T denom = sqrtf(r_v[ii]) + epsilon; r_p[ii] = r_p[ii] - (lr_corrected * r_m[ii] / denom); r_p[ii] = r_p[ii] - (lr * decay * r_p[ii]); + } else if (mode == ADAM_MODE_2) { + // Adapted to be mathematically equivalent to torch AdamW + r_p[ii] = r_p[ii] - (r_p[ii] * lr * decay); + r_m[ii] = beta1 * r_m[ii] + (1 - beta1) * r_g[ii]; + r_v[ii] = beta2 * r_v[ii] + (1 - beta2) * r_g[ii] * r_g[ii]; + MATH_T denom = (sqrtf(r_v[ii]) / sqrtf(bias_correction2)) + epsilon; + r_p[ii] = r_p[ii] - (lr * r_m[ii]) / (bias_correction1 * denom); } } #pragma unroll @@ -128,7 +138,7 @@ void multi_tensor_adam_cuda(int chunk_size, using namespace at; // Handle bias correction mode - double bias_correction1 = 1.0, bias_correction2 = 1.0; + float bias_correction1 = 1.0, bias_correction2 = 1.0; float lr_corrected = lr; if (bias_correction == 1) { bias_correction1 = 1 - std::pow(beta1, step); @@ -150,6 +160,8 @@ void multi_tensor_adam_cuda(int chunk_size, epsilon, lr, lr_corrected, + bias_correction1, + bias_correction2, (adamMode_t)mode, weight_decay);) diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py index 0964e47f21..be24d7c091 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py @@ -30,7 +30,7 @@ from onnxruntime.training.ortmodule import (ORTModule, _graph_execution_manager) import onnxruntime.training.ortmodule as ortmodule_module -from onnxruntime.training.optim import FusedAdam +from onnxruntime.training.optim import FusedAdam, AdamWMode from transformers import AdamW import _test_helpers @@ -4428,6 +4428,57 @@ def test_ortmodule_fused_adam_optimizer_correctness(): for pt_param, ort_param in zip(pt_model.parameters(), ort_model.parameters()): _test_helpers.assert_values_are_close(pt_param, ort_param, atol=1e-4, rtol=1e-5) +def test_ortmodule_fused_adam_optimizer_correctness_torch(): + + torch.manual_seed(8888) + + device = 'cuda' + N, D_in, H, D_out = 4, 4, 8, 4 + + pt_model = NeuralNetSinglePositionalArgument(D_in, H, D_out).to(device) + adamw_optimizer = torch.optim.AdamW(pt_model.parameters(), lr=1e-3) + + ort_model = ORTModule(copy.deepcopy(pt_model)) + ort_fused_adam_optimizer = FusedAdam(ort_model.parameters(), lr=1e-3, + adam_w_mode=AdamWMode.ADAMW_TORCH, + weight_decay=0.01, + eps=1e-8) + + def run_step(model, x): + prediction = model(x) + loss = prediction.sum() + loss.backward() + + return loss + + def run_optim_step(optimizer): + optimizer.step() + optimizer.zero_grad() + + ga_steps = 2 + pt_model.zero_grad() + ort_model.zero_grad() + + for step in range(1000): + x1 = torch.randn(N, D_in, device=device, dtype=torch.float32) + x2 = copy.deepcopy(x1) + + pt_loss = run_step(pt_model, x1) + ort_loss = run_step(ort_model, x2) + + for pt_param, ort_param in zip(pt_model.parameters(), ort_model.parameters()): + ort_param.grad = copy.deepcopy(pt_param.grad) + + _test_helpers.assert_values_are_close(pt_loss, ort_loss, atol=1e-4, rtol=1e-5) + _test_helpers.assert_gradients_match_and_reset_gradient(ort_model, pt_model, atol=1e-4, rtol=1e-5, reset_gradient=False) + + if (step+1) % ga_steps == 0: + run_optim_step(adamw_optimizer) + run_optim_step(ort_fused_adam_optimizer) + + for pt_param, ort_param in zip(pt_model.parameters(), ort_model.parameters()): + _test_helpers.assert_values_are_close(pt_param, ort_param, atol=1e-4, rtol=1e-5) + def test_sigmoid_grad(): class NeuralNetSigmoid(torch.nn.Module): def __init__(self, input_size, hidden_size, num_classes): From bfabef081dfb16226b056f33440d98ee7861f9d5 Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Fri, 21 Jan 2022 14:31:34 -0800 Subject: [PATCH 19/59] Remove unused pipeline orttraining-linux-gpu-perf-test-ci-pipeline.yml and unused send_perf_metrics tool. (#10326) --- .gitignore | 4 - ...aining-linux-gpu-perf-test-ci-pipeline.yml | 90 ---------- tools/perf_util/pom.xml | 56 ------ .../java/com/msft/send_perf_metrics/App.java | 168 ------------------ .../com/msft/send_perf_metrics/JdbcUtil.java | 19 -- 5 files changed, 337 deletions(-) delete mode 100644 tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml delete mode 100644 tools/perf_util/pom.xml delete mode 100644 tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java delete mode 100644 tools/perf_util/src/main/java/com/msft/send_perf_metrics/JdbcUtil.java diff --git a/.gitignore b/.gitignore index be2375d256..d27dedbc2b 100644 --- a/.gitignore +++ b/.gitignore @@ -49,10 +49,6 @@ java/gradle java/.gradle java/hs_*.log onnxruntime/python/version_info.py -/tools/perf_util/target/classes/com/msft/send_perf_metrics -/tools/perf_util/send_perf_metrics.iml -/tools/perf_util/target/classes -/tools/perf_util/src/main/resources /orttraining/orttraining/eager/ort_aten.g.cpp /orttraining/orttraining/eager/ort_customops.g.cpp /csharp/**/packages diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml deleted file mode 100644 index bb56fb7c34..0000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml +++ /dev/null @@ -1,90 +0,0 @@ -trigger: none - -jobs: -- job: Onnxruntime_Linux_GPU_Training_Perf_Test - - timeoutInMinutes: 120 - - variables: - - group: 'ortperf' # variable group - - steps: - - checkout: self - clean: true - submodules: recursive - - - template: templates/run-docker-build-steps.yml - parameters: - RunDockerBuildArgs: > - -o ubuntu20.04 -d gpu - -t onnxruntime_perf_test_image - -x " - --config RelWithDebInfo - --enable_training - --update --build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70 - " - DisplayName: 'Build performance tests' - - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/bert-data" -d "/bert_data" - displayName: 'Mount bert-data' - condition: succeededOrFailed() # ensure all tests are run - - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/gpt2-data" -d "/gpt2_data" - displayName: 'Mount gpt2 test data' - condition: succeededOrFailed() # ensure all tests are run - - - script: > - docker run --gpus all --rm --name onnxruntime-gpu-perf - --volume $(Build.SourcesDirectory):/onnxruntime_src - --volume $(Build.BinariesDirectory):/build - --volume /bert_data/bert_models:/build/bert_models:ro - --volume /bert_data:/build/bert_data:ro - -e NIGHTLY_BUILD onnxruntime_perf_test_image - /usr/bin/python3 /onnxruntime_src/orttraining/tools/ci_test/run_bert_perf_test.py - --binary_dir /build/RelWithDebInfo - --training_data_root /build/bert_data - --model_root /build/bert_models - displayName: 'Run bert performance tests' - condition: succeededOrFailed() - timeoutInMinutes: 120 - - - script: > - docker run --gpus all --rm --name onnxruntime-gpu-perf - --volume $(Build.SourcesDirectory):/onnxruntime_src - --volume $(Build.BinariesDirectory):/build - --volume /gpt2_data/gpt2_models:/build/gpt2_models:ro - --volume /gpt2_data:/build/gpt2_data:ro - -e NIGHTLY_BUILD onnxruntime_perf_test_image - /usr/bin/python3 /onnxruntime_src/orttraining/tools/ci_test/run_gpt2_perf_test.py - --binary_dir /build/RelWithDebInfo - --training_data_root /build/gpt2_data - --model_root /build/gpt2_models - displayName: 'Run gpt-2 performance tests' - condition: succeededOrFailed() - timeoutInMinutes: 120 - - # generate jdbc.properties - - script: > - mkdir -p $(Build.SourcesDirectory)/tools/perf_util/src/main/resources && - printf "url=jdbc:mysql://onnxruntimedashboard.mysql.database.azure.com/onnxruntime?serverTimezone=UTC&useUnicode=true&characterEncoding=UTF-8\nuser=powerbi@onnxruntimedashboard\npassword_env=ORT_PERF_PASSWORD" - > $(Build.SourcesDirectory)/tools/perf_util/src/main/resources/jdbc.properties - displayName: 'Create resource file' - - - script: > - mvn package - displayName: 'Maven build' - workingDirectory: $(Build.SourcesDirectory)/tools/perf_util - - # process json files - - script: > - java -cp target/send_perf_metrics-0.0.1-SNAPSHOT-jar-with-dependencies.jar com.msft.send_perf_metrics.App "$(Build.SourcesDirectory)/orttraining/tools/ci_test/results" - env: - ORT_PERF_PASSWORD: $(ortperf) - displayName: 'Populate perf metrics' - workingDirectory: $(Build.SourcesDirectory)/tools/perf_util - - - template: templates/component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - - - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/perf_util/pom.xml b/tools/perf_util/pom.xml deleted file mode 100644 index b5e129f4af..0000000000 --- a/tools/perf_util/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - 4.0.0 - - com.msft - send_perf_metrics - 0.0.1-SNAPSHOT - jar - - send_perf_metrics - http://maven.apache.org - - - - - maven-assembly-plugin - 3.1.1 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - - - UTF-8 - 1.8 - 1.8 - - - - - - com.googlecode.json-simple - json-simple - 1.1.1 - - - - mysql - mysql-connector-java - 8.0.22 - - - diff --git a/tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java b/tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java deleted file mode 100644 index a0a04211f6..0000000000 --- a/tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java +++ /dev/null @@ -1,168 +0,0 @@ -package com.msft.send_perf_metrics; - -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; - -import java.io.*; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.text.SimpleDateFormat; -import java.util.*; - -public class App { - - static String exec_command(Path source_dir, String... commands) throws Exception { - ProcessBuilder sb = new ProcessBuilder(commands).directory(source_dir.toFile()).redirectErrorStream(true); - Process p = sb.start(); - if (p.waitFor() != 0) - throw new RuntimeException("execute " + String.join(" ", commands) + " failed"); - try (BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()))) { - return r.readLine(); - } - } - - public static void main(String[] args) throws Exception { - - final Path source_dir = Paths.get(args[0]); - final List perf_metrics = new ArrayList(); - Files.walkFileTree(source_dir, new SimpleFileVisitor() { - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - String dirname = dir.getFileName().toString(); - if (dirname != "." && dirname.startsWith(".")) - return FileVisitResult.SKIP_SUBTREE; - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - String filename = file.getFileName().toString(); - - if (!filename.startsWith(".") && filename.endsWith(".json")) { - perf_metrics.add(file); - System.out.println(filename); - } - return FileVisitResult.CONTINUE; - } - - }); - - final Path cwd_dir = Paths.get(System.getProperty("user.dir")); - // git rev-parse HEAD - String commit_id = exec_command(cwd_dir, "git", "rev-parse", "HEAD"); - String date = exec_command(cwd_dir, "git", "show", "-s", "--format=%ci", commit_id); - final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z"); - java.util.Date commitDate = sdf.parse(date); - final SimpleDateFormat simple_date_format = new SimpleDateFormat("yyyy-MM-dd"); - String batch_id = simple_date_format.format(commitDate); - System.out.println(String.format("Commit change date: %s", batch_id)); - - // collect all json files list - processPerfMetrics(perf_metrics, commit_id, batch_id); - - // TODO - add e2e tests later, run it w/ process command - } - - private static void processPerfMetrics(final List perf_metrics, String commit_id, - String batch_id) throws Exception { - try { - Connection conn = JdbcUtil.GetConn(); - System.out.println("MySQL DB connection established.\n"); - // go thru each json file - JSONParser jsonParser = new JSONParser(); - for (Path metrics_json : perf_metrics) { - try (FileReader reader = new FileReader(metrics_json.toAbsolutePath().toString())) { - // Read JSON file - Object obj = jsonParser.parse(reader); - loadMetricsIntoMySQL(conn, commit_id, batch_id, (JSONObject) obj); - } - } - } catch (Exception e) { - e.printStackTrace(); - throw e; - } - } - - - static private void loadMetricsIntoMySQL(java.sql.Connection conn, String commit_id, String batch_id, - JSONObject json_object) throws Exception { - - // field name -> json value - Map field_mapping = new LinkedHashMap(); - Set update_on_duplicate_fields = - new LinkedHashSet<> (Arrays.asList("AvgTimePerBatch", "Throughput", "StabilizedThroughput", "EndToEndThroughput", "TotalTime", "AvgCPU", "Memory")); - - field_mapping.put("BatchId", batch_id); - field_mapping.put("CommitId", commit_id.substring(0, 8)); - json_object.forEach((key, value) -> { - if (key.equals("DerivedProperties")) { - JSONObject properties = (JSONObject) json_object.get("DerivedProperties"); - properties.forEach((sub_key, sub_value) -> { - field_mapping.put((String)sub_key, sub_value); - }); - } else { - field_mapping.put((String)key, value); - } - }); - - // building sql statement - StringBuilder sb = new StringBuilder("INSERT INTO perf_test_training_data ("); - field_mapping.forEach((key, value) -> { - sb.append(key).append(","); - }); - sb.append("Time) values ("); - for(int i = 0; i < field_mapping.size(); i++) { - sb.append("?,"); - } - sb.append("Now()) ON DUPLICATE KEY UPDATE "); - update_on_duplicate_fields.forEach((key) -> { - if(field_mapping.get(key) != null) { - sb.append(key).append("=?,"); - } - }); - - try (java.sql.PreparedStatement st = conn.prepareStatement(sb.substring(0, sb.length() - 1))) { - int i = 0; // param index - for (Map.Entry entry : field_mapping.entrySet()) { - setSqlParam(++i, st, entry.getValue()); - } - - // update section - for(String key : update_on_duplicate_fields) { - Object value = field_mapping.get(key); - if(value != null) { - setSqlParam(++i, st, value); - } - } - - st.executeUpdate(); - } catch (Exception e) { - e.printStackTrace(); - throw e; - } - - } - - static void setSqlParam(int param_index, PreparedStatement st, Object value) throws Exception { - if (value instanceof String) { - st.setString(param_index, (String) value); - } else if (value instanceof Long) { - st.setInt(param_index, (int) (long) value); - } else if (value instanceof Double) { - st.setFloat(param_index, (float) (double) value); - } else if (value instanceof Boolean) { - st.setBoolean(param_index, (Boolean) value); - } else { - throw new Exception("Unsupported data type:" + value.getClass().getName()); - } - } - -} diff --git a/tools/perf_util/src/main/java/com/msft/send_perf_metrics/JdbcUtil.java b/tools/perf_util/src/main/java/com/msft/send_perf_metrics/JdbcUtil.java deleted file mode 100644 index d7132f7f26..0000000000 --- a/tools/perf_util/src/main/java/com/msft/send_perf_metrics/JdbcUtil.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.msft.send_perf_metrics; - -import java.sql.DriverManager; -import java.util.Map; -import java.util.Properties; - -public class JdbcUtil { - static java.sql.Connection GetConn() throws Exception { - try (java.io.InputStream in = App.class.getResourceAsStream("/jdbc.properties")) { - if (in == null) - throw new RuntimeException("Error reading jdbc properties"); - Properties props = new Properties(); - props.load(in); - // loading password via env variable - return DriverManager.getConnection(props.getProperty("url"), props.getProperty("user"), - System.getenv(props.getProperty("password_env"))); - } - } -} From 6876641c1e6f270398a2345e2245b20a4e04454d Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Fri, 21 Jan 2022 19:35:58 -0800 Subject: [PATCH 20/59] Pin version of post to dashboard scripts' dependencies and update them to work with recent version. (#10353) --- .../binary-size-checks-pipeline.yml | 1 + .../azure-pipelines/templates/c-api-cpu.yml | 15 ++++----------- .../github/linux/upload_code_coverage_data.sh | 3 ++- .../windows/post_binary_sizes_to_dashboard.py | 6 ++++-- .../windows/post_code_coverage_to_dashboard.py | 8 +++++--- .../windows/post_to_dashboard/requirements.txt | 2 ++ 6 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 tools/ci_build/github/windows/post_to_dashboard/requirements.txt diff --git a/tools/ci_build/github/azure-pipelines/binary-size-checks-pipeline.yml b/tools/ci_build/github/azure-pipelines/binary-size-checks-pipeline.yml index 4ed1b8e3e5..522bbaf52d 100644 --- a/tools/ci_build/github/azure-pipelines/binary-size-checks-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/binary-size-checks-pipeline.yml @@ -65,6 +65,7 @@ jobs: echo "File not found: ${BINARY_SIZE_DATA_FILE}" exit 1 fi + /usr/bin/python3 -m pip install -r $(Build.SourcesDirectory)/tools/ci_build/github/windows/post_to_dashboard/requirements.txt && \ /usr/bin/python3 $(Build.SourcesDirectory)/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py \ --commit_hash=$(Build.SourceVersion) \ --size_data_file="${BINARY_SIZE_DATA_FILE}" \ diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 84002ad381..2c42f6b255 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -857,19 +857,10 @@ jobs: parameters: packageFolder: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - - task: PowerShell@2 - displayName: 'Get Current Date' - inputs: - targetType: 'inline' - script: | - $date = $(Get-Date -Format "yyyy-MM-dd") - Write-Host "##vso[task.setvariable variable=CurrentDate]$date" - - task: CmdLine@2 displayName: 'Post binary sizes to the dashboard database using command line' inputs: script: | - python.exe -m pip install azure-kusto-data[pandas] azure-kusto-ingest[pandas] echo changing directory to artifact download path cd $(Build.BinariesDirectory)/nuget-artifact/final-package echo processing nupkg @@ -888,8 +879,9 @@ jobs: 7z.exe l -slt %%~ni.zip runtimes\osx.10.14-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo osx,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x86,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt - ) - ) + ) + ) + - task: AzureCLI@2 displayName: 'Azure CLI' inputs: @@ -897,6 +889,7 @@ jobs: scriptLocation: inlineScript scriptType: batch inlineScript: | + python.exe -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_to_dashboard\requirements.txt && ^ python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=$(Build.SourceVersion) --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/linux/upload_code_coverage_data.sh b/tools/ci_build/github/linux/upload_code_coverage_data.sh index 0130c71fbb..2f63e4c2fe 100755 --- a/tools/ci_build/github/linux/upload_code_coverage_data.sh +++ b/tools/ci_build/github/linux/upload_code_coverage_data.sh @@ -1,5 +1,6 @@ #!/bin/bash # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -set -x +set -x -e +/usr/bin/env python3 -m pip install -r $BUILD_SOURCESDIRECTORY/tools/ci_build/github/windows/post_to_dashboard/requirements.txt $BUILD_SOURCESDIRECTORY/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py --commit_hash=$BUILD_SOURCEVERSION --report_file $1 --report_url $2 --branch $BUILD_SOURCEBRANCHNAME --arch $3 --os $4 --build_config $5 \ No newline at end of file diff --git a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py index 16cd8294ec..6468366d68 100644 --- a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py +++ b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py @@ -9,10 +9,12 @@ import os import datetime # ingest from dataframe import pandas -from azure.kusto.data import KustoConnectionStringBuilder +from azure.kusto.data import ( + DataFormat, + KustoConnectionStringBuilder, +) from azure.kusto.ingest import ( IngestionProperties, - DataFormat, ReportLevel, QueuedIngestClient, ) diff --git a/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py b/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py index e5cd8a8b28..182c216033 100755 --- a/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py +++ b/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py @@ -14,10 +14,12 @@ import sys import datetime # ingest from dataframe import pandas -from azure.kusto.data import KustoConnectionStringBuilder +from azure.kusto.data import ( + DataFormat, + KustoConnectionStringBuilder, +) from azure.kusto.ingest import ( IngestionProperties, - DataFormat, ReportLevel, QueuedIngestClient, ) @@ -25,7 +27,7 @@ from azure.kusto.ingest import ( def parse_arguments(): parser = argparse.ArgumentParser( - description="ONNXRuntime test coverge report uploader for dashboard") + description="ONNXRuntime test coverage report uploader for dashboard") parser.add_argument("--report_url", type=str, help="URL to the LLVM json report") parser.add_argument( "--report_file", type=str, help="Path to the local JSON/TXT report", required=True) diff --git a/tools/ci_build/github/windows/post_to_dashboard/requirements.txt b/tools/ci_build/github/windows/post_to_dashboard/requirements.txt new file mode 100644 index 0000000000..b8c00a610b --- /dev/null +++ b/tools/ci_build/github/windows/post_to_dashboard/requirements.txt @@ -0,0 +1,2 @@ +azure-kusto-data[pandas]==3.0.1 +azure-kusto-ingest[pandas]==3.0.1 From 42db893607b3a4789cc2a13924269986a781ca9c Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Sat, 22 Jan 2022 13:29:07 -0800 Subject: [PATCH 21/59] Add ThresholdedRelu to ROCm EP. (#9480) Sources were already hipified and compiled, but ROCm EP registration was missing. --- onnxruntime/core/providers/rocm/rocm_execution_provider.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/rocm/rocm_execution_provider.cc b/onnxruntime/core/providers/rocm/rocm_execution_provider.cc index 33000f4a5f..6aead1c46b 100644 --- a/onnxruntime/core/providers/rocm/rocm_execution_provider.cc +++ b/onnxruntime/core/providers/rocm/rocm_execution_provider.cc @@ -1588,9 +1588,9 @@ static Status RegisterRocmKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - // BuildKernelCreateInfo, - // BuildKernelCreateInfo, - // BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, // BuildKernelCreateInfo, BuildKernelCreateInfo, From 3dfadf9031e6a80f487719e6603c2cc2f75fe65b Mon Sep 17 00:00:00 2001 From: PeixuanZuo <94887879+PeixuanZuo@users.noreply.github.com> Date: Mon, 24 Jan 2022 15:34:48 +0800 Subject: [PATCH 22/59] [FIX] Add condition in amd ci pipeline yaml to stop test in time when onnxruntime build failed (#10335) * [FIX] Add condition in amd ci pipeline yaml to stop test in time when onnxruntime build failed. --- .../orttraining-pai-ci-pipeline.yml | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml index d92e371852..be6938a3ea 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml @@ -12,6 +12,8 @@ jobs: value: 44 - name: render value: 109 + - name: onnxruntimeBuildSucceeded + value: false # generated from tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile container: @@ -50,17 +52,21 @@ jobs: --skip_tests displayName: 'Build onnxruntime' + - bash: |- + echo "##vso[task.setvariable variable=onnxruntimeBuildSucceeded]true" + displayName: 'Set Onnxruntime Build Succeeded' + - script: |- cd ./build/RelWithDebInfo &&\ ../../tools/ci_build/github/pai/pai_test_launcher.sh displayName: 'Run onnxruntime unit tests' - + - script: |- cd ./build/RelWithDebInfo export PYTHONPATH=$PWD python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install displayName: 'Compile torch extensions into build directory' - condition: succeededOrFailed() # ensure all tests are run + condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed - script: |- cd ./build/RelWithDebInfo @@ -85,7 +91,7 @@ jobs: ci-pipeline-actual.json \ ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm4.3.1.json displayName: 'Run Python Hugging-Face BERT-L test' - condition: succeededOrFailed() # ensure all tests are run + condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed - script: |- cd ./build/RelWithDebInfo @@ -111,7 +117,7 @@ jobs: ci-pipeline-actual.json \ ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm4.3.1.json displayName: 'Run Python Hugging-Face GPT2 test' - condition: succeededOrFailed() # ensure all tests are run + condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed # - script: |- # cd ./build/RelWithDebInfo @@ -191,7 +197,7 @@ jobs: ci-pipeline-actual.json \ ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm4.3.1.json displayName: 'Run Python Hugging-Face DistilBERT test' - condition: succeededOrFailed() # ensure all tests are run + condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed #- script: |- # cd ./build/RelWithDebInfo @@ -250,7 +256,7 @@ jobs: --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ --target_dir training_e2e_test_data \ --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 - condition: succeededOrFailed() # ensure all tests are run + condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed displayName: 'Download onnxruntime_training_data.zip data' - script: |- @@ -259,7 +265,7 @@ jobs: --model_root training_e2e_test_data/models \ --gpu_sku MI100_32G displayName: 'Run C++ BERT-L batch size test' - condition: succeededOrFailed() # ensure all tests are run + condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed - script: |- python orttraining/tools/ci_test/run_bert_perf_test.py \ @@ -268,7 +274,7 @@ jobs: --training_data_root training_e2e_test_data/data \ --gpu_sku MI100_32G displayName: 'Run C++ BERT-L performance test' - condition: succeededOrFailed() # ensure all tests are run + condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed - script: |- python orttraining/tools/ci_test/run_convergence_test.py \ @@ -277,4 +283,4 @@ jobs: --training_data_root training_e2e_test_data/data \ --gpu_sku MI100_32G displayName: 'Run C++ BERT-L convergence test' - condition: succeededOrFailed() # ensure all tests are run + condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed From 5df15c56444fe1b5f770eaca38d8031b5ffe151e Mon Sep 17 00:00:00 2001 From: wejoncy <247153481@qq.com> Date: Tue, 25 Jan 2022 02:17:56 +0800 Subject: [PATCH 23/59] additional options of NNAPI for ORT_PERF_TOOL (#10351) * additional options of NNAPI for ORT_PERF_TOOL * reuse current key '-i' * fix * fix * _MSC_VER won't be defined when build with NDK * fix * fix --- .../test/perftest/command_args_parser.cc | 10 ++++++++-- onnxruntime/test/perftest/ort_test_session.cc | 20 ++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 7d1a4e8b47..2667060331 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -65,8 +65,8 @@ namespace perftest { "\t [Usage]: -e -i '| |'\n\n" "\t [Example] [For OpenVINO EP] -e openvino -i \"device_type|CPU_FP32 enable_vpu_fast_compile|true num_of_threads|5 use_compiled_network|true blob_dump_path|\"\"\"\n" "\t [TensorRT only] [trt_max_partition_iterations]: Maximum iterations for TensorRT parser to get capability.\n" - "\t [TensorRT only] [trt_min_subgraph_size]: Minimum size of TensorRT subgraphs.\n" - "\t [TensorRT only] [trt_max_workspace_size]: Set TensorRT maximum workspace size in byte.\n" + "\t [TensorRT only] [trt_min_subgraph_size]: Minimum size of TensorRT subgraphs.\n" + "\t [TensorRT only] [trt_max_workspace_size]: Set TensorRT maximum workspace size in byte.\n" "\t [TensorRT only] [trt_fp16_enable]: Enable TensorRT FP16 precision.\n" "\t [TensorRT only] [trt_int8_enable]: Enable TensorRT INT8 precision.\n" "\t [TensorRT only] [trt_int8_calibration_table_name]: Specify INT8 calibration table name.\n" @@ -79,6 +79,12 @@ namespace perftest { "\t [TensorRT only] [trt_force_sequential_engine_build]: Force TensorRT engines to be built sequentially.\n" "\t [Usage]: -e -i '| |'\n\n" "\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n" + "\t [NNAPI only] [NNAPI_FLAG_USE_FP16]: Use fp16 relaxation in NNAPI EP..\n" + "\t [NNAPI only] [NNAPI_FLAG_USE_NCHW]: Use the NCHW layout in NNAPI EP.\n" + "\t [NNAPI only] [NNAPI_FLAG_CPU_DISABLED]: Prevent NNAPI from using CPU devices.\n" + "\t [NNAPI only] [NNAPI_FLAG_CPU_ONLY]: Using CPU only in NNAPI EP.\n" + "\t [Usage]: -e -i ' '\n\n" + "\t [Example] [For NNAPI EP] -e nnapi -i \" NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED \"\n" "\t-h: help\n"); } #ifdef _WIN32 diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 551709ea62..75a2c178ea 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -323,7 +323,25 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #endif } else if (provider_name == onnxruntime::kNnapiExecutionProvider) { #ifdef USE_NNAPI - Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Nnapi(session_options, 0)); + uint32_t nnapi_flags = 0; + std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; + std::istringstream ss(ov_string); + std::string key; + while (ss >> key) { + if (key == "NNAPI_FLAG_USE_FP16") { + nnapi_flags |= NNAPI_FLAG_USE_FP16; + } else if (key == "NNAPI_FLAG_USE_NCHW") { + nnapi_flags |= NNAPI_FLAG_USE_NCHW; + } else if (key == "NNAPI_FLAG_CPU_DISABLED") { + nnapi_flags |= NNAPI_FLAG_CPU_DISABLED; + } else if (key == "NNAPI_FLAG_CPU_ONLY") { + nnapi_flags |= NNAPI_FLAG_CPU_ONLY; + } else if (key.empty()) { + } else { + ORT_THROW("[ERROR] [NNAPI] wrong key type entered. Choose from the following runtime key options that are available for NNAPI. ['NNAPI_FLAG_USE_FP16', 'NNAPI_FLAG_USE_NCHW', 'NNAPI_FLAG_CPU_DISABLED', 'NNAPI_FLAG_CPU_ONLY'] \n"); + } + } + Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Nnapi(session_options, nnapi_flags)); #else ORT_THROW("NNAPI is not supported in this build\n"); #endif From 7e092a7e3f8f9f590de0ac5548035e8fa4d512f8 Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Mon, 24 Jan 2022 10:40:46 -0800 Subject: [PATCH 24/59] Reduce number of memory allocations based on a customer profiling case (#10193) Add abseil and inlined containers typedefs Introduce TensorShapeVector for shape building. Use gsl::span to make interfaces accept different types of vector like args. Introduce InineShapeVectorT for shape capacity typed instantiations Refactor cuda slice along with provider shared interfaces Refactor Concat, Conv, Pad Build with Conv Einsum and ConvTranspose refactored. Remove TesnorShape::GetDimsAsVector() Refactor SliceIterator and SliceIteratorBase Refactor broadcast Refactor Pads for twice as long Remove memory planner intermediate shapes vector Refactor orttraining Fix passing TenshroShapeVector to tests Remove abseil copy and submodule, use FetchContent_Declare/Fetch Path with separate command Make RocmAsyncBuffer accept anything convertible to span. Adjust Linux GPU pipeline. --- .gitmodules | 1 - cmake/external/abseil-cpp.cmake | 34 +++ cmake/onnxruntime_common.cmake | 4 +- cmake/onnxruntime_framework.cmake | 4 + cmake/onnxruntime_providers.cmake | 4 +- .../abseil/Fix_Nvidia_Build_Break.patch | 26 ++ .../onnxruntime/core/framework/allocator.h | 8 +- .../core/framework/op_node_proto_helper.h | 26 +- .../onnxruntime/core/framework/tensor_shape.h | 50 ++- onnxruntime/contrib_ops/cpu/expand_dims.h | 2 +- .../contrib_ops/cpu/maxpool_with_mask.h | 18 +- onnxruntime/contrib_ops/cpu/nchwc_ops.cc | 23 +- onnxruntime/contrib_ops/cpu/nchwc_ops.h | 4 +- .../cpu/quantization/nhwc_max_pool.cc | 4 +- .../cpu/quantization/qlinear_concat.cc | 2 +- .../cpu/quantization/qlinear_pool.cc | 36 +-- onnxruntime/contrib_ops/cpu/unique.cc | 54 ++-- .../contrib_ops/cuda/bert/attention.cc | 2 +- onnxruntime/contrib_ops/cuda/grid_sample.cc | 2 +- .../quantization/attention_quantization.cc | 2 +- onnxruntime/core/codegen/mti/mti_tvm_utils.cc | 4 +- onnxruntime/core/codegen/mti/mti_tvm_utils.h | 5 +- .../core/framework/allocation_planner.cc | 9 +- onnxruntime/core/framework/execution_frame.cc | 7 +- .../core/framework/inlined_containers.h | 40 +++ .../core/framework/op_node_proto_helper.cc | 17 +- .../core/framework/ort_stl_allocator.h | 19 +- .../framework/orttraining_partial_executor.cc | 5 +- .../core/framework/parallel_executor.cc | 5 +- .../core/framework/sequential_executor.cc | 5 +- onnxruntime/core/framework/session_state.cc | 30 +- onnxruntime/core/framework/session_state.h | 8 +- .../core/language_interop_ops/pyop/pyop.cc | 6 +- .../optimizer/transpose_optimizer/api_impl.cc | 6 +- onnxruntime/core/providers/acl/nn/conv.cc | 10 +- onnxruntime/core/providers/armnn/nn/conv.cc | 2 +- .../builders/impl/reshape_op_builder.cc | 2 +- .../core/providers/cpu/controlflow/scan.h | 10 +- .../core/providers/cpu/controlflow/scan_8.cc | 8 +- .../core/providers/cpu/controlflow/scan_9.cc | 44 +-- .../providers/cpu/controlflow/scan_utils.cc | 12 +- .../providers/cpu/controlflow/scan_utils.h | 6 +- .../core/providers/cpu/cpu_provider_shared.cc | 31 +- .../core/providers/cpu/cpu_provider_shared.h | 37 +-- .../core/providers/cpu/generator/random.h | 8 +- .../math/einsum_utils/einsum_auxiliary_ops.cc | 40 +-- .../math/einsum_utils/einsum_auxiliary_ops.h | 12 +- .../einsum_compute_preprocessor.cc | 4 +- .../einsum_compute_preprocessor.h | 4 +- .../einsum_typed_compute_processor.cc | 54 ++-- .../einsum_typed_compute_processor.h | 4 +- .../providers/cpu/math/element_wise_ops.h | 37 ++- .../core/providers/cpu/ml/onehotencoder.cc | 2 +- onnxruntime/core/providers/cpu/nn/conv.cc | 21 +- onnxruntime/core/providers/cpu/nn/conv.h | 2 +- .../core/providers/cpu/nn/conv_attributes.h | 64 ++-- .../cpu/nn/conv_transpose_attributes.h | 34 +-- onnxruntime/core/providers/cpu/nn/pool.cc | 16 +- .../core/providers/cpu/nn/pool_attributes.h | 26 +- .../core/providers/cpu/nn/pool_functors.h | 24 +- .../cpu/quantization/conv_integer.cc | 12 +- .../providers/cpu/quantization/qlinearconv.cc | 12 +- .../providers/cpu/reduction/reduction_ops.cc | 112 +++---- .../providers/cpu/reduction/reduction_ops.h | 54 ++-- .../core/providers/cpu/rnn/rnn_helpers.h | 2 +- .../cpu/sequence/concat_from_sequence.cc | 2 +- .../providers/cpu/sequence/sequence_ops.cc | 4 +- .../core/providers/cpu/tensor/concat.cc | 23 +- .../core/providers/cpu/tensor/concatbase.h | 8 +- onnxruntime/core/providers/cpu/tensor/copy.cc | 22 +- onnxruntime/core/providers/cpu/tensor/copy.h | 34 +-- .../core/providers/cpu/tensor/onehot.cc | 6 +- .../core/providers/cpu/tensor/onehot.h | 2 +- onnxruntime/core/providers/cpu/tensor/pad.cc | 43 +-- .../core/providers/cpu/tensor/padbase.h | 14 +- .../core/providers/cpu/tensor/reshape.h | 8 +- .../providers/cpu/tensor/reshape_helper.h | 4 +- .../core/providers/cpu/tensor/slice.cc | 112 ++++--- onnxruntime/core/providers/cpu/tensor/slice.h | 28 +- .../cpu/tensor/slice_compute_metadata.h | 14 +- .../core/providers/cpu/tensor/slice_helper.h | 71 +++-- .../core/providers/cpu/tensor/split.cc | 2 +- .../core/providers/cpu/tensor/squeeze.h | 18 +- onnxruntime/core/providers/cpu/tensor/tile.cc | 2 +- .../core/providers/cpu/tensor/transpose.cc | 34 +-- .../core/providers/cpu/tensor/transpose.h | 12 +- .../core/providers/cpu/tensor/unsqueeze.cc | 9 +- .../core/providers/cpu/tensor/unsqueeze.h | 2 +- .../core/providers/cpu/tensor/upsample.cc | 4 +- .../core/providers/cpu/tensor/upsample.h | 4 +- onnxruntime/core/providers/cpu/tensor/utils.h | 86 +++--- .../core/providers/cuda/controlflow/scan.cc | 2 +- onnxruntime/core/providers/cuda/cuda_kernel.h | 4 + .../core/providers/cuda/cudnn_common.cc | 9 +- .../core/providers/cuda/cudnn_common.h | 2 +- .../math/einsum_utils/einsum_auxiliary_ops.cc | 4 +- .../math/einsum_utils/einsum_auxiliary_ops.h | 2 +- onnxruntime/core/providers/cuda/math/topk.cc | 2 +- onnxruntime/core/providers/cuda/nn/conv.cc | 62 ++-- onnxruntime/core/providers/cuda/nn/conv.h | 36 ++- .../core/providers/cuda/nn/conv_transpose.cc | 24 +- .../providers/cuda/nn/max_pool_with_index.cu | 33 +- .../providers/cuda/nn/max_pool_with_index.h | 8 +- onnxruntime/core/providers/cuda/nn/pool.cc | 34 +-- .../non_max_suppression_impl.cu | 4 +- .../providers/cuda/reduction/reduction_ops.cc | 80 ++--- .../providers/cuda/reduction/reduction_ops.h | 10 +- .../core/providers/cuda/rnn/cudnn_rnn_base.cc | 10 +- .../core/providers/cuda/tensor/concat.cc | 9 +- .../core/providers/cuda/tensor/expand.cc | 20 +- .../core/providers/cuda/tensor/onehot.cc | 2 +- onnxruntime/core/providers/cuda/tensor/pad.cc | 12 +- .../core/providers/cuda/tensor/reshape.h | 11 +- .../core/providers/cuda/tensor/resize_impl.cu | 2 +- .../core/providers/cuda/tensor/resize_impl.h | 2 +- .../core/providers/cuda/tensor/sequence_op.h | 2 +- .../core/providers/cuda/tensor/slice.cc | 24 +- .../core/providers/cuda/tensor/slice.h | 6 +- .../core/providers/cuda/tensor/split.cc | 4 +- .../core/providers/cuda/tensor/squeeze.cc | 4 +- .../core/providers/cuda/tensor/tile.cc | 2 +- .../core/providers/cuda/tensor/transpose.cc | 21 +- .../core/providers/cuda/tensor/transpose.h | 4 +- .../providers/cuda/tensor/transpose_impl.cu | 14 +- .../providers/cuda/tensor/transpose_impl.h | 10 +- .../core/providers/cuda/tensor/upsample.cc | 6 +- .../core/providers/cuda/tensor/upsample.h | 2 +- .../providers/dnnl/subgraph/dnnl_reduce.cc | 16 +- .../providers/dnnl/subgraph/dnnl_reshape.cc | 8 +- .../dnnl/subgraph/dnnl_subgraph_primitive.h | 21 ++ .../nnapi_builtin/builders/op_builder.cc | 18 +- .../nuphar/compiler/nuphar_codegen_ctx.cc | 2 +- .../nuphar/compiler/nuphar_op_ir_builder.cc | 4 +- .../nuphar/mti_x86/math/reduce_ops.cc | 3 +- .../providers/nuphar/mti_x86/nn/pool_ops.cc | 2 +- .../core/providers/rocm/miopen_common.cc | 8 +- .../core/providers/rocm/miopen_common.h | 2 +- onnxruntime/core/providers/rocm/nn/conv.cc | 59 ++-- onnxruntime/core/providers/rocm/nn/conv.h | 33 +- .../core/providers/rocm/nn/conv_transpose.cc | 18 +- .../providers/rocm/reduction/reduction_ops.cc | 221 +++++++------- .../providers/rocm/reduction/reduction_ops.h | 16 +- onnxruntime/core/providers/rocm/rocm_kernel.h | 2 +- .../provider_bridge_provider.cc | 28 +- .../shared_library/provider_interfaces.h | 1 + .../shared_library/provider_wrappedtypes.h | 26 ++ .../core/session/provider_bridge_ort.cc | 3 + .../python/onnxruntime_pybind_iobinding.cc | 9 +- .../python/onnxruntime_pybind_ortvalue.cc | 2 +- .../test/contrib_ops/unique_op_test.cc | 12 +- onnxruntime/test/eager/ort_invoker_test.cc | 2 +- .../test/framework/sparse_kernels_test.cc | 4 +- .../test/framework/tensor_shape_test.cc | 8 +- .../test/onnx/microbenchmark/activation.cc | 9 +- .../providers/cpu/controlflow/scan_test.cc | 17 +- .../cpu/math/element_wise_ops_test.cc | 2 +- .../cpu/reduction/reduction_ops_test.cc | 286 +++++++++--------- .../test/providers/cpu/tensor/cast_op_test.cc | 2 +- .../test/providers/cpu/tensor/copy_test.cc | 62 ++-- .../test/providers/provider_test_utils.cc | 18 +- .../test/providers/provider_test_utils.h | 154 ++++++++-- .../graph/optimizer/adam_optimizer_builder.cc | 2 +- .../graph/optimizer/lamb_optimizer_builder.cc | 2 +- .../core/graph/optimizer_builder.cc | 13 +- .../core/graph/optimizer_builder.h | 63 +++- .../graph/zero_optimizer_graph_builder.cc | 4 +- .../core/optimizer/megatron_transformer.cc | 8 +- .../orttraining/core/session/tensor_helper.cc | 6 +- .../core/session/training_session.h | 2 +- orttraining/orttraining/eager/ort_tensor.cpp | 7 +- .../models/runner/training_util.cc | 4 +- .../test/gradient/gradient_checker.cc | 32 +- .../test/gradient/gradient_checker.h | 4 +- .../test/gradient/gradient_op_test_utils.cc | 2 +- .../test/gradient/gradient_ops_test.cc | 37 +-- .../cpu/activation/activation_op_test.cc | 8 +- .../training_ops/cpu/nn/dropout_op_test.cc | 6 +- .../cpu/tensor/gather_grad_op_test.cc | 12 +- .../training_ops/cpu/aten_ops/aten_op.cc | 6 +- .../training_ops/cpu/aten_ops/aten_op.h | 2 +- .../cpu/loss/softmax_cross_entropy_loss.cc | 6 +- .../cpu/loss/softmax_cross_entropy_loss.h | 2 +- .../training_ops/cpu/nn/conv_grad.cc | 8 +- .../training_ops/cpu/tensor/concat.cc | 2 +- .../training_ops/cpu/tensor/slice_grad.cc | 18 +- .../training_ops/cpu/tensor/slice_grad.h | 8 +- .../training_ops/cpu/tensor/split.cc | 2 +- .../training_ops/cuda/communication/recv.cc | 6 +- .../loss/softmax_cross_entropy_loss_impl.cc | 4 +- .../cuda/loss/softmaxcrossentropy_impl.cc | 2 +- .../training_ops/cuda/math/div_grad.cc | 16 +- .../training_ops/cuda/nn/conv_grad.cc | 14 +- .../training_ops/cuda/nn/conv_grad.h | 4 +- .../cuda/reduction/reduction_ops.cc | 4 +- .../training_ops/cuda/tensor/concat.cc | 8 +- .../training_ops/cuda/tensor/slice_grad.cc | 6 +- .../training_ops/cuda/tensor/slice_grad.h | 4 +- .../training_ops/cuda/tensor/split.cc | 4 +- .../training_ops/rocm/math/div_grad.cc | 18 +- .../training_ops/rocm/nn/conv_grad.cc | 16 +- .../training_ops/rocm/nn/conv_grad.h | 4 +- .../rocm/reduction/reduction_ops.cc | 4 +- .../azure-pipelines/linux-gpu-ci-pipeline.yml | 1 + 203 files changed, 2094 insertions(+), 1594 deletions(-) create mode 100644 cmake/external/abseil-cpp.cmake create mode 100644 cmake/patches/abseil/Fix_Nvidia_Build_Break.patch create mode 100644 onnxruntime/core/framework/inlined_containers.h diff --git a/.gitmodules b/.gitmodules index 3cb897233e..5d22836c28 100644 --- a/.gitmodules +++ b/.gitmodules @@ -75,7 +75,6 @@ [submodule "cmake/external/pytorch_cpuinfo"] path = cmake/external/pytorch_cpuinfo url = https://github.com/pytorch/cpuinfo.git - [submodule "cmake/external/onnx-tensorrt"] path = cmake/external/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git diff --git a/cmake/external/abseil-cpp.cmake b/cmake/external/abseil-cpp.cmake new file mode 100644 index 0000000000..ff322f5e5a --- /dev/null +++ b/cmake/external/abseil-cpp.cmake @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +include(FetchContent) + +set(abseil_URL https://github.com/abseil/abseil-cpp.git) +set(abseil_TAG 9336be04a242237cd41a525bedfcf3be1bb55377) + +# Pass to build +set(ABSL_PROPAGATE_CXX_STD 1) +set(BUILD_TESTING 0) + +FetchContent_Declare( + abseil_cpp + PREFIX "${CMAKE_CURRENT_BINARY_DIR}/abseil-cpp" + BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/external/abseil-cpp" + GIT_REPOSITORY ${abseil_URL} + GIT_TAG ${abseil_TAG} +) + +FetchContent_MakeAvailable(abseil_cpp) +FetchContent_GetProperties(abseil_cpp SOURCE_DIR) + +# Patching separately because repeated builds would fail the whole FetchContent_* command +# instead of just patching +execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace ${PROJECT_SOURCE_DIR}/patches/abseil/Fix_Nvidia_Build_Break.patch + WORKING_DIRECTORY "${abseil_cpp_SOURCE_DIR}" + ) + +include_directories("${abseil_cpp_SOURCE_DIR}") + +list(APPEND onnxruntime_EXTERNAL_LIBRARIES absl::inlined_vector absl::flat_hash_set absl::flat_hash_map absl::base absl::throw_delegate absl::raw_hash_set absl::hash absl::city absl::low_level_hash absl::raw_logging_internal) +list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES absl::inlined_vector absl::flat_hash_set absl::flat_hash_map absl::base absl::throw_delegate absl::raw_hash_set absl::hash absl::city absl::low_level_hash absl::raw_logging_internal) + diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index 9b8b5bf818..a6e62802f0 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -108,6 +108,8 @@ if (onnxruntime_USE_MIMALLOC) endif() endif() +include(external/abseil-cpp.cmake) + onnxruntime_add_include_to_target(onnxruntime_common date_interface wil) target_include_directories(onnxruntime_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} @@ -115,7 +117,7 @@ target_include_directories(onnxruntime_common PUBLIC ${OPTIONAL_LITE_INCLUDE_DIR}) -target_link_libraries(onnxruntime_common safeint_interface Boost::mp11) +target_link_libraries(onnxruntime_common safeint_interface Boost::mp11 absl::throw_delegate) if(NOT WIN32) target_include_directories(onnxruntime_common PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/external/nsync/public") diff --git a/cmake/onnxruntime_framework.cmake b/cmake/onnxruntime_framework.cmake index 60f547e395..82f1e75dbc 100644 --- a/cmake/onnxruntime_framework.cmake +++ b/cmake/onnxruntime_framework.cmake @@ -68,6 +68,10 @@ if (onnxruntime_USE_MIMALLOC) target_link_libraries(onnxruntime_framework mimalloc-static) endif() +if (onnxruntime_BUILD_WEBASSEMBLY) + target_link_libraries(onnxruntime_framework absl::raw_hash_set absl::hash absl::city) +endif() + set_target_properties(onnxruntime_framework PROPERTIES FOLDER "ONNXRuntime") # need onnx to build to create headers that this project includes add_dependencies(onnxruntime_framework ${onnxruntime_EXTERNAL_DEPENDENCIES}) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 462791f2b9..85c0ce3294 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -426,7 +426,7 @@ if (onnxruntime_USE_CUDA) endif() add_dependencies(onnxruntime_providers_cuda onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES} ${onnxruntime_tvm_dependencies}) - target_link_libraries(onnxruntime_providers_cuda PRIVATE cublas cudnn curand cufft ${ONNXRUNTIME_PROVIDERS_SHARED}) + target_link_libraries(onnxruntime_providers_cuda PRIVATE cublas cudnn curand cufft absl::throw_delegate ${ONNXRUNTIME_PROVIDERS_SHARED}) target_include_directories(onnxruntime_providers_cuda PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) # ${CMAKE_CURRENT_BINARY_DIR} is so that #include "onnxruntime_config.h" inside tensor_shape.h is found set_target_properties(onnxruntime_providers_cuda PROPERTIES LINKER_LANGUAGE CUDA) @@ -1236,7 +1236,7 @@ if (onnxruntime_USE_ROCM) endif() add_dependencies(onnxruntime_providers_rocm onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES}) - target_link_libraries(onnxruntime_providers_rocm PRIVATE ${ONNXRUNTIME_ROCM_LIBS} ${ONNXRUNTIME_PROVIDERS_SHARED}) + target_link_libraries(onnxruntime_providers_rocm PRIVATE ${ONNXRUNTIME_ROCM_LIBS} ${ONNXRUNTIME_PROVIDERS_SHARED} absl::throw_delegate) # During transition to separate hipFFT repo, put hipfft/include early target_include_directories(onnxruntime_providers_rocm PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${eigen_INCLUDE_DIRS} PUBLIC ${onnxruntime_ROCM_HOME}/hipfft/include ${onnxruntime_ROCM_HOME}/include ${onnxruntime_ROCM_HOME}/hipcub/include ${onnxruntime_ROCM_HOME}/hiprand/include ${onnxruntime_ROCM_HOME}/rocrand/include) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/rocm DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) diff --git a/cmake/patches/abseil/Fix_Nvidia_Build_Break.patch b/cmake/patches/abseil/Fix_Nvidia_Build_Break.patch new file mode 100644 index 0000000000..1ab35ca454 --- /dev/null +++ b/cmake/patches/abseil/Fix_Nvidia_Build_Break.patch @@ -0,0 +1,26 @@ +diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h +index 98c26af..b4f1c4a 100644 +--- a/absl/container/internal/inlined_vector.h ++++ b/absl/container/internal/inlined_vector.h +@@ -925,8 +925,8 @@ auto Storage::Swap(Storage* other_storage_ptr) -> void { + inlined_ptr->GetSize()); + } + ABSL_INTERNAL_CATCH_ANY { +- allocated_ptr->SetAllocation( +- {allocated_storage_view.data, allocated_storage_view.capacity}); ++ allocated_ptr->SetAllocation(Allocation{ ++ allocated_storage_view.data, allocated_storage_view.capacity}); + ABSL_INTERNAL_RETHROW; + } + +@@ -934,8 +934,8 @@ auto Storage::Swap(Storage* other_storage_ptr) -> void { + inlined_ptr->GetInlinedData(), + inlined_ptr->GetSize()); + +- inlined_ptr->SetAllocation( +- {allocated_storage_view.data, allocated_storage_view.capacity}); ++ inlined_ptr->SetAllocation(Allocation{allocated_storage_view.data, ++ allocated_storage_view.capacity}); + } + + swap(GetSizeAndIsAllocated(), other_storage_ptr->GetSizeAndIsAllocated()); diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h index 0aa6701e7d..27d06e2ed0 100644 --- a/include/onnxruntime/core/framework/allocator.h +++ b/include/onnxruntime/core/framework/allocator.h @@ -148,7 +148,7 @@ class IAllocator { size_t alloc_size = count_or_bytes; // if T is not void, 'count_or_bytes' == number of items so allow for that - if (!std::is_void::value) { + ORT_IF_CONSTEXPR(!std::is_void::value) { // sizeof(void) isn't valid, but the compiler isn't smart enough to ignore that this line isn't // reachable if T is void. use std::conditional to 'use' void* in the sizeof call if (!CalcMemSizeForArray( @@ -157,9 +157,11 @@ class IAllocator { } } + // allocate + T* p = static_cast(use_reserve ? allocator->Reserve(alloc_size) : allocator->Alloc(alloc_size)); return IAllocatorUniquePtr{ - static_cast(use_reserve ? allocator->Reserve(alloc_size) : allocator->Alloc(alloc_size)), // allocate - [=](T* ptr) { // capture 'allocator' by value so it's always valid + p, + [allocator = std::move(allocator)](T* ptr) { // capture 'allocator' by value so it's always valid allocator->Free(ptr); }}; } diff --git a/include/onnxruntime/core/framework/op_node_proto_helper.h b/include/onnxruntime/core/framework/op_node_proto_helper.h index 84b1b7dfa7..41250bd42c 100644 --- a/include/onnxruntime/core/framework/op_node_proto_helper.h +++ b/include/onnxruntime/core/framework/op_node_proto_helper.h @@ -5,6 +5,7 @@ #ifndef SHARED_PROVIDER #include "core/common/status.h" +#include "core/framework/tensor_shape.h" #include "core/graph/graph_viewer.h" #include "gsl/gsl" #endif @@ -74,15 +75,6 @@ class OpNodeProtoHelper { return GetAttrs(name, tmp).IsOK() ? tmp : default_value; } - /** - Get repeated attributes - */ - template - MUST_USE_RESULT Status GetAttrs(const std::string& name, std::vector& values) const; - - template - MUST_USE_RESULT Status GetAttrs(const std::string& name, gsl::span values) const; - /// /// Return a gsl::span that points to an array of primitive types held by AttributeProto /// This function allows to avoid copying big attributes locally into a kernel and operate on @@ -97,6 +89,22 @@ class OpNodeProtoHelper { template MUST_USE_RESULT Status GetAttrsAsSpan(const std::string& name, gsl::span& values) const; + MUST_USE_RESULT Status GetAttrs(const std::string& name, TensorShapeVector& out) const; + + MUST_USE_RESULT TensorShapeVector GetAttrsOrDefault(const std::string& name, const TensorShapeVector& default_value = TensorShapeVector{}) const { + TensorShapeVector tmp; + return GetAttrs(name, tmp).IsOK() ? tmp : default_value; + } + + /** + Get repeated attributes + */ + template + MUST_USE_RESULT Status GetAttrs(const std::string& name, std::vector& values) const; + + template + MUST_USE_RESULT Status GetAttrs(const std::string& name, gsl::span values) const; + MUST_USE_RESULT Status GetAttrsStringRefs(const std::string& name, std::vector>& refs) const; diff --git a/include/onnxruntime/core/framework/tensor_shape.h b/include/onnxruntime/core/framework/tensor_shape.h index 408fafe81f..01de9e43ee 100644 --- a/include/onnxruntime/core/framework/tensor_shape.h +++ b/include/onnxruntime/core/framework/tensor_shape.h @@ -10,6 +10,18 @@ #include #include "onnxruntime_config.h" +#ifdef _MSC_VER +#pragma warning(push) +// C4127: conditional expression is constant +#pragma warning(disable : 4127) +#endif + +#include + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + namespace onnxruntime { #ifdef __GNUC__ #pragma GCC diagnostic push @@ -17,6 +29,27 @@ namespace onnxruntime { #pragma GCC diagnostic ignored "-Wnull-dereference" #endif #endif + +constexpr size_t kTensorShapeSmallBufferElementsSize = 5; + +// Use this type to build a shape and then create TensorShape. +using TensorShapeVector = absl::InlinedVector; + +// Use this for inlined shape size where different types are needed. +template +using InlinedShapeVectorT = absl::InlinedVector; + +inline TensorShapeVector ToShapeVector(const gsl::span& span) { + TensorShapeVector out; + out.reserve(span.size()); + out.assign(span.cbegin(), span.cend()); + return out; +} + +inline gsl::span ToConstSpan(const TensorShapeVector& vec) { + return gsl::make_span(vec); +} + class TensorShape { // We use negative numbers for unknown symbolic dimension. Each negative // number represents a unique symbolic dimension. @@ -25,13 +58,17 @@ class TensorShape { TensorShape(const TensorShape& other) : TensorShape(other.GetDims()) {} TensorShape& operator=(const TensorShape& other); + TensorShape& operator=(const gsl::span& dims) { + *this = TensorShape(dims); + return *this; + } TensorShape(TensorShape&& other) noexcept { operator=(std::move(other)); } TensorShape& operator=(TensorShape&& other) noexcept; TensorShape(gsl::span dims); - TensorShape(const std::vector& dims) : TensorShape(gsl::make_span(dims)) {} - TensorShape(const std::initializer_list& dims) : TensorShape(gsl::make_span(dims.begin(), dims.end())) {} + TensorShape(const TensorShapeVector& dims) : TensorShape(gsl::make_span(dims)) {} + TensorShape(std::initializer_list dims) : TensorShape(gsl::make_span(dims.begin(), dims.end())) {} TensorShape(const int64_t* dimension_sizes, size_t dimension_count) : TensorShape(gsl::span(dimension_sizes, dimension_count)) {} TensorShape(const std::vector& dims, size_t start, size_t end) : TensorShape(gsl::span(&dims[start], end - start)) {} @@ -47,7 +84,7 @@ class TensorShape { int64_t& operator[](size_t idx) { return values_[idx]; } bool operator==(const TensorShape& other) const noexcept { return GetDims() == other.GetDims(); } - bool operator!=(const TensorShape& other) const noexcept { return GetDims() != other.GetDims(); } + bool operator!=(const TensorShape& other) const noexcept { return !(*this == other); } size_t NumDimensions() const noexcept { return values_.size(); @@ -73,7 +110,10 @@ class TensorShape { Return underlying vector representation. */ gsl::span GetDims() const { return values_; } - std::vector GetDimsAsVector() const { return std::vector(values_.begin(), values_.end()); } + + TensorShapeVector AsShapeVector() const { + return ToShapeVector(values_); + } /** * Return the total number of elements. Returns 1 for an empty (rank 0) TensorShape. @@ -135,7 +175,7 @@ class TensorShape { void Allocate(size_t size); gsl::span values_; - int64_t small_buffer_[5]; + int64_t small_buffer_[kTensorShapeSmallBufferElementsSize]; std::unique_ptr allocated_buffer_; friend struct ProviderHostImpl; // So that the shared provider interface can access Allocate diff --git a/onnxruntime/contrib_ops/cpu/expand_dims.h b/onnxruntime/contrib_ops/cpu/expand_dims.h index 71d5f99c9d..a96c22c4d7 100644 --- a/onnxruntime/contrib_ops/cpu/expand_dims.h +++ b/onnxruntime/contrib_ops/cpu/expand_dims.h @@ -30,7 +30,7 @@ class ExpandDims final : public OpKernel { if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); const TensorShape& X_shape = X->Shape(); - std::vector expanded_shape(X_shape.GetDimsAsVector()); + TensorShapeVector expanded_shape(X_shape.AsShapeVector()); int64_t X_NumDims = X_shape.Size(); ORT_ENFORCE(axis <= X_NumDims && axis >= -X_NumDims, "Axis must be within range [", -X_NumDims, ", ", X_NumDims, "].", " Axis is ", axis); diff --git a/onnxruntime/contrib_ops/cpu/maxpool_with_mask.h b/onnxruntime/contrib_ops/cpu/maxpool_with_mask.h index 0d3ae02b1f..8c18688767 100644 --- a/onnxruntime/contrib_ops/cpu/maxpool_with_mask.h +++ b/onnxruntime/contrib_ops/cpu/maxpool_with_mask.h @@ -26,8 +26,8 @@ struct MaxpoolWithMask1DTask final { int64_t stride_h; int64_t height; int64_t total_mask_channels; - const std::vector& kernel_shape; - const std::vector& pads; + const TensorShapeVector& kernel_shape; + const TensorShapeVector& pads; TensorOpCost Cost() { double loop_count = static_cast(pooled_height * kernel_shape[0]); return TensorOpCost{loop_count, loop_count, loop_count}; @@ -73,8 +73,8 @@ struct MaxpoolWithMask2DTask final { int64_t height; int64_t width; int64_t total_mask_channels; - const std::vector& kernel_shape; - const std::vector& pads; + const TensorShapeVector& kernel_shape; + const TensorShapeVector& pads; TensorOpCost Cost() { double loop_count = static_cast(pooled_height * kernel_shape[0]); return TensorOpCost{loop_count, loop_count, loop_count}; @@ -133,8 +133,8 @@ struct MaxpoolWithMask3DTask final { int64_t width; int64_t depth; int64_t total_mask_channels; - const std::vector& kernel_shape; - const std::vector& pads; + const TensorShapeVector& kernel_shape; + const TensorShapeVector& pads; TensorOpCost Cost() { double loop_count = static_cast(pooled_height * kernel_shape[0]); return TensorOpCost{loop_count, loop_count, loop_count}; @@ -204,10 +204,10 @@ class MaxpoolWithMask : public OpKernel, public PoolBase { // ONNXRUNTIME_RETURN_IF_NOT((x_shape[2] == m_shape[2]) && (x_shape[3] == m_shape[3]), " Input shape and mask shape // mismatch: ", x_shape, " vs ", m_shape); - std::vector pads = pool_attrs_.pads; - std::vector kernel_shape = pool_attrs_.kernel_shape; + TensorShapeVector pads = pool_attrs_.pads; + TensorShapeVector kernel_shape = pool_attrs_.kernel_shape; - std::vector output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); + TensorShapeVector output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); Tensor* Y = context->Output(0, TensorShape(output_dims)); const float* X_data = X->template Data(); diff --git a/onnxruntime/contrib_ops/cpu/nchwc_ops.cc b/onnxruntime/contrib_ops/cpu/nchwc_ops.cc index c88ffd2dce..ba18f48fa1 100644 --- a/onnxruntime/contrib_ops/cpu/nchwc_ops.cc +++ b/onnxruntime/contrib_ops/cpu/nchwc_ops.cc @@ -5,11 +5,12 @@ #include "core/mlas/inc/mlas.h" namespace onnxruntime { +using ConvPadVector = ConvAttributes::ConvPadVector; namespace contrib { Status ReorderInput::Compute(OpKernelContext* context) const { const auto* X = context->Input(0); - const auto& X_shape = X->Shape().GetDims(); + const auto X_shape = X->Shape().GetDims(); const auto X_rank = X_shape.size(); ORT_ENFORCE(X_rank == 4); @@ -24,7 +25,7 @@ Status ReorderInput::Compute(OpKernelContext* context) const { const int64_t nchwc_block_size = static_cast(MlasNchwcGetBlockSize()); const int64_t nchwc_channels = (channels + nchwc_block_size - 1) & ~(nchwc_block_size - 1); - std::vector Y_shape(X_rank); + TensorShapeVector Y_shape(X_rank); Y_shape[0] = batch_count; Y_shape[1] = nchwc_channels; int64_t spatial_size = 1; @@ -127,7 +128,7 @@ Status ReorderOutput::Compute(OpKernelContext* context) const { ORT_ENFORCE(channels_ <= X_shape[1]); // Build the output shape in NCHW or NHWC order. - std::vector Y_shape(X_rank); + TensorShapeVector Y_shape(X_rank); Y_shape[0] = X_shape[0]; Y_shape[channels_last_ ? X_rank - 1 : 1] = channels_; auto* Y_spatial_dims = Y_shape.data() + (channels_last_ ? 1 : 2); @@ -162,26 +163,26 @@ Status NchwcConv::Compute(OpKernelContext* context) const { const size_t nchwc_block_size = MlasNchwcGetBlockSize(); ORT_ENFORCE((static_cast(X_shape[1]) < nchwc_block_size) || ((X_shape[1] % nchwc_block_size) == 0)); - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W_shape, kernel_shape)); if (kernel_shape.size() != 2) { return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Unsupported convolution size."); } - std::vector pads(conv_attrs_.pads); + ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(kernel_shape.size() * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(kernel_shape.size(), 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(kernel_shape.size(), 1); } - std::vector Y_dims; + TensorShapeVector Y_dims; Y_dims.insert(Y_dims.begin(), {X_shape[0], W_shape[0]}); TensorShape input_shape = X->Shape().Slice(2); ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShape(input_shape, kernel_shape, strides, dilations, pads, Y_dims)); @@ -224,8 +225,8 @@ Status NchwcPoolBase::NchwcPool(OpKernelContext* context, MLAS_POOLING_KIND kind ORT_ENFORCE(X_shape.NumDimensions() == 4); ORT_ENFORCE((X_shape[1] % MlasNchwcGetBlockSize()) == 0); - std::vector pads = pool_attrs_.pads; - std::vector output_dims = pool_attrs_.SetOutputSize(X_shape, X_shape[1], &pads); + TensorShapeVector pads = pool_attrs_.pads; + TensorShapeVector output_dims = pool_attrs_.SetOutputSize(X_shape, X_shape[1], &pads); auto* Y = context->Output(0, output_dims); MlasNchwcPool( @@ -285,7 +286,7 @@ std::vector NchwcUpsample::ComputeInterpolation(int64_t input_length, Status NchwcUpsample::Compute(OpKernelContext* context) const { const auto* X = context->Input(0); - const auto& X_shape = X->Shape().GetDims(); + const auto X_shape = X->Shape().GetDims(); ORT_ENFORCE(X_shape.size() == 4); ORT_ENFORCE((X_shape[1] % MlasNchwcGetBlockSize()) == 0); diff --git a/onnxruntime/contrib_ops/cpu/nchwc_ops.h b/onnxruntime/contrib_ops/cpu/nchwc_ops.h index 7e8e735694..4827d70489 100644 --- a/onnxruntime/contrib_ops/cpu/nchwc_ops.h +++ b/onnxruntime/contrib_ops/cpu/nchwc_ops.h @@ -90,7 +90,7 @@ class NchwcUpsample final : public OpKernel { public: NchwcUpsample(const OpKernelInfo& info) : OpKernel(info) { - ORT_ENFORCE(info.GetAttrs("scales", scales_).IsOK()); + ORT_ENFORCE(info.GetAttrs("scales", scales_).IsOK()); ORT_ENFORCE(scales_.size() == 4); // Batch and channel dimensions cannot scale and spatial scaling must be positive. ORT_ENFORCE(scales_[0] == 1 && scales_[1] == 1 && scales_[2] >= 1 && scales_[3] >= 1); @@ -126,7 +126,7 @@ class NchwcUpsample final : public OpKernel { int64_t output_length, int64_t scale) const; - std::vector scales_; + TensorShapeVector scales_; TransformationMode transformation_mode_; bool nearest_mode_; }; diff --git a/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc b/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc index a6f297c7b5..f4b3898028 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc @@ -40,8 +40,8 @@ Status NhwcMaxPool::Compute(OpKernelContext* context) const { const size_t spatial_dims = input_rank - 2; // Compute the output size and effective padding for this pooling operation. - std::vector output_dims({N}); - std::vector pads = pool_attrs_.pads; + TensorShapeVector output_dims({N}); + TensorShapeVector pads = pool_attrs_.pads; int64_t kernel_size = 1; int64_t input_image_size = 1; int64_t output_image_size = 1; diff --git a/onnxruntime/contrib_ops/cpu/quantization/qlinear_concat.cc b/onnxruntime/contrib_ops/cpu/quantization/qlinear_concat.cc index 76288daa8f..da4c65375a 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/qlinear_concat.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/qlinear_concat.cc @@ -98,7 +98,7 @@ Status QLinearConcat::Compute(OpKernelContext* ctx) const { std::vector> dynamic_lookup_tables(input_count); std::vector dynamic_table_attrs(input_count, 0); - std::vector input_tensors(input_count); + InlinedTensorsVector input_tensors(input_count); for (auto input_index = 0; input_index < input_count; ++input_index) { auto tuple_start = 2 + input_index * 3; input_tensors[input_index] = ctx->Input(static_cast(tuple_start)); diff --git a/onnxruntime/contrib_ops/cpu/quantization/qlinear_pool.cc b/onnxruntime/contrib_ops/cpu/quantization/qlinear_pool.cc index f06a80512a..d60e77f7a1 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/qlinear_pool.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/qlinear_pool.cc @@ -34,7 +34,7 @@ static inline T8Bits quantize_value(float y, float y_scale, T8Bits y_zero_point) return static_cast(std::max(min_8bits, std::min(static_cast(std::nearbyintf(y / y_scale + y_zero_point)), max_8bits))); } -static void SwitchDimsNchwNhwc(std::vector& dims, bool from_nchw_to_nhwc) { +static void SwitchDimsNchwNhwc(TensorShapeVector& dims, bool from_nchw_to_nhwc) { if (from_nchw_to_nhwc) { int64_t channel = dims[1]; dims.erase(dims.begin() + 1); @@ -56,8 +56,8 @@ struct QLinearPool1DTask final { int64_t pooled_height; int64_t stride_h; int64_t height; - const std::vector& kernel_shape; - const std::vector& pads; + const TensorShapeVector& kernel_shape; + const TensorShapeVector& pads; const PoolProcessContext& pool_context_; const PoolAttributes& pool_attrs_; @@ -104,8 +104,8 @@ struct QLinearPoolNhwc1DTask final { int64_t pooled_height; int64_t stride_h; int64_t height; - const std::vector& kernel_shape; - const std::vector& pads; + const TensorShapeVector& kernel_shape; + const TensorShapeVector& pads; const PoolProcessContext& pool_context_; const PoolAttributes& pool_attrs_; @@ -170,8 +170,8 @@ struct QLinearPool2DTask final { int64_t stride_w; int64_t height; int64_t width; - const std::vector& kernel_shape; - const std::vector& pads; + const TensorShapeVector& kernel_shape; + const TensorShapeVector& pads; const PoolProcessContext& pool_context_; const PoolAttributes& pool_attrs_; @@ -233,8 +233,8 @@ struct QLinearPoolNhwc2DTask final { int64_t stride_w; int64_t height; int64_t width; - const std::vector& kernel_shape; - const std::vector& pads; + const TensorShapeVector& kernel_shape; + const TensorShapeVector& pads; const PoolProcessContext& pool_context_; const PoolAttributes& pool_attrs_; @@ -325,8 +325,8 @@ struct QLinearPool3DTask final { int64_t height; int64_t width; int64_t depth; - const std::vector& kernel_shape; - const std::vector& pads; + const TensorShapeVector& kernel_shape; + const TensorShapeVector& pads; const PoolProcessContext& pool_context_; const PoolAttributes& pool_attrs_; @@ -401,8 +401,8 @@ struct QLinearPoolNhwc3DTask final { int64_t height; int64_t width; int64_t depth; - const std::vector& kernel_shape; - const std::vector& pads; + const TensorShapeVector& kernel_shape; + const TensorShapeVector& pads; const PoolProcessContext& pool_context_; const PoolAttributes& pool_attrs_; @@ -537,16 +537,16 @@ Status QLinearAveragePool::ComputeImpl(OpKernelContext* context) const { T8Bits y_zero_point = (tensor_y_zero_point ? *(tensor_y_zero_point->Data()) : (T8Bits)0); ORT_RETURN_IF_NOT(x_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3."); - std::vector pads = pool_attrs_.pads; - std::vector strides = pool_attrs_.strides; - std::vector kernel_shape = pool_attrs_.kernel_shape; + TensorShapeVector pads = pool_attrs_.pads; + TensorShapeVector strides = pool_attrs_.strides; + TensorShapeVector kernel_shape = pool_attrs_.kernel_shape; if (channels_last_) { - std::vector x_dims = x_shape.GetDimsAsVector(); + TensorShapeVector x_dims = x_shape.AsShapeVector(); SwitchDimsNchwNhwc(x_dims, false); x_shape = TensorShape(x_dims); } - std::vector output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); + TensorShapeVector output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); int64_t batch_count = x_shape[0]; const int64_t channels = x_shape[1]; diff --git a/onnxruntime/contrib_ops/cpu/unique.cc b/onnxruntime/contrib_ops/cpu/unique.cc index be8eda75ee..8e5c0cef7f 100644 --- a/onnxruntime/contrib_ops/cpu/unique.cc +++ b/onnxruntime/contrib_ops/cpu/unique.cc @@ -8,6 +8,7 @@ #pragma warning(disable : 4996) #endif #include "unique.h" +#include "core/framework/inlined_containers.h" #include "core/providers/cpu/tensor/utils.h" namespace onnxruntime { @@ -30,40 +31,41 @@ Status Unique::Compute(OpKernelContext* ctx) const { // obtain raw input data const float* input_data = input->Data(); - size_t num_elements = static_cast(input->Shape().Size()); + const auto num_elements = input->Shape().Size(); // 'idx' output has same output shape as input Tensor* output_idx = ctx->Output(1, input->Shape()); int64_t* output_idx_data = output_idx->template MutableData(); - // container to hold the unique elements (in the order it was first seen) - std::vector unique_elements; - // number of unique elements is atmost number of elements in the raw data - unique_elements.reserve(num_elements); + struct ElementData { + int64_t input_pos_; // original index + int64_t output_pos_; + int64_t count_; // number of times encountered + }; - // containers to store other metadata needed for other output tensors - std::unordered_map mapped_indices; - std::unordered_map element_counts; + // XXX: Refactoring for less memory allocations. unordered_map + // used originally for float uniqueness, is this correct? + using IndexingMap = InlinedHashMap; + IndexingMap mapped_indices(num_elements); // processing - for (size_t i = 0; i < num_elements; ++i) { - float temp = input_data[i]; + for (int64_t i = 0; i < num_elements; ++i) { + float value = input_data[i]; - const auto iter = mapped_indices.find(temp); - if (iter == mapped_indices.end()) { - // element is being seen for the first time - element_counts[temp] = 1; - output_idx_data[i] = mapped_indices[temp] = unique_elements.size(); - unique_elements.push_back(temp); + const auto original_index = i; + const auto num_unique = static_cast(mapped_indices.size()); + auto insert_result = mapped_indices.emplace(value, ElementData{original_index, num_unique, 1}); + if (insert_result.second) { + output_idx_data[i] = num_unique; } else { - // element has been seen before - output_idx_data[i] = iter->second; - ++element_counts[temp]; + // Seen before + output_idx_data[i] = insert_result.first->second.output_pos_; + insert_result.first->second.count_++; } } // 'uniques' output - TensorShape output_shape({static_cast(unique_elements.size())}); + TensorShape output_shape({static_cast(mapped_indices.size())}); Tensor* output_uniques = ctx->Output(0, output_shape); float* output_uniques_data = output_uniques->template MutableData(); @@ -71,16 +73,12 @@ Status Unique::Compute(OpKernelContext* ctx) const { Tensor* output_counts = ctx->Output(2, output_shape); int64_t* output_counts_data = output_counts->template MutableData(); - size_t iter = 0; - for (const float& e : unique_elements) { + for (const auto& e : mapped_indices) { // 'uniques' data - output_uniques_data[iter] = e; - + const auto output_pos = e.second.output_pos_; + output_uniques_data[output_pos] = e.first; // 'counts' data - const auto iter_map = element_counts.find(e); - output_counts_data[iter] = static_cast(iter_map->second); - - ++iter; + output_counts_data[output_pos] = e.second.count_; } return Status::OK(); diff --git a/onnxruntime/contrib_ops/cuda/bert/attention.cc b/onnxruntime/contrib_ops/cuda/bert/attention.cc index b0ad840219..5560e2d91f 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention.cc +++ b/onnxruntime/contrib_ops/cuda/bert/attention.cc @@ -55,7 +55,7 @@ Status Attention::ComputeInternal(OpKernelContext* context) const { int head_size = hidden_size / num_heads_; - std::vector output_shape(3); + TensorShapeVector output_shape(3); output_shape[0] = shape[0]; output_shape[1] = shape[1]; output_shape[2] = static_cast(hidden_size); diff --git a/onnxruntime/contrib_ops/cuda/grid_sample.cc b/onnxruntime/contrib_ops/cuda/grid_sample.cc index 6a431dbad2..d366fdd417 100644 --- a/onnxruntime/contrib_ops/cuda/grid_sample.cc +++ b/onnxruntime/contrib_ops/cuda/grid_sample.cc @@ -60,7 +60,7 @@ Status GridSample::ComputeInternal(OpKernelContext* context) const { ORT_ENFORCE(dims_grid[0] == dims_input[0], "Grid batch size ", dims_grid[0], " does not match input batch size ", dims_input[0]); ORT_ENFORCE(dims_grid[3] == 2, "Last dimension of grid: ", dims_grid[3], ", expect 2"); - std::vector dims_output(4); + TensorShapeVector dims_output(4); dims_output[0] = dims_input[0]; dims_output[1] = dims_input[1]; dims_output[2] = dims_grid[1]; diff --git a/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc b/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc index 5c26013937..cfa116f539 100644 --- a/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc +++ b/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc @@ -118,7 +118,7 @@ Status QAttention::ComputeInternal(OpKernelContext* context) const { const int hidden_size = static_cast(bias_shape.GetDims()[0]) / 3; const int head_size = hidden_size / num_heads_; - std::vector output_shape(3); + TensorShapeVector output_shape(3); output_shape[0] = shape[0]; output_shape[1] = shape[1]; output_shape[2] = static_cast(hidden_size); diff --git a/onnxruntime/core/codegen/mti/mti_tvm_utils.cc b/onnxruntime/core/codegen/mti/mti_tvm_utils.cc index 0afc25d627..8e73629c05 100644 --- a/onnxruntime/core/codegen/mti/mti_tvm_utils.cc +++ b/onnxruntime/core/codegen/mti/mti_tvm_utils.cc @@ -11,7 +11,7 @@ namespace onnxruntime { namespace tvm_codegen { -tvm::Array ToTvmArray(const std::vector& shape) { +tvm::Array ToTvmArray(gsl::span shape) { tvm::Array arr; for (size_t i = 0; i < shape.size(); ++i) { arr.push_back(tvm::Expr(static_cast(shape[i]))); @@ -19,7 +19,7 @@ tvm::Array ToTvmArray(const std::vector& shape) { return arr; } -tvm::Array ToTvmArrayInt(const std::vector& shape) { +tvm::Array ToTvmArrayInt(gsl::span shape) { tvm::Array arr; for (size_t i = 0; i < shape.size(); ++i) { arr.push_back(shape[i]); diff --git a/onnxruntime/core/codegen/mti/mti_tvm_utils.h b/onnxruntime/core/codegen/mti/mti_tvm_utils.h index 487c6415a7..c2a14106c1 100644 --- a/onnxruntime/core/codegen/mti/mti_tvm_utils.h +++ b/onnxruntime/core/codegen/mti/mti_tvm_utils.h @@ -5,15 +5,16 @@ #include #include +#include #include #include "core/codegen/mti/common.h" namespace onnxruntime { namespace tvm_codegen { -tvm::Array ToTvmArray(const std::vector& shape); +tvm::Array ToTvmArray(gsl::span shape); -tvm::Array ToTvmArrayInt(const std::vector& shape); +tvm::Array ToTvmArrayInt(gsl::span shape); // Helper function to compute sub shape size to axis (not included) tvm::Expr SizeToDimension(const tvm::Array& shape, int64_t axis); diff --git a/onnxruntime/core/framework/allocation_planner.cc b/onnxruntime/core/framework/allocation_planner.cc index bd2e0c8aa5..9698f72437 100644 --- a/onnxruntime/core/framework/allocation_planner.cc +++ b/onnxruntime/core/framework/allocation_planner.cc @@ -11,6 +11,7 @@ #include "core/framework/data_types.h" #include "core/framework/kernel_def_builder.h" #include "core/framework/mldata_type_utils.h" +#include "core/framework/inlined_containers.h" #include "core/framework/op_kernel.h" #include "core/framework/session_state.h" #include "core/framework/tensorprotoutils.h" @@ -497,8 +498,10 @@ class PlannerImpl { Status ComputeUseCounts() { // Note: for every ml-value, its definition must appear before all its uses in a topological sort of a valid model - std::unordered_set graph_inputs; - for (auto& graph_input : graph_viewer_.GetInputsIncludingInitializers()) { + using GraphInputsSet = InlinedHashSet; + const auto& graph_inputs_nodes = graph_viewer_.GetInputsIncludingInitializers(); + GraphInputsSet graph_inputs(graph_inputs_nodes.size()); + for (auto& graph_input : graph_inputs_nodes) { graph_inputs.insert(graph_input->Name()); } @@ -522,7 +525,7 @@ class PlannerImpl { UseCount(initializer_name)++; } - std::unordered_set set_node_arg_has_explicit_consumer; + InlinedHashSet set_node_arg_has_explicit_consumer; for (SequentialExecutionPlan::NodeExecutionPlan& step : plan_.execution_plan) { auto pnode = graph_viewer_.GetNode(step.node_index); diff --git a/onnxruntime/core/framework/execution_frame.cc b/onnxruntime/core/framework/execution_frame.cc index 2ab88c0314..284013ffb4 100644 --- a/onnxruntime/core/framework/execution_frame.cc +++ b/onnxruntime/core/framework/execution_frame.cc @@ -371,23 +371,18 @@ ExecutionFrame::ExecutionFrame(const std::vector& feed_mlvalue_idxs, const // and we have execution plan generated, try to setup // memory pattern optimization. if (session_state.GetEnableMemoryPattern() && session_state.GetExecutionPlan()) { - std::vector > input_shapes; bool all_tensors = true; // Reserve mem to avoid re-allocation. - input_shapes.reserve(feeds.size()); for (const auto& feed : feeds) { if (!feed.IsTensor()) { all_tensors = false; break; } - auto& tensor = feed.Get(); - - input_shapes.push_back(std::cref(tensor.Shape())); } //if there are some traditional ml value type in inputs disable the memory pattern optimization. if (all_tensors) { - mem_patterns_ = session_state.GetMemoryPatternGroup(input_shapes, feed_mlvalue_idxs, inferred_shapes_); + mem_patterns_ = session_state.GetMemoryPatternGroup(feeds, feed_mlvalue_idxs, inferred_shapes_); // if no existing patterns, generate one in this executionframe if (!mem_patterns_) { planner_ = std::make_unique(*session_state.GetExecutionPlan()); diff --git a/onnxruntime/core/framework/inlined_containers.h b/onnxruntime/core/framework/inlined_containers.h new file mode 100644 index 0000000000..dfe35555e5 --- /dev/null +++ b/onnxruntime/core/framework/inlined_containers.h @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#ifdef _MSC_VER +#pragma warning(push) +// C4127: conditional expression is constant +#pragma warning(disable : 4127) +#endif + +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +namespace onnxruntime { + +// Use InlinedVector for small arrays that can fit on a stack. +// Use TensorShapeVector for shapes. +template +using InlinedVector = absl::InlinedVector; + +// InlinedHashSet and InlinedHashMap are preferred +// hash based containers. They store their values in the +// buckets array that is allocated in one shot. It eliminates +// per-node new/delete calls. Always call reserve() on any set/map +// be it a std container or not. +template +using InlinedHashSet = absl::flat_hash_set; + +template +using InlinedHashMap = absl::flat_hash_map; + +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/op_node_proto_helper.cc b/onnxruntime/core/framework/op_node_proto_helper.cc index cd3dfc6797..7268e957b2 100644 --- a/onnxruntime/core/framework/op_node_proto_helper.cc +++ b/onnxruntime/core/framework/op_node_proto_helper.cc @@ -73,7 +73,6 @@ inline constexpr int ArrayTypeToAttributeType() { return AttributeProto_AttributeType_STRINGS; } - #define ORT_DEFINE_GET_ATTR(IMPL_T, T, type) \ template <> \ template <> \ @@ -152,8 +151,9 @@ inline constexpr int ArrayTypeToAttributeType() { ORT_DEFINE_GET_ATTRS(ProtoHelperNodeContext, type, list) \ ORT_DEFINE_GET_ATTRS(InferenceContext, type, list) -#define ORT_DEFINE_GET_ATTRS_SPAN_SPECIALIZATION(type, list) \ - ORT_DEFINE_GET_ATTRS_AS_SPAN(ProtoHelperNodeContext, type, list) +#define ORT_DEFINE_GET_ATTRS_SPAN_SPECIALIZATION(type, list) \ + ORT_DEFINE_GET_ATTRS_AS_SPAN(ProtoHelperNodeContext, type, list) \ + ORT_DEFINE_GET_ATTRS_AS_SPAN(InferenceContext, type, list) #else #define ORT_DEFINE_GET_ATTR_SPECIALIZATIONS(type, list) \ @@ -180,6 +180,17 @@ ORT_DEFINE_GET_ATTRS_SPECIALIZATIONS(GraphProto, graphs) ORT_DEFINE_GET_ATTRS_SPAN_SPECIALIZATION(float, floats) ORT_DEFINE_GET_ATTRS_SPAN_SPECIALIZATION(int64_t, ints) +template +MUST_USE_RESULT Status OpNodeProtoHelper::GetAttrs(const std::string& name, TensorShapeVector& out) const { + gsl::span span; + Status status = this->GetAttrsAsSpan(name, span); + if (status.IsOK()) { + out.reserve(span.size()); + out.assign(span.cbegin(), span.cend()); + } + return status; +} + template MUST_USE_RESULT Status OpNodeProtoHelper::GetAttrsStringRefs( const std::string& name, diff --git a/onnxruntime/core/framework/ort_stl_allocator.h b/onnxruntime/core/framework/ort_stl_allocator.h index 62c313de94..85becb7d06 100644 --- a/onnxruntime/core/framework/ort_stl_allocator.h +++ b/onnxruntime/core/framework/ort_stl_allocator.h @@ -1,16 +1,22 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#pragma once + +#include +#include "core/framework/allocator.h" + namespace onnxruntime { -// An STL wrapper for ORT allocators. This enables overriding the +// An STL wrapper for ORT allocators. This enables overriding the // std::allocator used in STL containers for better memory performance. template class OrtStlAllocator { - template friend class OrtStlAllocator; + template + friend class OrtStlAllocator; AllocatorPtr allocator_; -public: + public: typedef T value_type; using propagate_on_container_copy_assignment = std::true_type; using propagate_on_container_move_assignment = std::true_type; @@ -41,11 +47,10 @@ public: template bool operator==(const OrtStlAllocator& lhs, const OrtStlAllocator& rhs) noexcept { - return lhs.allocator_ == rhs.allocator_; + return lhs.allocator_ == rhs.allocator_; } template bool operator!=(const OrtStlAllocator& lhs, const OrtStlAllocator& rhs) noexcept { - return lhs.allocator_ != rhs.allocator_; + return lhs.allocator_ != rhs.allocator_; } - -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/framework/orttraining_partial_executor.cc b/onnxruntime/core/framework/orttraining_partial_executor.cc index a457e16287..bfec6d6703 100644 --- a/onnxruntime/core/framework/orttraining_partial_executor.cc +++ b/onnxruntime/core/framework/orttraining_partial_executor.cc @@ -484,21 +484,18 @@ Status PartialExecutor::Execute(const SessionState& session_state, const std::ve #endif if (frame.HasMemoryPatternPlanner()) { - std::vector> input_shapes; bool all_tensors = true; for (const auto& feed : feeds) { if (!(feed.IsTensor())) { all_tensors = false; break; } - auto& tensor = feed.Get(); - input_shapes.push_back(std::cref(tensor.Shape())); } if (all_tensors) { auto mem_patterns = std::make_unique(); ORT_RETURN_IF_ERROR(frame.GeneratePatterns(mem_patterns.get())); - ORT_RETURN_IF_ERROR(session_state.UpdateMemoryPatternGroupCache(input_shapes, std::move(mem_patterns))); + ORT_RETURN_IF_ERROR(session_state.UpdateMemoryPatternGroupCache(feeds, std::move(mem_patterns))); } } diff --git a/onnxruntime/core/framework/parallel_executor.cc b/onnxruntime/core/framework/parallel_executor.cc index 1f782f38b1..f0576b3143 100644 --- a/onnxruntime/core/framework/parallel_executor.cc +++ b/onnxruntime/core/framework/parallel_executor.cc @@ -82,21 +82,18 @@ Status ParallelExecutor::Execute(const SessionState& session_state, const std::v VLOGS(logger, 1) << "Done execution."; if (root_frame_->HasMemoryPatternPlanner()) { - std::vector> input_shapes; bool all_tensors = true; for (const auto& feed : feeds) { if (!(feed.IsTensor())) { all_tensors = false; break; } - auto& tensor = feed.Get(); - input_shapes.push_back(std::cref(tensor.Shape())); } if (all_tensors) { auto mem_patterns = std::make_unique(); ORT_RETURN_IF_ERROR(root_frame_->GeneratePatterns(mem_patterns.get())); - ORT_RETURN_IF_ERROR(session_state.UpdateMemoryPatternGroupCache(input_shapes, std::move(mem_patterns))); + ORT_RETURN_IF_ERROR(session_state.UpdateMemoryPatternGroupCache(feeds, std::move(mem_patterns))); } } diff --git a/onnxruntime/core/framework/sequential_executor.cc b/onnxruntime/core/framework/sequential_executor.cc index 0b5346e2ea..2391c2ab57 100644 --- a/onnxruntime/core/framework/sequential_executor.cc +++ b/onnxruntime/core/framework/sequential_executor.cc @@ -462,21 +462,18 @@ Status SequentialExecutor::Execute(const SessionState& session_state, const std: #endif if (frame.HasMemoryPatternPlanner()) { - std::vector> input_shapes; bool all_tensors = true; for (const auto& feed : feeds) { if (!(feed.IsTensor())) { all_tensors = false; break; } - auto& tensor = feed.Get(); - input_shapes.push_back(std::cref(tensor.Shape())); } if (all_tensors) { auto mem_patterns = std::make_unique(); ORT_RETURN_IF_ERROR(frame.GeneratePatterns(mem_patterns.get())); - ORT_RETURN_IF_ERROR(session_state.UpdateMemoryPatternGroupCache(input_shapes, std::move(mem_patterns))); + ORT_RETURN_IF_ERROR(session_state.UpdateMemoryPatternGroupCache(feeds, std::move(mem_patterns))); } } diff --git a/onnxruntime/core/framework/session_state.cc b/onnxruntime/core/framework/session_state.cc index eaf9f901fe..e6709305a8 100644 --- a/onnxruntime/core/framework/session_state.cc +++ b/onnxruntime/core/framework/session_state.cc @@ -424,10 +424,10 @@ Status SessionState::PrepackConstantInitializedTensors(std::unordered_map>& shapes) { +static int64_t CalculateMemoryPatternsKey(const gsl::span& tensor_inputs) { int64_t key = 0; - for (auto shape : shapes) { - for (auto dim : shape.get().GetDims()) key ^= dim; + for (const auto& input : tensor_inputs) { + for (auto dim : input.Get().Shape().GetDims()) key ^= dim; } return key; } @@ -467,13 +467,13 @@ Status TryResolveShape( const NodeArg* arg, const std::unordered_map& symbolic_dimensions, size_t& is_resolved, // indicate whether resolve successfully or not. - std::vector& resolved_shape) { + TensorShapeVector& resolved_shape) { if (!arg->Shape()) { is_resolved = 0; return Status::OK(); } - std::vector shape; + TensorShapeVector shape; SafeInt safe_size = 1; for (auto& dim : arg->Shape()->dim()) { @@ -515,7 +515,7 @@ void TryCalculateSizeFromResolvedShape(int ml_value_idx, std::unordered_map>& input_shape, +Status SessionState::GeneratePatternGroupCache(const gsl::span& tensor_inputs, const std::vector& feed_mlvalue_idxs, MemoryPatternGroup* output, std::unordered_map& resolved_shapes) const { @@ -523,7 +523,7 @@ Status SessionState::GeneratePatternGroupCache(const std::vectorort_value_name_idx_map_.GetName(feed_mlvalue_idxs[i], name)); - feeds.insert({name, input_shape[i]}); + feeds.insert({name, tensor_inputs[i].Get().Shape()}); } std::unordered_map map; ORT_RETURN_IF_ERROR(ResolveDimParams(*graph_viewer_, feeds, map)); @@ -550,14 +550,14 @@ Status SessionState::GeneratePatternGroupCache(const std::vectorOutputDefs()[i]; size_t is_resolved = 0; - std::vector resolved_shape; + TensorShapeVector resolved_shape; // Tensors whose shape cannot be resolved statically will be allocated at runtime. if (TryResolveShape(arg, map, is_resolved, resolved_shape).IsOK()) { // Store all valid resolved shapes. They will be queried in, for example, // Recv operator to bypass the dependency of output shapes on inputs. if (is_resolved != 0) { - resolved_shapes[ml_value_idx] = resolved_shape; + resolved_shapes[ml_value_idx] = gsl::make_span(resolved_shape); } } else { LOGS(logger_, INFO) << "[Static memory planning] Could not resolve shape for tensor with ML index " @@ -651,18 +651,18 @@ Status SessionState::GeneratePatternGroupCache(const std::vector>& input_shapes, +const MemoryPatternGroup* SessionState::GetMemoryPatternGroup(const gsl::span& tensor_inputs, const std::vector& feed_mlvalue_idxs, std::unordered_map& inferred_shapes) const { - int64_t key = CalculateMemoryPatternsKey(input_shapes); + int64_t key = CalculateMemoryPatternsKey(tensor_inputs); std::lock_guard lock(mem_patterns_lock_); auto it = mem_patterns_.find(key); if (it == mem_patterns_.end()) { #ifdef ENABLE_TRAINING auto mem_patterns = std::make_unique(); - if (GeneratePatternGroupCache(input_shapes, feed_mlvalue_idxs, mem_patterns.get(), inferred_shapes).IsOK()) { - key = CalculateMemoryPatternsKey(input_shapes); + if (GeneratePatternGroupCache(tensor_inputs, feed_mlvalue_idxs, mem_patterns.get(), inferred_shapes).IsOK()) { + key = CalculateMemoryPatternsKey(tensor_inputs); auto ptr = mem_patterns.get(); mem_patterns_[key] = std::move(mem_patterns); shape_patterns_[key] = inferred_shapes; @@ -703,9 +703,9 @@ void SessionState::ResolveMemoryPatternFlag() { } } -Status SessionState::UpdateMemoryPatternGroupCache(const std::vector>& input_shapes, +Status SessionState::UpdateMemoryPatternGroupCache(const gsl::span& tensor_inputs, std::unique_ptr mem_patterns) const { - int64_t key = CalculateMemoryPatternsKey(input_shapes); + int64_t key = CalculateMemoryPatternsKey(tensor_inputs); std::lock_guard lock(mem_patterns_lock_); auto it = mem_patterns_.find(key); diff --git a/onnxruntime/core/framework/session_state.h b/onnxruntime/core/framework/session_state.h index 45443b6612..4c25d0a7af 100644 --- a/onnxruntime/core/framework/session_state.h +++ b/onnxruntime/core/framework/session_state.h @@ -204,17 +204,19 @@ class SessionState { /** Get cached memory pattern based on input shapes + Must be called only when all values contain tensors */ const MemoryPatternGroup* GetMemoryPatternGroup( - const std::vector>& input_shapes, + const gsl::span& tensor_inputs, const std::vector& feed_mlvalue_idxs, std::unordered_map& inferred_shapes) const; /** Set generated memory pattern with a given input shapes. Const as it's an internal cache update only. + All inputs must represent Tensors */ - Status UpdateMemoryPatternGroupCache(const std::vector>& input_shape, + Status UpdateMemoryPatternGroupCache(const gsl::span& tensor_inputs, std::unique_ptr mem_patterns) const; bool GetUseDeterministicCompute() const { return use_deterministic_compute_; } @@ -380,7 +382,7 @@ class SessionState { #ifdef ENABLE_TRAINING Status GeneratePatternGroupCache( - const std::vector>& input_shape, + const gsl::span& inputs, const std::vector& feed_mlvalue_idxs, MemoryPatternGroup* output, std::unordered_map& inferred_shapes) const; diff --git a/onnxruntime/core/language_interop_ops/pyop/pyop.cc b/onnxruntime/core/language_interop_ops/pyop/pyop.cc index 9ff325714c..0a8bf18a7d 100644 --- a/onnxruntime/core/language_interop_ops/pyop/pyop.cc +++ b/onnxruntime/core/language_interop_ops/pyop/pyop.cc @@ -268,9 +268,11 @@ void PyCustomKernel::Compute(OrtKernelContext* context) { for (size_t i = 0; i < inputs_count; ++i) { auto ort_value = ort_.KernelContext_GetInput(context, i); - inputs.push_back(const_cast(ort_value)->Get().DataRaw()); + inputs.push_back(ort_value->Get().DataRaw()); inputs_type.push_back(GetType(ort_value)); - inputs_dim.push_back(const_cast(ort_value)->Get().Shape().GetDimsAsVector()); + auto tensor_dims = ort_value->Get().Shape().GetDims(); + std::vector shape(tensor_dims.cbegin(), tensor_dims.cend()); + inputs_dim.push_back(std::move(shape)); } std::string err; diff --git a/onnxruntime/core/optimizer/transpose_optimizer/api_impl.cc b/onnxruntime/core/optimizer/transpose_optimizer/api_impl.cc index 2baa762321..ab1d3d13d2 100644 --- a/onnxruntime/core/optimizer/transpose_optimizer/api_impl.cc +++ b/onnxruntime/core/optimizer/transpose_optimizer/api_impl.cc @@ -153,7 +153,11 @@ std::optional> ApiValueInfo::Shape() const { } TensorShape shape = utils::GetTensorShapeFromTensorShapeProto(*shape_proto); - return shape.GetDimsAsVector(); + const auto dims = shape.GetDims(); + std::vector result; + result.reserve(dims.size()); + result.assign(dims.cbegin(), dims.cend()); + return result; } api::DataType ApiValueInfo::DType() const { diff --git a/onnxruntime/core/providers/acl/nn/conv.cc b/onnxruntime/core/providers/acl/nn/conv.cc index 1aed263847..fb3c590c98 100644 --- a/onnxruntime/core/providers/acl/nn/conv.cc +++ b/onnxruntime/core/providers/acl/nn/conv.cc @@ -86,23 +86,23 @@ Status Conv::Compute(OpKernelContext* context) const { ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X, W)); - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape)); - std::vector pads(conv_attrs_.pads); + ConvAttributes::ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(kernel_shape.size() * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(kernel_shape.size(), 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(kernel_shape.size(), 1); } - std::vector Y_dims; + TensorShapeVector Y_dims; Y_dims.insert(Y_dims.begin(), {N, M}); TensorShape input_shape = X->Shape().Slice(2); ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShape(input_shape, kernel_shape, strides, dilations, pads, Y_dims)); diff --git a/onnxruntime/core/providers/armnn/nn/conv.cc b/onnxruntime/core/providers/armnn/nn/conv.cc index 56f487ed8c..70620435ff 100644 --- a/onnxruntime/core/providers/armnn/nn/conv.cc +++ b/onnxruntime/core/providers/armnn/nn/conv.cc @@ -121,7 +121,7 @@ Status Conv::Compute(OpKernelContext* context) const { std::vector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape)); - std::vector pads(conv_attrs_.pads); + ConvAttributes::ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(kernel_shape.size() * 2, 0); } diff --git a/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc index 7d1dded377..19bff2e97d 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc @@ -57,7 +57,7 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, : target_shape_tensor.int64_data().data(); const auto size = target_shape_tensor.dims()[0]; - std::vector target_shape{raw_target_shape, raw_target_shape + size}; + TensorShapeVector target_shape{raw_target_shape, raw_target_shape + size}; std::vector input_shape; ORT_RETURN_IF_NOT(GetShape(*input_defs[0], input_shape, logger), "Cannot get shape"); ReshapeHelper helper(TensorShape(input_shape), target_shape); diff --git a/onnxruntime/core/providers/cpu/controlflow/scan.h b/onnxruntime/core/providers/cpu/controlflow/scan.h index 1130e9ee26..c45b251045 100644 --- a/onnxruntime/core/providers/cpu/controlflow/scan.h +++ b/onnxruntime/core/providers/cpu/controlflow/scan.h @@ -43,7 +43,7 @@ struct Info { // Provide as needed when Scan is being run by a non-CPU based ExecutionProvider struct DeviceHelpers { using ZeroData = std::function; - using Transpose = std::function& permutations, + using Transpose = std::function& permutations, const Tensor& input, Tensor& output)>; using CreateConstSlicer = std::function(const OrtValue& ort_value, int64_t slice_dimension /*=0*/, @@ -89,10 +89,10 @@ class Scan : public controlflow::IControlFlowKernel { private: int64_t num_scan_inputs_; - std::vector input_directions_; - std::vector output_directions_; - std::vector input_axes_; - std::vector output_axes_; + TensorShapeVector input_directions_; + TensorShapeVector output_directions_; + TensorShapeVector input_axes_; + TensorShapeVector output_axes_; // Info and FeedsFetchesManager re-used for each subgraph execution. std::unique_ptr info_; diff --git a/onnxruntime/core/providers/cpu/controlflow/scan_8.cc b/onnxruntime/core/providers/cpu/controlflow/scan_8.cc index 53e7ae0e69..e0d180c8cc 100644 --- a/onnxruntime/core/providers/cpu/controlflow/scan_8.cc +++ b/onnxruntime/core/providers/cpu/controlflow/scan_8.cc @@ -88,7 +88,7 @@ class Scan8Impl { Scan8Impl(OpKernelContextInternal& context, const SessionState& session_state, const Scan<8>::Info& info, - const std::vector& directions, + const gsl::span& directions, const scan::detail::DeviceHelpers& device_helpers); // Initialize by validating all the inputs, and allocating the output tensors @@ -117,7 +117,7 @@ class Scan8Impl { int64_t batch_size_ = -1; int64_t max_sequence_len_ = -1; - const std::vector& directions_; + const gsl::span directions_; const Tensor* sequence_lens_tensor_; std::vector sequence_lens_; @@ -142,7 +142,7 @@ void Scan<8>::Init(const OpKernelInfo& info) { ReadDirections(info, "directions", input_directions_, num_scan_inputs_); - device_helpers_.transpose_func = [](const std::vector&, const Tensor&, Tensor&) -> Status { + device_helpers_.transpose_func = [](const gsl::span&, const Tensor&, Tensor&) -> Status { ORT_NOT_IMPLEMENTED("Scan<8> spec does not support transpose of output. This should never be called."); }; @@ -191,7 +191,7 @@ Status Scan<8>::Compute(OpKernelContext* ctx) const { Scan8Impl::Scan8Impl(OpKernelContextInternal& context, const SessionState& session_state, const Scan<8>::Info& info, - const std::vector& directions, + const gsl::span& directions, const scan::detail::DeviceHelpers& device_helpers) : context_(context), session_state_(session_state), diff --git a/onnxruntime/core/providers/cpu/controlflow/scan_9.cc b/onnxruntime/core/providers/cpu/controlflow/scan_9.cc index c387a3da4c..8bd03e6e47 100644 --- a/onnxruntime/core/providers/cpu/controlflow/scan_9.cc +++ b/onnxruntime/core/providers/cpu/controlflow/scan_9.cc @@ -102,10 +102,10 @@ class ScanImpl { ScanImpl(OpKernelContextInternal& context, const SessionState& session_state, const Scan<9>::Info& info, - const std::vector& input_directions, - const std::vector& output_directions, - const std::vector& input_axes, - const std::vector& output_axes, + const gsl::span& input_directions, + const gsl::span& output_directions, + const gsl::span& input_axes, + const gsl::span& output_axes, const scan::detail::DeviceHelpers& device_helpers); // Initialize by validating all the inputs, and allocating the output tensors @@ -138,11 +138,11 @@ class ScanImpl { int64_t sequence_len_ = -1; - const std::vector& input_directions_; - const std::vector& output_directions_; - const std::vector& input_axes_from_attribute_; - const std::vector& output_axes_from_attribute_; - std::vector input_axes_; + gsl::span input_directions_; + gsl::span output_directions_; + gsl::span input_axes_from_attribute_; + gsl::span output_axes_from_attribute_; + TensorShapeVector input_axes_; // inputs for graph. either original input value or transposed input if an axis other than 0 was specified std::vector inputs_; @@ -170,22 +170,22 @@ void Scan<9>::Init(const OpKernelInfo& info) { ReadDirections(info, "scan_input_directions", input_directions_, num_scan_inputs_); ReadDirections(info, "scan_output_directions", output_directions_, num_scan_outputs); - if (info.GetAttrs("scan_input_axes", input_axes_).IsOK()) { + if (info.GetAttrs("scan_input_axes", input_axes_).IsOK()) { ORT_ENFORCE(gsl::narrow_cast(input_axes_.size()) == num_scan_inputs_, "Number of entries in 'scan_input_axes' was ", input_axes_.size(), " but expected ", num_scan_inputs_); } else { - input_axes_ = std::vector(num_scan_inputs_, 0); + input_axes_.resize(num_scan_inputs_, 0); } - if (info.GetAttrs("scan_output_axes", output_axes_).IsOK()) { + if (info.GetAttrs("scan_output_axes", output_axes_).IsOK()) { ORT_ENFORCE(gsl::narrow_cast(output_axes_.size()) == num_scan_outputs, "Number of entries in 'scan_output_axes' was ", output_axes_.size(), " but expected ", num_scan_outputs); } else { - output_axes_ = std::vector(num_scan_outputs, 0); + output_axes_.resize(num_scan_outputs, 0); } - device_helpers_.transpose_func = [](const std::vector& permutations, const Tensor& input, + device_helpers_.transpose_func = [](const gsl::span& permutations, const Tensor& input, Tensor& output) -> Status { return TransposeBase::DoTranspose(permutations, input, output); }; @@ -236,10 +236,10 @@ Status Scan<9>::Compute(OpKernelContext* ctx) const { ScanImpl::ScanImpl(OpKernelContextInternal& context, const SessionState& session_state, const Scan<9>::Info& info, - const std::vector& input_directions, - const std::vector& output_directions, - const std::vector& input_axes, - const std::vector& output_axes, + const gsl::span& input_directions, + const gsl::span& output_directions, + const gsl::span& input_axes, + const gsl::span& output_axes, const scan::detail::DeviceHelpers& device_helpers) : context_(context), session_state_(session_state), @@ -345,8 +345,8 @@ Status ScanImpl::SetupInputs() { auto& input_tensor = *context_.Input(i + info_.num_loop_state_variables); const auto& input_shape = input_tensor.Shape(); - std::vector permutations; - std::vector new_shape; + InlinedShapeVectorT permutations; + TensorShapeVector new_shape; CalculateTransposedShapeForInput(input_shape, sequence_dim, permutations, new_shape); if (!alloc) { @@ -478,8 +478,8 @@ Status ScanImpl::TransposeOutput() { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid value in scan_output_axes for output ", i, " of ", axis, ". Output tensor rank was ", output_rank); - std::vector permutations; - std::vector new_shape; + InlinedShapeVectorT permutations; + TensorShapeVector new_shape; CalculateTransposedShapeForOutput(temporary_output_tensor.Shape(), axis, permutations, new_shape); Tensor* output = context_.Output(output_index, new_shape); diff --git a/onnxruntime/core/providers/cpu/controlflow/scan_utils.cc b/onnxruntime/core/providers/cpu/controlflow/scan_utils.cc index e5b25b56a4..ea674953f4 100644 --- a/onnxruntime/core/providers/cpu/controlflow/scan_utils.cc +++ b/onnxruntime/core/providers/cpu/controlflow/scan_utils.cc @@ -61,8 +61,8 @@ Info::Info(const Node& node, const GraphViewer& subgraph_in, int num_scan_inputs } void ReadDirections(const OpKernelInfo& info, const std::string& attr_name, - std::vector& directions, size_t num_entries) { - if (info.GetAttrs(attr_name, directions).IsOK()) { + TensorShapeVector& directions, size_t num_entries) { + if (info.GetAttrs(attr_name, directions).IsOK()) { ORT_ENFORCE(directions.size() == num_entries, "Number of entries in '", attr_name, "' was ", directions.size(), " but expected ", num_entries); @@ -73,7 +73,7 @@ void ReadDirections(const OpKernelInfo& info, const std::string& attr_name, ORT_ENFORCE(valid, "Invalid values in '", attr_name, "'. 0 == forward. 1 == reverse."); } else { // default to forward if we know how many entries there should be - directions = std::vector(num_entries, static_cast(ScanDirection::kForward)); + directions = TensorShapeVector(num_entries, static_cast(ScanDirection::kForward)); } } @@ -97,7 +97,7 @@ Status AllocateOutput(OpKernelContextInternal& context, const GraphViewer& subgr TensorShape output_shape = onnxruntime::utils::GetTensorShapeFromTensorShapeProto(*graph_output_shape); auto graph_output_dims(output_shape.GetDims()); - std::vector scan_output_dims; + TensorShapeVector scan_output_dims; scan_output_dims.reserve(graph_output_dims.size() + 2); // v8 has batch size. v9 and later do not. @@ -300,7 +300,7 @@ OrtValue AllocateTensorInMLValue(const MLDataType data_type, const TensorShape& }; void CalculateTransposedShapeForInput(const TensorShape& original_shape, int64_t axis, - std::vector& permutations, std::vector& transposed_shape) { + InlinedShapeVectorT& permutations, TensorShapeVector& transposed_shape) { int64_t rank = original_shape.NumDimensions(); const auto& dims = original_shape.GetDims(); @@ -319,7 +319,7 @@ void CalculateTransposedShapeForInput(const TensorShape& original_shape, int64_t } void CalculateTransposedShapeForOutput(const TensorShape& original_shape, int64_t axis, - std::vector& permutations, std::vector& transposed_shape) { + InlinedShapeVectorT& permutations, TensorShapeVector& transposed_shape) { int64_t rank = original_shape.NumDimensions(); const auto& dims = original_shape.GetDims(); diff --git a/onnxruntime/core/providers/cpu/controlflow/scan_utils.h b/onnxruntime/core/providers/cpu/controlflow/scan_utils.h index 4eee03b6f2..dbd11bd9b8 100644 --- a/onnxruntime/core/providers/cpu/controlflow/scan_utils.h +++ b/onnxruntime/core/providers/cpu/controlflow/scan_utils.h @@ -162,7 +162,7 @@ class OutputIterator { }; void ReadDirections(const OpKernelInfo& info, const std::string& attr_name, - std::vector& directions, size_t num_entries); + TensorShapeVector& directions, size_t num_entries); Status AllocateOutput(OpKernelContextInternal& context, const GraphViewer& subgraph, int output_index, bool is_loop_state_var, int64_t batch_size, int64_t sequence_len, @@ -196,7 +196,7 @@ e.g. if shape is {2, 3, 4} and axis 1 is chosen the permutations will be {1, 0, if axis 2 is chosen the permutations will be {2, 0, 1} and the output shape will be {4, 2, 3} */ void CalculateTransposedShapeForInput(const TensorShape& original_shape, int64_t axis, - std::vector& permutations, std::vector& transposed_shape); + InlinedShapeVectorT& permutations, TensorShapeVector& transposed_shape); /** Calculate the transpose permutations and shape by shifting the chosen axis FROM the first dimension. @@ -205,7 +205,7 @@ e.g. if shape is {4, 2, 3} and axis 2 is chosen, dimension 0 will move to dimens the permutations will be {1, 2, 0} and output shape will be {2, 3, 4} */ void CalculateTransposedShapeForOutput(const TensorShape& original_shape, int64_t axis, - std::vector& permutations, std::vector& transposed_shape); + InlinedShapeVectorT& permutations, TensorShapeVector& transposed_shape); } // namespace detail } // namespace scan diff --git a/onnxruntime/core/providers/cpu/cpu_provider_shared.cc b/onnxruntime/core/providers/cpu/cpu_provider_shared.cc index c38b096419..d330274dbb 100644 --- a/onnxruntime/core/providers/cpu/cpu_provider_shared.cc +++ b/onnxruntime/core/providers/cpu/cpu_provider_shared.cc @@ -72,7 +72,8 @@ struct ProviderHostCPUImpl : ProviderHostCPU { std::vector& split_sizes) override { return p->SplitBase::PrepareForCompute(input_shape, num_outputs, axis, before_dims, after_dims_including_split_axis, after_dims_excluding_split, split_sizes); } // From cpu/tensor/concatbase.h (direct) - Status ConcatBase__PrepareForCompute(const ConcatBase* p, OpKernelContext* ctx, const std::vector& input_tensors, Prepare& prepare) override { return p->ConcatBase::PrepareForCompute(ctx, input_tensors, prepare); } + Status ConcatBase__PrepareForCompute(const ConcatBase* p, OpKernelContext* ctx, const ConcatBase_InlinedTensorsVector& input_tensors, Prepare& prepare) override { + return p->ConcatBase::PrepareForCompute(ctx, reinterpret_cast(input_tensors), prepare); } // GatherElements (direct) Status GatherElements__ValidateInputShapes(const TensorShape& input_data_shape, const TensorShape& indices_shape, int64_t axis) override { return GatherElements::ValidateInputShapes(input_data_shape, indices_shape, axis); } @@ -88,28 +89,28 @@ struct ProviderHostCPUImpl : ProviderHostCPU { // From onehot.h (direct) Status ValidateInputs(const Tensor* depth, const Tensor* values) override { return onnxruntime::ValidateInputs(depth, values); } - Status PrepareOutputShape(const Tensor* indices, const int64_t depth_val, const int64_t axis, int64_t& prefix_dim_size, int64_t& suffix_dim_size, std::vector& output_shape) override { return onnxruntime::PrepareOutputShape(indices, depth_val, axis, prefix_dim_size, suffix_dim_size, output_shape); } + Status PrepareOutputShape(const Tensor* indices, const int64_t depth_val, const int64_t axis, int64_t& prefix_dim_size, int64_t& suffix_dim_size, TensorShapeVector& output_shape) override { return onnxruntime::PrepareOutputShape(indices, depth_val, axis, prefix_dim_size, suffix_dim_size, output_shape); } // From cpu/tensor/slice.h (direct) - Status SliceBase__PrepareForCompute(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, + Status SliceBase__PrepareForCompute(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, SliceOp__PrepareForComputeMetadata& compute_metadata) override { return SliceBase::PrepareForCompute(raw_starts, raw_ends, raw_axes, reinterpret_cast(compute_metadata)); } - Status SliceBase__PrepareForCompute(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, - const std::vector& raw_steps, + Status SliceBase__PrepareForCompute(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, + const gsl::span& raw_steps, SliceOp__PrepareForComputeMetadata& compute_metadata) override { return SliceBase::PrepareForCompute(raw_starts, raw_ends, raw_axes, raw_steps, reinterpret_cast(compute_metadata)); } Status SliceBase__FillVectorsFromInput(const Tensor& start_tensor, const Tensor& ends_tensor, const Tensor* axes_tensor, const Tensor* steps_tensor, - std::vector& input_starts, - std::vector& input_ends, - std::vector& input_axes, - std::vector& input_steps) override { return SliceBase::FillVectorsFromInput(start_tensor, ends_tensor, axes_tensor, steps_tensor, input_starts, input_ends, input_axes, input_steps); } + TensorShapeVector& input_starts, + TensorShapeVector& input_ends, + TensorShapeVector& input_axes, + TensorShapeVector& input_steps) override { return SliceBase::FillVectorsFromInput(start_tensor, ends_tensor, axes_tensor, steps_tensor, input_starts, input_ends, input_axes, input_steps); } // If (direct) void If__Init(If* p, const OpKernelInfo& info) override { p->If::Init(info); } @@ -173,13 +174,13 @@ struct ProviderHostCPUImpl : ProviderHostCPU { Status contrib__PassThrough__Compute(const contrib::PassThrough* p, OpKernelContext* context) override { return p->PassThrough::Compute(context); } void contrib__VerifyLogitWeightAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, const TensorShape* weight_shape) override { contrib::VerifyLogitWeightAndLabelShape(logit_shape, label_shape, weight_shape); } void contrib__GetNDCFromLogitAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, int64_t& N_D, int64_t& C) override { contrib::GetNDCFromLogitAndLabelShape(logit_shape, label_shape, N_D, C); } - void contrib__GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, std::vector& new_shape, std::vector& permutations) override { contrib::GetPermutationAndShape(ncd_to_ndc, tensor_shape, new_shape, permutations); } + void contrib__GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, TensorShapeVector& new_shape, std::vector& permutations) override { contrib::GetPermutationAndShape(ncd_to_ndc, tensor_shape, new_shape, permutations); } Status contrib__PrepareForTrainingCompute(const TensorShape& input_shape, int num_outputs, int64_t& axis, int& before_dims, int& after_dims_including_split_axis, int& after_dims_excluding_split, std::vector& split_sizes) override { return contrib::PrepareForTrainingCompute(input_shape, num_outputs, axis, before_dims, after_dims_including_split_axis, after_dims_excluding_split, split_sizes); } Status contrib__YieldOp__Compute(const contrib::YieldOp* p, OpKernelContext* context) override { return p->YieldOp::Compute(context); } // From aten_op.h (direct) bool contrib__IsATenOperatorExecutorInitialized() override { return contrib::IsATenOperatorExecutorInitialized(); } - Status contrib__ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const std::vector& axes, bool keepdims) override { return contrib::ExecuteReduceSumATenOp(p_ctx, axes, keepdims); } + Status contrib__ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const gsl::span& axes, bool keepdims) override { return contrib::ExecuteReduceSumATenOp(p_ctx, axes, keepdims); } #endif #endif }; diff --git a/onnxruntime/core/providers/cpu/cpu_provider_shared.h b/onnxruntime/core/providers/cpu/cpu_provider_shared.h index 602169256d..f2142c3893 100644 --- a/onnxruntime/core/providers/cpu/cpu_provider_shared.h +++ b/onnxruntime/core/providers/cpu/cpu_provider_shared.h @@ -9,6 +9,7 @@ class AttentionBase; } // namespace contrib class GatherBase__Prepare; +class ConcatBase_InlinedTensorsVector; class SliceOp__PrepareForComputeMetadata; // Directly maps to SliceOp::PrepareForComputeMetadata class UnsqueezeBase__Prepare; // Directly maps to UnsqueezeBase::Prepare @@ -38,7 +39,7 @@ struct ProviderHostCPU { int& after_dims_including_split_axis, int& after_dims_excluding_split, std::vector& split_sizes) = 0; // From cpu/tensor/concatbase.h - virtual Status ConcatBase__PrepareForCompute(const ConcatBase* p, OpKernelContext* ctx, const std::vector& input_tensors, Prepare& prepare) = 0; + virtual Status ConcatBase__PrepareForCompute(const ConcatBase* p, OpKernelContext* ctx, const ConcatBase_InlinedTensorsVector& input_tensors, Prepare& prepare) = 0; // GatherElements virtual Status GatherElements__ValidateInputShapes(const TensorShape& input_data_shape, const TensorShape& indices_shape, int64_t axis) = 0; @@ -53,27 +54,27 @@ struct ProviderHostCPU { // From onehot.h virtual Status ValidateInputs(const Tensor* depth, const Tensor* values) = 0; - virtual Status PrepareOutputShape(const Tensor* indices, const int64_t depth_val, const int64_t axis, int64_t& prefix_dim_size, int64_t& suffix_dim_size, std::vector& output_shape) = 0; + virtual Status PrepareOutputShape(const Tensor* indices, const int64_t depth_val, const int64_t axis, int64_t& prefix_dim_size, int64_t& suffix_dim_size, TensorShapeVector& output_shape) = 0; // From cpu/tensor/slice.h - virtual Status SliceBase__PrepareForCompute(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, + virtual Status SliceBase__PrepareForCompute(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, SliceOp__PrepareForComputeMetadata& compute_metadata) = 0; - virtual Status SliceBase__PrepareForCompute(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, - const std::vector& raw_steps, + virtual Status SliceBase__PrepareForCompute(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, + const gsl::span& raw_steps, SliceOp__PrepareForComputeMetadata& compute_metadata) = 0; virtual Status SliceBase__FillVectorsFromInput(const Tensor& start_tensor, const Tensor& ends_tensor, const Tensor* axes_tensor, const Tensor* steps_tensor, - std::vector& input_starts, - std::vector& input_ends, - std::vector& input_axes, - std::vector& input_steps) = 0; + TensorShapeVector& input_starts, + TensorShapeVector& input_ends, + TensorShapeVector& input_axes, + TensorShapeVector& input_steps) = 0; virtual Status Einsum__Compute(const Einsum* p, OpKernelContext* context) = 0; @@ -135,13 +136,13 @@ struct ProviderHostCPU { virtual Status contrib__PassThrough__Compute(const contrib::PassThrough* p, OpKernelContext* context) = 0; virtual void contrib__VerifyLogitWeightAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, const TensorShape* weight_shape) = 0; virtual void contrib__GetNDCFromLogitAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, int64_t& N_D, int64_t& C) = 0; - virtual void contrib__GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, std::vector& new_shape, std::vector& permutations) = 0; + virtual void contrib__GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, TensorShapeVector& new_shape, std::vector& permutations) = 0; virtual Status contrib__PrepareForTrainingCompute(const TensorShape& input_shape, int num_outputs, int64_t& axis, int& before_dims, int& after_dims_including_split_axis, int& after_dims_excluding_split, std::vector& split_sizes) = 0; virtual Status contrib__YieldOp__Compute(const contrib::YieldOp* p, OpKernelContext* context) = 0; // From aten_op.h virtual bool contrib__IsATenOperatorExecutorInitialized() = 0; - virtual Status contrib__ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const std::vector& axes, bool keepdims) = 0; + virtual Status contrib__ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const gsl::span& axes, bool keepdims) = 0; #endif #endif }; @@ -167,7 +168,7 @@ inline Status CheckROIAlignValidInput(const Tensor* X_ptr, const Tensor* rois_pt inline Status ValidateInputs(const Tensor* depth, const Tensor* values) { return g_host_cpu.ValidateInputs(depth, values); } inline Status PrepareOutputShape(const Tensor* indices, const int64_t depth_val, const int64_t axis, int64_t& prefix_dim_size, int64_t& suffix_dim_size, - std::vector& output_shape) { return g_host_cpu.PrepareOutputShape(indices, depth_val, axis, prefix_dim_size, suffix_dim_size, output_shape); } + TensorShapeVector& output_shape) { return g_host_cpu.PrepareOutputShape(indices, depth_val, axis, prefix_dim_size, suffix_dim_size, output_shape); } struct EinsumComputePreprocessor { static void operator delete(void* p) { g_host_cpu.EinsumComputePreprocessor__operator_delete(reinterpret_cast(p)); } @@ -206,12 +207,12 @@ inline void wait_event_in_tensor(const Tensor& event_id_tensor) { return g_host_ inline void VerifyLogitWeightAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, const TensorShape* weight_shape) { g_host_cpu.contrib__VerifyLogitWeightAndLabelShape(logit_shape, label_shape, weight_shape); } inline void GetNDCFromLogitAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, int64_t& N_D, int64_t& C) { g_host_cpu.contrib__GetNDCFromLogitAndLabelShape(logit_shape, label_shape, N_D, C); } -inline void GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, std::vector& new_shape, std::vector& permutations) { g_host_cpu.contrib__GetPermutationAndShape(ncd_to_ndc, tensor_shape, new_shape, permutations); } +inline void GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, TensorShapeVector& new_shape, std::vector& permutations) { g_host_cpu.contrib__GetPermutationAndShape(ncd_to_ndc, tensor_shape, new_shape, permutations); } inline Status PrepareForTrainingCompute(const TensorShape& input_shape, int num_outputs, int64_t& axis, int& before_dims, int& after_dims_including_split_axis, int& after_dims_excluding_split, std::vector& split_sizes) { return g_host_cpu.contrib__PrepareForTrainingCompute(input_shape, num_outputs, axis, before_dims, after_dims_including_split_axis, after_dims_excluding_split, split_sizes); } // From aten_op.h inline bool IsATenOperatorExecutorInitialized() { return g_host_cpu.contrib__IsATenOperatorExecutorInitialized(); } -inline Status ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const std::vector& axes, bool keepdims) { return g_host_cpu.contrib__ExecuteReduceSumATenOp(p_ctx, axes, keepdims); } +inline Status ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const gsl::span& axes, bool keepdims) { return g_host_cpu.contrib__ExecuteReduceSumATenOp(p_ctx, axes, keepdims); } } // namespace contrib #endif // ENABLE_TRAINING #endif // USE_CUDA || USE_ROCM diff --git a/onnxruntime/core/providers/cpu/generator/random.h b/onnxruntime/core/providers/cpu/generator/random.h index 8dee437f91..3bf482d36e 100644 --- a/onnxruntime/core/providers/cpu/generator/random.h +++ b/onnxruntime/core/providers/cpu/generator/random.h @@ -34,8 +34,8 @@ class RandomNormal final : public OpKernel { ORT_ENFORCE(ONNX_NAMESPACE::TensorProto::DataType_IsValid(dtype_) && dtype_ != ONNX_NAMESPACE::TensorProto::UNDEFINED, "Invalid dtype of ", dtype_); - std::vector shape; - ORT_ENFORCE(info.GetAttrs("shape", shape).IsOK()); + TensorShapeVector shape; + ORT_ENFORCE(info.GetAttrs("shape", shape).IsOK()); shape_ = TensorShape(shape); } @@ -110,8 +110,8 @@ class RandomUniform final : public OpKernel { ORT_ENFORCE(ONNX_NAMESPACE::TensorProto::DataType_IsValid(dtype_) && dtype_ != ONNX_NAMESPACE::TensorProto::UNDEFINED, "Invalid dtype of ", dtype_); - std::vector shape; - ORT_ENFORCE(info.GetAttrs("shape", shape).IsOK()); + TensorShapeVector shape; + ORT_ENFORCE(info.GetAttrs("shape", shape).IsOK()); shape_ = TensorShape(shape); } diff --git a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.cc b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.cc index fb923bbfc3..43210934c4 100644 --- a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.cc +++ b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.cc @@ -23,7 +23,7 @@ Status DataCopy(const Tensor& input, Tensor& output, void* /*einsum_cuda_assets* } // CPU specific Transpose helper -Status Transpose(const std::vector& permutation, const Tensor& input, +Status Transpose(const gsl::span& permutation, const Tensor& input, Tensor& output, const TensorShape* input_shape_override, void* /*einsum_cuda_assets*/) { return TransposeBase::DoTranspose(permutation, input, output, input_shape_override); } @@ -112,7 +112,7 @@ static std::unique_ptr DiagonalInnermostDims(const Tensor& input, ORT_ENFORCE(input_dims[rank - 2] == input_dims[rank - 1], "The innermost dims should have the same dim value to parse the diagonal elements"); - std::vector output_dims; + TensorShapeVector output_dims; output_dims.reserve(rank); int64_t batch_size = 1; // Flatten the outermost dims - this will be the number of iterations @@ -135,7 +135,7 @@ static std::unique_ptr DiagonalInnermostDims(const Tensor& input, // Pass in allocator as that will be used as an allocator deleter by the framework // and it will de-allocate the memory for this intermediate tensor when it goes out of scope - std::unique_ptr output = std::make_unique(input.DataType(), output_dims, allocator); + std::unique_ptr output = std::make_unique(input.DataType(), output_dims, std::move(allocator)); switch (element_size_in_bytes) { case 4: @@ -237,7 +237,7 @@ std::unique_ptr Diagonal(const Tensor& input, int64_t dim_1, int64_t dim } // Make copy of the output dims - auto output_dims = output->Shape().GetDimsAsVector(); + auto output_dims = output->Shape().AsShapeVector(); // Unsqueeze the reduced dim auto iter = output_dims.begin() + second_dim; @@ -252,7 +252,7 @@ std::unique_ptr Diagonal(const Tensor& input, int64_t dim_1, int64_t dim } // namespace DeviceHelpers // This helps decide if we need to apply (and pay the cost) of a Transpose -bool IsTransposeRequired(size_t input_rank, const std::vector& permutation) { +bool IsTransposeRequired(size_t input_rank, const gsl::span& permutation) { ORT_ENFORCE(input_rank == permutation.size(), "The rank of the input must match permutation size for Transpose"); // No transpose required for scalars @@ -274,12 +274,12 @@ bool IsTransposeRequired(size_t input_rank, const std::vector& permutati // The following are thin wrappers over device specific helpers std::unique_ptr Transpose(const Tensor& input, const TensorShape& input_shape_override, - const std::vector& permutation, AllocatorPtr allocator, + const gsl::span& permutation, AllocatorPtr allocator, void* einsum_cuda_assets, const DeviceHelpers::Transpose& device_transpose_func) { auto input_rank = input_shape_override.NumDimensions(); ORT_ENFORCE(input_rank == permutation.size(), "Length of permutation must match the rank of the input to be permutated"); - std::vector output_dims; + TensorShapeVector output_dims; output_dims.reserve(input_rank); for (const auto& dim : permutation) { @@ -301,8 +301,8 @@ std::unique_ptr Transpose(const Tensor& input, const TensorShape& input_ } template -std::unique_ptr MatMul(const Tensor& input_1, const std::vector& input_shape_1_override, - const Tensor& input_2, const std::vector& input_shape_2_override, +std::unique_ptr MatMul(const Tensor& input_1, const gsl::span& input_shape_1_override, + const Tensor& input_2, const gsl::span& input_shape_2_override, AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets, const DeviceHelpers::MatMul& device_matmul_func) { // Sanity checks before the actual MatMul @@ -320,7 +320,7 @@ std::unique_ptr MatMul(const Tensor& input_1, const std::vector size_t right_offset = K * N; size_t output_offset = M * N; - std::vector output_dims; + TensorShapeVector output_dims; output_dims.reserve(3); output_dims.push_back(static_cast(batches)); output_dims.push_back(static_cast(M)); @@ -363,8 +363,8 @@ template Status DeviceHelpers::CpuDeviceHelpers::MatMul( void* einsum_cuda_assets); template std::unique_ptr MatMul( - const Tensor& input_1, const std::vector& input_shape_1_override, - const Tensor& input_2, const std::vector& input_shape_2_override, + const Tensor& input_1, const gsl::span& input_shape_1_override, + const Tensor& input_2, const gsl::span& input_shape_2_override, AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets, const DeviceHelpers::MatMul& device_matmul_func); @@ -387,8 +387,8 @@ template Status DeviceHelpers::CpuDeviceHelpers::MatMul( void* einsum_cuda_assets); template std::unique_ptr MatMul( - const Tensor& input_1, const std::vector& input_shape_1_override, - const Tensor& input_2, const std::vector& input_shape_2_override, + const Tensor& input_1, const gsl::span& input_shape_1_override, + const Tensor& input_2, const gsl::span& input_shape_2_override, AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets, const DeviceHelpers::MatMul& device_matmul_func); @@ -412,8 +412,8 @@ template Status DeviceHelpers::CpuDeviceHelpers::MatMul( void* einsum_cuda_assets); template std::unique_ptr MatMul( - const Tensor& input_1, const std::vector& input_shape_1_override, - const Tensor& input_2, const std::vector& input_shape_2_override, + const Tensor& input_1, const gsl::span& input_shape_1_override, + const Tensor& input_2, const gsl::span& input_shape_2_override, AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets, const DeviceHelpers::MatMul& device_matmul_func); @@ -443,8 +443,8 @@ template std::unique_ptr DeviceHelpers::CpuDeviceHelpers::ReduceSum MatMul( - const Tensor& input_1, const std::vector& input_shape_1_override, - const Tensor& input_2, const std::vector& input_shape_2_override, + const Tensor& input_1, const gsl::span& input_shape_1_override, + const Tensor& input_2, const gsl::span& input_shape_2_override, AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets, const DeviceHelpers::MatMul& device_matmul_func); @@ -455,8 +455,8 @@ template std::unique_ptr ReduceSum( // MLFloat16 template std::unique_ptr MatMul( - const Tensor& input_1, const std::vector& input_shape_1_override, - const Tensor& input_2, const std::vector& input_shape_2_override, + const Tensor& input_1, const gsl::span& input_shape_1_override, + const Tensor& input_2, const gsl::span& input_shape_2_override, AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets, const DeviceHelpers::MatMul& device_matmul_func); diff --git a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.h b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.h index 2409ac3b2f..a858a49e3e 100644 --- a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.h +++ b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.h @@ -28,7 +28,7 @@ namespace DeviceHelpers { using DataCopy = std::function; // Transpose op - Transposes given input based on data in `permutation` -using Transpose = std::function& permutation, const Tensor& input, +using Transpose = std::function& permutation, const Tensor& input, Tensor& output, const TensorShape* input_shape_override, void* einsum_cuda_assets)>; @@ -63,7 +63,7 @@ namespace CpuDeviceHelpers { Status DataCopy(const Tensor& input, Tensor& output, void* einsum_cuda_assets); -Status Transpose(const std::vector& permutation, const Tensor& input, +Status Transpose(const gsl::span& permutation, const Tensor& input, Tensor& output, const TensorShape* input_shape_override, void* einsum_cuda_assets); template @@ -85,19 +85,19 @@ std::unique_ptr Diagonal(const Tensor& input, int64_t dim_1, int64_t dim } // namespace DeviceHelpers // This helps decide if we need to apply (and pay the cost) of a Transpose -bool IsTransposeRequired(size_t input_rank, const std::vector& permutation); +bool IsTransposeRequired(size_t input_rank, const gsl::span& permutation); // Thin wrapper over the Transpose op to be called from Einsum that does some checks and invokes the device specific helper std::unique_ptr Transpose(const Tensor& input, const TensorShape& input_shape_override, - const std::vector& permutation, AllocatorPtr allocator, void* einsum_cuda_assets, + const gsl::span& permutation, AllocatorPtr allocator, void* einsum_cuda_assets, const DeviceHelpers::Transpose& device_transpose_func); // Thin wrapper over the MatMul op to be called from Einsum that does some checks and invokes the device specific helper // Not using the MatMulHelper for checks and to compute output dims as it adds a lot of checking overhead involving transposes of the inputs // In our case, we have a more simplistic version which doesn't need to have those checks template -std::unique_ptr MatMul(const Tensor& input_1, const std::vector& input_1_shape_override, - const Tensor& input_2, const std::vector& input_2_shape_override, +std::unique_ptr MatMul(const Tensor& input_1, const gsl::span& input_1_shape_override, + const Tensor& input_2, const gsl::span& input_2_shape_override, AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets, const DeviceHelpers::MatMul& device_matmul_func); diff --git a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.cc b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.cc index ab6944b011..04754ee1e7 100644 --- a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.cc +++ b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.cc @@ -32,7 +32,7 @@ Status EinsumComputePreprocessor::Run() { return Status::OK(); } -const std::vector& EinsumComputePreprocessor::GetOutputDims() const { +const TensorShapeVector& EinsumComputePreprocessor::GetOutputDims() const { return output_dims_; } @@ -424,7 +424,7 @@ Status EinsumComputePreprocessor::PreprocessInputs() { std::vector subscript_indices_to_input_index(num_subscript_indices_, -1); // This is the input dims after re-ordering so that all inputs have same axes order - std::vector homogenized_input_dims(num_subscript_indices_, 1); + TensorShapeVector homogenized_input_dims(num_subscript_indices_, 1); // Preprocessed dim rank may not be the same as original input rank if we need to parse diagonals along the way // (which reduces rank in the preprocessed input by 1 for each diagonal we parse) diff --git a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.h b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.h index 1ab64b3e8d..172df97597 100644 --- a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.h +++ b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.h @@ -117,7 +117,7 @@ class EinsumComputePreprocessor final { Status Run(); // Get the output dims of the op's output - const std::vector& GetOutputDims() const; + const TensorShapeVector& GetOutputDims() const; // Pre-process inputs if needed - preprocessing includes - // 1) Parsing diagonals from raw inputs @@ -204,7 +204,7 @@ class EinsumComputePreprocessor final { std::vector subscript_indices_to_dim_value_; // Holds the final calculated output dimensions - std::vector output_dims_; + TensorShapeVector output_dims_; // All subscript indices in the equation for each input std::vector> input_subscript_indices_; diff --git a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.cc b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.cc index 1c11b71ff1..8645f19545 100644 --- a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.cc +++ b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.cc @@ -7,10 +7,10 @@ namespace onnxruntime { template void EinsumTypedComputeProcessor::FinalizeOutput(const Tensor& candidate_output, - const std::vector& ordered_subscript_indices_in_candidate) { + const gsl::span& ordered_subscript_indices_in_candidate) { const std::vector& subscript_indices_to_output_indices = einsum_compute_preprocessor_.GetMappedSubscriptIndicesToOutputindices(); - const auto& output_dims = einsum_compute_preprocessor_.GetOutputDims(); + const auto output_dims = einsum_compute_preprocessor_.GetOutputDims(); TensorShape output_shape = TensorShape(output_dims); const auto output_rank = output_dims.size(); Tensor& output = *context_->Output(0, output_dims); @@ -18,12 +18,12 @@ void EinsumTypedComputeProcessor::FinalizeOutput(const Tensor& candidate_outp ORT_ENFORCE(candidate_output.Shape().Size() == output_shape.Size(), "Einsum op: The candidate output cannot be reshaped into the op's output"); - const auto& candidate_output_dims = candidate_output.Shape().GetDims(); + const auto candidate_output_dims = candidate_output.Shape().GetDims(); const auto candidate_output_rank = candidate_output_dims.size(); // This vector holds the shape of the candidate_output after removing the dims that have // been reduced in the final output - std::vector candidate_output_shape_without_reduced_dims; + TensorShapeVector candidate_output_shape_without_reduced_dims; candidate_output_shape_without_reduced_dims.reserve(candidate_output_rank); // reserve upper bound // Identify the permutation required by the op's output @@ -70,9 +70,9 @@ void EinsumTypedComputeProcessor::FinalizeOutput(const Tensor& candidate_outp } } -static bool IsTransposeReshapeForEinsum(const std::vector& perm, +static bool IsTransposeReshapeForEinsum(const gsl::span& perm, gsl::span input_dims, - std::vector& new_shape) { + TensorShapeVector& new_shape) { // As long as the dims with values > 1 stay in the same order, it's a reshape. // Example: Shape=(1,1,1024,4096) -> perm=(2,0,3,1). size_t last_permuted_axis = 0; @@ -83,7 +83,7 @@ static bool IsTransposeReshapeForEinsum(const std::vector& perm, return false; last_permuted_axis = perm[i]; } - new_shape = std::vector(input_dims.begin(), input_dims.end()); + new_shape.assign(input_dims.cbegin(), input_dims.cend()); for (size_t i = 0; i < perm.size(); ++i) { new_shape[i] = input_dims[perm[i]]; } @@ -95,7 +95,7 @@ std::unique_ptr EinsumTypedComputeProcessor::PairwiseOperandProcess(c const TensorShape& left_shape_override, const Tensor& right, const TensorShape& right_shape_override, - const std::vector& reduce_dims, + const gsl::span& reduce_dims, bool is_final_pair) { // Use the provided dim overrides instead of the actual shapes of the operands ORT_ENFORCE(left.Shape().Size() == left_shape_override.Size(), @@ -123,14 +123,14 @@ std::unique_ptr EinsumTypedComputeProcessor::PairwiseOperandProcess(c // lro: dim indices that are present in left, right, and reduce_dims // lo: dim indices that are present in left and reduce_dims // ro: dim indices that are present in right and reduce_dims - std::vector lro; - lro.reserve(8); // Reserve an arbitrary amount of space for this vector (not bound to see a tensor of rank > 8) + InlinedShapeVectorT lro; + lro.reserve(kTensorShapeSmallBufferElementsSize); // Reserve an arbitrary amount of space for this vector (not bound to see a tensor of rank > kTensorShapeSmallBufferElementsSize) - std::vector lo; - lo.reserve(8); // Reserve an arbitrary amount of space for this vector (not bound to see a tensor of rank > 8) + InlinedShapeVectorT lo; + lo.reserve(kTensorShapeSmallBufferElementsSize); // Reserve an arbitrary amount of space for this vector (not bound to see a tensor of rank > kTensorShapeSmallBufferElementsSize) - std::vector ro; - ro.reserve(8); // Reserve an arbitrary amount of space for this vector (not bound to see a tensor of rank > 8) + InlinedShapeVectorT ro; + ro.reserve(kTensorShapeSmallBufferElementsSize); // Reserve an arbitrary amount of space for this vector (not bound to see a tensor of rank > kTensorShapeSmallBufferElementsSize) // Maintain sizes to create reshaped "views" int64_t lro_size = 1; @@ -192,14 +192,14 @@ std::unique_ptr EinsumTypedComputeProcessor::PairwiseOperandProcess(c } // Permutate the left operand so that the axes order go like this: [lro, lo, reduce_dims, ro] - std::vector reshaped_dims; - std::vector left_permutation; + TensorShapeVector reshaped_dims; + InlinedShapeVectorT left_permutation; left_permutation.reserve(lro.size() + lo.size() + reduce_dims.size() + ro.size()); left_permutation.insert(left_permutation.end(), lro.begin(), lro.end()); left_permutation.insert(left_permutation.end(), lo.begin(), lo.end()); left_permutation.insert(left_permutation.end(), reduce_dims.begin(), reduce_dims.end()); left_permutation.insert(left_permutation.end(), ro.begin(), ro.end()); - if (EinsumOp::IsTransposeRequired(current_left ? current_left->Shape().GetDims().size() : left_dims.size(), + if (EinsumOp::IsTransposeRequired(current_left ? current_left->Shape().NumDimensions() : left_dims.size(), left_permutation)) { if (current_left && IsTransposeReshapeForEinsum(left_permutation, current_left->Shape().GetDims(), @@ -219,7 +219,7 @@ std::unique_ptr EinsumTypedComputeProcessor::PairwiseOperandProcess(c } // Permutate the right operand so that the axes order go like this: [lro, reduce_dims, ro, lo] - std::vector right_permutation; + InlinedShapeVectorT right_permutation; right_permutation.reserve(lro.size() + lo.size() + reduce_dims.size() + ro.size()); right_permutation.insert(right_permutation.end(), lro.begin(), lro.end()); right_permutation.insert(right_permutation.end(), reduce_dims.begin(), reduce_dims.end()); @@ -249,7 +249,7 @@ std::unique_ptr EinsumTypedComputeProcessor::PairwiseOperandProcess(c // dim_value of `lo` dims, // `1` for each of the `reduce_dims`, // dim_value of `ro` dims] - std::vector output_dims; + TensorShapeVector output_dims; output_dims.reserve(lro.size() + lo.size() + reduce_dims.size() + ro.size()); for (size_t i = 0; i < lro.size(); ++i) { output_dims.push_back(left_dims[lro[i]]); @@ -266,14 +266,14 @@ std::unique_ptr EinsumTypedComputeProcessor::PairwiseOperandProcess(c output_dims.push_back(right_dims[ro[i]]); } - std::vector current_subscript_order; + TensorShapeVector current_subscript_order; // Calculate output permutation // After the MatMul op, the because the two operands have been permutated, // the output is permutated as well with respect to the original ordering of the axes. // The permutated order will be the dims in: [lro, lo, reduced_dims, ro] // Hence invert the permutation by a permutation that puts the axes in the same ordering - std::vector output_permutation; + InlinedShapeVectorT output_permutation; if (!is_final_pair) { // If this is not the final pair, we need to permutate the result to match the pre-fixed order for the next iteration output_permutation.resize(lro.size() + lo.size() + reduce_dims.size() + ro.size(), 0); size_t iter = 0; @@ -298,8 +298,8 @@ std::unique_ptr EinsumTypedComputeProcessor::PairwiseOperandProcess(c } // Multiply the mutated inputs - auto output = EinsumOp::MatMul(current_left ? *current_left : left, {lro_size, lo_size, reduced_size}, - current_right ? *current_right : right, {lro_size, reduced_size, ro_size}, + auto output = EinsumOp::MatMul(current_left ? *current_left : left, TensorShapeVector{lro_size, lo_size, reduced_size}, + current_right ? *current_right : right, TensorShapeVector{lro_size, reduced_size, ro_size}, allocator_, tp_, einsum_ep_assets_, device_matmul_func_); output->Reshape(output_dims); @@ -353,9 +353,9 @@ Status EinsumTypedComputeProcessor::Run() { std::unique_ptr result; { - std::vector reduced_dims; - std::vector preserved_dims; // dims which were not reduced - std::vector preserved_shape; // shape pertaining to only the dims that were preserved (not reduced) + TensorShapeVector reduced_dims; + TensorShapeVector preserved_dims; // dims which were not reduced + TensorShapeVector preserved_shape; // shape pertaining to only the dims that were preserved (not reduced) reduced_dims.reserve(num_subscript_labels); // num_subscript_labels is the upper bound. No harm in over-reserving. preserved_dims.reserve(num_subscript_labels); // num_subscript_labels is the upper bound. No harm in over-reserving. @@ -395,7 +395,7 @@ Status EinsumTypedComputeProcessor::Run() { bool is_final_pair = false; // Keep processing each input pair-wise for (int input = 1; input < num_inputs; ++input) { - std::vector reduced_dims; + TensorShapeVector reduced_dims; reduced_dims.reserve(num_subscript_labels); // num_subscript_labels is the upper bound. No harm in over-reserving by a small margin. for (int64_t dim = 0; dim < num_subscript_labels; ++dim) { if (mapped_indices_to_last_input_index[dim] == input) { diff --git a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.h b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.h index 1b9945faef..f858019c63 100644 --- a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.h +++ b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_typed_compute_processor.h @@ -45,14 +45,14 @@ class EinsumTypedComputeProcessor { const TensorShape& left_shape_override, const Tensor& right, const TensorShape& right_shape_override, - const std::vector& reduce_dims, + const gsl::span& reduce_dims, bool is_final_pair); // Here we take a "candidate output"(candidate output is a tensor that is a permutation and / or a reshape away from the final output), // and after a few operations to get it to the required output structure, copy it to the op's output // The candidate output might contain dims that may not be part of the op's output (i.e.) the dims will have to be unsqueezed void FinalizeOutput(const Tensor& candidate_output, - const std::vector& ordered_subscript_indices_in_candidate); + const gsl::span& ordered_subscript_indices_in_candidate); // Private members - OpKernelContext* context_; diff --git a/onnxruntime/core/providers/cpu/math/element_wise_ops.h b/onnxruntime/core/providers/cpu/math/element_wise_ops.h index a94d6b25c3..34cfee949f 100644 --- a/onnxruntime/core/providers/cpu/math/element_wise_ops.h +++ b/onnxruntime/core/providers/cpu/math/element_wise_ops.h @@ -4,6 +4,7 @@ #pragma once #include "core/common/common.h" +#include "core/framework/inlined_containers.h" #include "core/framework/op_kernel.h" #include "core/util/math_cpuonly.h" #include "core/providers/cpu/element_wise_ranged_transform.h" @@ -464,7 +465,7 @@ auto MakeEigenArrayMap(const Tensor& t) -> ConstEigenVectorArrayMap { } struct BroadcastIterator { - size_t Current() const { return index_; } + size_t Current() const noexcept { return index_; } size_t AdvanceBy(size_t delta) { size_t index = index_; @@ -506,6 +507,18 @@ struct BroadcastIterator { count_ *= axis; } + void AllocateCounters() { + counters_.resize(counts_.size(), 0); + } + + ptrdiff_t GetCountsFront() const { + return counts_.front(); + } + + ptrdiff_t GetDeltasFront() const { + return deltas_.front(); + } + void Append(ptrdiff_t axis, ptrdiff_t largest) { ORT_ENFORCE(axis == 1 || axis == largest, "Attempting to broadcast an axis by a dimension other than 1. ", axis, " by ", largest); @@ -532,12 +545,14 @@ struct BroadcastIterator { counts_.push_back(1); } - std::vector counters_; - std::vector deltas_; - std::vector counts_; + private: + using BroadCastVector = InlinedVector; + + BroadCastVector counters_; + BroadCastVector deltas_; + BroadCastVector counts_; ptrdiff_t count_{1}; // Running total count of entries in tensor, used while building up the entries - private: size_t index_{}; }; @@ -640,14 +655,14 @@ struct Broadcaster { } // Allocate the counters - iterator1_.counters_.resize(iterator1_.counts_.size(), 0); - iterator2_.counters_.resize(iterator2_.counts_.size(), 0); + iterator1_.AllocateCounters(); + iterator2_.AllocateCounters(); } - size_t GetSpanSize() const { return std::min(iterator1_.counts_.front(), iterator2_.counts_.front()); } + size_t GetSpanSize() const { return std::min(iterator1_.GetCountsFront(), iterator2_.GetCountsFront()); } BroadcastIterator iterator1_, iterator2_; - std::vector output_shape_; + TensorShapeVector output_shape_; }; struct InputBroadcaster { @@ -676,8 +691,8 @@ struct InputBroadcaster { // before calling any methods that require input 1 to have data. bool HaveTwoTensors() const { return input_tensor1_ != nullptr; } - bool IsInput0Scalar() const { return broadcaster_.iterator1_.deltas_.front() == 0; } - bool IsInput1Scalar() const { return broadcaster_.iterator2_.deltas_.front() == 0; } + bool IsInput0Scalar() const { return broadcaster_.iterator1_.GetDeltasFront() == 0; } + bool IsInput1Scalar() const { return broadcaster_.iterator2_.GetDeltasFront() == 0; } size_t Input0ElementSize() const { return input0_element_size_; } size_t Input1ElementSize() const { return input1_element_size_; } diff --git a/onnxruntime/core/providers/cpu/ml/onehotencoder.cc b/onnxruntime/core/providers/cpu/ml/onehotencoder.cc index 36201a5d1e..b2f5ffec68 100644 --- a/onnxruntime/core/providers/cpu/ml/onehotencoder.cc +++ b/onnxruntime/core/providers/cpu/ml/onehotencoder.cc @@ -73,7 +73,7 @@ common::Status OneHotEncoderOp::Compute(OpKernelContext* context) const { const auto* X = context->Input(0); const TensorShape& input_shape = X->Shape(); - auto output_shape=input_shape.GetDimsAsVector(); + auto output_shape=input_shape.AsShapeVector(); output_shape.push_back(num_categories_); Tensor* Y = context->Output(0, TensorShape(output_shape)); diff --git a/onnxruntime/core/providers/cpu/nn/conv.cc b/onnxruntime/core/providers/cpu/nn/conv.cc index 65950da13b..5d123681ca 100644 --- a/onnxruntime/core/providers/cpu/nn/conv.cc +++ b/onnxruntime/core/providers/cpu/nn/conv.cc @@ -21,6 +21,7 @@ #include "core/util/math_cpuonly.h" namespace onnxruntime { +using ConvPadVector = ConvAttributes::ConvPadVector; template Status Conv::Compute(OpKernelContext* context) const { @@ -32,23 +33,23 @@ Status Conv::Compute(OpKernelContext* context) const { const int64_t M = W->Shape()[0]; ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X, W)); - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape)); - std::vector pads(conv_attrs_.pads); + ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(kernel_shape.size() * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(kernel_shape.size(), 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(kernel_shape.size(), 1); } - std::vector Y_dims({N, M}); + TensorShapeVector Y_dims({N, M}); TensorShape input_shape = X->Shape().Slice(2); ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShape(input_shape, kernel_shape, strides, dilations, pads, Y_dims)); Tensor* Y = context->Output(0, Y_dims); @@ -161,23 +162,23 @@ Status Conv::Compute(OpKernelContext* context) const { const int64_t M = W->Shape()[0]; ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X, W)); - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape)); - std::vector pads(conv_attrs_.pads); + ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(kernel_shape.size() * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(kernel_shape.size(), 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(kernel_shape.size(), 1); } - std::vector Y_dims({N, M}); + TensorShapeVector Y_dims({N, M}); TensorShape input_shape = X->Shape().Slice(2); ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShape(input_shape, kernel_shape, strides, dilations, pads, Y_dims)); Tensor* Y = context->Output(0, TensorShape(Y_dims)); diff --git a/onnxruntime/core/providers/cpu/nn/conv.h b/onnxruntime/core/providers/cpu/nn/conv.h index 6d8d4ad92e..5ed5d2ca91 100644 --- a/onnxruntime/core/providers/cpu/nn/conv.h +++ b/onnxruntime/core/providers/cpu/nn/conv.h @@ -24,7 +24,7 @@ class Conv : public OpKernel { template <> class Conv : public OpKernel { public: - Conv(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info) { + Conv(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info) { activation_.ActivationKind = MlasIdentityActivation; } diff --git a/onnxruntime/core/providers/cpu/nn/conv_attributes.h b/onnxruntime/core/providers/cpu/nn/conv_attributes.h index 393a5ea992..ca1387b933 100644 --- a/onnxruntime/core/providers/cpu/nn/conv_attributes.h +++ b/onnxruntime/core/providers/cpu/nn/conv_attributes.h @@ -8,12 +8,16 @@ #include "core/providers/common.h" #include "core/util/math.h" #endif + +#include "core/framework/inlined_containers.h" #include "core/framework/op_node_proto_helper.h" namespace onnxruntime { // A helper struct holding attributes for Conv-family ops struct ConvAttributes { + using ConvPadVector = InlinedVector; + explicit ConvAttributes(const OpKernelInfo& info) { std::string auto_pad_str; auto status = info.GetAttr("auto_pad", &auto_pad_str); @@ -21,14 +25,15 @@ struct ConvAttributes { auto_pad = StringToAutoPadType(auto_pad_str); } - kernel_shape_specified = info.GetAttrs("kernel_shape", kernel_shape_).IsOK(); + kernel_shape_specified = info.GetAttrs("kernel_shape", kernel_shape_).IsOK(); - status = info.GetAttrs("strides", strides); + status = info.GetAttrs("strides", strides); if (!status.IsOK() || strides.empty()) { strides.resize(kernel_shape_.size(), 1); } - status = info.GetAttrs("pads", pads); + gsl::span pads_span; + status = info.GetAttrsAsSpan("pads", pads_span); if (!status.IsOK()) { // If pads are not explicitly provided, fill the container with all zeros // so that we can compute and fill in pad values downstream @@ -37,9 +42,10 @@ struct ConvAttributes { // Pads are explicitly provided, make sure that auto_pad is NOTSET ORT_ENFORCE(auto_pad == AutoPadType::NOTSET, "A Conv/ConvTranspose node has both 'auto_pad' and 'pads' attributes"); + pads.assign(pads_span.cbegin(), pads_span.cend()); } - status = info.GetAttrs("dilations", dilations); + status = info.GetAttrs("dilations", dilations); if (!status.IsOK() || dilations.empty()) { dilations.resize(kernel_shape_.size(), 1); } @@ -56,16 +62,16 @@ struct ConvAttributes { ORT_ENFORCE(info.GetAttr("auto_pad", &auto_pad_str).IsOK()); auto_pad = StringToAutoPadType(auto_pad_str); ORT_ENFORCE(info.GetAttr("group", &group).IsOK()); - ORT_ENFORCE(info.GetAttrs("kernel_shape", kernel_shape_).IsOK()); - ORT_ENFORCE(info.GetAttrs("strides", strides).IsOK()); - ORT_ENFORCE(info.GetAttrs("pads", pads).IsOK()); - ORT_ENFORCE(info.GetAttrs("dilations", dilations).IsOK()); + ORT_ENFORCE(info.GetAttrs("kernel_shape", kernel_shape_).IsOK()); + ORT_ENFORCE(info.GetAttrs("strides", strides).IsOK()); + ORT_ENFORCE(info.GetAttrs("pads", pads).IsOK()); + ORT_ENFORCE(info.GetAttrs("dilations", dilations).IsOK()); #endif } ~ConvAttributes() = default; - Status ComputeKernelShape(const TensorShape& weight_shape, std::vector& kernel_shape) const { + Status ComputeKernelShape(const TensorShape& weight_shape, TensorShapeVector& kernel_shape) const { if (kernel_shape_specified) { kernel_shape = kernel_shape_; if (kernel_shape.size() + 2 != weight_shape.NumDimensions()) { @@ -82,7 +88,7 @@ struct ConvAttributes { } } else { auto weight_dims = weight_shape.GetDims(); - kernel_shape = std::vector(weight_dims.begin() + 2, weight_dims.end()); + kernel_shape.assign(weight_dims.begin() + 2, weight_dims.end()); } return Status::OK(); @@ -120,11 +126,11 @@ struct ConvAttributes { } Status InferOutputShape(const TensorShape& input_shape, - const std::vector& kernel_shape, - const std::vector& strides_p, - const std::vector& dilations_p, - std::vector& pads_p, - std::vector& output_shape, + const gsl::span& kernel_shape, + const gsl::span& strides_p, + const gsl::span& dilations_p, + ConvPadVector& pads_p, + TensorShapeVector& output_shape, bool force_symmetric_auto_padding = false) const { size_t rank = input_shape.NumDimensions(); @@ -168,16 +174,16 @@ struct ConvAttributes { // and to collect metadata regarding the portion of the output (with "adjusted" pads) // to be sliced off to make the output correspond to the "actual" asymmetric paddings Status InferOutputShapeWithAdjustedPads(const TensorShape& input_shape, - const std::vector& kernel_shape, - const std::vector& strides_p, - const std::vector& dilations_p, - std::vector& pads_p, - std::vector& output_shape, - std::vector& output_shape_with_revised_pads, + const gsl::span& kernel_shape, + const gsl::span& strides_p, + const gsl::span& dilations_p, + ConvPadVector& pads_p, + TensorShapeVector& output_shape, + TensorShapeVector& output_shape_with_revised_pads, bool& post_slicing_needed, - std::vector& slice_starts, - std::vector& slice_ends, - std::vector& slice_axes) const { + TensorShapeVector& slice_starts, + TensorShapeVector& slice_ends, + TensorShapeVector& slice_axes) const { size_t rank = input_shape.NumDimensions(); // Make sure all "metadata" containers have the right number of elements if (rank > strides_p.size()) @@ -299,14 +305,14 @@ struct ConvAttributes { AutoPadType auto_pad = AutoPadType::NOTSET; int64_t group; bool kernel_shape_specified; - std::vector strides; - std::vector pads; - std::vector dilations; + TensorShapeVector strides; + ConvPadVector pads; + TensorShapeVector dilations; std::string activation; - float alpha; + float alpha = 1.0f; private: - std::vector kernel_shape_; // must use ComputeKernelShape(...), instead of kernel_shape_ + TensorShapeVector kernel_shape_; // must use ComputeKernelShape(...), instead of kernel_shape_ }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h b/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h index 621a522344..ee74877da8 100644 --- a/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h +++ b/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h @@ -24,8 +24,8 @@ namespace onnxruntime { struct ConvTransposeAttributes : public ConvAttributes { explicit ConvTransposeAttributes(const OpKernelInfo& info) : ConvAttributes(info), - output_padding(info.GetAttrsOrDefault("output_padding")), - output_shape(info.GetAttrsOrDefault("output_shape")) { + output_padding(info.GetAttrsOrDefault("output_padding")), + output_shape(info.GetAttrsOrDefault("output_shape")) { } struct Prepare { @@ -37,10 +37,10 @@ struct ConvTransposeAttributes : public ConvAttributes { int64_t num_input_channels; int64_t num_output_channels; TensorShape input_shape; - std::vector kernel_shape; - std::vector pads; - std::vector dilations; - std::vector strides; + TensorShapeVector kernel_shape; + ConvPadVector pads; + TensorShapeVector dilations; + TensorShapeVector strides; }; Status PrepareForCompute(OpKernelContext* context, bool has_bias, Prepare& p, @@ -84,14 +84,14 @@ struct ConvTransposeAttributes : public ConvAttributes { " group: ", group); } - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(ComputeKernelShape(F_Shape, kernel_shape)); - std::vector local_output_padding(output_padding); + TensorShapeVector local_output_padding(output_padding); if (local_output_padding.empty()) { local_output_padding.resize(kernel_shape.size(), 0); } - std::vector local_pads; + ConvPadVector local_pads; local_pads.reserve(2 * (input_shape.NumDimensions())); if (dynamic_padding) { for (int64_t i = 0; i < Pads->Shape().SizeFromDimension(0); ++i) { @@ -103,16 +103,16 @@ struct ConvTransposeAttributes : public ConvAttributes { if (local_pads.empty()) { local_pads.resize(kernel_shape.size() * 2, 0); } - std::vector local_dilations(dilations); + TensorShapeVector local_dilations(dilations); if (local_dilations.empty()) { local_dilations.resize(kernel_shape.size(), 1); } - std::vector local_strides(strides); + TensorShapeVector local_strides(strides); if (local_strides.empty()) { local_strides.resize(kernel_shape.size(), 1); } - std::vector Y_dims; + TensorShapeVector Y_dims; ComputePadsAndOutputShape(input_shape, num_output_channels, kernel_shape, local_strides, local_dilations, local_output_padding, N, &local_pads, &Y_dims); @@ -135,9 +135,9 @@ struct ConvTransposeAttributes : public ConvAttributes { } void ComputePadsAndOutputShape(TensorShape input_shape, int64_t output_channel, - const std::vector& kernel_shape, const std::vector& p_strides, - const std::vector& p_dilations, const std::vector& p_output_padding, const int64_t N, - std::vector* p_pads, std::vector* output_shape_p) const { + const TensorShapeVector& kernel_shape, const TensorShapeVector& p_strides, + const TensorShapeVector& p_dilations, const TensorShapeVector& p_output_padding, const int64_t N, + ConvPadVector* p_pads, TensorShapeVector* output_shape_p) const { size_t output_shape_size = output_shape.size(); output_shape_p->insert(output_shape_p->begin(), {N, output_channel}); @@ -165,8 +165,8 @@ struct ConvTransposeAttributes : public ConvAttributes { } } - const std::vector output_padding; - const std::vector output_shape; + TensorShapeVector output_padding; + TensorShapeVector output_shape; private: int64_t ComputeTotalPad(int64_t in_size, int64_t stride, int64_t adj, diff --git a/onnxruntime/core/providers/cpu/nn/pool.cc b/onnxruntime/core/providers/cpu/nn/pool.cc index b78b6dca01..82e70bcee6 100644 --- a/onnxruntime/core/providers/cpu/nn/pool.cc +++ b/onnxruntime/core/providers/cpu/nn/pool.cc @@ -53,8 +53,8 @@ Status Pool::Compute(OpKernelContext* context) const { ORT_RETURN_IF_NOT(x_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3."); - std::vector pads = pool_attrs_.pads; - std::vector kernel_shape = pool_attrs_.kernel_shape; + auto pads = pool_attrs_.pads; + auto kernel_shape = pool_attrs_.kernel_shape; if (pool_attrs_.global_pooling) { const auto& input_dims = x_shape.GetDims(); @@ -62,7 +62,7 @@ Status Pool::Compute(OpKernelContext* context) const { pads.assign(kernel_shape.size(), 0); } - std::vector output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); + auto output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); Tensor* Y = context->Output(0, output_dims); const auto* X_data = X->template Data(); @@ -127,8 +127,8 @@ Status PoolBase::Compute(OpKernelContext* context, MLAS_POOLING_KIND kind) const "kernel_shape num_dims is not compatible with X num_dims."); } - std::vector pads = pool_attrs_.pads; - std::vector output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); + auto pads = pool_attrs_.pads; + auto output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); TensorShape output_shape(output_dims); Tensor* Y = context->Output(0, output_shape); @@ -189,10 +189,10 @@ Status MaxPoolV8::ComputeImpl(OpKernelContext* context) const { ORT_RETURN_IF_NOT(x_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3."); - std::vector pads = pool_attrs_.pads; - std::vector kernel_shape = pool_attrs_.kernel_shape; + auto pads = pool_attrs_.pads; + auto kernel_shape = pool_attrs_.kernel_shape; - std::vector output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); + auto output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); Tensor* Y = context->Output(0, output_dims); Tensor* I = context->Output(1, output_dims); diff --git a/onnxruntime/core/providers/cpu/nn/pool_attributes.h b/onnxruntime/core/providers/cpu/nn/pool_attributes.h index 87f07e55f7..392936f0b8 100644 --- a/onnxruntime/core/providers/cpu/nn/pool_attributes.h +++ b/onnxruntime/core/providers/cpu/nn/pool_attributes.h @@ -32,18 +32,18 @@ struct PoolAttributes { return; } - ORT_ENFORCE(info.GetAttrs("kernel_shape", kernel_shape).IsOK(), + ORT_ENFORCE(info.GetAttrs("kernel_shape", kernel_shape).IsOK(), "No kernel shape is set."); std::string auto_padding; ORT_ENFORCE(info.GetAttr("auto_pad", &auto_padding).IsOK()); auto_pad = StringToAutoPadType(auto_padding); - if (!info.GetAttrs("pads", pads).IsOK() || pads.empty()) { + if (!info.GetAttrs("pads", pads).IsOK() || pads.empty()) { pads.resize(kernel_shape.size() * 2, 0); } - if (!info.GetAttrs("strides", strides).IsOK() || strides.empty()) { + if (!info.GetAttrs("strides", strides).IsOK() || strides.empty()) { strides.resize(kernel_shape.size(), 1); } @@ -52,7 +52,7 @@ struct PoolAttributes { } default_dilations = false; - if (!info.GetAttrs("dilations", dilations).IsOK() || dilations.empty()) { + if (!info.GetAttrs("dilations", dilations).IsOK() || dilations.empty()) { dilations.resize(kernel_shape.size(), 1); default_dilations = true; } else { @@ -87,20 +87,20 @@ struct PoolAttributes { bool count_include_pad{}; int64_t storage_order{0}; // MaxPool_8 only. 0 is row major, and 1 is column major. Default is 0. int64_t ceil_mode{0}; // Introduced in MaxPool_10 - std::vector kernel_shape; - std::vector pads; - std::vector strides; - std::vector dilations; // Introduced in MaxPool_10 + TensorShapeVector kernel_shape; + TensorShapeVector pads; + TensorShapeVector strides; + TensorShapeVector dilations; // Introduced in MaxPool_10 // default_dilations is true if dilations is not set or all dilations are 1 bool default_dilations; AutoPadType auto_pad; - std::vector SetOutputSize(const TensorShape& input_shape, + TensorShapeVector SetOutputSize(const TensorShape& input_shape, int64_t output_channel, - std::vector* actual_pads) const { + TensorShapeVector* actual_pads) const { ORT_ENFORCE(input_shape.Size() > 0 || input_shape[0] == 0, "Invalid input shape. Only N can be zero. Got:", input_shape); - std::vector output_dims; + TensorShapeVector output_dims; int64_t N = input_shape[0]; InferOutputSize(input_shape.GetDims(), &output_dims, actual_pads); @@ -110,8 +110,8 @@ struct PoolAttributes { } void InferOutputSize(gsl::span input_dims, - std::vector* output_dims, - std::vector* actual_pads) const { + TensorShapeVector* output_dims, + TensorShapeVector* actual_pads) const { ORT_ENFORCE(input_dims.size() >= 2); if (global_pooling) { output_dims->assign(input_dims.size() - 2, 1); diff --git a/onnxruntime/core/providers/cpu/nn/pool_functors.h b/onnxruntime/core/providers/cpu/nn/pool_functors.h index 16d3da07a1..90104cb56f 100644 --- a/onnxruntime/core/providers/cpu/nn/pool_functors.h +++ b/onnxruntime/core/providers/cpu/nn/pool_functors.h @@ -15,8 +15,8 @@ struct Pool1DTask final { int64_t pooled_height; int64_t stride_h; int64_t height; - const std::vector& kernel_shape; - const std::vector& pads; + gsl::span kernel_shape; + gsl::span pads; const PoolProcessContext& pool_context_; const PoolAttributes& pool_attrs_; TensorOpCost Cost() { @@ -63,8 +63,8 @@ struct Pool2DTask final { int64_t stride_w; int64_t height; int64_t width; - const std::vector& kernel_shape; - const std::vector& pads; + gsl::span kernel_shape; + gsl::span pads; const PoolProcessContext& pool_context_; const PoolAttributes& pool_attrs_; @@ -125,8 +125,8 @@ struct Pool3DTask final { int64_t height; int64_t width; int64_t depth; - const std::vector& kernel_shape; - const std::vector& pads; + gsl::span kernel_shape; + gsl::span pads; const PoolProcessContext& pool_context_; const PoolAttributes& pool_attrs_; @@ -191,8 +191,8 @@ struct MaxPool1DTask final { int64_t pooled_height; int64_t stride_h; int64_t height; - const std::vector& kernel_shape; - const std::vector& pads; + gsl::span kernel_shape; + gsl::span pads; TensorOpCost Cost() { double loop_count = static_cast(pooled_height * kernel_shape[0]); return TensorOpCost{loop_count, loop_count, loop_count}; @@ -242,8 +242,8 @@ struct MaxPool2DTask final { int64_t stride_w; int64_t height; int64_t width; - const std::vector& kernel_shape; - const std::vector& pads; + gsl::span kernel_shape; + gsl::span pads; int64_t storage_order; TensorOpCost Cost() { @@ -313,8 +313,8 @@ struct MaxPool3DTask { int64_t height; int64_t width; int64_t depth; - const std::vector& kernel_shape; - const std::vector& pads; + gsl::span kernel_shape; + gsl::span pads; int64_t storage_order; void operator()(std::ptrdiff_t begin, std::ptrdiff_t end) const { diff --git a/onnxruntime/core/providers/cpu/quantization/conv_integer.cc b/onnxruntime/core/providers/cpu/quantization/conv_integer.cc index d03569f89f..dd960b2b9c 100644 --- a/onnxruntime/core/providers/cpu/quantization/conv_integer.cc +++ b/onnxruntime/core/providers/cpu/quantization/conv_integer.cc @@ -11,6 +11,8 @@ namespace onnxruntime { +using ConvPadVector = ConvAttributes::ConvPadVector; + class ConvInteger : public OpKernel { public: explicit ConvInteger(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info) {} @@ -53,23 +55,23 @@ Status ConvInteger::Compute(OpKernelContext* context) const { const int64_t M = W->Shape()[0]; ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X, W)); - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape)); - std::vector pads(conv_attrs_.pads); + ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(kernel_shape.size() * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(kernel_shape.size(), 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(kernel_shape.size(), 1); } - std::vector Y_dims({N, M}); + TensorShapeVector Y_dims({N, M}); TensorShape input_shape = X->Shape().Slice(2); ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShape(input_shape, kernel_shape, strides, dilations, pads, Y_dims)); Tensor* Y = context->Output(0, TensorShape(Y_dims)); diff --git a/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc b/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc index 2b81637f7f..26a70a0033 100644 --- a/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc +++ b/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc @@ -13,6 +13,8 @@ namespace onnxruntime { +using ConvPadVector = ConvAttributes::ConvPadVector; + template class QLinearConv : public OpKernel { public: @@ -440,20 +442,20 @@ Status QLinearConv::Compute(OpKernelContext* context) const { ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X->Shape(), W_shape, channels_last_)); - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W_shape, kernel_shape)); const size_t kernel_rank = kernel_shape.size(); - std::vector pads(conv_attrs_.pads); + ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(kernel_rank * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(kernel_rank, 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(kernel_rank, 1); } @@ -462,7 +464,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { const size_t spatial_dim_start = channels_last_ ? 1 : 2; const size_t spatial_dim_end = spatial_dim_start + kernel_rank; - std::vector Y_dims({N}); + TensorShapeVector Y_dims({N}); if (!channels_last_) { Y_dims.push_back(M); } diff --git a/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc b/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc index dc85ed54f0..73c9edf7ab 100644 --- a/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc +++ b/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/framework/inlined_containers.h" #include "core/providers/cpu/reduction/reduction_ops.h" #include "core/providers/common.h" //TODO: fix the warnings @@ -247,28 +248,28 @@ static void ValidateMustBeOverloaded() { ORT_ENFORCE(false, "must be overloaded."); } -static void ValidateFastReduceKR(const std::vector& fast_shape, const Tensor& output) { +static void ValidateFastReduceKR(const gsl::span& fast_shape, const Tensor& output) { ORT_ENFORCE(fast_shape.size() == 2, "Only works on matrices with two dimensions."); ORT_ENFORCE(fast_shape[0] == output.Shape().Size(), "Output size mismatch."); } -static void ValidateFastReduceRK(const std::vector& fast_shape, const Tensor& output) { +static void ValidateFastReduceRK(const gsl::span& fast_shape, const Tensor& output) { ORT_ENFORCE(fast_shape.size() == 2, "Only works on matrices with two dimensions."); ORT_ENFORCE(fast_shape[1] == output.Shape().Size(), "Output size mismatch."); } -static void ValidateFastReduceKRK(const std::vector& fast_shape, const Tensor& output) { +static void ValidateFastReduceKRK(const gsl::span& fast_shape, const Tensor& output) { ORT_ENFORCE(fast_shape.size() == 3, "Only works on matrices with two dimensions."); ORT_ENFORCE(fast_shape[0] * fast_shape[2] == output.Shape().Size(), "Output size mismatch."); } -void ReduceAggregatorBase::FastReduceKR(const Tensor&, const std::vector&, Tensor&, concurrency::ThreadPool*) { +void ReduceAggregatorBase::FastReduceKR(const Tensor&, const gsl::span&, Tensor&, concurrency::ThreadPool*) { ValidateMustBeOverloaded(); } -void ReduceAggregatorBase::FastReduceRK(const Tensor&, const std::vector&, Tensor&, concurrency::ThreadPool*) { +void ReduceAggregatorBase::FastReduceRK(const Tensor&, const gsl::span&, Tensor&, concurrency::ThreadPool*) { ValidateMustBeOverloaded(); } -void ReduceAggregatorBase::FastReduceKRK(const Tensor&, const std::vector&, Tensor&, concurrency::ThreadPool*) { +void ReduceAggregatorBase::FastReduceKRK(const Tensor&, const gsl::span&, Tensor&, concurrency::ThreadPool*) { ValidateMustBeOverloaded(); } @@ -276,7 +277,7 @@ void NoTransposePrepareForReduce(const TensorShape& new_input_shape, gsl::span reduced_axes, ResultsNoTransposePrepareForReduce& results) { // Common initialisation for the indices. - auto cumulative_shape = new_input_shape.GetDimsAsVector(); + auto cumulative_shape = new_input_shape.AsShapeVector(); cumulative_shape[cumulative_shape.size() - 1] = 1; for (int i = static_cast(cumulative_shape.size()) - 2; i >= 0; --i) { cumulative_shape[i] = cumulative_shape[i + 1] * new_input_shape[i + 1]; @@ -307,7 +308,7 @@ void NoTransposePrepareForReduce(const TensorShape& new_input_shape, results.projected_index.resize(1, 0); } else { results.projected_index.resize(projection_size); - std::vector projected_indices(reduced_axes_size, 0); + TensorShapeVector projected_indices(reduced_axes_size, 0); int64_t current_index = 0; size_t current_pos = 0; int j; @@ -327,7 +328,7 @@ void NoTransposePrepareForReduce(const TensorShape& new_input_shape, } // Builds the list of indices for the unprojected sum. - std::vector unreduced_axes; + TensorShapeVector unreduced_axes; for (int64_t i = 0; i < static_cast(cumulative_shape.size()); ++i) { if (std::find(reduced_axes.begin(), reduced_axes.end(), i) != reduced_axes.end()) continue; @@ -340,7 +341,7 @@ void NoTransposePrepareForReduce(const TensorShape& new_input_shape, if (unprojection_size == 0) { return; } - std::vector unprojected_indices(unreduced_axes.size(), 0); + TensorShapeVector unprojected_indices(unreduced_axes.size(), 0); // The last index is usually an image size. // We differently process the last unprojected dimension. @@ -519,10 +520,10 @@ void NoTransposeReduce2Loops(Tensor* output, const TensorShape& new_input_shape, concurrency::ThreadPool::TryParallelFor(tp, count, cost, fn); } -void DropDimensions(const std::vector& input_shape, - const std::vector& axes, - std::vector& dropped_axes) { - auto dropped_dims = input_shape; +void DropDimensions(const gsl::span& input_shape, + const gsl::span& axes, + TensorShapeVector& dropped_axes) { + TensorShapeVector dropped_dims = ToShapeVector(input_shape); for (auto i : axes) { dropped_dims[i] = -1; } @@ -535,33 +536,34 @@ void DropDimensions(const std::vector& input_shape, FastReduceKind OptimizeShapeForFastReduce(gsl::span input_shape, gsl::span reduced_axes, - std::vector& fast_shape, - std::vector& fast_output_shape, - std::vector& fast_axes, + TensorShapeVector& fast_shape, + TensorShapeVector& fast_output_shape, + TensorShapeVector& fast_axes, bool keep_dims, bool noop_with_empty_axes) { if (input_shape.empty()) { - fast_shape = std::vector(input_shape.begin(), input_shape.end()); + fast_shape.assign(input_shape.begin(), input_shape.end()); fast_output_shape = fast_shape; - fast_axes = std::vector(reduced_axes.begin(), reduced_axes.end()); + fast_axes.assign(reduced_axes.begin(), reduced_axes.end()); return FastReduceKind::kNone; } - std::set axes; + InlinedHashSet axes; + const auto input_shape_size = gsl::narrow(input_shape.size()); if (reduced_axes.size() == 0 && !noop_with_empty_axes) { - for (int64_t i = 0; i < (int64_t)input_shape.size(); ++i) { + for (int64_t i = 0; i < input_shape_size; ++i) { axes.insert(i); } } else { - for (auto it = reduced_axes.begin(); it != reduced_axes.end(); ++it) { - axes.insert(HandleNegativeAxis(*it, static_cast(input_shape.size()))); + for (auto ax : reduced_axes) { + axes.insert(HandleNegativeAxis(ax, input_shape_size)); } } fast_output_shape.clear(); - fast_output_shape.reserve(input_shape.size()); + fast_output_shape.reserve(input_shape_size); bool empty_reduce = false; - std::vector reduce(input_shape.size()); - for (int64_t i = 0; i < (int64_t)input_shape.size(); ++i) { + InlinedShapeVectorT reduce(input_shape_size); + for (int64_t i = 0; i < input_shape_size; ++i) { reduce[i] = axes.find(i) != axes.end(); if (reduce[i]) { empty_reduce |= input_shape[i] == 0; @@ -584,11 +586,11 @@ FastReduceKind OptimizeShapeForFastReduce(gsl::span input_shape, } if (noop_with_empty_axes) { fast_axes.clear(); - fast_output_shape = std::vector(input_shape.begin(), input_shape.end()); + fast_output_shape.assign(input_shape.cbegin(), input_shape.cend()); return FastReduceKind::kK; } else { if (keep_dims) { - fast_output_shape.resize(input_shape.size(), 1); + fast_output_shape.resize(input_shape_size, 1); } else { fast_output_shape.clear(); } @@ -600,13 +602,13 @@ FastReduceKind OptimizeShapeForFastReduce(gsl::span input_shape, fast_shape.clear(); fast_axes.clear(); - fast_shape.reserve(input_shape.size()); + fast_shape.reserve(input_shape_size); fast_axes.reserve(reduced_axes.size()); fast_shape.push_back(input_shape[0]); if (reduce[0]) fast_axes.push_back(0); - for (size_t i = 1; i < input_shape.size(); ++i) { + for (int64_t i = 1; i < input_shape_size; ++i) { if (reduce[i] == reduce[i - 1]) { fast_shape[fast_shape.size() - 1] *= input_shape[i]; } else { @@ -635,7 +637,7 @@ void ValidateCommonFastReduce(const Tensor* axes_tensor) { } //template -bool CommonFastReduceCopy(OpKernelContext* ctx, std::vector& input_axes, bool noop_with_empty_axes) { +bool CommonFastReduceCopy(OpKernelContext* ctx, TensorShapeVector& input_axes, bool noop_with_empty_axes) { if (ctx->InputCount() == 2) { // second input holds the axes. const Tensor* axes_tensor = ctx->Input(1); @@ -655,25 +657,25 @@ bool CommonFastReduceCopy(OpKernelContext* ctx, std::vector& input_axes return false; } -typedef void fast_reduce_fct(const Tensor& input, const std::vector& fast_shape, +typedef void fast_reduce_fct(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp); bool CommonFastReduceSwitch(OpKernelContext* ctx, - const std::vector& axes_, + const gsl::span& axes_, int64_t keepdims_, bool noop_with_empty_axes, FastReduceKind& fast_kind, - std::vector& fast_shape, - std::vector& output_shape, - std::vector& fast_axes, + TensorShapeVector& fast_shape, + TensorShapeVector& output_shape, + TensorShapeVector& fast_axes, FastReduceKind which_fast_reduce, fast_reduce_fct* case_kr, fast_reduce_fct* case_rk, fast_reduce_fct* case_krk) { - std::vector axes; + TensorShapeVector axes; const Tensor* input = ctx->Input(0); auto reduced_dims = input->Shape().GetDims(); - std::vector input_axes; + TensorShapeVector input_axes; if (CommonFastReduceCopy(ctx, input_axes, noop_with_empty_axes)) { return true; @@ -727,13 +729,13 @@ bool CommonFastReduceSwitch(OpKernelContext* ctx, template bool CommonFastReduce(OpKernelContext* ctx, - const std::vector& axes_, + const gsl::span& axes_, int64_t keepdims_, bool noop_with_empty_axes, FastReduceKind& fast_kind, - std::vector& fast_shape, - std::vector& output_shape, - std::vector& fast_axes) { + TensorShapeVector& fast_shape, + TensorShapeVector& output_shape, + TensorShapeVector& fast_axes) { return CommonFastReduceSwitch(ctx, axes_, keepdims_, noop_with_empty_axes, fast_kind, fast_shape, output_shape, fast_axes, AGG::WhichFastReduce(), &AGG::FastReduceKR, &AGG::FastReduceRK, &AGG::FastReduceKRK); @@ -752,12 +754,12 @@ static void ValidateKeepDims(const Tensor* input, int64_t keepdims) { template void CommonReduce1Loop(OpKernelContext* ctx, - const std::vector& axes_, int64_t keepdims_, + const gsl::span& axes_, int64_t keepdims_, bool noop_with_empty_axes) { FastReduceKind fast_kind; - std::vector fast_shape; - std::vector output_shape; - std::vector fast_axes; + TensorShapeVector fast_shape; + TensorShapeVector output_shape; + TensorShapeVector fast_axes; if (CommonFastReduce(ctx, axes_, keepdims_, noop_with_empty_axes, fast_kind, fast_shape, output_shape, fast_axes)) { return; @@ -785,10 +787,10 @@ void CommonReduce1Loop(OpKernelContext* ctx, template void CommonReduce2Loops(OpKernelContext* ctx, - const std::vector& axes_, int64_t keepdims_, + const gsl::span& axes_, int64_t keepdims_, bool noop_with_empty_axes) { FastReduceKind fast_kind; - std::vector fast_shape, output_shape, fast_axes; + TensorShapeVector fast_shape, output_shape, fast_axes; if (CommonFastReduce(ctx, axes_, keepdims_, noop_with_empty_axes, fast_kind, fast_shape, output_shape, fast_axes)) { return; @@ -876,15 +878,15 @@ template std::unique_ptr ReduceSum::Impl(const Tensor& input, gsl::span reduce_axes, AllocatorPtr allocator, concurrency::ThreadPool* tp, bool keep_dims, const TensorShape* input_shape_override) { - std::vector axes; - std::vector output_shape, fast_shape, fast_axes; + TensorShapeVector axes; + TensorShapeVector output_shape, fast_shape, fast_axes; TensorShape new_input_shape = input_shape_override == nullptr ? input.Shape() : *input_shape_override; auto reduced_dims = new_input_shape.GetDims(); FastReduceKind fast_kind = OptimizeShapeForFastReduce( reduced_dims, reduce_axes, fast_shape, output_shape, fast_axes, keep_dims, false); - auto output = make_unique(input.DataType(), keep_dims ? output_shape : std::vector(), allocator); + auto output = std::make_unique(input.DataType(), keep_dims ? output_shape : TensorShapeVector(), allocator); if (fast_kind == FastReduceKind::kEmpty) { if (new_input_shape.Size() == 1) { @@ -973,16 +975,16 @@ template class ReduceSum; template class ReduceSum; template void CommonReduce1Loop>(OpKernelContext* ctx, - const std::vector& axes_, int64_t keepdims_, + const gsl::span& axes_, int64_t keepdims_, bool noop_with_empty_axes); template void CommonReduce1Loop>(OpKernelContext* ctx, - const std::vector& axes_, int64_t keepdims_, + const gsl::span& axes_, int64_t keepdims_, bool noop_with_empty_axes); template void CommonReduce1Loop>(OpKernelContext* ctx, - const std::vector& axes_, int64_t keepdims_, + const gsl::span& axes_, int64_t keepdims_, bool noop_with_empty_axes); template void CommonReduce1Loop>(OpKernelContext* ctx, - const std::vector& axes_, int64_t keepdims_, + const gsl::span& axes_, int64_t keepdims_, bool noop_with_empty_axes); } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/reduction/reduction_ops.h b/onnxruntime/core/providers/cpu/reduction/reduction_ops.h index 8acce4524d..ebfa9f36e0 100644 --- a/onnxruntime/core/providers/cpu/reduction/reduction_ops.h +++ b/onnxruntime/core/providers/cpu/reduction/reduction_ops.h @@ -61,19 +61,19 @@ constexpr TensorOpCost ParallelReduceFastCost(int64_t n_row, int64_t n_col, int6 */ FastReduceKind OptimizeShapeForFastReduce(gsl::span input_shape, gsl::span reduced_axes, - std::vector& fast_shape, - std::vector& fast_output_shape, - std::vector& fast_axes, + TensorShapeVector& fast_shape, + TensorShapeVector& fast_output_shape, + TensorShapeVector& fast_axes, bool keep_dims, bool noop_with_empty_axes = false); class ResultsNoTransposePrepareForReduce { public: - std::vector input_shape; - std::vector reduced_axes; - std::vector projected_index; + TensorShapeVector input_shape; + TensorShapeVector reduced_axes; + TensorShapeVector projected_index; int64_t last_loop_red_size; int64_t last_loop_red_inc; - std::vector unprojected_index; + TensorShapeVector unprojected_index; int64_t last_loop_size; int64_t last_loop_inc; @@ -151,9 +151,9 @@ class ReduceAggregatorBase { public: // Fast reduction: see OptimizeShapeForFastReduce's comment. static inline FastReduceKind WhichFastReduce() { return FastReduceKind::kNone; } - static void FastReduceKR(const Tensor&, const std::vector&, Tensor&, concurrency::ThreadPool*); - static void FastReduceRK(const Tensor&, const std::vector&, Tensor&, concurrency::ThreadPool*); - static void FastReduceKRK(const Tensor&, const std::vector&, Tensor&, concurrency::ThreadPool*); + static void FastReduceKR(const Tensor&, const gsl::span&, Tensor&, concurrency::ThreadPool*); + static void FastReduceRK(const Tensor&, const gsl::span&, Tensor&, concurrency::ThreadPool*); + static void FastReduceKRK(const Tensor&, const gsl::span&, Tensor&, concurrency::ThreadPool*); }; template @@ -191,7 +191,7 @@ class ReduceAggregatorSum : public ReduceAggregator { return FastReduceKind::kKR | FastReduceKind::kRK | FastReduceKind::kKRK; } - static void FastReduceKR(const Tensor& input, const std::vector& fast_shape, + static void FastReduceKR(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { const T* data = input.Data(); T* out = output.MutableData(); @@ -205,7 +205,7 @@ class ReduceAggregatorSum : public ReduceAggregator { }); } - static void FastReduceRK(const Tensor& input, const std::vector& fast_shape, + static void FastReduceRK(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { int64_t N = fast_shape[1]; const T* data = input.Data(); @@ -223,7 +223,7 @@ class ReduceAggregatorSum : public ReduceAggregator { }); } - static void FastReduceKRK(const Tensor& input, const std::vector& fast_shape, + static void FastReduceKRK(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { int64_t N = fast_shape[2]; const T* data = input.Data(); @@ -263,7 +263,7 @@ class ReduceAggregatorMean : public ReduceAggregatorSum { // Fast reduction // WhichFastReduce() already defined in ReduceAggregatorSum - static void FastReduceKR(const Tensor& input, const std::vector& fast_shape, + static void FastReduceKR(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { ReduceAggregatorSum::FastReduceKR(input, fast_shape, output, tp); // TODO: use MLAS or BLAS @@ -274,7 +274,7 @@ class ReduceAggregatorMean : public ReduceAggregatorSum { } } - static void FastReduceRK(const Tensor& input, const std::vector& fast_shape, + static void FastReduceRK(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { ReduceAggregatorSum::FastReduceRK(input, fast_shape, output, tp); // TODO: use MLAS or BLAS @@ -285,7 +285,7 @@ class ReduceAggregatorMean : public ReduceAggregatorSum { } } - static void FastReduceKRK(const Tensor& input, const std::vector& fast_shape, + static void FastReduceKRK(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { ReduceAggregatorSum::FastReduceKRK(input, fast_shape, output, tp); int64_t strideo = fast_shape[2]; @@ -317,7 +317,7 @@ class ReduceAggregatorMax : public ReduceAggregator { return FastReduceKind::kKR | FastReduceKind::kRK | FastReduceKind::kKRK; } - static void FastReduceKR(const Tensor& input, const std::vector& fast_shape, + static void FastReduceKR(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { const T* data = input.Data(); T* out = output.MutableData(); @@ -332,7 +332,7 @@ class ReduceAggregatorMax : public ReduceAggregator { }); } - static void FastReduceRK(const Tensor& input, const std::vector& fast_shape, + static void FastReduceRK(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { int64_t n_rows = fast_shape[0]; int64_t N = fast_shape[1]; @@ -353,7 +353,7 @@ class ReduceAggregatorMax : public ReduceAggregator { }); } - static void FastReduceKRK(const Tensor& input, const std::vector& fast_shape, + static void FastReduceKRK(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { const T* data = input.Data(); T* out = output.MutableData(); @@ -476,7 +476,7 @@ class ReduceAggregatorMin : public ReduceAggregator { return FastReduceKind::kKR | FastReduceKind::kRK | FastReduceKind::kKRK; } - static void FastReduceKR(const Tensor& input, const std::vector& fast_shape, + static void FastReduceKR(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { const T* data = input.Data(); T* out = output.MutableData(); @@ -491,7 +491,7 @@ class ReduceAggregatorMin : public ReduceAggregator { }); } - static void FastReduceRK(const Tensor& input, const std::vector& fast_shape, + static void FastReduceRK(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { int64_t n_rows = fast_shape[0]; int64_t N = fast_shape[1]; @@ -512,7 +512,7 @@ class ReduceAggregatorMin : public ReduceAggregator { }); } - static void FastReduceKRK(const Tensor& input, const std::vector& fast_shape, + static void FastReduceKRK(const Tensor& input, const gsl::span& fast_shape, Tensor& output, concurrency::ThreadPool* tp) { const T* data = input.Data(); T* out = output.MutableData(); @@ -598,7 +598,7 @@ class ReduceAggregatorLogSumExp : public ReduceAggregator { }; void NoTransposePrepareForReduce(const TensorShape& new_input_shape, - const std::vector& reduced_axes, + gsl::span reduced_axes, ResultsNoTransposePrepareForReduce& results); template @@ -614,13 +614,13 @@ void NoTransposeReduce2Loops(Tensor* output, const TensorShape& new_input_shape, template void CommonReduce1Loop(OpKernelContext* ctx, - const std::vector& axes_, int64_t keepdims_, + const gsl::span& axes_, int64_t keepdims_, bool noop_with_empty_axes = false); // Specific case for ReduceLogSumExp. template void CommonReduce2Loops(OpKernelContext* ctx, - const std::vector& axes_, int64_t keepdims_, + const gsl::span& axes_, int64_t keepdims_, bool noop_with_empty_axes = false); template @@ -628,7 +628,7 @@ class ReduceKernelBase { protected: ReduceKernelBase(const OpKernelInfo& info, optional keepdims_override = {}) { if (allow_multi_axes) { - axes_ = info.GetAttrsOrDefault("axes"); + axes_ = ToShapeVector(info.GetAttrsOrDefault("axes")); } else { auto v = info.GetAttrOrDefault("axis", 0); axes_.push_back(v); @@ -646,7 +646,7 @@ class ReduceKernelBase { select_last_index_ = (select_last_index != 0); } - std::vector axes_; + TensorShapeVector axes_; bool keepdims_; bool noop_with_empty_axes_; bool select_last_index_; diff --git a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h index c94cdf14ab..5551ad1cad 100644 --- a/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h +++ b/onnxruntime/core/providers/cpu/rnn/rnn_helpers.h @@ -56,7 +56,7 @@ gsl::span Allocate(std::shared_ptr allocator, size_t size, IAllocatorUniquePtr& unique_ptr, bool fill = false, TAlloc fill_value = TAlloc{}) { - unique_ptr = IAllocator::MakeUniquePtr(allocator, size); + unique_ptr = IAllocator::MakeUniquePtr(std::move(allocator), size); auto span = gsl::make_span(unique_ptr.get(), size); if (fill) { diff --git a/onnxruntime/core/providers/cpu/sequence/concat_from_sequence.cc b/onnxruntime/core/providers/cpu/sequence/concat_from_sequence.cc index 352cdb83e0..2e7ddbb98a 100644 --- a/onnxruntime/core/providers/cpu/sequence/concat_from_sequence.cc +++ b/onnxruntime/core/providers/cpu/sequence/concat_from_sequence.cc @@ -23,7 +23,7 @@ Status ConcatFromSequence::Compute(OpKernelContext* ctx) const { ORT_ENFORCE(X != nullptr, "Got nullptr for sequence input."); // Hold pointers to the input tensors to be used in the PrepareForCompute() step - std::vector input_tensor_pointers; + InlinedTensorsVector input_tensor_pointers; input_tensor_pointers.reserve(X->Size()); for (const auto& t : *X) { input_tensor_pointers.push_back(&t); diff --git a/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc b/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc index 9645028745..e2a51e0c0c 100644 --- a/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc +++ b/onnxruntime/core/providers/cpu/sequence/sequence_ops.cc @@ -506,7 +506,7 @@ Status SplitToSequence::ComputeImpl(OpKernelContext& context, const Tensor& inpu split_sizes)); // copy dimensions so we can update the selected axis in place - auto output_dimensions = input_shape.GetDimsAsVector(); + auto output_dimensions = input_shape.AsShapeVector(); std::vector tensors; int64_t input_offset = 0; const T* input_data = input.template Data(); @@ -540,7 +540,7 @@ Status SplitToSequence::ComputeImpl(OpKernelContext& context, const Tensor& inpu // if keep_dims = 0, reshape the tensor by dropping the dimension corresponding to 'axis' if (use_keep_dims && keepdims_ == 0) { - std::vector new_dims; + TensorShapeVector new_dims; new_dims.reserve(output_dimensions.size() - 1); for (int64_t idx = 0, end = static_cast(output_dimensions.size()); idx < end; ++idx) { if (idx != axis) { diff --git a/onnxruntime/core/providers/cpu/tensor/concat.cc b/onnxruntime/core/providers/cpu/tensor/concat.cc index 9ca662231d..1aaadb0b26 100644 --- a/onnxruntime/core/providers/cpu/tensor/concat.cc +++ b/onnxruntime/core/providers/cpu/tensor/concat.cc @@ -54,19 +54,19 @@ using EnabledDataTypes = ORT_OP_KERNEL_ARG_ENABLED_TYPE_LIST_ALL_OPSETS(kCpuExec // this method will be shared between 'Concat' (CPU and GPU) and // 'ConcatFromSequence' ('concat' and 'stack' modes) to validate inputs Status ConcatBase::PrepareForCompute(OpKernelContext* ctx, - const std::vector& input_tensors, + const InlinedTensorsVector& input_tensors, Prepare& p) const { int input_count = static_cast(input_tensors.size()); // Must have atleast one input to concat ORT_RETURN_IF_NOT(input_count >= 1, "Must have 1 or more inputs"); - std::vector reference_dims; + TensorShapeVector reference_dims; size_t reference_rank = 0; int reference_tensor_index = 0; - std::vector input_tensor_sizes; + InlinedVector input_tensor_sizes; input_tensor_sizes.reserve(input_count); bool all_inputs_are_empty = true; @@ -81,7 +81,7 @@ Status ConcatBase::PrepareForCompute(OpKernelContext* ctx, const auto& shape = input->Shape(); const auto num_elements = shape.Size(); if (num_elements > 0) { - reference_dims = shape.GetDimsAsVector(); + reference_dims = shape.AsShapeVector(); reference_rank = reference_dims.size(); reference_tensor_index = index; input_tensor_sizes.push_back(num_elements); @@ -97,7 +97,7 @@ Status ConcatBase::PrepareForCompute(OpKernelContext* ctx, // No shape/rank validations will be done (as all inputs are empty). // But the rest of the execution flow (filling in the Prepare instance - p) // can use this info. - reference_dims = input_tensors[0]->Shape().GetDimsAsVector(); + reference_dims = input_tensors[0]->Shape().AsShapeVector(); reference_rank = reference_dims.size(); } @@ -164,7 +164,7 @@ Status ConcatBase::PrepareForCompute(OpKernelContext* ctx, } // Calculate the shape of the output tensor - std::vector output_dims = reference_dims; + auto output_dims = reference_dims; if (!is_stack_) { // 'Concat' mode // While concatenating, the rank of the output is the same as the input rank(s) @@ -233,16 +233,17 @@ Status ConcatBase::PrepareForCompute(OpKernelContext* ctx, } namespace { -std::vector StridesForStack(const std::vector& full_strides, uint64_t axis) { +TensorShapeVector StridesForStack(const TensorShapeVector& full_strides, uint64_t axis) { // if we are stacking, skip the dimension that will be stacked along in the output strides // (the striding for that dimension is handled by the initial_output_offset) - auto num_dims = full_strides.size(); + const auto num_dims = full_strides.size(); - std::vector strides(num_dims - 1); + TensorShapeVector strides; + strides.reserve(num_dims - 1); for (size_t i = 0; i < num_dims - 1; i++) { auto read_i = (i >= axis) ? i + 1 : i; - strides[i] = full_strides[read_i]; + strides.push_back(full_strides[read_i]); } return strides; } @@ -291,7 +292,7 @@ Status Concat::Compute(OpKernelContext* ctx) const { auto input_count = Node().InputArgCount().front(); // Hold pointers to the input tensors to be used in the PrepareForCompute() step - std::vector input_tensors; + InlinedTensorsVector input_tensors; input_tensors.reserve(input_count); for (int i = 0; i < input_count; ++i) { input_tensors.push_back(ctx->Input(i)); diff --git a/onnxruntime/core/providers/cpu/tensor/concatbase.h b/onnxruntime/core/providers/cpu/tensor/concatbase.h index 2f62ec9f5a..9d3d95ea1e 100644 --- a/onnxruntime/core/providers/cpu/tensor/concatbase.h +++ b/onnxruntime/core/providers/cpu/tensor/concatbase.h @@ -2,16 +2,19 @@ // Licensed under the MIT License. #pragma once +#include "core/framework/inlined_containers.h" + namespace onnxruntime { // structure to hold some inputs and some metadata to be used during Compute() struct Prepare { + static constexpr size_t kExpectedNumberOfInputs = 5; struct InputInfo { const Tensor* tensor; int64_t axis_pitch; int64_t num_elements; }; - std::vector inputs; + InlinedVector inputs; int64_t output_num_elements; int64_t output_axis_pitch; Tensor* output_tensor; @@ -23,7 +26,8 @@ class ConcatBase { public: // the core method that will be invoked by the 'Concat' (CPU and GPU) // and 'ConcatFromSequence' kernels - Status PrepareForCompute(OpKernelContext* ctx, const std::vector& input_tensors, + using InlinedTensorsVector = InlinedVector; + Status PrepareForCompute(OpKernelContext* ctx, const InlinedTensorsVector& input_tensors, Prepare& p) const; protected: diff --git a/onnxruntime/core/providers/cpu/tensor/copy.cc b/onnxruntime/core/providers/cpu/tensor/copy.cc index ecfa96671a..7bdeac0aa0 100644 --- a/onnxruntime/core/providers/cpu/tensor/copy.cc +++ b/onnxruntime/core/providers/cpu/tensor/copy.cc @@ -8,9 +8,9 @@ namespace onnxruntime { -std::vector StridesForTensor(const Tensor& tensor) { - auto shape = tensor.Shape(); - auto strides = std::vector(shape.NumDimensions()); +TensorShapeVector StridesForTensor(const Tensor& tensor) { + const auto& shape = tensor.Shape(); + TensorShapeVector strides(shape.NumDimensions()); int64_t running_size = 1; for (auto i = shape.NumDimensions(); i > 0; i--) { strides[i - 1] = running_size; @@ -29,8 +29,8 @@ namespace { * strides[dim + 1] * shape[dim + 1] = strides[dim] (for all tensors) */ inline bool CanCoalesce( - std::initializer_list>>& tensors_strides, - const std::vector& shape, + std::initializer_list>& tensors_strides, + const gsl::span& shape, std::size_t dim, std::size_t ndim) { auto shape_dim = shape[dim]; @@ -40,7 +40,7 @@ inline bool CanCoalesce( } for (const auto& cur_stride : tensors_strides) { - std::vector& strides = cur_stride.get(); + auto& strides = cur_stride.get(); if (shape_ndim * strides[ndim] != strides[dim]) { return false; } @@ -52,10 +52,10 @@ inline bool CanCoalesce( Copy the stride from ndim to dim in all tensors. */ inline void CopyStride( - std::initializer_list>>& tensors_strides, + std::initializer_list>& tensors_strides, std::size_t dim, std::size_t ndim) { for (const auto& cur_stride : tensors_strides) { - std::vector& strides = cur_stride.get(); + auto& strides = cur_stride.get(); strides[dim] = strides[ndim]; } } @@ -65,8 +65,8 @@ inline void CopyStride( /* Coalesce contiguous dimensions in the tensors. Operates inplace on the function arguments. */ -void CoalesceDimensions(std::initializer_list>>&& tensors_strides, - std::vector& shape) { +void CoalesceDimensions(std::initializer_list>&& tensors_strides, + TensorShapeVector& shape) { const std::size_t dims = shape.size(); // the current dimension is the one we are attempting to "coalesce onto" @@ -92,7 +92,7 @@ void CoalesceDimensions(std::initializer_list& strides = cur_stride.get(); + auto& strides = cur_stride.get(); strides.resize(current_dim + 1); } } diff --git a/onnxruntime/core/providers/cpu/tensor/copy.h b/onnxruntime/core/providers/cpu/tensor/copy.h index cc14e02282..34e63edf7c 100644 --- a/onnxruntime/core/providers/cpu/tensor/copy.h +++ b/onnxruntime/core/providers/cpu/tensor/copy.h @@ -13,9 +13,9 @@ namespace onnxruntime { void CoalesceDimensions( - std::initializer_list>>&& tensors_strides, std::vector& shape); + std::initializer_list>&& tensors_strides, TensorShapeVector& shape); -std::vector StridesForTensor(const Tensor& tensor); +TensorShapeVector StridesForTensor(const Tensor& tensor); namespace strided_copy_detail { @@ -52,7 +52,7 @@ void Copy1D(T* dst, int64_t dst_stride, const T* src, int64_t src_stride, std::p } struct NdCounter { - NdCounter(const std::vector& shape, std::ptrdiff_t first, std::ptrdiff_t last) + NdCounter(const TensorShapeVector& shape, std::ptrdiff_t first, std::ptrdiff_t last) : dims(shape.size()), last_dim_size(shape[dims - 1]), current_offset(first), @@ -98,22 +98,22 @@ struct NdCounter { const int64_t last_dim_size; ptrdiff_t current_offset; const ptrdiff_t last; - std::vector current_index; - const std::vector& shape; + TensorShapeVector current_index; + const TensorShapeVector& shape; }; } // namespace strided_copy_detail template void StridedCopy(concurrency::ThreadPool* thread_pool, T* dst, - const std::vector& dst_strides_in, + const TensorShapeVector& dst_strides_in, const TensorShape& copy_shape_in, const T* src, - const std::vector& src_strides_in) { + const TensorShapeVector& src_strides_in) { // Coalesce dimensions - std::vector dst_strides = dst_strides_in; - std::vector src_strides = src_strides_in; - std::vector copy_shape(copy_shape_in.GetDimsAsVector()); + TensorShapeVector dst_strides(dst_strides_in); + TensorShapeVector src_strides(src_strides_in); + TensorShapeVector copy_shape = copy_shape_in.AsShapeVector(); CoalesceDimensions({dst_strides, src_strides}, copy_shape); ORT_ENFORCE(dst_strides.size() == src_strides.size() && @@ -190,9 +190,9 @@ void StridedCopy(concurrency::ThreadPool* thread_pool, }); } else { // enforce that the lambda doesn't change anything - const std::vector& const_dst_strides = dst_strides; - const std::vector& const_src_strides = src_strides; - const std::vector& const_copy_shape = copy_shape; + const TensorShapeVector& const_dst_strides = dst_strides; + const TensorShapeVector& const_src_strides = src_strides; + const TensorShapeVector& const_copy_shape = copy_shape; concurrency::ThreadPool::TryParallelFor( thread_pool, num_iterations, @@ -230,10 +230,10 @@ template inline bool StridedCopyIfEnabled(concurrency::ThreadPool* thread_pool, Tensor& dst, std::ptrdiff_t dst_offset, - const std::vector& dst_strides, + const TensorShapeVector& dst_strides, const TensorShape& copy_shape, const Tensor& src, - const std::vector& src_strides) { + const TensorShapeVector& src_strides) { constexpr bool enabled = utils::HasTypeWithSameSize(); if constexpr (enabled) { // T doesn't necessarily match the data type in src or dst so use reinterpret_cast. @@ -255,10 +255,10 @@ template Status DispatchStridedCopy(concurrency::ThreadPool* thread_pool, Tensor& dst, std::ptrdiff_t dst_offset, - const std::vector& dst_strides, + const TensorShapeVector& dst_strides, const TensorShape& copy_shape, const Tensor& src, - const std::vector& src_strides) { + const TensorShapeVector& src_strides) { ORT_ENFORCE(dst.DataType() == src.DataType(), "src and dst types must match"); bool supported = false; diff --git a/onnxruntime/core/providers/cpu/tensor/onehot.cc b/onnxruntime/core/providers/cpu/tensor/onehot.cc index 42e20284c5..49dc8b627a 100644 --- a/onnxruntime/core/providers/cpu/tensor/onehot.cc +++ b/onnxruntime/core/providers/cpu/tensor/onehot.cc @@ -87,11 +87,11 @@ Status ValidateInputs(const Tensor* depth, const Tensor* values) { Status PrepareOutputShape(const Tensor* indices, const int64_t depth_val, const int64_t axis, int64_t& prefix_dim_size, int64_t& suffix_dim_size, - std::vector& output_shape) { + TensorShapeVector& output_shape) { const auto& indices_shape = indices->Shape(); const auto indices_dims = indices_shape.GetDims(); const auto indices_num_dims = indices_shape.NumDimensions(); - output_shape = indices_shape.GetDimsAsVector(); + output_shape = indices_shape.AsShapeVector(); // output rank is always 1 more than the input rank as a new dimension is added to the input shape const auto output_rank = static_cast(indices_num_dims) + 1; @@ -160,7 +160,7 @@ Status OneHotOp::Compute(OpKernelContext* p_op_ke // prepare output shape int64_t prefix_dim_size, suffix_dim_size; - std::vector output_shape; + TensorShapeVector output_shape; ORT_RETURN_IF_ERROR(PrepareOutputShape(indices, depth_val, axis_, prefix_dim_size, suffix_dim_size, output_shape)); // allocate output diff --git a/onnxruntime/core/providers/cpu/tensor/onehot.h b/onnxruntime/core/providers/cpu/tensor/onehot.h index f2419d12e3..ba210b904c 100644 --- a/onnxruntime/core/providers/cpu/tensor/onehot.h +++ b/onnxruntime/core/providers/cpu/tensor/onehot.h @@ -12,7 +12,7 @@ Status ValidateInputs(const Tensor* depth, const Tensor* values); Status PrepareOutputShape(const Tensor* indices, const int64_t depth_val, const int64_t axis, int64_t& prefix_dim_size, int64_t& suffix_dim_size, - std::vector& output_shape); + TensorShapeVector& output_shape); template class OneHotOp final : public OpKernel { diff --git a/onnxruntime/core/providers/cpu/tensor/pad.cc b/onnxruntime/core/providers/cpu/tensor/pad.cc index 1cf7d7e59c..d2ea2aa964 100644 --- a/onnxruntime/core/providers/cpu/tensor/pad.cc +++ b/onnxruntime/core/providers/cpu/tensor/pad.cc @@ -124,6 +124,9 @@ ONNX_CPU_OPERATOR_KERNEL( BuildKernelDefConstraintsFromTypeList()), Pad); + +using PadsVector = PadBase::PadsVector; + // This is the general padding method to n-dimensionally do edge or reflection padding (based on the inputDelta values) template static void PadAxis(T* output, T* input, ptrdiff_t input_delta, ptrdiff_t input_pitch, @@ -204,7 +207,7 @@ template static Status PadInputWithDimValueOfZero(OpKernelContext* ctx, const Mode& mode, const TensorShape& input_shape, - std::vector& output_dims, + TensorShapeVector& output_dims, T value) { TensorShape output_shape(output_dims); ORT_RETURN_IF_ERROR(PadBase::HandleDimValueZero(mode, input_shape, output_shape)); @@ -224,8 +227,8 @@ static Status PadInputWithDimValueOfZero(OpKernelContext* ctx, // Flatten no padding inner most Axis, so one memcpy cover multiple Axis. // For example, for a shape of [1,224,224,3] with padding [0,3,3,0,0,3,3,0], can be flatten as // [1,224,224*3] with padding [0,3,3*3,0,3,3*3]. -static void FlattenInnerShape(const std::vector& input_dims, const std::vector& pads, - const std::vector& slices, std::vector& reshaped_dims) { +static void FlattenInnerShape(const TensorShapeVector& input_dims, const PadsVector& pads, + const PadsVector& slices, TensorShapeVector& reshaped_dims) { size_t dims_count = input_dims.size(); size_t inner_axis = dims_count - 1; size_t inner_size = 1; @@ -244,15 +247,15 @@ static void FlattenInnerShape(const std::vector& input_dims, const std: } while (inner_axis-- > 0); - reshaped_dims.resize(inner_axis + 1); - std::copy(input_dims.begin(), input_dims.begin() + inner_axis + 1, reshaped_dims.begin()); + reshaped_dims.reserve(inner_axis + 1); + std::copy(input_dims.cbegin(), input_dims.cbegin() + inner_axis + 1, std::back_inserter(reshaped_dims)); // Flatten inner axis. reshaped_dims[inner_axis] = inner_size; } -static void ReshapePads(const std::vector& src_pad, size_t src_dim_count, size_t new_dim_count, - size_t inner_no_pad_size, std::vector& reshaped_pad) { +static void ReshapePads(const PadsVector& src_pad, size_t src_dim_count, size_t new_dim_count, + size_t inner_no_pad_size, PadsVector& reshaped_pad) { size_t inner_axis = new_dim_count - 1; std::copy(src_pad.begin(), src_pad.begin() + inner_axis, reshaped_pad.begin()); std::copy(src_pad.begin() + src_dim_count, src_pad.begin() + src_dim_count + inner_axis, @@ -265,8 +268,8 @@ static void ReshapePads(const std::vector& src_pad, size_t src_dim_coun template static Status PadImpl(OpKernelContext* ctx, - const std::vector& pads, - const std::vector& slices, + const PadsVector& pads, + const PadsVector& slices, const Mode& mode, T value) { if (!utils::HasTypeWithSameSize()) { @@ -275,7 +278,7 @@ static Status PadImpl(OpKernelContext* ctx, const auto& input_tensor = *ctx->Input(0); const auto& orig_input_shape = input_tensor.Shape(); - std::vector output_dims(orig_input_shape.GetDimsAsVector()); + auto output_dims(orig_input_shape.AsShapeVector()); size_t data_rank = output_dims.size(); // make copy of raw_pads as it may be mutated below @@ -283,7 +286,7 @@ static Status PadImpl(OpKernelContext* ctx, ORT_ENFORCE(data_rank * 2 == pads.size(), "'pads' has wrong number of values"); // Reshape input dims - std::vector reshaped_input_dims; + TensorShapeVector reshaped_input_dims; FlattenInnerShape(output_dims, pads, slices, reshaped_input_dims); // Reshape padding @@ -292,13 +295,13 @@ static Status PadImpl(OpKernelContext* ctx, size_t inner_no_pad_size = output_dims[inner_axis] > 0 ? reshaped_input_dims[inner_axis] / output_dims[inner_axis] : 0; - std::vector reshaped_pad(2 * new_dims_count), reshaped_slice(2 * new_dims_count); + PadsVector reshaped_pad(2 * new_dims_count), reshaped_slice(2 * new_dims_count); ReshapePads(pads, data_rank, new_dims_count, inner_no_pad_size, reshaped_pad); ReshapePads(slices, data_rank, new_dims_count, inner_no_pad_size, reshaped_slice); - std::vector reshaped_output_dims = reshaped_input_dims; - std::vector input_starts; - std::vector input_extents; + TensorShapeVector reshaped_output_dims = reshaped_input_dims; + TensorShapeVector input_starts; + TensorShapeVector input_extents; // Calculate output dimensions, and handle any negative padding input_starts.reserve(new_dims_count); @@ -473,10 +476,10 @@ Status Pad::Compute(OpKernelContext* ctx) const { const Tensor& input_tensor = *ctx->Input(0); MLDataType data_type = input_tensor.DataType(); const auto element_size = data_type->Size(); - std::vector pads; - std::vector slices; - const std::vector* pads_to_use; - const std::vector* slices_to_use; + PadsVector pads; + PadsVector slices; + const PadsVector* pads_to_use; + const PadsVector* slices_to_use; PadValue value; // kOnnxDomain Pad opset >= 11 (Or) kMsDomain opset == 1 @@ -502,7 +505,7 @@ Status Pad::Compute(OpKernelContext* ctx) const { } // Separate out any negative pads into the slices array - slices = std::vector(pads.size(), 0); + slices.assign(pads.size(), 0); for (size_t index = 0; index < pads.size(); index++) { if (pads[index] < 0) { slices[index] = pads[index]; diff --git a/onnxruntime/core/providers/cpu/tensor/padbase.h b/onnxruntime/core/providers/cpu/tensor/padbase.h index 6da4a8fa5c..8787488b84 100644 --- a/onnxruntime/core/providers/cpu/tensor/padbase.h +++ b/onnxruntime/core/providers/cpu/tensor/padbase.h @@ -3,6 +3,8 @@ #pragma once +#include "core/framework/inlined_containers.h" + namespace onnxruntime { enum class Mode : int { @@ -13,6 +15,9 @@ enum class Mode : int { class PadBase { public: + // Pads and slices are usually about twice the shapes involved + using PadsVector = InlinedVector; + // Update the output_shape to make it consistent with numpy handling where there are one or more dimensions // in the input_shape with a value of zero. static Status HandleDimValueZero(const Mode& mode, const TensorShape& input_shape, TensorShape& output_shape); @@ -42,9 +47,10 @@ class PadBase { } if (!is_dynamic_) { - if (!info.GetAttrs("pads", pads_).IsOK()) + gsl::span pads_span; + if (!info.GetAttrsAsSpan("pads", pads_span).IsOK()) ORT_THROW("Invalid 'pads' attribute value"); - + pads_.assign(pads_span.cbegin(), pads_span.cend()); // Separate out any negative pads_ into the slices_ array slices_.resize(pads_.size(), 0); for (size_t index = 0; index < pads_.size(); index++) { @@ -59,8 +65,8 @@ class PadBase { ~PadBase() = default; Mode mode_{Mode::Constant}; - std::vector pads_; // After construction, only >=0 values are in here - std::vector slices_; // All of the negative padding values are separated out into slices_ + PadsVector pads_; // After construction, only >=0 values are in here + PadsVector slices_; // All of the negative padding values are separated out into slices_ const float value_; // will always be float (when 'value' parsed from attribute - opset 10 and below) // flag used to differentiate the cases where some input values to the op are diff --git a/onnxruntime/core/providers/cpu/tensor/reshape.h b/onnxruntime/core/providers/cpu/tensor/reshape.h index c7358107b9..b074bfe933 100644 --- a/onnxruntime/core/providers/cpu/tensor/reshape.h +++ b/onnxruntime/core/providers/cpu/tensor/reshape.h @@ -25,7 +25,7 @@ class Reshape final : public OpKernel { "A shape tensor must be a vector tensor."); auto nDims = static_cast(shapeTensor->Shape()[0]); const auto* data = shapeTensor->template Data(); - std::vector shape(data, data + nDims); + TensorShapeVector shape(data, data + nDims); const auto* X = context->Input(0); const TensorShape& X_shape = X->Shape(); @@ -46,12 +46,12 @@ class Reshape final : public OpKernel { class Reshape_1 final : public OpKernel { public: explicit Reshape_1(const OpKernelInfo& info) : OpKernel(info) { - Status status = info.GetAttrs("shape", shape_); + Status status = info.GetAttrs("shape", shape_); ORT_ENFORCE(status.IsOK(), "Attribute shape is not set."); } Status Compute(OpKernelContext* context) const override { - std::vector shape = shape_; + TensorShapeVector shape = shape_; const auto* X = context->Input(0); const TensorShape& X_shape = X->Shape(); @@ -65,7 +65,7 @@ class Reshape_1 final : public OpKernel { } private: - std::vector shape_; + TensorShapeVector shape_; }; } //namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/reshape_helper.h b/onnxruntime/core/providers/cpu/tensor/reshape_helper.h index b8fd8ea14f..67c6c69ebd 100644 --- a/onnxruntime/core/providers/cpu/tensor/reshape_helper.h +++ b/onnxruntime/core/providers/cpu/tensor/reshape_helper.h @@ -3,12 +3,14 @@ #pragma once +#include "core/framework/tensor_shape.h" + namespace onnxruntime { // Verify and convert unknown dim during reshape class ReshapeHelper { public: - ReshapeHelper(const TensorShape& input_shape, std::vector& requested_shape, bool allow_zero = false) { + ReshapeHelper(const TensorShape& input_shape, TensorShapeVector& requested_shape, bool allow_zero = false) { auto nDims = requested_shape.size(); ptrdiff_t unknown_dim = -1; int64_t size = 1; diff --git a/onnxruntime/core/providers/cpu/tensor/slice.cc b/onnxruntime/core/providers/cpu/tensor/slice.cc index 3d01feb787..247c97821b 100644 --- a/onnxruntime/core/providers/cpu/tensor/slice.cc +++ b/onnxruntime/core/providers/cpu/tensor/slice.cc @@ -16,6 +16,7 @@ using namespace ::onnxruntime::common; namespace onnxruntime { + namespace op_kernel_type_control { // we're using one set of types for all opsets ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPE_LIST_ALL_OPSETS( @@ -77,12 +78,12 @@ ONNX_CPU_OPERATOR_KERNEL( // Updates starts and steps to match flattened_output_dims if it is. // e.g. if input shape is { 2, 2, 2 }, output shape is { 1, 2, 2 }, and the 'steps' value for the last two dims is 1, // we are keeping all the data of the inner most two dimensions so can combine those into dims of { 1, 4 } -static void FlattenOutputDims(gsl::span input_dimensions, - gsl::span output_dims, - std::vector& starts, - std::vector& ends, - std::vector& steps, - std::vector*& flattened_output_dims) { +static void FlattenOutputDims(const gsl::span& input_dimensions, + const gsl::span& output_dims, + TensorShapeVector& starts, + TensorShapeVector& ends, + TensorShapeVector& steps, + TensorShapeVector*& flattened_output_dims) { int num_to_combine = 0; for (int64_t i = static_cast(starts.size()) - 1; i >= 0; --i) { // if we're keeping all the data for the dimension and not reversing the direction we can potentially combine it @@ -94,7 +95,7 @@ static void FlattenOutputDims(gsl::span input_dimensions, if (num_to_combine > 1) { auto num_dims = output_dims.size() - num_to_combine + 1; - *flattened_output_dims = std::vector(output_dims.begin(), output_dims.end()); + flattened_output_dims->assign(output_dims.cbegin(), output_dims.cend()); flattened_output_dims->resize(num_dims); int64_t dim_value = 1; @@ -118,9 +119,9 @@ static void FlattenOutputDims(gsl::span input_dimensions, } // Slice V1-9 & DynamicSlice -Status SliceBase::PrepareForCompute(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, +Status SliceBase::PrepareForCompute(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, SliceOp::PrepareForComputeMetadata& compute_metadata) { ORT_RETURN_IF_ERROR(SliceOp::PrepareForComputeHelper(raw_starts, raw_ends, raw_axes, compute_metadata)); FlattenOutputDims(compute_metadata.input_dimensions_, compute_metadata.output_dims_, compute_metadata.starts_, @@ -129,10 +130,10 @@ Status SliceBase::PrepareForCompute(const std::vector& raw_starts, } // DynamicSlice & Slice V10 -Status SliceBase::PrepareForCompute(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, - const std::vector& raw_steps, +Status SliceBase::PrepareForCompute(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, + const gsl::span& raw_steps, SliceOp::PrepareForComputeMetadata& compute_metadata) { ORT_RETURN_IF_ERROR(SliceOp::PrepareForComputeHelper(raw_starts, raw_ends, raw_axes, raw_steps, compute_metadata)); FlattenOutputDims(compute_metadata.input_dimensions_, compute_metadata.output_dims_, compute_metadata.starts_, @@ -141,15 +142,41 @@ Status SliceBase::PrepareForCompute(const std::vector& raw_starts, return Status::OK(); } +namespace { +template +void CopyData(const Tensor& start_tensor, + const Tensor& ends_tensor, + const Tensor* axes_tensor, + const Tensor* steps_tensor, + TensorShapeVector& input_starts, + TensorShapeVector& input_ends, + TensorShapeVector& input_axes, + TensorShapeVector& input_steps) { + auto start_data = start_tensor.DataAsSpan(); + std::copy(start_data.cbegin(), start_data.cend(), std::back_inserter(input_starts)); + auto ends_data = ends_tensor.DataAsSpan(); + std::copy(ends_data.cbegin(), ends_data.cend(), std::back_inserter(input_ends)); + if (nullptr != axes_tensor) { + auto axes_data = axes_tensor->DataAsSpan(); + std::copy(axes_data.cbegin(), axes_data.cend(), std::back_inserter(input_axes)); + } + // Slice V10 + if (nullptr != steps_tensor) { + auto steps_data = steps_tensor->DataAsSpan(); + std::copy(steps_data.cbegin(), steps_data.cend(), std::back_inserter(input_steps)); + } +} +} // namespace + // Slice V10 & DynamicSlice Status SliceBase::FillVectorsFromInput(const Tensor& start_tensor, const Tensor& ends_tensor, const Tensor* axes_tensor, const Tensor* steps_tensor, - std::vector& input_starts, - std::vector& input_ends, - std::vector& input_axes, - std::vector& input_steps) { + TensorShapeVector& input_starts, + TensorShapeVector& input_ends, + TensorShapeVector& input_axes, + TensorShapeVector& input_steps) { ORT_RETURN_IF_NOT(start_tensor.Shape().NumDimensions() == 1, "Starts must be a 1-D array"); ORT_RETURN_IF_NOT(ends_tensor.Shape().NumDimensions() == 1, "Ends must be a 1-D array"); ORT_RETURN_IF_NOT(start_tensor.Shape() == ends_tensor.Shape(), "Starts and ends shape mismatch"); @@ -158,40 +185,24 @@ Status SliceBase::FillVectorsFromInput(const Tensor& start_tensor, ORT_RETURN_IF_NOT(nullptr == steps_tensor || start_tensor.Shape() == steps_tensor->Shape(), "Starts and steps shape mismatch"); - const auto& size = start_tensor.Shape().Size(); - input_starts.resize(size); - input_ends.resize(size); + const auto size = start_tensor.Shape().Size(); + input_starts.reserve(size); + input_ends.reserve(size); if (nullptr != axes_tensor) - input_axes.resize(size); + input_axes.reserve(size); // Slice V10 if (nullptr != steps_tensor) - input_steps.resize(size); + input_steps.reserve(size); // check for type reduction of supported indices types constexpr bool int32_enabled = utils::HasType(); constexpr bool int64_enabled = utils::HasType(); if (int32_enabled && start_tensor.IsDataType()) { - std::copy(start_tensor.Data(), start_tensor.Data() + size, input_starts.begin()); - std::copy(ends_tensor.Data(), ends_tensor.Data() + size, input_ends.begin()); - if (nullptr != axes_tensor) - std::copy(axes_tensor->Data(), axes_tensor->Data() + size, input_axes.begin()); - // Slice V10 - if (nullptr != steps_tensor) - std::copy(steps_tensor->Data(), steps_tensor->Data() + size, input_steps.begin()); - } - - else if (int64_enabled && start_tensor.IsDataType()) { - std::copy(start_tensor.Data(), start_tensor.Data() + size, input_starts.begin()); - std::copy(ends_tensor.Data(), ends_tensor.Data() + size, input_ends.begin()); - if (nullptr != axes_tensor) - std::copy(axes_tensor->Data(), axes_tensor->Data() + size, input_axes.begin()); - // Slice V10 - if (nullptr != steps_tensor) - std::copy(steps_tensor->Data(), steps_tensor->Data() + size, input_steps.begin()); - } - - else { + CopyData(start_tensor, ends_tensor, axes_tensor, steps_tensor, input_starts, input_ends, input_axes, input_steps); + } else if (int64_enabled && start_tensor.IsDataType()) { + CopyData(start_tensor, ends_tensor, axes_tensor, steps_tensor, input_starts, input_ends, input_axes, input_steps); + } else { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Data type for starts and ends inputs' is not supported in this build. Got ", start_tensor.DataType()); @@ -232,10 +243,10 @@ static Status SliceImpl(OpKernelContext* ctx, if (compute_metadata.p_flattened_output_dims_) { // if we have flattened output dims we need to also flatten the input dims. // as we're combining the innermost dims and keeping all values we can just copy the size of the last dim - auto flattened_input_dims=input_tensor.Shape().GetDimsAsVector(); + auto flattened_input_dims = input_tensor.Shape().AsShapeVector(); flattened_input_dims.resize(compute_metadata.p_flattened_output_dims_->size()); flattened_input_dims.back() = compute_metadata.p_flattened_output_dims_->back(); - TensorShape input_shape(std::move(flattened_input_dims)); + TensorShape input_shape(flattened_input_dims); auto input_iterator2 = SliceIterator(input_tensor, input_shape, compute_metadata.starts_, *compute_metadata.p_flattened_output_dims_, compute_metadata.steps_); @@ -275,13 +286,14 @@ Status SliceBase::Compute(OpKernelContext* ctx) const { // Slice V10 & DynamicSlice if (dynamic_) { - std::vector input_starts; - std::vector input_ends; - std::vector input_axes; - std::vector input_steps; + TensorShapeVector input_starts; + TensorShapeVector input_ends; + TensorShapeVector input_axes; + TensorShapeVector input_steps; ORT_RETURN_IF_ERROR(FillVectorsFromInput(*ctx->Input(1), *ctx->Input(2), ctx->Input(3), ctx->Input(4), - input_starts, input_ends, input_axes, input_steps)); + input_starts, input_ends, + input_axes, input_steps)); ORT_RETURN_IF_ERROR(PrepareForCompute(input_starts, input_ends, input_axes, input_steps, compute_metadata)); } diff --git a/onnxruntime/core/providers/cpu/tensor/slice.h b/onnxruntime/core/providers/cpu/tensor/slice.h index f9549257ca..0ff724aae1 100644 --- a/onnxruntime/core/providers/cpu/tensor/slice.h +++ b/onnxruntime/core/providers/cpu/tensor/slice.h @@ -16,16 +16,16 @@ class SliceBase { // static methods that can be used from other ops if needed public: // compute output_dims without steps (Slice V1-9 & DynamicSlice) - static Status PrepareForCompute(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, + static Status PrepareForCompute(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, SliceOp::PrepareForComputeMetadata& compute_metadata); // compute output_dims with steps (Slice V10) - static Status PrepareForCompute(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, - const std::vector& raw_steps, + static Status PrepareForCompute(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, + const gsl::span& raw_steps, SliceOp::PrepareForComputeMetadata& compute_metadata); // Slice V10 & DynamicSlice @@ -33,10 +33,10 @@ class SliceBase { const Tensor& ends_tensor, const Tensor* axes_tensor, const Tensor* steps_tensor, - std::vector& input_starts, - std::vector& input_ends, - std::vector& input_axes, - std::vector& input_steps); + TensorShapeVector& input_starts, + TensorShapeVector& input_ends, + TensorShapeVector& input_axes, + TensorShapeVector& input_steps); protected: SliceBase(const OpKernelInfo& info, bool dynamic = false) @@ -55,9 +55,9 @@ class SliceBase { Status Compute(OpKernelContext* context) const; protected: - const std::vector& StartsAttribute() const { return attr_starts_; } - const std::vector& EndsAttribute() const { return attr_ends_; } - const std::vector& AxesAttribute() const { return attr_axes_; } + gsl::span StartsAttribute() const { return attr_starts_; } + gsl::span EndsAttribute() const { return attr_ends_; } + gsl::span AxesAttribute() const { return attr_axes_; } private: bool dynamic_; diff --git a/onnxruntime/core/providers/cpu/tensor/slice_compute_metadata.h b/onnxruntime/core/providers/cpu/tensor/slice_compute_metadata.h index ad7b32ab56..5f80bd1033 100644 --- a/onnxruntime/core/providers/cpu/tensor/slice_compute_metadata.h +++ b/onnxruntime/core/providers/cpu/tensor/slice_compute_metadata.h @@ -6,6 +6,8 @@ #include #include +#include +#include "core/framework/tensor_shape.h" namespace onnxruntime { @@ -21,12 +23,12 @@ struct PrepareForComputeMetadata { } gsl::span input_dimensions_; - std::vector starts_; - std::vector ends_; - std::vector steps_; - std::vector output_dims_; - std::vector flattened_output_dims_; - std::vector* p_flattened_output_dims_ = &flattened_output_dims_; + TensorShapeVector starts_; + TensorShapeVector ends_; + TensorShapeVector steps_; + TensorShapeVector output_dims_; + TensorShapeVector flattened_output_dims_; + TensorShapeVector* p_flattened_output_dims_ = &flattened_output_dims_; }; } // namespace SliceOp diff --git a/onnxruntime/core/providers/cpu/tensor/slice_helper.h b/onnxruntime/core/providers/cpu/tensor/slice_helper.h index a6c09fe24d..61b6d4a63b 100644 --- a/onnxruntime/core/providers/cpu/tensor/slice_helper.h +++ b/onnxruntime/core/providers/cpu/tensor/slice_helper.h @@ -5,34 +5,45 @@ // for Slice op, which can be called from other ops or EPs. #pragma once #include "core/providers/cpu/tensor/slice_compute_metadata.h" +#include "core/framework/inlined_containers.h" +#include "core/framework/ort_stl_allocator.h" namespace onnxruntime { namespace SliceOp { // compute output_dims without steps (Slice V1-9 & DynamicSlice) // Please note this will not Flatten the output shape -inline Status PrepareForComputeHelper(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, +inline Status PrepareForComputeHelper(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, SliceOp::PrepareForComputeMetadata& compute_metadata) { // Initialize axes to the provided axes attribute or to the default sequence - std::vector axes(raw_axes); - if (axes.empty()) { + TensorShapeVector axes; + if (raw_axes.empty()) { //axes are omitted, they are set to[0, ..., ndim - 1] - axes.resize(raw_starts.size()); - std::iota(axes.begin(), axes.end(), 0); + axes.reserve(raw_starts.size()); + for (int64_t i = 0, limit = raw_starts.size(); i < limit; ++i) { + axes.push_back(i); + } + } else { + axes.reserve(raw_axes.size()); + axes.assign(raw_axes.cbegin(), raw_axes.cend()); } // Iterate through the provided axes and override the start/end ranges - std::unordered_set unique_axes; - const auto& dimension_count = compute_metadata.input_dimensions_.size(); - for (size_t axis_index = 0, axes_count = axes.size(); axis_index < axes_count; ++axis_index) { + using AxesSet = InlinedHashSet; + const auto axes_count = axes.size(); + AxesSet unique_axes(axes_count); + + const auto dimension_count = compute_metadata.input_dimensions_.size(); + for (size_t axis_index = 0; axis_index < axes_count; ++axis_index) { const auto axis = HandleNegativeAxis(axes[axis_index], dimension_count); // handle negative and enforce axis is valid if (axis >= static_cast(dimension_count) || axis < 0) return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "'axes' has an axis outside of the tensor dimension count"); - if (unique_axes.find(axis) != unique_axes.end()) + auto p = unique_axes.insert(axis); + if (!p.second) return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "'axes' has duplicates"); - unique_axes.insert(axis); + const auto dim_value = compute_metadata.input_dimensions_[axis]; // process start @@ -60,30 +71,36 @@ inline Status PrepareForComputeHelper(const std::vector& raw_starts, // compute output_dims with steps (Slice V10) // Please note this will not Flatten the output shape -inline Status PrepareForComputeHelper(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, - const std::vector& raw_steps, +inline Status PrepareForComputeHelper(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, + const gsl::span& raw_steps, SliceOp::PrepareForComputeMetadata& compute_metadata) { // Initialize axes to the provided axes attribute or to the default sequence - std::vector axes(raw_axes); - - if (axes.empty()) { - // axes are omitted, they are set to[0, ..., ndim - 1] - axes.resize(raw_starts.size()); - std::iota(axes.begin(), axes.end(), 0); + TensorShapeVector axes; + if (raw_axes.empty()) { + //axes are omitted, they are set to[0, ..., ndim - 1] + axes.reserve(raw_starts.size()); + for (int64_t i = 0, limit = raw_starts.size(); i < limit; ++i) { + axes.push_back(i); + } + } else { + axes.assign(raw_axes.cbegin(), raw_axes.cend()); } // Iterate through the provided axes and override the start/end/steps ranges - std::unordered_set unique_axes; - const auto& dimension_count = compute_metadata.input_dimensions_.size(); - for (size_t axis_index = 0, axes_count = axes.size(); axis_index < axes_count; ++axis_index) { + using AxesSet = InlinedHashSet; + const auto axes_count = axes.size(); + AxesSet unique_axes(axes_count); + + const auto dimension_count = compute_metadata.input_dimensions_.size(); + for (size_t axis_index = 0; axis_index < axes_count; ++axis_index) { const auto axis = axes[axis_index] < 0 ? axes[axis_index] + static_cast(dimension_count) : axes[axis_index]; if (axis >= static_cast(dimension_count) || axis < 0) return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "'axes' has an axis outside of the tensor dimension count"); - if (unique_axes.find(axis) != unique_axes.end()) + auto p = unique_axes.insert(axis); + if (!p.second) return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "'axes' has duplicates"); - unique_axes.insert(axis); const auto dim_value = compute_metadata.input_dimensions_[axis]; // process step diff --git a/onnxruntime/core/providers/cpu/tensor/split.cc b/onnxruntime/core/providers/cpu/tensor/split.cc index 345c9d2be3..4baef6b0cf 100644 --- a/onnxruntime/core/providers/cpu/tensor/split.cc +++ b/onnxruntime/core/providers/cpu/tensor/split.cc @@ -173,7 +173,7 @@ Status Split::ComputeImpl(OpKernelContext& context, const Tensor& input) const { split_sizes)); // copy dimensions so we can update the selected axis in place - auto output_dimensions = input_shape.GetDimsAsVector(); + auto output_dimensions = input_shape.AsShapeVector(); int64_t input_offset = 0; const T* input_data = input.template Data(); diff --git a/onnxruntime/core/providers/cpu/tensor/squeeze.h b/onnxruntime/core/providers/cpu/tensor/squeeze.h index 31189d5f4e..c097c3f269 100644 --- a/onnxruntime/core/providers/cpu/tensor/squeeze.h +++ b/onnxruntime/core/providers/cpu/tensor/squeeze.h @@ -16,11 +16,11 @@ namespace onnxruntime { class SqueezeBase { protected: explicit SqueezeBase(const OpKernelInfo& info) { - std::vector axes; + TensorShapeVector axes; size_t numInputs = info.GetInputCount(); if (numInputs == 1) { // Parse attribute 'axes' - Status status = info.GetAttrs("axes", axes); + Status status = info.GetAttrs("axes", axes); // Handle out of order and repeating dims when 'axes' exists. if (status.IsOK()) { @@ -31,15 +31,15 @@ class SqueezeBase { } } - static std::vector ComputeOutputShape( + static TensorShapeVector ComputeOutputShape( const TensorShape& input_shape, - const std::vector& axes) { + const TensorShapeVector& axes) { size_t j = 0; - std::vector output_shape; + TensorShapeVector output_shape; auto num_dimensions = input_shape.NumDimensions(); // Handle negtive axis, then resort and uniq. - std::vector axes_corrected(axes.size()); + TensorShapeVector axes_corrected(axes.size()); for (size_t i = 0; i < axes.size(); i++) { axes_corrected[i] = HandleNegativeAxis(axes[i], num_dimensions); } @@ -59,7 +59,7 @@ class SqueezeBase { return output_shape; } - std::vector axes_; + TensorShapeVector axes_; }; class Squeeze final : public OpKernel, public SqueezeBase { @@ -70,7 +70,7 @@ class Squeeze final : public OpKernel, public SqueezeBase { const auto* X = context->Input(0); const TensorShape& X_shape = X->Shape(); - std::vector axes; + TensorShapeVector axes; size_t num_inputs = context->InputCount(); if (num_inputs == 2) { //axes is an input const Tensor* axes_tensor = context->Input(1); @@ -84,7 +84,7 @@ class Squeeze final : public OpKernel, public SqueezeBase { axes.assign(axes_.begin(), axes_.end()); } - std::vector output_shape = ComputeOutputShape(X_shape, axes); + TensorShapeVector output_shape = ComputeOutputShape(X_shape, axes); Tensor* Y = context->Output(0, TensorShape(output_shape)); diff --git a/onnxruntime/core/providers/cpu/tensor/tile.cc b/onnxruntime/core/providers/cpu/tensor/tile.cc index 523f99f918..b3073faa07 100644 --- a/onnxruntime/core/providers/cpu/tensor/tile.cc +++ b/onnxruntime/core/providers/cpu/tensor/tile.cc @@ -154,7 +154,7 @@ Status Tile::Compute(OpKernelContext* ctx) const { // Calculate the shape of the output tensor const auto* repeats = repeats_tensor.template Data(); - std::vector output_dims = input_shape.GetDimsAsVector(); + auto output_dims = input_shape.AsShapeVector(); for (size_t axis = 0; axis < input_rank; axis++) { output_dims[axis] *= repeats[axis]; } diff --git a/onnxruntime/core/providers/cpu/tensor/transpose.cc b/onnxruntime/core/providers/cpu/tensor/transpose.cc index de6a1b33cf..437ba45682 100644 --- a/onnxruntime/core/providers/cpu/tensor/transpose.cc +++ b/onnxruntime/core/providers/cpu/tensor/transpose.cc @@ -72,7 +72,7 @@ struct MultiIndex { * element_size is the size of the tensor element (sizeof(float), sizeof(double)). */ static void IncrementIndexAndComputeOffsetSetup(MultiIndex& mindex, size_t num_axes, gsl::span target_dims, - const std::vector& stride, size_t element_size) { + const gsl::span& stride, size_t element_size) { mindex.Init(num_axes); size_t naxes = 0; for (size_t i = 0; i < num_axes; ++i) { @@ -133,7 +133,7 @@ static inline void DoTransposeSingleBlock(size_t num_elts_in_block, const std::s // DoTranspose: copies source tensor to target, transposing elements. // The stride vector indicates the transposition. static void DoTransposeImpl(int64_t num_axes, gsl::span target_dims, - size_t num_blocks, size_t num_elts_in_block, const std::vector& stride, + size_t num_blocks, size_t num_elts_in_block, const gsl::span& stride, const uint8_t* source, uint8_t* target, size_t element_size) { size_t blocksize = num_elts_in_block * element_size; MultiIndex mindex; @@ -149,7 +149,7 @@ static void DoTransposeImpl(int64_t num_axes, gsl::span target_di } static void DoTransposeImpl(int64_t num_axes, gsl::span target_dims, - size_t num_blocks, size_t num_elts_in_block, const std::vector& stride, + size_t num_blocks, size_t num_elts_in_block, const gsl::span& stride, const std::string* source, std::string* target) { ORT_ENFORCE(num_axes > 0, "Transpose not implemented for empty tensors."); MultiIndex mindex; @@ -172,7 +172,7 @@ inline void CopyPrim(uint8_t* target, const uint8_t* source) { // The function does not check num_axes > 0 but this is expected. template static bool TypedDoTransposeEltWise(int64_t num_axes, gsl::span target_dims, size_t num_blocks, - const std::vector& stride, const uint8_t* source, uint8_t* target) { + const gsl::span& stride, const uint8_t* source, uint8_t* target) { constexpr bool enabled = utils::HasTypeWithSameSize(); if (enabled) { @@ -195,7 +195,7 @@ static bool TypedDoTransposeEltWise(int64_t num_axes, gsl::span t // copies source tensor to target, transposing elements. // The stride vector indicates the transposition. Status DoTransposeEltWise(int64_t num_axes, gsl::span target_dims, size_t num_blocks, - const std::vector& stride, const uint8_t* source, uint8_t* target, + const gsl::span& stride, const uint8_t* source, uint8_t* target, size_t element_size) { bool enabled = false; switch (element_size) { @@ -222,7 +222,7 @@ Status DoTransposeEltWise(int64_t num_axes, gsl::span target_dims } static void DoTransposeEltWise(int64_t num_axes, gsl::span target_dims, size_t num_blocks, - const std::vector& stride, const std::string* source, std::string* target) { + const gsl::span& stride, const std::string* source, std::string* target) { ORT_ENFORCE(num_axes > 0, "Transpose not implemented for empty tensors."); MultiIndex mindex; IncrementIndexAndComputeOffsetSetup(mindex, num_axes, target_dims, stride, 1); @@ -238,7 +238,7 @@ static void DoTransposeEltWise(int64_t num_axes, gsl::span target } // `input_shape_override` overrides the shape of `input` for compute purposes. -static Status DoUntypedTranspose(const std::vector& permutations, const Tensor& input, Tensor& output, +static Status DoUntypedTranspose(const gsl::span& permutations, const Tensor& input, Tensor& output, const TensorShape* input_shape_override = nullptr) { const auto& input_shape = input_shape_override ? *input_shape_override : input.Shape(); const auto& input_dims = input_shape.GetDims(); @@ -247,7 +247,7 @@ static Status DoUntypedTranspose(const std::vector& permutations, const const auto element_size = input.DataType()->Size(); const bool is_string_type = input.IsDataTypeString(); - std::vector stride(rank); + InlinedShapeVectorT stride(rank); for (size_t i = 0; i < rank; i++) { size_t inpdim = permutations[i]; if (inpdim + 1 < rank) @@ -393,7 +393,7 @@ SimpleTransposeSingleAxisOutwards(const T* input_data, T* output_data, } // `input_shape_override` overrides the shape of `input` for compute purposes. -void TransposeSingleAxisOutwards(const std::vector& permutations, const Tensor& input, Tensor& output, +void TransposeSingleAxisOutwards(const gsl::span& permutations, const Tensor& input, Tensor& output, int64_t from, int64_t to, const TensorShape* input_shape_override = nullptr) { ORT_UNUSED_PARAMETER(permutations); @@ -503,7 +503,7 @@ SimpleTransposeSingleAxisInwards(const T* input_data, T* output_data, // moving a single axis inwards where the read/write size is a power of 2 and between 8 and 64 bits. // `input_shape_override` overrides the shape of `input` for compute purposes. -void TransposeSingleAxisInwards(const std::vector& permutations, const Tensor& input, Tensor& output, +void TransposeSingleAxisInwards(const gsl::span& permutations, const Tensor& input, Tensor& output, int64_t from, int64_t to, const TensorShape* input_shape_override = nullptr) { ORT_UNUSED_PARAMETER(permutations); @@ -572,7 +572,7 @@ void TransposeSingleAxisInwards(const std::vector& permutations, const T } // `input_shape_override` overrides the shape of `input` for compute purposes. -void SingleAxisTranspose(const std::vector& permutations, const Tensor& input, Tensor& output, +void SingleAxisTranspose(const gsl::span& permutations, const Tensor& input, Tensor& output, size_t from, size_t to, const TensorShape* input_shape_override = nullptr) { if (from > to) { TransposeSingleAxisOutwards(permutations, input, output, from, to, input_shape_override); @@ -581,7 +581,7 @@ void SingleAxisTranspose(const std::vector& permutations, const Tensor& } } -bool IsMovingSingleAxis(const std::vector& permutations, size_t& from, size_t& to) { +bool IsMovingSingleAxis(const gsl::span& permutations, size_t& from, size_t& to) { // if a single axis moved to an outer dimension, the values should be one lower than the index until the slot the // axis was moved from, and equal to the index after that. // e.g. axis 3 moves out to 1 would be: 0, 3, 1, 2, 4 @@ -652,7 +652,7 @@ bool IsMovingSingleAxis(const std::vector& permutations, size_t& from, s } // namespace -bool IsTransposeReshape(const std::vector& perm, gsl::span input_dims) { +bool IsTransposeReshape(const gsl::span& perm, gsl::span input_dims) { // As long as the dims with values > 1 stay in the same order, it's a reshape. // Example: Shape=(1,1,1024,4096) -> perm=(2,0,3,1). size_t last_permuted_axis = 0; @@ -667,7 +667,7 @@ bool IsTransposeReshape(const std::vector& perm, gsl::span& permutations, const Tensor& input, Tensor& output, +Status TransposeBase::DoTranspose(const gsl::span& permutations, const Tensor& input, Tensor& output, const TensorShape* input_shape_override) { Status status = Status::OK(); @@ -708,9 +708,9 @@ Status Transpose::Compute(OpKernelContext* ctx) const { auto input_dims = input_shape.GetDims(); size_t rank = input_dims.size(); - std::vector output_dims(rank); - const std::vector* p_perm; - std::vector default_perm(rank); + TensorShapeVector output_dims(rank); + const InlinedShapeVectorT* p_perm; + InlinedShapeVectorT default_perm(rank); Status status = ComputeOutputShape(X, output_dims, default_perm, p_perm); if (!status.IsOK()) return status; diff --git a/onnxruntime/core/providers/cpu/tensor/transpose.h b/onnxruntime/core/providers/cpu/tensor/transpose.h index 76c97fec1f..61cfe8280c 100644 --- a/onnxruntime/core/providers/cpu/tensor/transpose.h +++ b/onnxruntime/core/providers/cpu/tensor/transpose.h @@ -17,11 +17,11 @@ namespace onnxruntime { empty dimensions can change place, not empty dimensions must be in the same order in the permuted tenosr. */ -bool IsTransposeReshape(const std::vector& perm, gsl::span input_dims); +bool IsTransposeReshape(const gsl::span& perm, gsl::span input_dims); // Public function for element-wise transpose, primarily to unit test any out of bounds access Status DoTransposeEltWise(int64_t num_axes, gsl::span target_dims, size_t num_blocks, - const std::vector& stride, const uint8_t* source, uint8_t* target, + const gsl::span& stride, const uint8_t* source, uint8_t* target, size_t element_size); class TransposeBase { @@ -30,7 +30,7 @@ class TransposeBase { Transpose the input Tensor into the output Tensor using the provided permutations. Both Tensors must have the same data type. `input_shape_override` overrides the shape of `input` for compute purposes. */ - static Status DoTranspose(const std::vector& permutations, const Tensor& input, Tensor& output, + static Status DoTranspose(const gsl::span& permutations, const Tensor& input, Tensor& output, const TensorShape* input_shape_override = nullptr); protected: @@ -58,8 +58,8 @@ class TransposeBase { } } - Status ComputeOutputShape(const Tensor& X, std::vector& output_dims, std::vector& default_perm, - const std::vector*& p_perm) const { + Status ComputeOutputShape(const Tensor& X, TensorShapeVector& output_dims, InlinedShapeVectorT& default_perm, + const InlinedShapeVectorT*& p_perm) const { size_t rank = X.Shape().NumDimensions(); const auto& input_dims = X.Shape().GetDims(); @@ -93,7 +93,7 @@ class TransposeBase { } bool perm_specified_ = false; - std::vector perm_; + InlinedShapeVectorT perm_; }; class Transpose final : public OpKernel, public TransposeBase { diff --git a/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc b/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc index 930385406c..ac6f37a353 100644 --- a/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc +++ b/onnxruntime/core/providers/cpu/tensor/unsqueeze.cc @@ -41,7 +41,7 @@ Status UnsqueezeBase::PrepareCompute(OpKernelContext* ctx, Prepare& p) const { ORT_ENFORCE(X != nullptr); auto& input_tensor = *X; - std::vector axes; + TensorShapeVector axes; size_t num_inputs = ctx->InputCount(); if (num_inputs == 2) { //axes is an input const Tensor* axes_tensor = ctx->Input(1); @@ -49,16 +49,15 @@ Status UnsqueezeBase::PrepareCompute(OpKernelContext* ctx, Prepare& p) const { ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 0 || axes_tensor->Shape().NumDimensions() == 1, "An axes tensor must be a scalar or a 1-D tensor."); - auto num_axes_elements = static_cast(axes_tensor->Shape().Size()); - const auto* data = axes_tensor->template Data(); - axes.assign(data, data + num_axes_elements); + auto data_span = axes_tensor->template DataAsSpan(); + axes.assign(data_span.cbegin(), data_span.cend()); } else { axes.assign(axes_.begin(), axes_.end()); } // New dimension count is the current dimensions + the number of entries in axes // Initialize output_dims to 0 in each axis initially - std::vector output_dims(axes.size() + input_tensor.Shape().NumDimensions(), 0); + TensorShapeVector output_dims(axes.size() + input_tensor.Shape().NumDimensions(), 0); // Set all axes indices to 1 in output_dims and check for duplicates for (int64_t axis : axes) { diff --git a/onnxruntime/core/providers/cpu/tensor/unsqueeze.h b/onnxruntime/core/providers/cpu/tensor/unsqueeze.h index 9324edcdb5..665299aa35 100644 --- a/onnxruntime/core/providers/cpu/tensor/unsqueeze.h +++ b/onnxruntime/core/providers/cpu/tensor/unsqueeze.h @@ -28,7 +28,7 @@ class UnsqueezeBase { } private: - std::vector axes_; + TensorShapeVector axes_; }; class Unsqueeze final : public OpKernel, public UnsqueezeBase { diff --git a/onnxruntime/core/providers/cpu/tensor/upsample.cc b/onnxruntime/core/providers/cpu/tensor/upsample.cc index 55422654ff..81ac2cd325 100644 --- a/onnxruntime/core/providers/cpu/tensor/upsample.cc +++ b/onnxruntime/core/providers/cpu/tensor/upsample.cc @@ -1027,7 +1027,7 @@ template Status Upsample::BaseCompute(OpKernelContext* context, const std::vector& roi, const std::vector& scales, - const std::vector& output_dims) const { + const gsl::span& output_dims) const { const auto* X = context->Input(0); ORT_ENFORCE(X != nullptr); auto dims = X->Shape().GetDims(); @@ -1152,7 +1152,7 @@ Status Upsample::Compute(OpKernelContext* context) const { const auto* X = context->Input(0); ORT_ENFORCE(X != nullptr); - std::vector output_dims(X->Shape().GetDims().size()); + TensorShapeVector output_dims(X->Shape().GetDims().size()); // Get roi data // Initialize the roi array to all zeros as this will be the most common case diff --git a/onnxruntime/core/providers/cpu/tensor/upsample.h b/onnxruntime/core/providers/cpu/tensor/upsample.h index dd9dc489de..7ce78a7f3e 100644 --- a/onnxruntime/core/providers/cpu/tensor/upsample.h +++ b/onnxruntime/core/providers/cpu/tensor/upsample.h @@ -359,7 +359,7 @@ class UpsampleBase { void ComputeOutputShape(const std::vector& scales, gsl::span input_dims, - std::vector& output_dims) const { + TensorShapeVector& output_dims) const { for (std::size_t i = 0; i < input_dims.size(); i++) { output_dims[i] = static_cast(scales[i] * input_dims[i]); } @@ -375,7 +375,7 @@ class Upsample : public UpsampleBase, public OpKernel { Status Compute(OpKernelContext* context) const override; Status BaseCompute(OpKernelContext* context, const std::vector& roi, const std::vector& scales, - const std::vector& output_dims) const; + const gsl::span& output_dims) const; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/tensor/utils.h b/onnxruntime/core/providers/cpu/tensor/utils.h index 1334bf91e7..4a2f6a4b78 100644 --- a/onnxruntime/core/providers/cpu/tensor/utils.h +++ b/onnxruntime/core/providers/cpu/tensor/utils.h @@ -10,19 +10,19 @@ #include "core/common/safeint.h" namespace onnxruntime { -struct TensorPitches : std::vector { +struct TensorPitches : TensorShapeVector { TensorPitches(const Tensor& tensor, size_t rank = 0) : TensorPitches(tensor.Shape(), rank) {} TensorPitches(const TensorShape& shape, size_t rank = 0) : TensorPitches(shape.GetDims(), rank) {} - TensorPitches(const std::vector& dims, size_t rank = 0) - : std::vector(std::max(rank, dims.size()), 0) { + TensorPitches(const TensorShapeVector& dims, size_t rank = 0) + : TensorShapeVector(std::max(rank, dims.size()), 0) { Calculate(gsl::span(data(), size()), gsl::make_span(dims)); } - TensorPitches(gsl::span dims, size_t rank = 0) - : std::vector(std::max(rank, dims.size()), 0) { + TensorPitches(const gsl::span& dims, size_t rank = 0) + : TensorShapeVector(std::max(rank, dims.size()), 0) { Calculate(gsl::span(data(), size()), dims); } - static bool Calculate(gsl::span p, gsl::span dims) { + static bool Calculate(const gsl::span& p, const gsl::span& dims) { // The pitches is the size of the next inner axis. Aka the amount to move by one of the next inner axis. // For a tensor with shape(2,3,4,5) the values would be: (3*4*5, 4*5, 5, 1) // Note that the outermost '2' is never used, as you never need to move by the entire size of the outermost axis @@ -94,7 +94,7 @@ struct TensorAxisCounters { const Tensor& tensor_; bool running_{true}; size_t axis_; - std::vector indices_; // There is no index for innermost axis since it's a special case + TensorShapeVector indices_; // There is no index for innermost axis since it's a special case }; struct ExtentAxisCounters { @@ -129,16 +129,16 @@ struct ExtentAxisCounters { private: bool running_{true}; size_t axis_; - std::vector indices_; // There is no index for innermost axis since it's a special case + TensorShapeVector indices_; // There is no index for innermost axis since it's a special case gsl::span extents_; // The extents of each axis }; // A std::vector that holds the number of entries to skip to go to the next axis start given an extent // and optionally steps along each axis: // This is used by the SliceIterator to iterate over a slice of a tensor -struct SliceSkips : std::vector { +struct SliceSkips : TensorShapeVector { SliceSkips(const TensorShape& input_shape, gsl::span extents, gsl::span steps) - : std::vector(input_shape.NumDimensions(), 0) { + : TensorShapeVector(input_shape.NumDimensions(), 0) { auto dims = input_shape.GetDims(); ORT_ENFORCE(dims.size() == extents.size() && dims.size() >= steps.size()); @@ -175,25 +175,6 @@ struct SliceIteratorBase { private: enum class byte : unsigned char {}; - protected: - SliceIteratorBase(const Tensor& tensor, gsl::span starts, - gsl::span extents, gsl::span steps) - : tensor_(tensor), extents_(extents), skips_(tensor_.Shape(), extents, steps), indices_(extents.size(), 0) { - auto dims = tensor_.Shape().GetDims(); - Init(dims, starts, steps); - } - - // This construct takes a explicit tensor_shape which might be different from the shape defined in input tensor. - // The explicit tensor_shape usually has inner most axis flattened. For example, given shape[1,4,4,2], if last axis - // does not have padding or slice, then it will be flattened as [1,4,8] for better performance (One inner most copy instead of 4). - // Also supports arbitrary positive and negative stepping along individual axes - SliceIteratorBase(const Tensor& tensor, const TensorShape& tensor_shape, gsl::span starts, - gsl::span extents, gsl::span steps) - : tensor_(tensor), extents_(extents), skips_(tensor_shape, extents, steps), indices_(extents.size(), 0) { - auto dims = tensor_shape.GetDims(); - Init(dims, starts, steps); - } - // Initialize initial skip and inner_extent. void Init(gsl::span dims, gsl::span starts, gsl::span steps) { ORT_ENFORCE(dims.size() == starts.size() && @@ -214,6 +195,35 @@ struct SliceIteratorBase { : 1); } + protected: + SliceIteratorBase(const Tensor& tensor, gsl::span starts, + gsl::span extents, gsl::span steps) + : is_string_tensor_(tensor.IsDataTypeString()), + input_{reinterpret_cast(tensor.DataRaw())}, + element_size_(tensor.DataType()->Size()), + extents_(extents), + skips_(tensor.Shape(), extents, steps), + indices_(extents.size(), 0) { + auto dims = tensor.Shape().GetDims(); + Init(dims, starts, steps); + } + + // This construct takes a explicit tensor_shape which might be different from the shape defined in input tensor. + // The explicit tensor_shape usually has inner most axis flattened. For example, given shape[1,4,4,2], if last axis + // does not have padding or slice, then it will be flattened as [1,4,8] for better performance (One inner most copy instead of 4). + // Also supports arbitrary positive and negative stepping along individual axes + SliceIteratorBase(const Tensor& tensor, const TensorShape& tensor_shape, gsl::span starts, + gsl::span extents, gsl::span steps) + : is_string_tensor_(tensor.IsDataTypeString()), + input_{reinterpret_cast(tensor.DataRaw())}, + element_size_(tensor.DataType()->Size()), + extents_(extents), + skips_(tensor_shape, extents, steps), + indices_(extents.size(), 0) { + auto dims = tensor_shape.GetDims(); + Init(dims, starts, steps); + } + void AdvanceOverInnerExtent() { size_t axis = skips_.size() - 1; input_ += skips_[axis] * element_size_; @@ -304,17 +314,16 @@ struct SliceIteratorBase { return out; } - const Tensor& tensor_; - const bool is_string_tensor_{tensor_.IsDataTypeString()}; + const bool is_string_tensor_; // we do everything in this class using bytes to minimize binary size - const byte* input_{reinterpret_cast(tensor_.DataRaw())}; - const int64_t element_size_ = tensor_.DataType()->Size(); + const byte* input_; + const int64_t element_size_; gsl::span extents_; size_t inner_counter_{}, inner_extent_; ptrdiff_t inner_step_; SliceSkips skips_; - std::vector indices_; // There is no index for innermost axis since it's a special case + TensorShapeVector indices_; // There is no index for innermost axis since it's a special case }; // This provides easy sequential iteration over a subset of a tensor given a span of starts, extents & optionally steps @@ -384,8 +393,8 @@ template struct WritableSliceIterator { WritableSliceIterator(Tensor& tensor, gsl::span starts, gsl::span extents, gsl::span steps) - : tensor_(tensor), input_(tensor_.template MutableData()), extents_(extents), skips_(tensor_.Shape(), extents, steps), indices_(extents.size(), 0) { - auto dims = tensor_.Shape().GetDims(); + : input_(tensor.template MutableData()), extents_(extents), skips_(tensor.Shape(), extents, steps), indices_(extents.size(), 0) { + auto dims = tensor.Shape().GetDims(); Init(dims, starts, steps); } @@ -395,7 +404,7 @@ struct WritableSliceIterator { // Also supports arbitrary positive and negative stepping along individual axes WritableSliceIterator(Tensor& tensor, const TensorShape& tensor_shape, gsl::span starts, gsl::span extents, gsl::span steps) - : tensor_(tensor), input_(tensor_.template MutableData()), extents_(extents), skips_(tensor_shape, extents, steps), indices_(extents.size(), 0) { + : input_(tensor.template MutableData()), extents_(extents), skips_(tensor_shape, extents, steps), indices_(extents.size(), 0) { auto dims = tensor_shape.GetDims(); Init(dims, starts, steps); } @@ -505,12 +514,11 @@ struct WritableSliceIterator { } private: - Tensor& tensor_; T* input_; gsl::span extents_; size_t inner_counter_{}, inner_extent_, inner_step_; SliceSkips skips_; - std::vector indices_; // There is no index for innermost axis since it's a special case + TensorShapeVector indices_; // There is no index for innermost axis since it's a special case }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/controlflow/scan.cc b/onnxruntime/core/providers/cuda/controlflow/scan.cc index abd9375b9a..037f4e65bf 100644 --- a/onnxruntime/core/providers/cuda/controlflow/scan.cc +++ b/onnxruntime/core/providers/cuda/controlflow/scan.cc @@ -34,7 +34,7 @@ template <> Scan<9>::Scan(const OpKernelInfo& info) : onnxruntime::Scan<9>(info) { scan::detail::DeviceHelpers helpers; - helpers.transpose_func = [this](const std::vector& permutations, const Tensor& input, Tensor& output) { + helpers.transpose_func = [this](const gsl::span& permutations, const Tensor& input, Tensor& output) { // TODO: We construct a Transpose kernel on each call as doing so is fairly lightweight. // We could potentially keep a single instance and reuse it if that isn't performant enough. const OpKernelInfo& info = OpKernel::Info(); diff --git a/onnxruntime/core/providers/cuda/cuda_kernel.h b/onnxruntime/core/providers/cuda/cuda_kernel.h index cb3232ffd5..8bf39ce19e 100644 --- a/onnxruntime/core/providers/cuda/cuda_kernel.h +++ b/onnxruntime/core/providers/cuda/cuda_kernel.h @@ -92,6 +92,10 @@ class CudaKernel : public OpKernel { memcpy(CpuPtr(), vec.data(), vec.size() * sizeof(T)); } + CudaAsyncBuffer(const CudaKernel* op_kernel, const TensorShapeVector& vec) : CudaAsyncBuffer(op_kernel, vec.size()) { + memcpy(CpuPtr(), vec.data(), vec.size() * sizeof(T)); + } + void AllocCpuPtr(size_t count) { cpu_pinned_copy_ = op_kernel_->AllocateBufferOnCPUPinned(count); if (cpu_pinned_copy_ == nullptr) diff --git a/onnxruntime/core/providers/cuda/cudnn_common.cc b/onnxruntime/core/providers/cuda/cudnn_common.cc index d85ca2d7cf..c789031057 100644 --- a/onnxruntime/core/providers/cuda/cudnn_common.cc +++ b/onnxruntime/core/providers/cuda/cudnn_common.cc @@ -5,6 +5,7 @@ #include "gsl/gsl" #include "shared_inc/cuda_call.h" #include "core/providers/cpu/tensor/utils.h" +#include "core/framework/inlined_containers.h" namespace onnxruntime { namespace cuda { @@ -31,8 +32,8 @@ Status CudnnTensor::Set(gsl::span input_dims, cudnnDataType_t dat int rank = gsl::narrow_cast(input_dims.size()); TensorPitches pitches(input_dims); - std::vector dims(rank); - std::vector strides(rank); + InlinedVector dims(rank); + InlinedVector strides(rank); for (int i = 0; i < rank; i++) { dims[i] = gsl::narrow_cast(input_dims[i]); strides[i] = gsl::narrow_cast(pitches[i]); @@ -94,12 +95,12 @@ CudnnFilterDescriptor::~CudnnFilterDescriptor() { } } -Status CudnnFilterDescriptor::Set(const std::vector& filter_dims, cudnnDataType_t data_type) { +Status CudnnFilterDescriptor::Set(gsl::span filter_dims, cudnnDataType_t data_type) { if (!desc_) CUDNN_RETURN_IF_ERROR(cudnnCreateFilterDescriptor(&desc_)); int rank = gsl::narrow_cast(filter_dims.size()); - std::vector w_dims(rank); + InlinedShapeVectorT w_dims(rank); for (int i = 0; i < rank; i++) { w_dims[i] = gsl::narrow_cast(filter_dims[i]); } diff --git a/onnxruntime/core/providers/cuda/cudnn_common.h b/onnxruntime/core/providers/cuda/cudnn_common.h index f8cee8fa89..de720521be 100644 --- a/onnxruntime/core/providers/cuda/cudnn_common.h +++ b/onnxruntime/core/providers/cuda/cudnn_common.h @@ -56,7 +56,7 @@ class CudnnFilterDescriptor final { ~CudnnFilterDescriptor(); ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CudnnFilterDescriptor); - Status Set(const std::vector& filter_dims, cudnnDataType_t data_typ); + Status Set(gsl::span filter_dims, cudnnDataType_t data_typ); operator cudnnFilterDescriptor_t() const { return desc_; } diff --git a/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.cc b/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.cc index d8ee94d383..22e7cbfe7c 100644 --- a/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.cc +++ b/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.cc @@ -34,7 +34,7 @@ Status DataCopy(const Tensor& input, Tensor& output, void* einsum_cuda_assets) { } // CUDA EP specific Transpose helper -Status Transpose(const std::vector& permutation, const Tensor& input, +Status Transpose(const gsl::span& permutation, const Tensor& input, Tensor& output, const TensorShape* input_shape_override, void* einsum_cuda_assets) { return cuda::Transpose::DoTranspose(static_cast(einsum_cuda_assets)->cuda_ep_->GetDeviceProp(), static_cast(static_cast(einsum_cuda_assets)->cuda_ep_->GetComputeStream()), @@ -108,7 +108,7 @@ std::unique_ptr Diagonal(const Tensor& input, int64_t dim_1, int64_t dim } // Make a copy - we are going to mutate the dims - std::vector output_dims = input_shape.GetDimsAsVector(); + TensorShapeVector output_dims = input_shape.AsShapeVector(); // Remove the dim value in `second_dim` - // The diagonal values are stored along `first_dim` diff --git a/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.h b/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.h index 5db1468e1a..11971fb1b3 100644 --- a/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.h +++ b/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.h @@ -35,7 +35,7 @@ namespace DeviceHelpers { // These are CUDA EP specific device helper implementations namespace CudaDeviceHelpers { -Status Transpose(const std::vector& permutation, const Tensor& input, +Status Transpose(const gsl::span& permutation, const Tensor& input, Tensor& output, const TensorShape* input_shape_override, void* einsum_cuda_assets); Status DataCopy(const Tensor& input, Tensor& output, void* einsum_cuda_assets); diff --git a/onnxruntime/core/providers/cuda/math/topk.cc b/onnxruntime/core/providers/cuda/math/topk.cc index bb68c8f0f3..cb44041b16 100644 --- a/onnxruntime/core/providers/cuda/math/topk.cc +++ b/onnxruntime/core/providers/cuda/math/topk.cc @@ -73,7 +73,7 @@ Status TopK::ComputeInternal(OpKernelContext* ctx) const { return Status::OK(); } - auto elem_nums = tensor_X->Shape().GetDimsAsVector(); + auto elem_nums = tensor_X->Shape().AsShapeVector(); auto dimension = elem_nums[axis]; for (auto i = static_cast(elem_nums.size()) - 2; i >= 0; --i) { elem_nums[i] *= elem_nums[i + 1]; diff --git a/onnxruntime/core/providers/cuda/nn/conv.cc b/onnxruntime/core/providers/cuda/nn/conv.cc index f224006bcb..c4c4272c46 100644 --- a/onnxruntime/core/providers/cuda/nn/conv.cc +++ b/onnxruntime/core/providers/cuda/nn/conv.cc @@ -71,10 +71,10 @@ size_t GetMaxWorkspaceSize(const CudnnConvState& Status SliceOutUnwantedOutputSection(cudaStream_t stream, const void* input_data, gsl::span input_dims, void* output_data, - gsl::span output_dims, - std::vector starts, - const std::vector& ends, - const std::vector& axes, + const gsl::span& output_dims, + const gsl::span& starts, + const gsl::span& ends, + const gsl::span& axes, size_t element_size) { SliceOp::PrepareForComputeMetadata compute_metadata(input_dims); @@ -91,13 +91,13 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const //set X const Tensor* X = context->Input(0); const TensorShape& x_shape = X->Shape(); - const auto x_dims = x_shape.GetDims(); + const auto x_dims = x_shape.AsShapeVector(); s_.x_data = reinterpret_cast(X->template Data()); s_.element_size = X->DataType()->Size(); //set W const Tensor* W = context->Input(1); const TensorShape& w_shape = W->Shape(); - auto w_dims = w_shape.GetDimsAsVector(); + auto w_dims = w_shape.AsShapeVector(); s_.w_data = reinterpret_cast(W->template Data()); //set B if (context->InputCount() >= 3) { @@ -115,13 +115,13 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const s_.z_data = nullptr; } bool input_dims_changed = (s_.last_x_dims != x_dims); - bool w_dims_changed = (s_.last_w_dims != w_dims); + bool w_dims_changed = (s_.last_w_dims.AsShapeVector() != w_dims); if (input_dims_changed || w_dims_changed) { if (input_dims_changed) - s_.last_x_dims = x_dims; + s_.last_x_dims = gsl::make_span(x_dims); if (w_dims_changed) { - s_.last_w_dims = w_dims; + s_.last_w_dims = gsl::make_span(w_dims); s_.cached_benchmark_results.clear(); } @@ -130,45 +130,45 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X, W)); - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape)); auto rank = kernel_shape.size(); - std::vector pads(conv_attrs_.pads); + ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(rank * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(rank, 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(rank, 1); } - std::vector y_dims; + TensorShapeVector y_dims; y_dims.reserve(2 + rank); // rank indicates number of feature dimensions - so add 2 to account for 'N' and 'C' y_dims.insert(y_dims.begin(), {N, M}); - std::vector y_dims_with_adjusted_pads; + TensorShapeVector y_dims_with_adjusted_pads; y_dims_with_adjusted_pads.reserve(2 + rank); // rank indicates number of feature dimensions - so add 2 to account for 'N' and 'C' y_dims_with_adjusted_pads.insert(y_dims_with_adjusted_pads.begin(), {N, M}); bool post_slicing_required = false; - std::vector slice_starts; + TensorShapeVector slice_starts; slice_starts.reserve(rank); - std::vector slice_ends; + TensorShapeVector slice_ends; slice_ends.reserve(rank); - std::vector slice_axes; + TensorShapeVector slice_axes; slice_axes.reserve(rank); ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShapeWithAdjustedPads(x_shape.Slice(2), kernel_shape, strides, dilations, pads, y_dims, y_dims_with_adjusted_pads, post_slicing_required, slice_starts, slice_ends, slice_axes)); ORT_ENFORCE(y_dims.size() == y_dims_with_adjusted_pads.size()); - s_.y_dims = y_dims; + s_.y_dims = gsl::make_span(y_dims); s_.y_dims_with_adjusted_pads = y_dims_with_adjusted_pads; s_.post_slicing_required = post_slicing_required; s_.slice_starts = slice_starts; @@ -188,8 +188,8 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const s_.y_data = reinterpret_cast(s_.Y->template MutableData()); } - std::vector x_dims_cudnn{x_dims.begin(), x_dims.end()}; - std::vector y_dims_cudnn = !post_slicing_required ? y_dims : y_dims_with_adjusted_pads; + TensorShapeVector x_dims_cudnn{x_dims.begin(), x_dims.end()}; + TensorShapeVector y_dims_cudnn = !post_slicing_required ? y_dims : y_dims_with_adjusted_pads; if (rank < 2) { // TODO: Explore padding the provided input shape [N, C, D] to [N, C, 1, D] // especially for EXHAUSTIVE algo search which may result in a better algo selection. @@ -219,12 +219,12 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const const Tensor* B = context->Input(2); const auto& b_shape = B->Shape(); ORT_RETURN_IF_NOT(b_shape.NumDimensions() == 1, "bias should be 1D"); - std::vector b_dims(2 + kernel_shape.size(), 1); + TensorShapeVector b_dims(2 + kernel_shape.size(), 1); b_dims[1] = b_shape[0]; ORT_RETURN_IF_ERROR(s_.b_tensor.Set(b_dims, CudnnTensor::GetDataType())); //s_.b_data = reinterpret_cast(B->template Data()); } else if (bias_expected) { - std::vector b_dims(2 + kernel_shape.size(), 1); + TensorShapeVector b_dims(2 + kernel_shape.size(), 1); b_dims[1] = w_dims[0]; auto malloc_size = b_dims[1] * sizeof(CudaT); ORT_RETURN_IF_ERROR(s_.b_tensor.Set(b_dims, CudnnTensor::GetDataType())); @@ -238,7 +238,7 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const if (!s_.cached_benchmark_results.contains(x_dims_cudnn)) { // set math type to tensor core before algorithm search - if (std::is_same::value) + ORT_IF_CONSTEXPR(std::is_same::value) CUDNN_RETURN_IF_ERROR(cudnnSetConvolutionMathType(s_.conv_desc, CUDNN_TENSOR_OP_MATH)); cudnnConvolutionFwdAlgoPerf_t perf; @@ -299,7 +299,7 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const s_.workspace_bytes = perf.memory; } else { //set Y - s_.Y = context->Output(0, TensorShape(s_.y_dims)); + s_.Y = context->Output(0, s_.y_dims); if (s_.Y->Shape().Size() == 0) { return Status::OK(); } @@ -362,18 +362,18 @@ CudnnConvolutionDescriptor::~CudnnConvolutionDescriptor() { Status CudnnConvolutionDescriptor::Set( size_t rank, - const std::vector& pads, - const std::vector& strides, - const std::vector& dilations, + const gsl::span& pads, + const gsl::span& strides, + const gsl::span& dilations, int groups, cudnnConvolutionMode_t mode, cudnnDataType_t data_type) { if (!desc_) CUDNN_RETURN_IF_ERROR(cudnnCreateConvolutionDescriptor(&desc_)); - std::vector pad_dims(rank); - std::vector stride_dims(rank); - std::vector dilation_dims(rank); + InlinedVector pad_dims(rank); + InlinedVector stride_dims(rank); + InlinedVector dilation_dims(rank); for (size_t i = 0; i < rank; i++) { pad_dims[i] = gsl::narrow_cast(pads[i]); stride_dims[i] = gsl::narrow_cast(strides[i]); diff --git a/onnxruntime/core/providers/cuda/nn/conv.h b/onnxruntime/core/providers/cuda/nn/conv.h index 845f089a07..135b189d4b 100644 --- a/onnxruntime/core/providers/cuda/nn/conv.h +++ b/onnxruntime/core/providers/cuda/nn/conv.h @@ -10,6 +10,9 @@ #include namespace onnxruntime { + +using ConvPadVector = ConvAttributes::ConvPadVector; + namespace cuda { class CudnnConvolutionDescriptor final { @@ -18,9 +21,9 @@ class CudnnConvolutionDescriptor final { ~CudnnConvolutionDescriptor(); Status Set(size_t rank, - const std::vector& pads, - const std::vector& strides, - const std::vector& dilations, + const gsl::span& pads, + const gsl::span& strides, + const gsl::span& dilations, int groups, cudnnConvolutionMode_t mode, cudnnDataType_t data_type); @@ -41,6 +44,15 @@ struct vector_hash { } }; +struct tensor_shape_vector_hash { + std::size_t operator()(const TensorShapeVector& values) const { + std::size_t seed = values.size(); + for (auto& val : values) + seed ^= std::hash()(val) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + return seed; + } +}; + template , typename KeyEqual = std::equal_to, @@ -119,7 +131,7 @@ struct CudnnConvState { // these would be recomputed if x/w dims change TensorShape y_dims; - std::vector y_dims_with_adjusted_pads; + TensorShapeVector y_dims_with_adjusted_pads; size_t workspace_bytes; decltype(AlgoPerfType().algo) algo; CudnnTensor x_tensor; @@ -143,13 +155,13 @@ struct CudnnConvState { decltype(AlgoPerfType().mathType) mathType; }; - lru_unordered_map, PerfResultParams, vector_hash> cached_benchmark_results{MAX_CACHED_ALGO_PERF_RESULTS}; + lru_unordered_map cached_benchmark_results{MAX_CACHED_ALGO_PERF_RESULTS}; // Some properties needed to support asymmetric padded Conv nodes bool post_slicing_required; - std::vector slice_starts; - std::vector slice_ends; - std::vector slice_axes; + TensorShapeVector slice_starts; + TensorShapeVector slice_ends; + TensorShapeVector slice_axes; // note that conv objects are shared between execution frames, and a lock is needed to avoid multi-thread racing OrtMutex mutex; @@ -196,10 +208,10 @@ Status SliceOutUnwantedOutputSection(cudaStream_t stream, const void* input_data, gsl::span input_dims, void* output_data, - gsl::span output_dims, - std::vector starts, - const std::vector& ends, - const std::vector& axes, + const gsl::span& output_dims, + const gsl::span& starts, + const gsl::span& ends, + const gsl::span& axes, size_t element_size); } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/conv_transpose.cc b/onnxruntime/core/providers/cuda/nn/conv_transpose.cc index dd1d7002cc..fde3b79781 100644 --- a/onnxruntime/core/providers/cuda/nn/conv_transpose.cc +++ b/onnxruntime/core/providers/cuda/nn/conv_transpose.cc @@ -6,7 +6,7 @@ namespace onnxruntime { namespace cuda { -// Op Set 11 for ConvTranspose only update document to clearify default dilations and strides value. +// Op Set 11 for ConvTranspose only update document to clarify default dilations and strides value. // which are already covered by op set 11 cpu version, so simply add declaration. #define REGISTER_KERNEL_TYPED(T) \ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ @@ -41,7 +41,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ const Tensor* X = context->Input(0); const TensorShape& x_shape = X->Shape(); - auto x_dims = x_shape.GetDimsAsVector(); + auto x_dims = x_shape.AsShapeVector(); auto x_data = reinterpret_cast(X->template Data()); auto x_dimensions = X->Shape().NumDimensions(); @@ -52,7 +52,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ } const Tensor* W = context->Input(1); const TensorShape& w_shape = W->Shape(); - std::vector w_dims = w_shape.GetDimsAsVector(); + TensorShapeVector w_dims = w_shape.AsShapeVector(); auto w_data = reinterpret_cast(W->template Data()); size_t num_inputs = OpKernel::Node().InputDefs().size(); @@ -67,21 +67,21 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ { std::lock_guard lock(s_.mutex); // TODO: add a global cache if need to handle cases for multiple frames running simultaneously with different batch_size - bool input_dims_changed = (s_.last_x_dims != x_dims); - bool w_dims_changed = (s_.last_w_dims != w_dims); + bool input_dims_changed = (s_.last_x_dims.AsShapeVector() != x_dims); + bool w_dims_changed = (s_.last_w_dims.AsShapeVector() != w_dims); if (input_dims_changed || w_dims_changed) { if (input_dims_changed) - s_.last_x_dims = x_dims; + s_.last_x_dims = gsl::make_span(x_dims); if (w_dims_changed) { - s_.last_w_dims = w_dims; + s_.last_w_dims = gsl::make_span(w_dims); s_.cached_benchmark_results.clear(); } ConvTransposeAttributes::Prepare p; ORT_RETURN_IF_ERROR(conv_transpose_attrs_.PrepareForCompute(context, has_bias, p, dynamic_padding)); - auto y_dims = p.Y->Shape().GetDimsAsVector(); + auto y_dims = p.Y->Shape().AsShapeVector(); if (x_dimensions == 3) { y_dims.insert(y_dims.begin() + 2, 1); p.kernel_shape.insert(p.kernel_shape.begin(), 1); @@ -90,7 +90,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ p.strides.insert(p.strides.begin(), 1); p.dilations.insert(p.dilations.begin(), 1); } - s_.y_dims = y_dims; + s_.y_dims = gsl::make_span(y_dims); if (w_dims_changed) ORT_RETURN_IF_ERROR(s_.w_desc.Set(w_dims, CudnnTensor::GetDataType())); @@ -114,7 +114,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ if (has_bias) { const auto& b_shape = p.B->Shape(); ORT_RETURN_IF_NOT(b_shape.NumDimensions() == 1, "bias should be 1D"); - std::vector b_dims(2 + p.kernel_shape.size()); + TensorShapeVector b_dims(2 + p.kernel_shape.size()); b_dims[0] = 1; // N b_dims[1] = b_shape[0]; // C for (size_t i = 0; i < p.kernel_shape.size(); i++) @@ -129,7 +129,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ IAllocatorUniquePtr algo_search_workspace = GetScratchBuffer(AlgoSearchWorkspaceSize); // set math type to tensor core before algorithm search - if (std::is_same::value) + if constexpr (std::is_same::value) CUDNN_RETURN_IF_ERROR(cudnnSetConvolutionMathType(s_.conv_desc, CUDNN_TENSOR_OP_MATH)); cudnnConvolutionBwdDataAlgoPerf_t perf; @@ -160,7 +160,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ // The following block will be executed in case there has been no change in the shapes of the // input and the filter compared to the previous run if (!y_data) { - auto y_dims = s_.y_dims.GetDimsAsVector(); + auto y_dims = s_.y_dims.AsShapeVector(); if (x_dimensions == 3) { y_dims.erase(y_dims.begin() + 2); } diff --git a/onnxruntime/core/providers/cuda/nn/max_pool_with_index.cu b/onnxruntime/core/providers/cuda/nn/max_pool_with_index.cu index 2409ee12e3..ef1155af12 100644 --- a/onnxruntime/core/providers/cuda/nn/max_pool_with_index.cu +++ b/onnxruntime/core/providers/cuda/nn/max_pool_with_index.cu @@ -91,15 +91,14 @@ void MaxPoolWithIndex( cudaStream_t stream, const TensorShape& input_shape, const TensorShape& output_shape, - const std::vector& kernel_shape, - const std::vector& stride_shape, - const std::vector& pads, - const std::vector& dilations, + const gsl::span& kernel_shape, + const gsl::span& stride_shape, + const gsl::span& pads, + const gsl::span& dilations, int64_t storage_order, const T* p_input, T* p_output, int64_t* p_indices) { - int64_t batchs = input_shape[0]; int64_t channels = input_shape[1]; int64_t height = input_shape[2]; @@ -163,18 +162,18 @@ void MaxPoolWithIndex( p_indices); } -#define INSTANTIATEMAXPOOLWITHINDEX(T) \ - template void MaxPoolWithIndex( \ - cudaStream_t stream, \ - const TensorShape& input_shape, \ - const TensorShape& output_shape, \ - const std::vector& kernel_shape, \ - const std::vector& stride_shape, \ - const std::vector& pads, \ - const std::vector& dilations, \ - int64_t storage_order, \ - const T* p_input, \ - T* p_output, \ +#define INSTANTIATEMAXPOOLWITHINDEX(T) \ + template void MaxPoolWithIndex( \ + cudaStream_t stream, \ + const TensorShape& input_shape, \ + const TensorShape& output_shape, \ + const gsl::span& kernel_shape, \ + const gsl::span& stride_shape, \ + const gsl::span& pads, \ + const gsl::span& dilations, \ + int64_t storage_order, \ + const T* p_input, \ + T* p_output, \ int64_t* p_indices); INSTANTIATEMAXPOOLWITHINDEX(float) diff --git a/onnxruntime/core/providers/cuda/nn/max_pool_with_index.h b/onnxruntime/core/providers/cuda/nn/max_pool_with_index.h index 3c2420b45b..e2b8b3f39c 100644 --- a/onnxruntime/core/providers/cuda/nn/max_pool_with_index.h +++ b/onnxruntime/core/providers/cuda/nn/max_pool_with_index.h @@ -12,10 +12,10 @@ void MaxPoolWithIndex( cudaStream_t stream, const TensorShape& input_shape, const TensorShape& output_shape, - const std::vector& kernel_shape, - const std::vector& stride_shape, - const std::vector& pads, - const std::vector& dilations, + const gsl::span& kernel_shape, + const gsl::span& stride_shape, + const gsl::span& pads, + const gsl::span& dilations, int64_t storage_order, const T* p_input, T* p_output, diff --git a/onnxruntime/core/providers/cuda/nn/pool.cc b/onnxruntime/core/providers/cuda/nn/pool.cc index 2560651339..1670e30438 100644 --- a/onnxruntime/core/providers/cuda/nn/pool.cc +++ b/onnxruntime/core/providers/cuda/nn/pool.cc @@ -83,16 +83,16 @@ class CudnnPoolingDescriptor final { CudnnPoolingDescriptor& operator=(const CudnnPoolingDescriptor&) = delete; Status Set(cudnnPoolingMode_t mode, - const std::vector& kernel_shape, - const std::vector& pads, - const std::vector& strides) { + const gsl::span& kernel_shape, + const gsl::span& pads, + const gsl::span& strides) { if (!desc_) CUDNN_RETURN_IF_ERROR(cudnnCreatePoolingDescriptor(&desc_)); int rank = gsl::narrow_cast(kernel_shape.size()); - std::vector window(rank); - std::vector padding(rank); - std::vector stride(rank); + InlinedShapeVectorT window(rank); + InlinedShapeVectorT padding(rank); + InlinedShapeVectorT stride(rank); for (int i = 0; i < rank; i++) { window[i] = gsl::narrow_cast(kernel_shape[i]); } @@ -131,9 +131,9 @@ Status Pool::ComputeInternal(OpKernelContext* context) const { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input dimension cannot be less than 3."); } - std::vector kernel_shape = pool_attrs_.kernel_shape; - std::vector pads = pool_attrs_.pads; - std::vector strides = pool_attrs_.strides; + auto kernel_shape = pool_attrs_.kernel_shape; + auto pads = pool_attrs_.pads; + auto strides = pool_attrs_.strides; if (pool_attrs_.global_pooling) { kernel_shape.assign(x_dims.begin() + 2, x_dims.end()); @@ -141,7 +141,7 @@ Status Pool::ComputeInternal(OpKernelContext* context) const { strides.assign(kernel_shape.size(), 1); } - std::vector y_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); + auto y_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); TensorShape y_shape(y_dims); Tensor* Y = context->Output(0, y_shape); // special case when there is a dim value of 0 in the shape. @@ -151,8 +151,8 @@ Status Pool::ComputeInternal(OpKernelContext* context) const { auto x_data = reinterpret_cast(X->template Data()); auto y_data = reinterpret_cast(Y->template MutableData()); - std::vector x_dims_cudnn(x_dims.begin(), x_dims.end()); - std::vector y_dims_cudnn(y_dims.begin(), y_dims.end()); + TensorShapeVector x_dims_cudnn(x_dims.cbegin(), x_dims.cend()); + TensorShapeVector y_dims_cudnn(y_dims); if (kernel_shape.size() < 2) { // cudnn only takes 4D or 5D input, so pad dimensions if needed x_dims_cudnn.push_back(1); @@ -171,7 +171,7 @@ Status Pool::ComputeInternal(OpKernelContext* context) const { CudnnPoolingDescriptor pooling_desc; ORT_RETURN_IF_ERROR(pooling_desc.Set(mode, kernel_shape, pads, strides)); - if (std::is_same::value || std::is_same::value) { + if constexpr (std::is_same::value || std::is_same::value) { // Cast to float back and forth using temp buffer const auto alpha = Consts::One; const auto beta = Consts::Zero; @@ -213,9 +213,9 @@ Status Pool>::ComputeInternal(OpKernelContext* context) const { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input dimension cannot be less than 3."); } - std::vector kernel_shape = this->pool_attrs_.kernel_shape; - std::vector pads = this->pool_attrs_.pads; - std::vector strides = this->pool_attrs_.strides; + auto kernel_shape = this->pool_attrs_.kernel_shape; + auto pads = this->pool_attrs_.pads; + auto strides = this->pool_attrs_.strides; if (this->pool_attrs_.global_pooling) { kernel_shape.assign(x_dims.begin() + 2, x_dims.end()); @@ -223,7 +223,7 @@ Status Pool>::ComputeInternal(OpKernelContext* context) const { strides.assign(kernel_shape.size(), 1); } - std::vector y_dims = this->pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); + auto y_dims = this->pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); Tensor* Y = context->Output(0, TensorShape(y_dims)); // special case when there is a dim value of 0 in the shape. diff --git a/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu b/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu index 4aa98f03bd..fbb0e9fbb7 100644 --- a/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu +++ b/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu @@ -175,8 +175,8 @@ __launch_bounds__(kNmsBlockDim* kNmsBlockDim, 4) __global__ // Variadic template helpers for Index selecting multiple arrays at the same // time template -__device__ inline void SelectHelper(const Index i_selected, - const Index i_original) {} +__device__ inline void SelectHelper(const Index /*i_selected */, + const Index /* i_original */) {} template __device__ inline void SelectHelper(const Index i_selected, diff --git a/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc b/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc index 8ce2c268ae..5b05548728 100644 --- a/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc +++ b/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc @@ -160,7 +160,7 @@ Status ReduceKernel::ReduceKernelShared( OutT* Y, const TensorShape& output_shape, cudnnReduceTensorOp_t cudnn_reduce_op, - std::vector& output_dims) const { + TensorShapeVector& output_dims) const { typedef typename ToCudaType::MappedType CudaT; typedef typename ToCudaType::MappedType CudaOutT; cudnnDataType_t cudnn_type_X = CudnnTensor::GetDataType(); @@ -196,16 +196,16 @@ Status ReduceKernel::ReduceKernelShared( } // CUDNN requires at least 3D input, so pad 1s if needed - std::vector input_dims_cudnn = input_shape.GetDimsAsVector(); - std::vector output_dims_cudnn = output_dims; + auto input_dims_cudnn = input_shape.AsShapeVector(); + auto output_dims_cudnn = output_dims; if (rank < 3) { - std::vector pads(3 - rank, 1); + TensorShapeVector pads(3 - rank, 1); input_dims_cudnn.insert(input_dims_cudnn.end(), pads.begin(), pads.end()); output_dims_cudnn.insert(output_dims_cudnn.end(), pads.begin(), pads.end()); } CudnnReduceDescriptor reduce_desc; - if (std::is_same::value) + ORT_IF_CONSTEXPR (std::is_same::value) ORT_RETURN_IF_ERROR(reduce_desc.Set(cudnn_reduce_op, CudnnTensor::GetDataType(), ReduceTensorIndices)); else ORT_RETURN_IF_ERROR(reduce_desc.Set(cudnn_reduce_op, cudnn_type_X, ReduceTensorIndices)); @@ -345,7 +345,7 @@ template Status ReduceKernel::ReduceKernelShared& output_dims) const; + TensorShapeVector& output_dims) const; template Status ReduceKernel::ReduceKernelShared( const float* X, @@ -353,7 +353,7 @@ template Status ReduceKernel::ReduceKernelShared& output_dims) const; + TensorShapeVector& output_dims) const; template Status ReduceKernel::ReduceKernelShared( const MLFloat16* X, @@ -361,7 +361,7 @@ template Status ReduceKernel::ReduceKernelShared& output_dims) const; + TensorShapeVector& output_dims) const; // `input_shape_override` (if provided) is the input shape for compute purposes Status PrepareForReduce(const Tensor* X, @@ -379,10 +379,10 @@ Status PrepareForReduce(const Tensor* X, return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "cuDNN only supports up to 8-D tensors in reduction"); } - const auto& input_dims = input_shape.GetDims(); + const auto input_dims = input_shape.GetDims(); std::vector reduced(rank, false); if (axes.size() > 0) { - prepare_reduce_metadata.output_dims = input_shape.GetDimsAsVector(); + prepare_reduce_metadata.output_dims = input_shape.AsShapeVector(); for (auto axis : axes) { axis = HandleNegativeAxis(axis, rank); ORT_ENFORCE(input_dims[axis] != 0, @@ -419,10 +419,10 @@ Status PrepareForReduce(const Tensor* X, } // CUDNN requires at least 3D input, so pad 1s if needed - prepare_reduce_metadata.input_dims_cudnn = input_shape.GetDimsAsVector(); + prepare_reduce_metadata.input_dims_cudnn = input_shape.AsShapeVector(); prepare_reduce_metadata.output_dims_cudnn = prepare_reduce_metadata.output_dims; if (rank < 3) { - std::vector pads(3 - rank, 1); + TensorShapeVector pads(3 - rank, 1); prepare_reduce_metadata.input_dims_cudnn.insert(prepare_reduce_metadata.input_dims_cudnn.end(), pads.begin(), pads.end()); prepare_reduce_metadata.output_dims_cudnn.insert(prepare_reduce_metadata.output_dims_cudnn.end(), pads.begin(), pads.end()); } @@ -444,9 +444,9 @@ Status ReduceComputeCore(CUDAExecutionProvider& cuda_ep, const Tensor& input, Pr int64_t input_count = prepare_reduce_metadata.input_count; int64_t output_count = prepare_reduce_metadata.output_count; - std::vector& output_dims = prepare_reduce_metadata.output_dims; - std::vector& input_dims_cudnn = prepare_reduce_metadata.input_dims_cudnn; - std::vector& output_dims_cudnn = prepare_reduce_metadata.output_dims_cudnn; + auto& output_dims = prepare_reduce_metadata.output_dims; + auto& input_dims_cudnn = prepare_reduce_metadata.input_dims_cudnn; + auto& output_dims_cudnn = prepare_reduce_metadata.output_dims_cudnn; cudaStream_t stream = static_cast(cuda_ep.GetComputeStream()); // special case when there is a dim value of 0 in the shape. if (input_count == 0) { @@ -701,7 +701,7 @@ template template Status ReduceKernel::ComputeImpl(OpKernelContext* ctx, cudnnReduceTensorOp_t cudnn_reduce_op) const { const Tensor* X = ctx->Input(0); - std::vector axes; + TensorShapeVector axes; size_t num_inputs = ctx->InputCount(); if (num_inputs == 2) { @@ -753,7 +753,7 @@ Status ReduceKernel::ComputeImpl(OpKernelContext* ctx, cudnnRe OpKernelContext * ctx, cudnnReduceTensorOp_t cudnn_reduce_op) const { \ typedef typename ToCudaType::MappedType CudaT; \ const Tensor* X = ctx->Input(0); \ - std::vector axes; \ + TensorShapeVector axes; \ size_t num_inputs = ctx->InputCount(); \ if (num_inputs == 2) { \ const Tensor* axes_tensor = ctx->Input(1); \ @@ -780,8 +780,8 @@ Status ReduceKernel::ComputeImpl(OpKernelContext* ctx, cudnnRe \ int64_t input_count = prepare_reduce_metadata.input_count; \ int64_t output_count = prepare_reduce_metadata.output_count; \ - std::vector& input_dims_cudnn = prepare_reduce_metadata.input_dims_cudnn; \ - std::vector& output_dims_cudnn = prepare_reduce_metadata.output_dims_cudnn; \ + auto& input_dims_cudnn = prepare_reduce_metadata.input_dims_cudnn; \ + auto& output_dims_cudnn = prepare_reduce_metadata.output_dims_cudnn; \ \ if (input_count == 0) { \ assert(Y->Shape().Size() == 0); \ @@ -807,24 +807,24 @@ Status ReduceKernel::ComputeImpl(OpKernelContext* ctx, cudnnRe cudnnDataType_t cudnn_type_X = CUDNN_DATA_FLOAT; \ IAllocatorUniquePtr temp_X = GetScratchBuffer(input_count); \ Impl_Cast(Stream(), reinterpret_cast(X->template Data()), temp_X.get(), X->Shape().Size()); \ - \ - ORT_RETURN_IF_ERROR(reduce_desc.Set(cudnn_reduce_op, cudnn_type_X, CUDNN_REDUCE_TENSOR_NO_INDICES)); \ - ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_cudnn, cudnn_type_X)); \ - ORT_RETURN_IF_ERROR(output_tensor.Set(output_dims_cudnn, cudnn_type_X)); \ - CUDNN_RETURN_IF_ERROR( \ - cudnnGetReductionIndicesSize(CudnnHandle(), reduce_desc, input_tensor, output_tensor, &indices_bytes)); \ - CUDNN_RETURN_IF_ERROR( \ - cudnnGetReductionWorkspaceSize(CudnnHandle(), reduce_desc, input_tensor, output_tensor, &workspace_bytes)); \ - IAllocatorUniquePtr indices_cuda = GetScratchBuffer(indices_bytes); \ - IAllocatorUniquePtr workspace_cuda = GetScratchBuffer(workspace_bytes); \ - \ - const auto one = Consts::One; \ - const auto zero = Consts::Zero; \ - auto temp_Y = GetScratchBuffer(output_count); \ - CUDNN_RETURN_IF_ERROR(cudnnReduceTensor(CudnnHandle(), reduce_desc, indices_cuda.get(), indices_bytes, \ - workspace_cuda.get(), workspace_bytes, &one, input_tensor, temp_X.get(), \ - &zero, output_tensor, temp_Y.get())); \ - \ + \ + ORT_RETURN_IF_ERROR(reduce_desc.Set(cudnn_reduce_op, cudnn_type_X, CUDNN_REDUCE_TENSOR_NO_INDICES)); \ + ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_cudnn, cudnn_type_X)); \ + ORT_RETURN_IF_ERROR(output_tensor.Set(output_dims_cudnn, cudnn_type_X)); \ + CUDNN_RETURN_IF_ERROR( \ + cudnnGetReductionIndicesSize(CudnnHandle(), reduce_desc, input_tensor, output_tensor, &indices_bytes)); \ + CUDNN_RETURN_IF_ERROR( \ + cudnnGetReductionWorkspaceSize(CudnnHandle(), reduce_desc, input_tensor, output_tensor, &workspace_bytes)); \ + IAllocatorUniquePtr indices_cuda = GetScratchBuffer(indices_bytes); \ + IAllocatorUniquePtr workspace_cuda = GetScratchBuffer(workspace_bytes); \ + \ + const auto one = Consts::One; \ + const auto zero = Consts::Zero; \ + auto temp_Y = GetScratchBuffer(output_count); \ + CUDNN_RETURN_IF_ERROR(cudnnReduceTensor(CudnnHandle(), reduce_desc, indices_cuda.get(), indices_bytes, \ + workspace_cuda.get(), workspace_bytes, &one, input_tensor, temp_X.get(), \ + &zero, output_tensor, temp_Y.get())); \ + \ Impl_Cast(Stream(), temp_Y.get(), reinterpret_cast(Y->template MutableData()), output_count); \ \ return Status::OK(); \ @@ -841,7 +841,7 @@ Status ReduceKernel::ComputeImpl OpKernelContext* ctx, cudnnReduceTensorOp_t cudnn_reduce_op) const { typedef typename ToCudaType::MappedType CudaT; const Tensor* X = ctx->Input(0); - std::vector axes; + TensorShapeVector axes; size_t num_inputs = ctx->InputCount(); if (num_inputs == 2) { const Tensor* axes_tensor = ctx->Input(1); @@ -868,8 +868,8 @@ Status ReduceKernel::ComputeImpl int64_t input_count = prepare_reduce_metadata.input_count; int64_t output_count = prepare_reduce_metadata.output_count; - std::vector& input_dims_cudnn = prepare_reduce_metadata.input_dims_cudnn; - std::vector& output_dims_cudnn = prepare_reduce_metadata.output_dims_cudnn; + auto& input_dims_cudnn = prepare_reduce_metadata.input_dims_cudnn; + auto& output_dims_cudnn = prepare_reduce_metadata.output_dims_cudnn; if (input_count == 0) { assert(Y->Shape().Size() == 0); diff --git a/onnxruntime/core/providers/cuda/reduction/reduction_ops.h b/onnxruntime/core/providers/cuda/reduction/reduction_ops.h index 406cae8e02..a962a5a97d 100644 --- a/onnxruntime/core/providers/cuda/reduction/reduction_ops.h +++ b/onnxruntime/core/providers/cuda/reduction/reduction_ops.h @@ -28,11 +28,11 @@ struct PrepareReduceMetadata { int64_t input_count; int64_t output_count; // This holds the output dims without any reduced dims squeezed (even if keep_dims == 1) - std::vector output_dims; + TensorShapeVector output_dims; // This holds the output dims with with reduced dims squeezed (if keep_dims == 1) - std::vector squeezed_output_dims; - std::vector input_dims_cudnn; - std::vector output_dims_cudnn; + TensorShapeVector squeezed_output_dims; + TensorShapeVector input_dims_cudnn; + TensorShapeVector output_dims_cudnn; }; template @@ -68,7 +68,7 @@ class ReduceKernel : public CudaKernel, public ReduceKernelBase& output_dims) const; + TensorShapeVector& output_dims) const; using ReduceKernelBase::axes_; using ReduceKernelBase::keepdims_; diff --git a/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc index 1dcbebf695..2e10f15f09 100644 --- a/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc +++ b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc @@ -86,10 +86,10 @@ Status CudnnRnnBase::ReorganizeWeights(const Tensor* W, const Tensor* R, cons // LSTM B[num_directions_, 8*hidden_size_] size_t number = W_lin_layer_id_.size(); int64_t w_size = num_directions_ * (number * hidden_size_ * (input_size + hidden_size_ + 2)); - std::vector dims_w({w_size, 1, 1}); + TensorShapeVector dims_w({w_size, 1, 1}); ORT_RETURN_IF_ERROR(target_w_desc.Set(dims_w, CudnnTensor::GetDataType())); - std::vector fake_dims_x({1, input_size, 1}); + TensorShapeVector fake_dims_x({1, input_size, 1}); CudnnTensor fake_x_desc; ORT_RETURN_IF_ERROR(fake_x_desc.Set(fake_dims_x, CudnnTensor::GetDataType())); @@ -165,9 +165,9 @@ Status CudnnRnnBase::ComputeInternal(OpKernelContext* ctx) const { int64_t input_size = X->Shape()[2]; // optional outputs - std::vector dims_Y({seq_length, num_directions_, batch_size, hidden_size_}); - std::vector dims_hxy({RNN_NUM_LAYERS * num_directions_, batch_size, hidden_size_}); - std::vector dims_yc{num_directions_, batch_size, hidden_size_}; + TensorShapeVector dims_Y({seq_length, num_directions_, batch_size, hidden_size_}); + TensorShapeVector dims_hxy({RNN_NUM_LAYERS * num_directions_, batch_size, hidden_size_}); + TensorShapeVector dims_yc{num_directions_, batch_size, hidden_size_}; Tensor* Y = ctx->Output(Output_Index::Y, dims_Y); Tensor* Y_h = ctx->Output(Output_Index::Y_h, dims_hxy); Tensor* Y_c = ctx->Output(Output_Index::Y_c, dims_yc); diff --git a/onnxruntime/core/providers/cuda/tensor/concat.cc b/onnxruntime/core/providers/cuda/tensor/concat.cc index 7b741fa0d7..8654a0c347 100644 --- a/onnxruntime/core/providers/cuda/tensor/concat.cc +++ b/onnxruntime/core/providers/cuda/tensor/concat.cc @@ -35,7 +35,7 @@ Status Concat::ComputeInternal(OpKernelContext* ctx) const { auto input_count = Node().InputArgCount().front(); // Hold pointers to the input tensors to be used in the PrepareForCompute() step - std::vector input_tensors; + InlinedTensorsVector input_tensors; input_tensors.reserve(input_count); for (int i = 0; i < input_count; ++i) { input_tensors.push_back(ctx->Input(i)); @@ -48,15 +48,16 @@ Status Concat::ComputeInternal(OpKernelContext* ctx) const { if (p.output_num_elements == 0) return Status::OK(); - std::vector concat_sizes(input_count); + std::vector concat_sizes; + concat_sizes.reserve(input_count); CudaAsyncBuffer input_ptr(this, input_count); gsl::span input_ptr_cpuspan = input_ptr.CpuSpan(); std::vector axis_dimension_input_output_mapping(p.output_tensor->Shape()[p.axis]); int index = 0; for (int i = 0; i < input_count; ++i) { - auto input = p.inputs[i]; - concat_sizes[i] = input.tensor->Shape()[p.axis]; + const auto& input = p.inputs[i]; + concat_sizes.push_back(input.tensor->Shape()[p.axis]); input_ptr_cpuspan[i] = input.tensor->DataRaw(); for (int j = 0; j < input.tensor->Shape()[p.axis]; ++j) { axis_dimension_input_output_mapping.at(index++) = i; diff --git a/onnxruntime/core/providers/cuda/tensor/expand.cc b/onnxruntime/core/providers/cuda/tensor/expand.cc index 411336acd4..d1b2834d88 100644 --- a/onnxruntime/core/providers/cuda/tensor/expand.cc +++ b/onnxruntime/core/providers/cuda/tensor/expand.cc @@ -5,18 +5,16 @@ #include "expand_impl.h" #include "core/providers/cpu/tensor/utils.h" -using std::vector; - namespace onnxruntime { namespace cuda { // Logically expanded y could just be a view of x. -static void CalcEffectiveDims(vector& x_dims, vector& y_dims) { - vector x_reverse; - vector y_reverse; +static void CalcEffectiveDims(TensorShapeVector& x_dims, TensorShapeVector& y_dims) { + TensorShapeVector x_reverse; + TensorShapeVector y_reverse; - int xi = gsl::narrow_cast(x_dims.size()) - 1; - for (int yi = gsl::narrow_cast(y_dims.size()) - 1; yi >= 0; --yi, --xi) { + int64_t xi = gsl::narrow_cast(x_dims.size()) - 1; + for (int64_t yi = gsl::narrow_cast(y_dims.size()) - 1; yi >= 0; --yi, --xi) { int64_t xdim = (xi >= 0) ? x_dims[xi] : 1; int64_t ydim = y_dims[yi]; if (xdim == ydim || xdim == 1) { @@ -36,7 +34,7 @@ static void CalcEffectiveDims(vector& x_dims, vector& y_dims) x_dims.push_back(1); y_dims.push_back(1); // compact the dims, remove (x=1, y=1), merge (x=1, y1*y2...) - for (int i = gsl::narrow_cast(y_reverse.size()) - 1; i >= 0; --i) { + for (int64_t i = gsl::narrow_cast(y_reverse.size()) - 1; i >= 0; --i) { if (x_reverse[i] == 1) { if (y_reverse[i] == 1) { continue; @@ -65,7 +63,7 @@ Status Expand::ComputeInternal(OpKernelContext* ctx) const { // new shape to be expanded to const auto* p_shape = input_shape_tensor.template Data(); - std::vector output_dims{p_shape, p_shape + input_shape_tensor.Shape().Size()}; + TensorShapeVector output_dims{p_shape, p_shape + input_shape_tensor.Shape().Size()}; TensorShape output_shape(output_dims); ORT_RETURN_IF_ERROR(ComputeOutputShape(Node().Name(), input_data_tensor.Shape(), output_dims, output_shape)); @@ -74,8 +72,8 @@ Status Expand::ComputeInternal(OpKernelContext* ctx) const { return Status::OK(); } - output_dims = output_shape.GetDimsAsVector(); - auto input_dims = input_data_tensor.Shape().GetDimsAsVector(); + output_dims = output_shape.AsShapeVector(); + auto input_dims = input_data_tensor.Shape().AsShapeVector(); CalcEffectiveDims(input_dims, output_dims); int rank = gsl::narrow_cast(output_dims.size()); diff --git a/onnxruntime/core/providers/cuda/tensor/onehot.cc b/onnxruntime/core/providers/cuda/tensor/onehot.cc index 1ca9c3216f..fd412fc6e0 100644 --- a/onnxruntime/core/providers/cuda/tensor/onehot.cc +++ b/onnxruntime/core/providers/cuda/tensor/onehot.cc @@ -49,7 +49,7 @@ Status OneHotOp::ComputeInternal(OpKernelContext* // prepare output shape int64_t prefix_dim_size, suffix_dim_size; - std::vector output_shape; + TensorShapeVector output_shape; ORT_RETURN_IF_ERROR(PrepareOutputShape(indices, depth_val, axis_, prefix_dim_size, suffix_dim_size, output_shape)); // allocate output diff --git a/onnxruntime/core/providers/cuda/tensor/pad.cc b/onnxruntime/core/providers/cuda/tensor/pad.cc index 3aa61407e5..e869d55a47 100644 --- a/onnxruntime/core/providers/cuda/tensor/pad.cc +++ b/onnxruntime/core/providers/cuda/tensor/pad.cc @@ -41,6 +41,8 @@ namespace cuda { .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ Pad); +using PadsVector = PadBase::PadsVector; + static bool IsNCHWInputWithPaddingAlongHAndW(size_t input_rank, const TArray& lower_pads, const TArray& upper_pads) { @@ -82,13 +84,13 @@ Status Pad::ComputeInternal(OpKernelContext* ctx) const { auto const& input_shape = input_tensor.Shape(); int32_t dimension_count = static_cast(input_shape.NumDimensions()); - const std::vector* p_pads = &pads_; - const std::vector* p_slices = &slices_; + const PadsVector* p_pads = &pads_; + const PadsVector* p_slices = &slices_; CudaT value = ToCudaType::FromFloat(value_); // kOnnxDomain Pad opset >= 11 (Or) kMsDomain opset == 1 - std::vector pads; - std::vector slices; + PadsVector pads; + PadsVector slices; if (is_dynamic_) { const Tensor& pads_tensor = *ctx->Input(1); const auto pads_tensor_dims = pads_tensor.Shape().GetDims(); @@ -132,7 +134,7 @@ Status Pad::ComputeInternal(OpKernelContext* ctx) const { TArray input_dims(input_shape.GetDims()); TArray input_strides(input_pitches); - std::vector output_dims(input_shape.GetDimsAsVector()); + auto output_dims(input_shape.AsShapeVector()); ORT_ENFORCE(dimension_count * 2 == p_pads->size(), "'pads' attribute has wrong number of values"); // Calculate output dimensions, and handle any negative padding diff --git a/onnxruntime/core/providers/cuda/tensor/reshape.h b/onnxruntime/core/providers/cuda/tensor/reshape.h index 533591f6b4..106c01df3a 100644 --- a/onnxruntime/core/providers/cuda/tensor/reshape.h +++ b/onnxruntime/core/providers/cuda/tensor/reshape.h @@ -22,9 +22,8 @@ class Reshape final : public CudaKernel { const Tensor* shapeTensor = context->Input(1); if (shapeTensor == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); if (shapeTensor->Shape().NumDimensions() != 1) return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "A shape tensor must be a vector tensor, got ", shapeTensor->Shape().NumDimensions(), " dimensions"); - size_t nDims = static_cast(shapeTensor->Shape()[0]); - const int64_t* data = shapeTensor->template Data(); - std::vector shape(data, data + nDims); + auto data_span = shapeTensor->template DataAsSpan(); + TensorShapeVector shape(data_span.cbegin(), data_span.cend()); const Tensor* X = context->Input(0); if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); const TensorShape& X_shape = X->Shape(); @@ -49,12 +48,12 @@ class Reshape final : public CudaKernel { class Reshape_1 final : public CudaKernel { public: Reshape_1(const OpKernelInfo& info) : CudaKernel(info) { - Status status = info.GetAttrs("shape", shape_); + Status status = info.GetAttrs("shape", shape_); ORT_ENFORCE(status.IsOK(), "Attribute shape is not set."); } Status ComputeInternal(OpKernelContext* context) const override { - std::vector shape = shape_; + TensorShapeVector shape = shape_; const Tensor* X = context->Input(0); const TensorShape& X_shape = X->Shape(); @@ -72,7 +71,7 @@ class Reshape_1 final : public CudaKernel { } private: - std::vector shape_; + TensorShapeVector shape_; }; } // namespace cuda diff --git a/onnxruntime/core/providers/cuda/tensor/resize_impl.cu b/onnxruntime/core/providers/cuda/tensor/resize_impl.cu index cb1512b5ab..10232e1f15 100644 --- a/onnxruntime/core/providers/cuda/tensor/resize_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/resize_impl.cu @@ -573,7 +573,7 @@ __global__ void _ResizeBiCubicKernel( } size_t CalcResizeBufferSize(const onnxruntime::UpsampleMode upsample_mode, - const std::vector& output_dims) { + const gsl::span& output_dims) { switch (upsample_mode) { case UpsampleMode::NN: return sizeof(int64_t) * output_dims.size() + sizeof(NearestMappingInfo) * static_cast(std::accumulate(output_dims.begin(), output_dims.end(), (int64_t)0)); diff --git a/onnxruntime/core/providers/cuda/tensor/resize_impl.h b/onnxruntime/core/providers/cuda/tensor/resize_impl.h index c2359c260c..5fb2786bb6 100644 --- a/onnxruntime/core/providers/cuda/tensor/resize_impl.h +++ b/onnxruntime/core/providers/cuda/tensor/resize_impl.h @@ -12,7 +12,7 @@ namespace onnxruntime { namespace cuda { size_t CalcResizeBufferSize(const onnxruntime::UpsampleMode upsample_mode, - const std::vector& output_dims); + const gsl::span& output_dims); template void ResizeImpl( diff --git a/onnxruntime/core/providers/cuda/tensor/sequence_op.h b/onnxruntime/core/providers/cuda/tensor/sequence_op.h index 7d337184d6..410ab3bf98 100644 --- a/onnxruntime/core/providers/cuda/tensor/sequence_op.h +++ b/onnxruntime/core/providers/cuda/tensor/sequence_op.h @@ -121,7 +121,7 @@ class ConcatFromSequence final : public CudaKernel, public ConcatBase { Status ComputeInternal(OpKernelContext* context) const override { const TensorSeq* X = context->Input(0); int64_t input_count = static_cast(X->Size()); - std::vector input_tensors; + InlinedTensorsVector input_tensors; for (int64_t i = 0; i < input_count; ++i) { input_tensors.push_back(&X->Get(i)); } diff --git a/onnxruntime/core/providers/cuda/tensor/slice.cc b/onnxruntime/core/providers/cuda/tensor/slice.cc index df768932ba..fb41f641bd 100644 --- a/onnxruntime/core/providers/cuda/tensor/slice.cc +++ b/onnxruntime/core/providers/cuda/tensor/slice.cc @@ -111,7 +111,7 @@ static Status ComputeSliceStrides(const TensorShape& input_shape, TArray& input_strides, TArray& output_strides, SliceOp::PrepareForComputeMetadata& compute_metadata) { - const auto& input_dimensions = input_shape.GetDims(); + const auto input_dimensions = input_shape.GetDims(); size_t dimension_count = input_dimensions.size(); // if we are able to flatten the output dims we updated 'starts' and 'steps' to match the smaller number of dims. // update dimension_count to match. @@ -130,7 +130,7 @@ static Status ComputeSliceStrides(const TensorShape& input_shape, aggregated_last_dim *= input_dimensions[i]; } - std::vector flattened_input_dims(input_dimensions.begin(), input_dimensions.end()); + TensorShapeVector flattened_input_dims(input_dimensions.begin(), input_dimensions.end()); flattened_input_dims.resize(dimension_count); flattened_input_dims.back() = aggregated_last_dim; ORT_ENFORCE(TensorPitches::Calculate(input_strides_span, flattened_input_dims)); @@ -138,10 +138,12 @@ static Status ComputeSliceStrides(const TensorShape& input_shape, ORT_ENFORCE(TensorPitches::Calculate(input_strides_span, input_dimensions)); } - TensorPitches original_output_strides( - compute_metadata.p_flattened_output_dims_ != nullptr ? compute_metadata.flattened_output_dims_ : compute_metadata.output_dims_); + const auto output_dims = gsl::make_span(compute_metadata.p_flattened_output_dims_ != nullptr + ? compute_metadata.flattened_output_dims_ + : compute_metadata.output_dims_); + TensorPitches original_output_strides(output_dims); output_strides.SetSize(gsl::narrow_cast(original_output_strides.size())); - for (int32_t i = 0; i < static_cast(original_output_strides.size()); ++i) { + for (int32_t i = 0, limit = static_cast(original_output_strides.size()); i < limit; ++i) { output_strides[i] = fast_divmod(gsl::narrow_cast(original_output_strides[i])); } @@ -154,7 +156,7 @@ Status Impl(cudaStream_t stream, void* output_data, SliceOp::PrepareForComputeMetadata& compute_metadata, size_t element_size) { - const auto& input_dimensions = input_shape.GetDims(); + const auto input_dimensions = input_shape.GetDims(); size_t dimension_count = input_dimensions.size(); TArray starts_buffer(compute_metadata.starts_); @@ -186,13 +188,13 @@ Status Slice::ComputeInternal(OpKernelContext* ctx) const { const Tensor* input_tensor = GetSlicedOrUnslicedTensor(ctx); ORT_ENFORCE(nullptr != input_tensor); const auto& input_shape = input_tensor->Shape(); - const auto& input_dimensions = input_shape.GetDims(); + const auto input_dimensions = input_shape.GetDims(); if (input_dimensions.empty()) return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Cannot slice scalars"); SliceOp::PrepareForComputeMetadata compute_metadata(input_dimensions); if (dynamic) { - std::vector input_starts, input_ends, input_axes, input_steps; + TensorShapeVector input_starts, input_ends, input_axes, input_steps; ORT_RETURN_IF_ERROR(FillInputVectors(ctx, input_starts, input_ends, input_axes, input_steps)); ORT_RETURN_IF_ERROR(PrepareForCompute(input_starts, input_ends, input_axes, input_steps, compute_metadata)); @@ -226,9 +228,9 @@ const Tensor* Slice::GetSlicedOrUnslicedTensor(OpKernelContext* ctx) co } template -Status Slice::FillInputVectors(OpKernelContext* ctx, std::vector& input_starts, - std::vector& input_ends, std::vector& input_axes, - std::vector& input_steps) const { +Status Slice::FillInputVectors(OpKernelContext* ctx, TensorShapeVector& input_starts, + TensorShapeVector& input_ends, TensorShapeVector& input_axes, + TensorShapeVector& input_steps) const { return FillVectorsFromInput(*ctx->Input(1), *ctx->Input(2), ctx->Input(3), ctx->Input(4), input_starts, input_ends, input_axes, input_steps); } diff --git a/onnxruntime/core/providers/cuda/tensor/slice.h b/onnxruntime/core/providers/cuda/tensor/slice.h index 827068aa27..444e37c216 100644 --- a/onnxruntime/core/providers/cuda/tensor/slice.h +++ b/onnxruntime/core/providers/cuda/tensor/slice.h @@ -29,9 +29,9 @@ class Slice : public CudaKernel, public SliceBase { private: virtual const Tensor* GetSlicedOrUnslicedTensor(OpKernelContext* ctx) const; - virtual Status FillInputVectors(OpKernelContext* ctx, std::vector& input_starts, - std::vector& input_ends, std::vector& input_axes, - std::vector& input_steps) const; + virtual Status FillInputVectors(OpKernelContext* ctx, TensorShapeVector& input_starts, + TensorShapeVector& input_ends, TensorShapeVector& input_axes, + TensorShapeVector& input_steps) const; virtual Status CallSliceImp(size_t element_size, size_t dimension_count, const TArray& starts_buffer, const TArray& steps_buffer, const TArray& input_strides, diff --git a/onnxruntime/core/providers/cuda/tensor/split.cc b/onnxruntime/core/providers/cuda/tensor/split.cc index 48a96427c9..ae6d2830e3 100644 --- a/onnxruntime/core/providers/cuda/tensor/split.cc +++ b/onnxruntime/core/providers/cuda/tensor/split.cc @@ -64,11 +64,11 @@ Status Split::ComputeInternal(OpKernelContext* ctx) const { auto input_data = input_tensor->DataRaw(); auto input_dims = input_shape.GetDims(); - std::vector output_dimensions{input_shape.GetDimsAsVector()}; + auto output_dimensions{input_shape.AsShapeVector()}; CudaAsyncBuffer output_ptr(this, num_outputs); gsl::span output_ptr_span = output_ptr.CpuSpan(); - std::vector axis_dimension_input_output_mapping(input_dims[axis]); + TensorShapeVector axis_dimension_input_output_mapping(input_dims[axis]); int index = 0; for (int i = 0; i < num_outputs; ++i) { // update size of dimension for axis we're splitting on diff --git a/onnxruntime/core/providers/cuda/tensor/squeeze.cc b/onnxruntime/core/providers/cuda/tensor/squeeze.cc index 94ee38cdb3..d29e249882 100644 --- a/onnxruntime/core/providers/cuda/tensor/squeeze.cc +++ b/onnxruntime/core/providers/cuda/tensor/squeeze.cc @@ -43,7 +43,7 @@ Status Squeeze::ComputeInternal(OpKernelContext* ctx) const { const Tensor* X = ctx->Input(0); const TensorShape& X_shape = X->Shape(); - std::vector axes; + TensorShapeVector axes; size_t num_inputs = ctx->InputCount(); if (num_inputs == 2) { //axes is an input const Tensor* axes_tensor = ctx->Input(1); @@ -57,7 +57,7 @@ Status Squeeze::ComputeInternal(OpKernelContext* ctx) const { axes.assign(axes_.begin(), axes_.end()); } - std::vector output_shape = ComputeOutputShape(X_shape, axes); + TensorShapeVector output_shape = ComputeOutputShape(X_shape, axes); Tensor* Y = ctx->Output(0, TensorShape(output_shape)); diff --git a/onnxruntime/core/providers/cuda/tensor/tile.cc b/onnxruntime/core/providers/cuda/tensor/tile.cc index 9fff9f6cba..b7815a5337 100644 --- a/onnxruntime/core/providers/cuda/tensor/tile.cc +++ b/onnxruntime/core/providers/cuda/tensor/tile.cc @@ -54,7 +54,7 @@ Status Tile::ComputeInternal(OpKernelContext* ctx) const { auto* repeats = repeats_tensor.template Data(); const auto& input_shape = input_tensor.Shape(); const auto input_dims = input_shape.GetDims(); - std::vector output_dims(input_shape.GetDimsAsVector()); + auto output_dims(input_shape.AsShapeVector()); for (auto axis = 0; axis < rank; axis++) output_dims[axis] *= repeats[axis]; TensorShape output_shape(output_dims); diff --git a/onnxruntime/core/providers/cuda/tensor/transpose.cc b/onnxruntime/core/providers/cuda/tensor/transpose.cc index d785c8422b..71eae4cf61 100644 --- a/onnxruntime/core/providers/cuda/tensor/transpose.cc +++ b/onnxruntime/core/providers/cuda/tensor/transpose.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/framework/inlined_containers.h" #include "core/providers/cuda/tensor/transpose.h" #include "core/providers/cuda/tensor/transpose_impl.h" #include "core/providers/cpu/tensor/utils.h" @@ -28,7 +29,7 @@ ONNX_OPERATOR_KERNEL_EX( Transpose); // special case acceleration using cublas matrix transpose -static std::tuple TryTransposeWithCublas(const std::vector& perm, const TensorShape& input_shape) { +static std::tuple TryTransposeWithCublas(const gsl::span& perm, const TensorShape& input_shape) { int M = 0; int N = 0; @@ -77,14 +78,14 @@ Status TransposeWithCublas(cudaStream_t stream, cublasHandle_t cublas_handle, co } Status Transpose::DoTranspose(const Transpose& transpose_kernel, - const std::vector& permutations, const Tensor& input, Tensor& output) { + const gsl::span& permutations, const Tensor& input, Tensor& output) { return Transpose::DoTranspose(transpose_kernel.GetDeviceProp(), transpose_kernel.Stream(), transpose_kernel.CublasHandle(), permutations, input, output); } Status Transpose::DoTranspose(const cudaDeviceProp& prop, cudaStream_t stream, const cublasHandle_t cublas_handle, - const std::vector& permutations, const Tensor& input, Tensor& output, + const gsl::span& permutations, const Tensor& input, Tensor& output, const TensorShape* input_shape_override) { // special case when there is a dim value of 0 in the shape. if (output.Shape().Size() == 0) @@ -97,9 +98,9 @@ Status Transpose::DoTranspose(const cudaDeviceProp& prop, // flatten the adjacent dimensions which are contiguous // for example: permutations[0, 2, 3, 1] -> [0, 2, 1], permutations[0, 3, 1, 2] -> [0, 2, 1] auto new_rank = rank; - std::vector new_permutations(permutations); - std::vector new_input_dims(input_dims.begin(), input_dims.end()); - std::vector new_output_dims(output_dims.begin(), output_dims.end()); + InlinedShapeVectorT new_permutations(permutations.cbegin(), permutations.cend()); + TensorShapeVector new_input_dims = ToShapeVector(input_dims); + TensorShapeVector new_output_dims = ToShapeVector(output_dims); // Remove all dims with value 1. std::vector dims_to_remove(new_rank, false); @@ -203,7 +204,7 @@ Status Transpose::DoTranspose(const cudaDeviceProp& prop, dim3 grid_size, block_size; if (CanDoTranspose3D(prop, new_rank, new_input_dims, new_permutations, grid_size, block_size)) { TensorPitches new_input_strides(new_input_dims); - return Transpose3DImpl(stream, element_size, new_input_dims, new_input_strides, + return Transpose3DImpl(stream, element_size, ToConstSpan(new_input_dims), ToConstSpan(new_input_strides), input.DataRaw(), output.MutableDataRaw(), output.Shape().Size(), grid_size, block_size); } @@ -263,9 +264,9 @@ Status Transpose::ComputeInternal(OpKernelContext* ctx) const { const TensorShape& input_shape = X.Shape(); int32_t rank = gsl::narrow_cast(input_shape.NumDimensions()); - std::vector output_dims(rank); - std::vector default_perm(rank); - const std::vector* p_perm = nullptr; + TensorShapeVector output_dims(rank); + InlinedShapeVectorT default_perm(rank); + const InlinedShapeVectorT* p_perm = nullptr; const auto& status = ComputeOutputShape(X, output_dims, default_perm, p_perm); if (!status.IsOK()) return status; diff --git a/onnxruntime/core/providers/cuda/tensor/transpose.h b/onnxruntime/core/providers/cuda/tensor/transpose.h index ab996ba151..a71b47ae95 100644 --- a/onnxruntime/core/providers/cuda/tensor/transpose.h +++ b/onnxruntime/core/providers/cuda/tensor/transpose.h @@ -19,13 +19,13 @@ class Transpose final : public CudaKernel, public TransposeBase { Status ComputeInternal(OpKernelContext* context) const override; static Status DoTranspose(const Transpose& transpose_kernel, - const std::vector& permutations, const Tensor& input, Tensor& output); + const gsl::span& permutations, const Tensor& input, Tensor& output); // `input_shape_override` (if provided) overrides the shape of `input` for compute purposes static Status DoTranspose(const cudaDeviceProp& prop, cudaStream_t stream, const cublasHandle_t cublas_handle, - const std::vector& permutations, + const gsl::span& permutations, const Tensor& input, Tensor& output, const TensorShape* input_shape_override = nullptr); }; diff --git a/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu b/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu index df155f948d..0b77a2012e 100644 --- a/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/transpose_impl.cu @@ -29,8 +29,8 @@ __global__ void Transpose3DKernel(const TArray input_shape, bool CanDoTranspose3D(const cudaDeviceProp& prop, int32_t rank, - const std::vector& input_dims, - const std::vector& permutations, + const gsl::span& input_dims, + const gsl::span& permutations, dim3& grid_size, dim3& block_size) { if (rank == 3 && // permutation is done in the last two dimensions. @@ -124,8 +124,8 @@ __global__ void Transpose4DKernelParallelizeMultipleElementsPerThreadInInnermost bool CanDoTranspose4DParallelizeMultipleElementsPerThreadInInnermostDim(const cudaDeviceProp& prop, size_t element_size, int32_t rank, - const std::vector& input_dims, - const std::vector& permutations, + const gsl::span& input_dims, + const gsl::span& permutations, dim3& grid_size, dim3& block_size) { if (rank == 4 && // the permutations is not on the last dimension. @@ -245,8 +245,8 @@ __global__ void Transpose4DKernelParallelizeOneElementPerThread( bool CanDoTranspose4DParallelizeOneElementPerThread(const cudaDeviceProp& prop, size_t element_size, int32_t rank, - const std::vector& input_dims, - const std::vector& permutations, + const gsl::span& input_dims, + const gsl::span& permutations, dim3& grid_size, dim3& block_size) { if (rank == 4) { // dims[3]: block.x @@ -261,7 +261,7 @@ bool CanDoTranspose4DParallelizeOneElementPerThread(const cudaDeviceProp& prop, // 2. block_size_y * num_block_ext >= input_dims[2] int64_t block_size_x = input_dims[3]; int64_t max_block_size_y = prop.maxThreadsPerBlock / block_size_x; - int64_t block_size_y = min(input_dims[2], max_block_size_y); + int64_t block_size_y = std::min(input_dims[2], max_block_size_y); int64_t num_block_ext = CeilDiv(input_dims[2], block_size_y); if (num_block_ext <= prop.maxGridSize[0]) { diff --git a/onnxruntime/core/providers/cuda/tensor/transpose_impl.h b/onnxruntime/core/providers/cuda/tensor/transpose_impl.h index a2b5038096..4e4d7d8bca 100644 --- a/onnxruntime/core/providers/cuda/tensor/transpose_impl.h +++ b/onnxruntime/core/providers/cuda/tensor/transpose_impl.h @@ -9,7 +9,7 @@ namespace onnxruntime { namespace cuda { bool CanDoTranspose3D(const cudaDeviceProp& prop, - int32_t rank, const std::vector& input_dims, const std::vector& permutations, + int32_t rank, const gsl::span& input_dims, const gsl::span& permutations, dim3& grid_size, dim3& block_size); Status Transpose3DImpl(cudaStream_t stream, size_t element_size, const TArray& input_shape, const TArray& input_strides, const void* input_data, void* output_data, int64_t N, @@ -18,8 +18,8 @@ Status Transpose3DImpl(cudaStream_t stream, size_t element_size, const TArray& input_dims, - const std::vector& permutations, + const gsl::span& input_dims, + const gsl::span& permutations, dim3& grid_size, dim3& block_size); Status Transpose4DParallelizeMultipleElementsPerThreadInInnermostDim(cudaStream_t stream, size_t element_size, const TArray& input_shape, @@ -30,8 +30,8 @@ Status Transpose4DParallelizeMultipleElementsPerThreadInInnermostDim(cudaStream_ bool CanDoTranspose4DParallelizeOneElementPerThread(const cudaDeviceProp& prop, size_t element_size, int32_t rank, - const std::vector& input_dims, - const std::vector& permutations, + const gsl::span& input_dims, + const gsl::span& permutations, dim3& grid_size, dim3& block_size); Status Transpose4DParallelizeOneElementPerThread(cudaStream_t stream, size_t element_size, const TArray& input_shape, diff --git a/onnxruntime/core/providers/cuda/tensor/upsample.cc b/onnxruntime/core/providers/cuda/tensor/upsample.cc index 507d959601..82a1251f48 100644 --- a/onnxruntime/core/providers/cuda/tensor/upsample.cc +++ b/onnxruntime/core/providers/cuda/tensor/upsample.cc @@ -41,7 +41,7 @@ template Status Upsample::BaseCompute(OpKernelContext* context, const std::vector& roi, const std::vector& scales, - const std::vector& output_dims) const { + const gsl::span& output_dims) const { const Tensor* X = context->Input(0); auto X_dims = X->Shape().GetDims(); int32_t rank = static_cast(X_dims.size()); @@ -122,7 +122,7 @@ Status Upsample::ComputeInternal(OpKernelContext* context) const { const Tensor* X = context->Input(0); ORT_ENFORCE(X != nullptr); - std::vector output_dims(X->Shape().GetDims().size()); + TensorShapeVector output_dims(X->Shape().GetDims().size()); std::vector roi_array(X->Shape().GetDims().size() * 2, 0.0f); if (!roi_cached_) { bool use_default_roi = true; @@ -137,7 +137,7 @@ Status Upsample::ComputeInternal(OpKernelContext* context) const { if (use_default_roi) { // default roi includes ensures all the values in that axis are included in the roi // normalized roi is thus : [start, end] = [0, 1] - const auto& input_dims = X->Shape().GetDims(); + const auto input_dims = X->Shape().GetDims(); size_t input_rank = input_dims.size(); roi_array.resize(input_rank * 2); for (size_t i = 0; i < input_rank; ++i) { diff --git a/onnxruntime/core/providers/cuda/tensor/upsample.h b/onnxruntime/core/providers/cuda/tensor/upsample.h index 06f98e821d..c8c5db4839 100644 --- a/onnxruntime/core/providers/cuda/tensor/upsample.h +++ b/onnxruntime/core/providers/cuda/tensor/upsample.h @@ -18,7 +18,7 @@ class Upsample : public UpsampleBase, public CudaKernel { Status ComputeInternal(OpKernelContext* context) const override; Status BaseCompute(OpKernelContext* context, const std::vector& roi, const std::vector& scales, - const std::vector& output_dims) const; + const gsl::span& output_dims) const; }; } // namespace cuda diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_reduce.cc b/onnxruntime/core/providers/dnnl/subgraph/dnnl_reduce.cc index 7be31bb49c..32ef68c8ba 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_reduce.cc +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_reduce.cc @@ -271,18 +271,24 @@ void DnnlReduce::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& node) { for (size_t i = 0; i < ndim; ++i) { if ((j < axes.size() && axes[j] == static_cast(i)) || (axes.size() == 0 && src_dims[i] == 1)) { - ORT_ENFORCE(src_dims[i] == 1, "Dimension of input ", i, " must be 1 instead of ", src_dims[i], - ". shape=", src_dims); + if (src_dims[i] != 1) { + auto dims_span = gsl::make_span(src_dims); + ORT_ENFORCE(src_dims[i] == 1, "Dimension of input ", i, " must be 1 instead of ", src_dims[i], + ". shape=", dims_span); + } ++j; continue; } if ((j < axes.size() && axes[j] == static_cast(i) && src_dims[i] == 0) || (axes.size() == 0 && src_dims[i] == 0)) { + if (!keepdims) { + auto dims = src_md.dims(); ORT_ENFORCE(keepdims, - "Can't reduce on dim with value of 0 if 'keepdims' is false. " - "Invalid output shape would be produced. input_shape:", - TensorShape(src_md.dims())); + "Can't reduce on dim with value of 0 if 'keepdims' is false. " + "Invalid output shape would be produced. input_shape:", + TensorShape(gsl::make_span(dims))); + } } output_shape.push_back(src_dims[i]); } diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_reshape.cc b/onnxruntime/core/providers/dnnl/subgraph/dnnl_reshape.cc index 8759c821c9..16090e86cf 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_reshape.cc +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_reshape.cc @@ -21,13 +21,15 @@ void DnnlReshape::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& node) { dnnl::memory::dims shape_dims = shape_mem.get_desc().dims(); int64_t* shape_data = (int64_t*)shape_mem.get_data_handle(); - dnnl::memory::dims reshape_shape(shape_data, shape_data + shape_dims[0]); // Reshape helper will take input data_dims shape and the reshape_shape and replace the -1 and 0s with the calculated // Output values. The Reshape helper also does a lot of error checking to make sure the Reshape is possible. - ReshapeHelper helper(TensorShape(data_dims), reshape_shape, GetAllowZero(node)); + const auto data_dims_span = gsl::span(data_dims.data(), data_dims.size()); + TensorShapeVector reshape_shape(shape_data, shape_data + shape_dims[0]); + ReshapeHelper helper(TensorShape(data_dims_span), reshape_shape, GetAllowZero(node)); + dnnl::memory::dims reshape_shape_dims(reshape_shape.cbegin(), reshape_shape.cend()); //the dnnl::memory::desc.reshape(shape) failed on some models so we instead create a new dnnl:memory::desc - dnnl::memory::desc reshaped_md(reshape_shape, node.Input(IN_DATA).Type(), sp.GetDnnlFormat(reshape_shape.size())); + dnnl::memory::desc reshaped_md(reshape_shape_dims, node.Input(IN_DATA).Type(), sp.GetDnnlFormat(reshape_shape.size())); dnnl::memory reshaped_mem = dnnl::memory(reshaped_md, dnnl_engine, nullptr); sp.AddReshape(data_mem, reshaped_mem); diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.h b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.h index 0e0b7eaf35..a94b2fe8e1 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.h +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.h @@ -116,4 +116,25 @@ class DnnlSubgraphPrimitive { }; } // namespace ort_dnnl + +inline std::ostream& operator<<(std::ostream& os, const dnnl::memory::dims& dims) { + std::copy(dims.cbegin(), dims.cend(), std::ostream_iterator(os, " ")); + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const gsl::span& span) { + std::copy(span.cbegin(), span.cend(), std::ostream_iterator(os, " ")); + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const gsl::span& span) { + std::copy(span.cbegin(), span.cend(), std::ostream_iterator(os, " ")); + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const TensorShape& shape) { + return os << shape.GetDims(); +} + } // namespace onnxruntime + diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index bff260cb67..63a83cfbc0 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -2512,7 +2512,7 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const const auto& operand_types(model_builder.GetOperandTypes()); const auto& inputs = node_unit.Inputs(); const auto& input_shape = shaper[inputs[0].node_arg.Name()]; - std::vector input_shape_64(input_shape.cbegin(), input_shape.cend()); + TensorShapeVector input_shape_64(input_shape.cbegin(), input_shape.cend()); SliceOp::PrepareForComputeMetadata compute_metadata(input_shape_64); { @@ -2520,12 +2520,12 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const // to be used in shared PrepareForCompute function to calculate the output shape // and normalize inputs, for example, input can be starts/ends/steps for certain axes, // PrepareForCompute can generate standard starts/ends/steps/axes for each axes - std::vector input_starts; - std::vector input_ends; - std::vector input_axes; - std::vector input_steps; + TensorShapeVector input_starts; + TensorShapeVector input_ends; + TensorShapeVector input_axes; + TensorShapeVector input_steps; - const auto CopyInputData = [&inputs, &model_builder](size_t input_idx, std::vector& data) { + const auto CopyInputData = [&inputs, &model_builder](size_t input_idx, TensorShapeVector& data) { data.clear(); // This is an optional input, return empty vector @@ -2591,7 +2591,7 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const // helper function to add begin/end/strides of ANEURALNETWORKS_STRIDED_SLICE const auto AddOperand = [&model_builder, &node_unit, &input_indices, &operand_indices]( - const char* name, const Shape& shape, const std::vector& param_raw_data) { + const char* name, const Shape& shape, const gsl::span& param_raw_data) { std::vector param_data; param_data.reserve(param_raw_data.size()); std::transform(param_raw_data.cbegin(), param_raw_data.cend(), @@ -2637,8 +2637,8 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const // the end, for example, dim = 5, and end = -1, the end will be normalized to 4, which will cause // incorrect result, so here we have to make the end = -dim - 1 such that it will not be treated as // an index counting from the end. - std::vector ends = compute_metadata.ends_; - for (size_t i = 0; i < ends.size(); ++i) { + auto ends = compute_metadata.ends_; + for (size_t i = 0, limit = ends.size(); i < limit; ++i) { if (ends[i] == -1) { ends[i] = -static_cast(input_shape[i] + 1); } diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc b/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc index 43e640d545..530b7602af 100644 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc +++ b/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc @@ -113,7 +113,7 @@ static const Tensor* Marshalling( // input const auto& tensor_shape = original_initializer->Shape(); - auto input_shape = tensor_shape.GetDimsAsVector(); + auto input_shape = tensor_shape.AsShapeVector(); if (input_shape.empty()) input_shape.push_back(1); const void* input_data = original_initializer->DataRaw(); diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_op_ir_builder.cc b/onnxruntime/core/providers/nuphar/compiler/nuphar_op_ir_builder.cc index 2859a42d0e..5c4540dba6 100644 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_op_ir_builder.cc +++ b/onnxruntime/core/providers/nuphar/compiler/nuphar_op_ir_builder.cc @@ -91,7 +91,7 @@ bool CreateScalarTensorFromInitializer(const Tensor* tensor, std::string normalized_name = NormalizeCppName(name); auto tvm_tensor = tvm::compute( - tvm_codegen::ToTvmArray(tensor->Shape().GetDimsAsVector()), + tvm_codegen::ToTvmArray(tensor->Shape().GetDims()), [&](const tvm::Array&) { return constant_scalar; }, @@ -120,7 +120,7 @@ const tvm::Tensor& GetOrCreateInitializer(const std::string& name, DLDataType dtype = tvm_codegen::ToTvmDLDataType(ONNXRUNTIME_data_type); HalideIR::Type halide_type((halideir_type_code_t)dtype.code, dtype.bits, dtype.lanes); std::string normalized_name = NormalizeCppName(name); - auto tvm_shape = tvm_codegen::ToTvmArray(tensor->Shape().GetDimsAsVector()); + auto tvm_shape = tvm_codegen::ToTvmArray(tensor->Shape().GetDims()); auto tvm_tensor = CreateInputPlaceholder(tvm_shape, halide_type, normalized_name, is_sliced); // create the layout info ctx_codegen.CreateWeightLayoutInfo(name, tvm_tensor); diff --git a/onnxruntime/core/providers/nuphar/mti_x86/math/reduce_ops.cc b/onnxruntime/core/providers/nuphar/mti_x86/math/reduce_ops.cc index 2bcb6f0010..7639ae2f9b 100644 --- a/onnxruntime/core/providers/nuphar/mti_x86/math/reduce_ops.cc +++ b/onnxruntime/core/providers/nuphar/mti_x86/math/reduce_ops.cc @@ -359,7 +359,8 @@ tvm::Tensor ReduceValueLowest_noPad(const tvm::Tensor& X, //[n1, w] for 2D or [w] for 1D auto head_tensor = tvm::compute(head_shape, l_head, name + "_head_reduce"); //[n1, 1] for 2D or [1] for 1D - return topi::CommReduce(head_tensor, tvm_codegen::ToTvmArrayInt({(int64_t)(input_shape_rank)-1}), func, true, true); + const auto rank_minus_one = gsl::narrow(input_shape_rank) - 1; + return topi::CommReduce(head_tensor, tvm_codegen::ToTvmArrayInt({rank_minus_one}), func, true, true); } tvm::Tensor ReduceSumV(const tvm::Tensor& X, const int32_t vector_size, const std::string& name) { diff --git a/onnxruntime/core/providers/nuphar/mti_x86/nn/pool_ops.cc b/onnxruntime/core/providers/nuphar/mti_x86/nn/pool_ops.cc index 724721c91e..b928341912 100644 --- a/onnxruntime/core/providers/nuphar/mti_x86/nn/pool_ops.cc +++ b/onnxruntime/core/providers/nuphar/mti_x86/nn/pool_ops.cc @@ -112,7 +112,7 @@ static tvm::Tensor MakePoolCommon(const tvm::Tensor& X, "kernel_shape num_dims is not compatible with X num_dims."); tvm::Array pooling_args; - auto add_args_fn = [&](const std::vector& v) { + auto add_args_fn = [&](const TensorShapeVector& v) { pooling_args.push_back(tvm::make_const(tvm::Int(64), static_cast(v.size()))); for (auto n : v) { pooling_args.push_back(tvm::make_const(tvm::Int(64), n)); diff --git a/onnxruntime/core/providers/rocm/miopen_common.cc b/onnxruntime/core/providers/rocm/miopen_common.cc index 4110b3f5e7..7b44b6069c 100644 --- a/onnxruntime/core/providers/rocm/miopen_common.cc +++ b/onnxruntime/core/providers/rocm/miopen_common.cc @@ -31,8 +31,8 @@ Status MiopenTensor::Set(gsl::span input_dims, miopenDataType_t d int rank = gsl::narrow_cast(input_dims.size()); TensorPitches pitches(input_dims); - std::vector dims(rank); - std::vector strides(rank); + InlinedShapeVectorT dims(rank); + InlinedShapeVectorT strides(rank); for (int i = 0; i < rank; i++) { dims[i] = gsl::narrow_cast(input_dims[i]); strides[i] = gsl::narrow_cast(pitches[i]); @@ -58,12 +58,12 @@ MiopenTensorDescriptor::~MiopenTensorDescriptor() { } } -Status MiopenTensorDescriptor::Set(const std::vector& filter_dims, miopenDataType_t data_type) { +Status MiopenTensorDescriptor::Set(gsl::span filter_dims, miopenDataType_t data_type) { if (!desc_) MIOPEN_RETURN_IF_ERROR(miopenCreateTensorDescriptor(&desc_)); int rank = gsl::narrow_cast(filter_dims.size()); - std::vector w_dims(rank); + InlinedShapeVectorT w_dims(rank); for (int i = 0; i < rank; i++) { w_dims[i] = gsl::narrow_cast(filter_dims[i]); } diff --git a/onnxruntime/core/providers/rocm/miopen_common.h b/onnxruntime/core/providers/rocm/miopen_common.h index 6d17c216b2..8140cad54b 100644 --- a/onnxruntime/core/providers/rocm/miopen_common.h +++ b/onnxruntime/core/providers/rocm/miopen_common.h @@ -44,7 +44,7 @@ class MiopenTensorDescriptor final { ~MiopenTensorDescriptor(); ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(MiopenTensorDescriptor); - Status Set(const std::vector& filter_dims, miopenDataType_t data_typ); + Status Set(gsl::span filter_dims, miopenDataType_t data_typ); operator miopenTensorDescriptor_t() const { return desc_; } diff --git a/onnxruntime/core/providers/rocm/nn/conv.cc b/onnxruntime/core/providers/rocm/nn/conv.cc index 7e5bea700f..234040d11d 100644 --- a/onnxruntime/core/providers/rocm/nn/conv.cc +++ b/onnxruntime/core/providers/rocm/nn/conv.cc @@ -67,12 +67,13 @@ size_t GetMaxWorkspaceSize(const MiopenConvState& s, } Status SliceOutUnwantedOutputSection(hipStream_t stream, - const void* input_data, gsl::span input_dims, + const void* input_data, + const gsl::span& input_dims, void* output_data, - gsl::span output_dims, - std::vector starts, - const std::vector& ends, - const std::vector& axes, + const gsl::span& output_dims, + const gsl::span& starts, + const gsl::span& ends, + const gsl::span& axes, size_t element_size) { SliceOp::PrepareForComputeMetadata compute_metadata(input_dims); @@ -95,7 +96,7 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const //set W const Tensor* W = context->Input(1); const TensorShape& w_shape = W->Shape(); - auto w_dims = w_shape.GetDimsAsVector(); + auto w_dims = w_shape.AsShapeVector(); s_.w_data = reinterpret_cast(W->template Data()); //set B if (context->InputCount() >= 3) { @@ -112,14 +113,14 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const } else { s_.z_data = nullptr; } - bool input_dims_changed = (s_.last_x_dims != x_dims); - bool w_dims_changed = (s_.last_w_dims != w_dims); + bool input_dims_changed = (s_.last_x_dims.GetDims() != x_dims); + bool w_dims_changed = (s_.last_w_dims.GetDims() != gsl::make_span(w_dims)); if (input_dims_changed || w_dims_changed) { if (input_dims_changed) s_.last_x_dims = x_dims; if (w_dims_changed) { - s_.last_w_dims = w_dims; + s_.last_w_dims = gsl::make_span(w_dims); s_.cached_benchmark_fwd_results.clear(); } @@ -128,45 +129,45 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X, W)); - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape)); auto rank = kernel_shape.size(); - std::vector pads(conv_attrs_.pads); + ConvAttributes::ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(rank * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(rank, 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(rank, 1); } - std::vector y_dims; + TensorShapeVector y_dims; y_dims.reserve(2 + rank); // rank indicates number of feature dimensions - so add 2 to account for 'N' and 'C' y_dims.insert(y_dims.begin(), {N, M}); - std::vector y_dims_with_adjusted_pads; + TensorShapeVector y_dims_with_adjusted_pads; y_dims_with_adjusted_pads.reserve(2 + rank); // rank indicates number of feature dimensions - so add 2 to account for 'N' and 'C' y_dims_with_adjusted_pads.insert(y_dims_with_adjusted_pads.begin(), {N, M}); bool post_slicing_required = false; - std::vector slice_starts; + TensorShapeVector slice_starts; slice_starts.reserve(rank); - std::vector slice_ends; + TensorShapeVector slice_ends; slice_ends.reserve(rank); - std::vector slice_axes; + TensorShapeVector slice_axes; slice_axes.reserve(rank); ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShapeWithAdjustedPads(x_shape.Slice(2), kernel_shape, strides, dilations, pads, y_dims, y_dims_with_adjusted_pads, post_slicing_required, slice_starts, slice_ends, slice_axes)); ORT_ENFORCE(y_dims.size() == y_dims_with_adjusted_pads.size()); - s_.y_dims = y_dims; + s_.y_dims = gsl::make_span(y_dims); s_.y_dims_with_adjusted_pads = y_dims_with_adjusted_pads; s_.post_slicing_required = post_slicing_required; s_.slice_starts = slice_starts; @@ -186,8 +187,8 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const s_.y_data = reinterpret_cast(s_.Y->template MutableData()); } - std::vector x_dims_miopen{x_dims.begin(), x_dims.end()}; - std::vector y_dims_miopen = !post_slicing_required ? y_dims : y_dims_with_adjusted_pads; + TensorShapeVector x_dims_miopen{x_dims.begin(), x_dims.end()}; + TensorShapeVector y_dims_miopen = !post_slicing_required ? y_dims : y_dims_with_adjusted_pads; if (rank < 2) { // TODO: Remove asym padding correction. x_dims_miopen.push_back(1); @@ -213,11 +214,11 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const const Tensor* B = context->Input(2); const auto& b_shape = B->Shape(); ORT_RETURN_IF_NOT(b_shape.NumDimensions() == 1, "bias should be 1D"); - std::vector b_dims(2 + kernel_shape.size(), 1); + TensorShapeVector b_dims(2 + kernel_shape.size(), 1); b_dims[1] = b_shape[0]; ORT_RETURN_IF_ERROR(s_.b_tensor.Set(b_dims, MiopenTensor::GetDataType())); } else if (bias_expected) { - std::vector b_dims(2 + kernel_shape.size(), 1); + TensorShapeVector b_dims(2 + kernel_shape.size(), 1); b_dims[1] = w_dims[0]; auto malloc_size = b_dims[1] * sizeof(HipT); ORT_RETURN_IF_ERROR(s_.b_tensor.Set(b_dims, MiopenTensor::GetDataType())); @@ -323,18 +324,18 @@ MiopenConvolutionDescriptor::~MiopenConvolutionDescriptor() { Status MiopenConvolutionDescriptor::Set( size_t rank, - const std::vector& pads, - const std::vector& strides, - const std::vector& dilations, + gsl::span pads, + gsl::span strides, + gsl::span dilations, int groups, miopenConvolutionMode_t mode, miopenDataType_t data_type) { if (!desc_) MIOPEN_RETURN_IF_ERROR(miopenCreateConvolutionDescriptor(&desc_)); - std::vector pad_dims(rank); - std::vector stride_dims(rank); - std::vector dilation_dims(rank); + InlinedShapeVectorT pad_dims(rank); + InlinedShapeVectorT stride_dims(rank); + InlinedShapeVectorT dilation_dims(rank); for (size_t i = 0; i < rank; i++) { pad_dims[i] = gsl::narrow_cast(pads[i]); stride_dims[i] = gsl::narrow_cast(strides[i]); diff --git a/onnxruntime/core/providers/rocm/nn/conv.h b/onnxruntime/core/providers/rocm/nn/conv.h index 4431aa2eee..5cb5f653b8 100644 --- a/onnxruntime/core/providers/rocm/nn/conv.h +++ b/onnxruntime/core/providers/rocm/nn/conv.h @@ -18,9 +18,9 @@ class MiopenConvolutionDescriptor final { ~MiopenConvolutionDescriptor(); Status Set(size_t rank, - const std::vector& pads, - const std::vector& strides, - const std::vector& dilations, + gsl::span pads, + gsl::span strides, + gsl::span dilations, int groups, miopenConvolutionMode_t mode, miopenDataType_t data_type); @@ -31,12 +31,11 @@ class MiopenConvolutionDescriptor final { miopenConvolutionDescriptor_t desc_; }; -template struct vector_hash { - std::size_t operator()(const std::vector& values) const { + std::size_t operator()(const TensorShapeVector& values) const { std::size_t seed = values.size(); for (auto& val : values) - seed ^= std::hash()(val) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + seed ^= std::hash()(val) + 0x9e3779b9 + (seed << 6) + (seed >> 2); return seed; } }; @@ -119,7 +118,7 @@ struct MiopenConvState { // these would be recomputed if x/w dims change TensorShape y_dims; - std::vector y_dims_with_adjusted_pads; + TensorShapeVector y_dims_with_adjusted_pads; size_t workspace_bytes; decltype(AlgoPerfType().bwd_data_algo) bwd_data_algo; decltype(AlgoPerfType().fwd_algo) fwd_algo; @@ -148,14 +147,14 @@ struct MiopenConvState { decltype(AlgoPerfType().memory) memory; }; - lru_unordered_map, PerfFwdResultParams, vector_hash> cached_benchmark_fwd_results{MAX_CACHED_ALGO_PERF_RESULTS}; - lru_unordered_map, PerfBwdResultParams, vector_hash> cached_benchmark_bwd_results{MAX_CACHED_ALGO_PERF_RESULTS}; + lru_unordered_map cached_benchmark_fwd_results{MAX_CACHED_ALGO_PERF_RESULTS}; + lru_unordered_map cached_benchmark_bwd_results{MAX_CACHED_ALGO_PERF_RESULTS}; // Some properties needed to support asymmetric padded Conv nodes bool post_slicing_required; - std::vector slice_starts; - std::vector slice_ends; - std::vector slice_axes; + TensorShapeVector slice_starts; + TensorShapeVector slice_ends; + TensorShapeVector slice_axes; // note that conv objects are shared between execution frames, and a lock is needed to avoid multi-thread racing OrtMutex mutex; @@ -200,12 +199,12 @@ class Conv : public RocmKernel { Status SliceOutUnwantedOutputSection(hipStream_t stream, const void* input_data, - gsl::span input_dims, + const gsl::span& input_dims, void* output_data, - gsl::span output_dims, - std::vector starts, - const std::vector& ends, - const std::vector& axes, + const gsl::span& output_dims, + const gsl::span& starts, + const gsl::span& ends, + const gsl::span& axes, size_t element_size); } // namespace rocm } // namespace onnxruntime diff --git a/onnxruntime/core/providers/rocm/nn/conv_transpose.cc b/onnxruntime/core/providers/rocm/nn/conv_transpose.cc index c39b92b987..d0b64e4ff3 100644 --- a/onnxruntime/core/providers/rocm/nn/conv_transpose.cc +++ b/onnxruntime/core/providers/rocm/nn/conv_transpose.cc @@ -42,7 +42,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ const Tensor* X = context->Input(0); const TensorShape& x_shape = X->Shape(); - auto x_dims = x_shape.GetDimsAsVector(); + auto x_dims = x_shape.AsShapeVector(); auto x_data = reinterpret_cast(X->template Data()); auto x_dimensions = X->Shape().NumDimensions(); @@ -53,7 +53,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ } const Tensor* W = context->Input(1); const TensorShape& w_shape = W->Shape(); - std::vector w_dims = w_shape.GetDimsAsVector(); + auto w_dims = w_shape.AsShapeVector(); auto w_data = reinterpret_cast(W->template Data()); size_t num_inputs = OpKernel::Node().InputDefs().size(); @@ -68,21 +68,21 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ { std::lock_guard lock(s_.mutex); // TODO: add a global cache if need to handle cases for multiple frames running simultaneously with different batch_size - bool input_dims_changed = (s_.last_x_dims != x_dims); - bool w_dims_changed = (s_.last_w_dims != w_dims); + bool input_dims_changed = (s_.last_x_dims.GetDims() != gsl::make_span(x_dims)); + bool w_dims_changed = (s_.last_w_dims.GetDims() != gsl::make_span(w_dims)); if (input_dims_changed || w_dims_changed) { if (input_dims_changed) - s_.last_x_dims = x_dims; + s_.last_x_dims = gsl::make_span(x_dims); if (w_dims_changed) { - s_.last_w_dims = w_dims; + s_.last_w_dims = gsl::make_span(w_dims); s_.cached_benchmark_bwd_results.clear(); } ConvTransposeAttributes::Prepare p; ORT_RETURN_IF_ERROR(conv_transpose_attrs_.PrepareForCompute(context, has_bias, p, dynamic_padding)); - auto y_dims = p.Y->Shape().GetDimsAsVector(); + auto y_dims = p.Y->Shape().AsShapeVector(); if (x_dimensions == 3) { y_dims.insert(y_dims.begin() + 2, 1); p.kernel_shape.insert(p.kernel_shape.begin(), 1); @@ -91,7 +91,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ p.strides.insert(p.strides.begin(), 1); p.dilations.insert(p.dilations.begin(), 1); } - s_.y_dims = y_dims; + s_.y_dims = gsl::make_span(y_dims); if (w_dims_changed) { @@ -159,7 +159,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ // The following block will be executed in case there has been no change in the shapes of the // input and the filter compared to the previous run if (!y_data) { - auto y_dims = s_.y_dims.GetDimsAsVector(); + auto y_dims = s_.y_dims.AsShapeVector(); if (x_dimensions == 3) { y_dims.erase(y_dims.begin() + 2); } diff --git a/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc b/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc index 928a6fe3d2..35f17076dd 100644 --- a/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc +++ b/onnxruntime/core/providers/rocm/reduction/reduction_ops.cc @@ -157,7 +157,7 @@ Status ReduceKernel::ReduceKernelShared( OutT* Y, const TensorShape& output_shape, miopenReduceTensorOp_t miopen_reduce_op, - std::vector& output_dims) const { + TensorShapeVector& output_dims) const { typedef typename ToHipType::MappedType HipT; typedef typename ToHipType::MappedType HipOutT; miopenDataType_t miopen_type_X = MiopenTensor::GetDataType(); @@ -193,19 +193,18 @@ Status ReduceKernel::ReduceKernelShared( } // MIOpen requires at least 3D input, so pad 1s if needed - std::vector input_dims_miopen = input_shape.GetDimsAsVector(); - std::vector output_dims_miopen = output_dims; + auto input_dims_miopen = input_shape.AsShapeVector(); + auto output_dims_miopen = output_dims; if (rank < 3) { - std::vector pads(3 - rank, 1); + TensorShapeVector pads(3 - rank, 1); input_dims_miopen.insert(input_dims_miopen.end(), pads.begin(), pads.end()); output_dims_miopen.insert(output_dims_miopen.end(), pads.begin(), pads.end()); } MiopenReduceDescriptor reduce_desc; - if (std::is_same::value) - ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, MiopenTensor::GetDataType(), ReduceTensorIndices)); - else - ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, miopen_type_X, ReduceTensorIndices)); + ORT_IF_CONSTEXPR(std::is_same::value) + ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, MiopenTensor::GetDataType(), ReduceTensorIndices)); + else ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, miopen_type_X, ReduceTensorIndices)); const auto one = ReduceConsts::One; const auto zero = ReduceConsts::Zero; @@ -352,7 +351,7 @@ template Status ReduceKernel::ReduceKernelShared& output_dims) const; + TensorShapeVector& output_dims) const; template Status ReduceKernel::ReduceKernelShared( const MLFloat16* X, @@ -360,12 +359,12 @@ template Status ReduceKernel::ReduceKernelShared& output_dims) const; + TensorShapeVector& output_dims) const; // `input_shape_override` (if provided) is the input shape for compute purposes Status PrepareForReduce(const Tensor* X, bool keepdims, - const std::vector& axes, + const gsl::span& axes, PrepareReduceMetadata& prepare_reduce_metadata, const TensorShape* input_shape_override) { ORT_ENFORCE(nullptr != X); @@ -378,11 +377,11 @@ Status PrepareForReduce(const Tensor* X, return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "MIOpen only supports up to 8-D tensors in reduction"); } - const auto& input_dims = input_shape.GetDims(); - std::vector reduced(rank, false); + const auto input_dims = input_shape.GetDims(); + InlinedShapeVectorT reduced(rank, false); prepare_reduce_metadata.output_dims.reserve(input_dims.size()); if (axes.size() > 0) { - prepare_reduce_metadata.output_dims = input_shape.GetDimsAsVector(); + prepare_reduce_metadata.output_dims = input_shape.AsShapeVector(); for (auto axis : axes) { axis = HandleNegativeAxis(axis, rank); ORT_ENFORCE(input_dims[axis] != 0, @@ -418,10 +417,10 @@ Status PrepareForReduce(const Tensor* X, } // MIOpen requires at least 3D input, so pad 1s if needed - prepare_reduce_metadata.input_dims_miopen = input_shape.GetDimsAsVector(); + prepare_reduce_metadata.input_dims_miopen = input_shape.AsShapeVector(); prepare_reduce_metadata.output_dims_miopen = prepare_reduce_metadata.output_dims; if (rank < 3) { - std::vector pads(3 - rank, 1); + TensorShapeVector pads(3 - rank, 1); prepare_reduce_metadata.input_dims_miopen.insert(prepare_reduce_metadata.input_dims_miopen.end(), pads.begin(), pads.end()); prepare_reduce_metadata.output_dims_miopen.insert(prepare_reduce_metadata.output_dims_miopen.end(), pads.begin(), pads.end()); } @@ -435,7 +434,7 @@ Status PrepareForReduce(const Tensor* X, template Status ReduceComputeCore(ROCMExecutionProvider& rocm_ep, const Tensor& input, PrepareReduceMetadata& prepare_reduce_metadata, /*out*/ Tensor& output, miopenReduceTensorOp_t miopen_reduce_op, - const std::vector& axes, + const gsl::span& axes, bool calculate_log, bool calculate_sqt, bool log_sum_exp, bool fast_reduction, const TensorShape* input_shape_override) { typedef typename ToHipType::MappedType HipT; @@ -443,9 +442,9 @@ Status ReduceComputeCore(ROCMExecutionProvider& rocm_ep, const Tensor& input, Pr int64_t input_count = prepare_reduce_metadata.input_count; int64_t output_count = prepare_reduce_metadata.output_count; - std::vector& output_dims = prepare_reduce_metadata.output_dims; - std::vector& input_dims_miopen = prepare_reduce_metadata.input_dims_miopen; - std::vector& output_dims_miopen = prepare_reduce_metadata.output_dims_miopen; + auto& output_dims = prepare_reduce_metadata.output_dims; + auto& input_dims_miopen = prepare_reduce_metadata.input_dims_miopen; + auto& output_dims_miopen = prepare_reduce_metadata.output_dims_miopen; hipStream_t stream = static_cast(rocm_ep.GetComputeStream()); // special case when there is a dim value of 0 in the shape. if (input_count == 0) { @@ -466,9 +465,9 @@ Status ReduceComputeCore(ROCMExecutionProvider& rocm_ep, const Tensor& input, Pr input_data = reinterpret_cast(input_data_buffer.get()); fast_divmod tmp_div; Impl_Mul(stream, static_cast(SimpleBroadcast::NoBroadcast), nullptr, - reinterpret_cast(input.template Data()), nullptr, - reinterpret_cast(input.template Data()), nullptr, tmp_div, tmp_div, - reinterpret_cast(input_data_buffer.get()), input_count); + reinterpret_cast(input.template Data()), nullptr, + reinterpret_cast(input.template Data()), nullptr, tmp_div, tmp_div, + reinterpret_cast(input_data_buffer.get()), input_count); input_data = reinterpret_cast(input_data_buffer.get()); } @@ -491,7 +490,7 @@ Status ReduceComputeCore(ROCMExecutionProvider& rocm_ep, const Tensor& input, Pr if (calculate_log) { Impl_Log(stream, reinterpret_cast(output.template Data()), - reinterpret_cast(output.template MutableData()), output_count); + reinterpret_cast(output.template MutableData()), output_count); } else if (miopen_reduce_op == MIOPEN_REDUCE_TENSOR_AVG) { float denominator_float = applicable_matrix_reduction == ApplicableMatrixReduction::Rows ? static_cast(m) @@ -735,88 +734,88 @@ Status ReduceKernel::ComputeImpl(OpKernelContext* ctx, miopenR calculate_log_, calculate_sqt_, log_sum_exp_, fast_reduction); } -#define SPECIALIZED_REDUCEKERNEL_COMPUTEIMPL(T) \ - template <> \ - template <> \ - Status ReduceKernel::ComputeImpl( \ - OpKernelContext * ctx, miopenReduceTensorOp_t miopen_reduce_op) const { \ - typedef typename ToHipType::MappedType HipT; \ - const Tensor* X = ctx->Input(0); \ - std::vector axes; \ - size_t num_inputs = ctx->InputCount(); \ - if (num_inputs == 2) { \ - const Tensor* axes_tensor = ctx->Input(1); \ - ORT_ENFORCE(axes_tensor != nullptr, "Axes input is null"); \ - ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 1, "An axes tensor must be a vector tensor."); \ - auto nDims = static_cast(axes_tensor->Shape()[0]); \ - const auto* data = axes_tensor->template Data(); \ - axes.assign(data, data + nDims); \ - } else { \ - axes.assign(axes_.begin(), axes_.end()); \ - } \ - \ - if (axes.empty() && noop_with_empty_axes_) { \ - auto* Y = ctx->Output(0, X->Shape()); \ - HIP_RETURN_IF_ERROR(hipMemcpyAsync(Y->template MutableData(), X->template Data(), X->SizeInBytes(), \ - hipMemcpyDeviceToDevice, Stream())); \ - return Status::OK(); \ - } \ - \ - PrepareReduceMetadata prepare_reduce_metadata; \ - ORT_RETURN_IF_ERROR(PrepareForReduce(X, keepdims_, axes, prepare_reduce_metadata)); \ - \ - Tensor* Y = ctx->Output(0, prepare_reduce_metadata.squeezed_output_dims); \ - \ - int64_t input_count = prepare_reduce_metadata.input_count; \ - int64_t output_count = prepare_reduce_metadata.output_count; \ - std::vector& input_dims_miopen = prepare_reduce_metadata.input_dims_miopen; \ - std::vector& output_dims_miopen = prepare_reduce_metadata.output_dims_miopen; \ - \ - if (input_count == 0) { \ - assert(Y->Shape().Size() == 0); \ - return Status::OK(); \ - } \ - \ - if (input_count == output_count) { \ - if (Y->template MutableData() != X->template Data()) { \ - HIP_RETURN_IF_ERROR(hipMemcpyAsync(Y->template MutableData(), X->template Data(), \ - input_count * sizeof(T), hipMemcpyDeviceToDevice, Stream())); \ - } \ - return Status::OK(); \ - } \ - \ - HIP_RETURN_IF_ERROR(hipMemsetAsync(Y->MutableDataRaw(), 0, Y->SizeInBytes(), Stream())); \ - \ - size_t indices_bytes = 0; \ - size_t workspace_bytes = 0; \ - MiopenTensor input_tensor; \ - MiopenTensor output_tensor; \ - MiopenReduceDescriptor reduce_desc; \ - \ - miopenDataType_t miopen_type_X = miopenFloat; \ - IAllocatorUniquePtr temp_X = GetScratchBuffer(input_count); \ - Impl_Cast(Stream(), reinterpret_cast(X->template Data()), temp_X.get(), X->Shape().Size()); \ - \ - ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, miopen_type_X, MIOPEN_REDUCE_TENSOR_NO_INDICES)); \ - ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_miopen, miopen_type_X)); \ - ORT_RETURN_IF_ERROR(output_tensor.Set(output_dims_miopen, miopen_type_X)); \ - MIOPEN_RETURN_IF_ERROR( \ - miopenGetReductionIndicesSize(MiopenHandle(), reduce_desc, input_tensor, output_tensor, &indices_bytes)); \ - MIOPEN_RETURN_IF_ERROR( \ - miopenGetReductionWorkspaceSize(MiopenHandle(), reduce_desc, input_tensor, output_tensor, &workspace_bytes)); \ - IAllocatorUniquePtr indices_rocm = GetScratchBuffer(indices_bytes); \ - IAllocatorUniquePtr workspace_rocm = GetScratchBuffer(workspace_bytes); \ - \ - const auto one = Consts::One; \ - const auto zero = Consts::Zero; \ - auto temp_Y = GetScratchBuffer(output_count); \ - MIOPEN_RETURN_IF_ERROR(miopenReduceTensor(MiopenHandle(), reduce_desc, indices_rocm.get(), indices_bytes, \ - workspace_rocm.get(), workspace_bytes, &one, input_tensor, temp_X.get(), \ - &zero, output_tensor, temp_Y.get())); \ - \ - Impl_Cast(Stream(), temp_Y.get(), reinterpret_cast(Y->template MutableData()), output_count); \ - \ - return Status::OK(); \ +#define SPECIALIZED_REDUCEKERNEL_COMPUTEIMPL(T) \ + template <> \ + template <> \ + Status ReduceKernel::ComputeImpl( \ + OpKernelContext * ctx, miopenReduceTensorOp_t miopen_reduce_op) const { \ + typedef typename ToHipType::MappedType HipT; \ + const Tensor* X = ctx->Input(0); \ + TensorShapeVector axes; \ + size_t num_inputs = ctx->InputCount(); \ + if (num_inputs == 2) { \ + const Tensor* axes_tensor = ctx->Input(1); \ + ORT_ENFORCE(axes_tensor != nullptr, "Axes input is null"); \ + ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 1, "An axes tensor must be a vector tensor."); \ + auto nDims = static_cast(axes_tensor->Shape()[0]); \ + const auto* data = axes_tensor->template Data(); \ + axes.assign(data, data + nDims); \ + } else { \ + axes.assign(axes_.begin(), axes_.end()); \ + } \ + \ + if (axes.empty() && noop_with_empty_axes_) { \ + auto* Y = ctx->Output(0, X->Shape()); \ + HIP_RETURN_IF_ERROR(hipMemcpyAsync(Y->template MutableData(), X->template Data(), X->SizeInBytes(), \ + hipMemcpyDeviceToDevice, Stream())); \ + return Status::OK(); \ + } \ + \ + PrepareReduceMetadata prepare_reduce_metadata; \ + ORT_RETURN_IF_ERROR(PrepareForReduce(X, keepdims_, axes, prepare_reduce_metadata)); \ + \ + Tensor* Y = ctx->Output(0, prepare_reduce_metadata.squeezed_output_dims); \ + \ + int64_t input_count = prepare_reduce_metadata.input_count; \ + int64_t output_count = prepare_reduce_metadata.output_count; \ + auto& input_dims_miopen = prepare_reduce_metadata.input_dims_miopen; \ + auto& output_dims_miopen = prepare_reduce_metadata.output_dims_miopen; \ + \ + if (input_count == 0) { \ + assert(Y->Shape().Size() == 0); \ + return Status::OK(); \ + } \ + \ + if (input_count == output_count) { \ + if (Y->template MutableData() != X->template Data()) { \ + HIP_RETURN_IF_ERROR(hipMemcpyAsync(Y->template MutableData(), X->template Data(), \ + input_count * sizeof(T), hipMemcpyDeviceToDevice, Stream())); \ + } \ + return Status::OK(); \ + } \ + \ + HIP_RETURN_IF_ERROR(hipMemsetAsync(Y->MutableDataRaw(), 0, Y->SizeInBytes(), Stream())); \ + \ + size_t indices_bytes = 0; \ + size_t workspace_bytes = 0; \ + MiopenTensor input_tensor; \ + MiopenTensor output_tensor; \ + MiopenReduceDescriptor reduce_desc; \ + \ + miopenDataType_t miopen_type_X = miopenFloat; \ + IAllocatorUniquePtr temp_X = GetScratchBuffer(input_count); \ + Impl_Cast(Stream(), reinterpret_cast(X->template Data()), temp_X.get(), X->Shape().Size()); \ + \ + ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, miopen_type_X, MIOPEN_REDUCE_TENSOR_NO_INDICES)); \ + ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_miopen, miopen_type_X)); \ + ORT_RETURN_IF_ERROR(output_tensor.Set(output_dims_miopen, miopen_type_X)); \ + MIOPEN_RETURN_IF_ERROR( \ + miopenGetReductionIndicesSize(MiopenHandle(), reduce_desc, input_tensor, output_tensor, &indices_bytes)); \ + MIOPEN_RETURN_IF_ERROR( \ + miopenGetReductionWorkspaceSize(MiopenHandle(), reduce_desc, input_tensor, output_tensor, &workspace_bytes)); \ + IAllocatorUniquePtr indices_rocm = GetScratchBuffer(indices_bytes); \ + IAllocatorUniquePtr workspace_rocm = GetScratchBuffer(workspace_bytes); \ + \ + const auto one = Consts::One; \ + const auto zero = Consts::Zero; \ + auto temp_Y = GetScratchBuffer(output_count); \ + MIOPEN_RETURN_IF_ERROR(miopenReduceTensor(MiopenHandle(), reduce_desc, indices_rocm.get(), indices_bytes, \ + workspace_rocm.get(), workspace_bytes, &one, input_tensor, temp_X.get(), \ + &zero, output_tensor, temp_Y.get())); \ + \ + Impl_Cast(Stream(), temp_Y.get(), reinterpret_cast(Y->template MutableData()), output_count); \ + \ + return Status::OK(); \ } SPECIALIZED_REDUCEKERNEL_COMPUTEIMPL(int32_t) @@ -828,7 +827,7 @@ namespace ReductionOps { template std::unique_ptr ReduceCompute(ROCMExecutionProvider& rocm_ep, miopenReduceTensorOp_t miopen_reduce_op, AllocatorPtr allocator, - const Tensor& input, const std::vector& axes, + const Tensor& input, gsl::span axes, bool keep_dims, bool calculate_log, bool calculate_sqt, bool log_sum_exp, bool fast_reduction, const TensorShape* input_shape_override) { PrepareReduceMetadata prepare_reduce_metadata; @@ -842,7 +841,7 @@ std::unique_ptr ReduceCompute(ROCMExecutionProvider& rocm_ep, miopenRedu ORT_THROW(ONNXRUNTIME, FAIL, "Failed to perform reduce op: ", status.ErrorMessage()); } - auto output = Tensor::Create(input.DataType(), prepare_reduce_metadata.squeezed_output_dims, allocator); + auto output = Tensor::Create(input.DataType(), prepare_reduce_metadata.squeezed_output_dims, std::move(allocator)); status = ReduceComputeCore(rocm_ep, input, prepare_reduce_metadata, *output, miopen_reduce_op, axes, calculate_log, calculate_sqt, log_sum_exp, fast_reduction, input_shape_override); @@ -859,21 +858,21 @@ std::unique_ptr ReduceCompute(ROCMExecutionProvider& rocm_ep, miopenRedu template std::unique_ptr ReduceCompute( ROCMExecutionProvider& rocm_ep, miopenReduceTensorOp_t miopen_reduce_op, AllocatorPtr allocator, - const Tensor& input, const std::vector& axes, + const Tensor& input, gsl::span axes, bool keep_dims, bool calculate_log, bool calculate_sqt, bool log_sum_exp, bool fast_reduction, const TensorShape* input_shape_override); // template std::unique_ptr ReduceCompute( // ROCMExecutionProvider& rocm_ep, miopenReduceTensorOp_t miopen_reduce_op, // AllocatorPtr allocator, -// const Tensor& input, const std::vector& axes, +// const Tensor& input, gsl::span axes, // bool keep_dims, bool calculate_log, bool calculate_sqt, bool log_sum_exp, // bool fast_reduction, const TensorShape* input_shape_override); template std::unique_ptr ReduceCompute( ROCMExecutionProvider& rocm_ep, miopenReduceTensorOp_t miopen_reduce_op, AllocatorPtr allocator, - const Tensor& input, const std::vector& axes, + const Tensor& input, gsl::span axes, bool keep_dims, bool calculate_log, bool calculate_sqt, bool log_sum_exp, bool fast_reduction, const TensorShape* input_shape_override); diff --git a/onnxruntime/core/providers/rocm/reduction/reduction_ops.h b/onnxruntime/core/providers/rocm/reduction/reduction_ops.h index 8561e3fd21..57169ce747 100644 --- a/onnxruntime/core/providers/rocm/reduction/reduction_ops.h +++ b/onnxruntime/core/providers/rocm/reduction/reduction_ops.h @@ -17,7 +17,7 @@ namespace ReductionOps { template std::unique_ptr ReduceCompute(ROCMExecutionProvider& rocm_ep, miopenReduceTensorOp_t miopen_reduce_op, AllocatorPtr allocator, - const Tensor& input, const std::vector& axes, + const Tensor& input, gsl::span axes, bool keep_dims, bool calculate_log, bool calculate_sqt, bool log_sum_exp, bool fast_reduction, const TensorShape* input_shape_override = nullptr); @@ -28,11 +28,11 @@ struct PrepareReduceMetadata { int64_t input_count; int64_t output_count; // This holds the output dims without any reduced dims squeezed (even if keep_dims == 1) - std::vector output_dims; + TensorShapeVector output_dims; // This holds the output dims with with reduced dims squeezed (if keep_dims == 1) - std::vector squeezed_output_dims; - std::vector input_dims_miopen; - std::vector output_dims_miopen; + TensorShapeVector squeezed_output_dims; + TensorShapeVector input_dims_miopen; + TensorShapeVector output_dims_miopen; }; template @@ -68,7 +68,7 @@ class ReduceKernel : public RocmKernel, public ReduceKernelBase& output_dims) const; + TensorShapeVector& output_dims) const; using ReduceKernelBase::axes_; using ReduceKernelBase::keepdims_; @@ -221,14 +221,14 @@ class ReduceLogSumExp final : public ReduceKernel { Status PrepareForReduce(const Tensor* X, bool keepdims, - const std::vector& axes, + const gsl::span& axes, PrepareReduceMetadata& prepare_reduce_metadata, const TensorShape* input_shape_override = nullptr); template Status ReduceComputeCore(ROCMExecutionProvider& rocm_ep, const Tensor& input, PrepareReduceMetadata& prepare_reduce_metadata, /*out*/ Tensor& output, miopenReduceTensorOp_t miopen_reduce_op, - const std::vector& axes, + const gsl::span& axes, bool calculate_log, bool calculate_sqt, bool log_sum_exp, bool fast_reduction, const TensorShape* input_shape_override = nullptr); diff --git a/onnxruntime/core/providers/rocm/rocm_kernel.h b/onnxruntime/core/providers/rocm/rocm_kernel.h index c66e0d12fb..bd0309853f 100644 --- a/onnxruntime/core/providers/rocm/rocm_kernel.h +++ b/onnxruntime/core/providers/rocm/rocm_kernel.h @@ -89,7 +89,7 @@ class RocmKernel : public OpKernel { } } - RocmAsyncBuffer(const RocmKernel* op_kernel, const std::vector& vec) : RocmAsyncBuffer(op_kernel, vec.size()) { + RocmAsyncBuffer(const RocmKernel* op_kernel, gsl::span vec) : RocmAsyncBuffer(op_kernel, vec.size()) { memcpy(CpuPtr(), vec.data(), vec.size() * sizeof(T)); } diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc index a1993e80b8..d1057c6d24 100644 --- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc +++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc @@ -9,6 +9,7 @@ #include "core/providers/shared/common.h" #include "core/framework/random_generator.h" +#include "core/framework/inlined_containers.h" #include "core/providers/cpu/controlflow/if.h" #include "core/providers/cpu/controlflow/loop.h" #include "core/providers/cpu/controlflow/scan.h" @@ -475,25 +476,25 @@ bool TileOp::IsTileMemcpy(const TensorShape& input_shape, const int64_t* repeats return g_host_cpu.TileOp__IsTileMemcpy(input_shape, repeats, rank, is_batched_memcpy, num_of_elements_per_batch, num_of_copies_per_batch, num_of_batch_copies); } -Status SliceBase::PrepareForCompute(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, +Status SliceBase::PrepareForCompute(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, SliceOp::PrepareForComputeMetadata& compute_metadata) { return g_host_cpu.SliceBase__PrepareForCompute(raw_starts, raw_ends, raw_axes, reinterpret_cast(compute_metadata)); } -Status SliceBase::PrepareForCompute(const std::vector& raw_starts, - const std::vector& raw_ends, - const std::vector& raw_axes, - const std::vector& raw_steps, +Status SliceBase::PrepareForCompute(const gsl::span& raw_starts, + const gsl::span& raw_ends, + const gsl::span& raw_axes, + const gsl::span& raw_steps, SliceOp::PrepareForComputeMetadata& compute_metadata) { return g_host_cpu.SliceBase__PrepareForCompute(raw_starts, raw_ends, raw_axes, raw_steps, reinterpret_cast(compute_metadata)); } Status SliceBase::FillVectorsFromInput(const Tensor& start_tensor, const Tensor& ends_tensor, const Tensor* axes_tensor, const Tensor* steps_tensor, - std::vector& input_starts, - std::vector& input_ends, - std::vector& input_axes, - std::vector& input_steps) { return g_host_cpu.SliceBase__FillVectorsFromInput(start_tensor, ends_tensor, axes_tensor, steps_tensor, input_starts, input_ends, input_axes, input_steps); } + TensorShapeVector& input_starts, + TensorShapeVector& input_ends, + TensorShapeVector& input_axes, + TensorShapeVector& input_steps) { return g_host_cpu.SliceBase__FillVectorsFromInput(start_tensor, ends_tensor, axes_tensor, steps_tensor, input_starts, input_ends, input_axes, input_steps); } Status SplitBase::PrepareForCompute(const TensorShape& input_shape, int num_outputs, int64_t& axis, int& before_dims, int& after_dims_including_split_axis, int& after_dims_excluding_split, @@ -507,8 +508,9 @@ Status ScatterNDBase::ValidateShapes(const TensorShape& input_shape, Status PadBase::HandleDimValueZero(const Mode& mode, const TensorShape& input_shape, TensorShape& output_shape) { return g_host_cpu.PadBase__HandleDimValueZero(mode, input_shape, output_shape); } -Status ConcatBase::PrepareForCompute(OpKernelContext* ctx, const std::vector& input_tensors, - Prepare& p) const { return g_host_cpu.ConcatBase__PrepareForCompute(this, ctx, input_tensors, p); } +Status ConcatBase::PrepareForCompute(OpKernelContext* ctx, const ConcatBase::InlinedTensorsVector& input_tensors, + Prepare& p) const { + return g_host_cpu.ConcatBase__PrepareForCompute(this, ctx, reinterpret_cast(input_tensors), p); } PhiloxGenerator& PhiloxGenerator::Default() { return g_host->PhiloxGenerator__Default(); } diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 862b082d6d..73615a5c19 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -704,6 +704,7 @@ struct ProviderHost { virtual Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) = 0; virtual Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) = 0; virtual Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) = 0; + virtual Status OpKernelInfo__GetAttrsAsSpan(const OpKernelInfo* p, const std::string& name, gsl::span& values) = 0; virtual const DataTransferManager& OpKernelInfo__GetDataTransferManager(const OpKernelInfo* p) noexcept = 0; virtual const KernelDef& OpKernelInfo__GetKernelDef(const OpKernelInfo* p) = 0; diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index 572f6f7ca1..7f960900da 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -797,6 +797,13 @@ struct OpKernelInfo final { return GetAttrs(name, tmp).IsOK() ? tmp : default_value; } + template + Status GetAttrsAsSpan(const std::string& name, gsl::span& out) const; + + Status GetAttrs(const std::string& name, TensorShapeVector& out) const; + + TensorShapeVector GetAttrsOrDefault(const std::string& name, const TensorShapeVector& default_value = TensorShapeVector{}) const; + bool TryGetConstantInput(int input_index, const Tensor** constant_input_value) const { return g_host->OpKernelInfo__TryGetConstantInput(this, input_index, constant_input_value); } const DataTransferManager& GetDataTransferManager() const noexcept { return g_host->OpKernelInfo__GetDataTransferManager(this); } @@ -826,6 +833,25 @@ template <> inline Status OpKernelInfo::GetAttrs(const std::string& name, std::vector& values) const { return g_host->OpKernelInfo__GetAttrs(this, name, values); } template <> inline Status OpKernelInfo::GetAttrs(const std::string& name, std::vector& values) const { return g_host->OpKernelInfo__GetAttrs(this, name, values); } +template <> +inline Status OpKernelInfo::GetAttrsAsSpan(const std::string& name, gsl::span& values) const { return g_host->OpKernelInfo__GetAttrsAsSpan(this, name, values); } + +inline Status OpKernelInfo::GetAttrs(const std::string& name, TensorShapeVector& out) const { + gsl::span span; + Status status = this->GetAttrsAsSpan(name, span); + if (status.IsOK()) { + out.reserve(span.size()); + out.assign(span.cbegin(), span.cend()); + } + return status; +} + +inline TensorShapeVector OpKernelInfo::GetAttrsOrDefault(const std::string& name, const TensorShapeVector& default_value) const { + TensorShapeVector tmp; + return GetAttrs(name, tmp).IsOK() ? tmp : default_value; +} + + class SessionState { public: diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 36ebf32f04..82fdc41126 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -803,6 +803,9 @@ struct ProviderHostImpl : ProviderHost { Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) override { return p->GetAttrs(name, values); } Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) override { return p->GetAttrs(name, values); } Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) override { return p->GetAttrs(name, values); } + Status OpKernelInfo__GetAttrsAsSpan(const OpKernelInfo* p, const std::string& name, gsl::span& values) override { + return p->GetAttrsAsSpan(name, values); + } const DataTransferManager& OpKernelInfo__GetDataTransferManager(const OpKernelInfo* p) noexcept override { return p->GetDataTransferManager(); } const KernelDef& OpKernelInfo__GetKernelDef(const OpKernelInfo* p) override { return p->GetKernelDef(); } diff --git a/onnxruntime/python/onnxruntime_pybind_iobinding.cc b/onnxruntime/python/onnxruntime_pybind_iobinding.cc index fb88844f4f..8d25292a82 100644 --- a/onnxruntime/python/onnxruntime_pybind_iobinding.cc +++ b/onnxruntime/python/onnxruntime_pybind_iobinding.cc @@ -59,7 +59,8 @@ void addIoBindingMethods(pybind11::module& m) { } }) // This binds input as a Tensor that wraps memory pointer along with the OrtMemoryInfo - .def("bind_input", [](SessionIOBinding* io_binding, const std::string& name, const OrtDevice& device, py::object& element_type, std::vector& shape, int64_t data_ptr) -> void { + .def("bind_input", [](SessionIOBinding* io_binding, const std::string& name, const OrtDevice& device, py::object& element_type, + const std::vector& shape, int64_t data_ptr) -> void { ORT_ENFORCE(data_ptr != 0, "Pointer to data memory is not valid"); PyArray_Descr* dtype; @@ -72,7 +73,7 @@ void addIoBindingMethods(pybind11::module& m) { OrtMemoryInfo info(GetDeviceName(device), OrtDeviceAllocator, device, device.Id()); auto ml_type = NumpyTypeToOnnxRuntimeType(type_num); OrtValue ml_value; - Tensor::InitOrtValue(ml_type, shape, reinterpret_cast(data_ptr), info, ml_value); + Tensor::InitOrtValue(ml_type, gsl::make_span(shape), reinterpret_cast(data_ptr), info, ml_value); auto status = io_binding->Get()->BindInput(name, ml_value); if (!status.IsOK()) { @@ -94,7 +95,7 @@ void addIoBindingMethods(pybind11::module& m) { } }) // This binds output to a pre-allocated memory as a Tensor - .def("bind_output", [](SessionIOBinding* io_binding, const std::string& name, const OrtDevice& device, py::object& element_type, std::vector& shape, int64_t data_ptr) -> void { + .def("bind_output", [](SessionIOBinding* io_binding, const std::string& name, const OrtDevice& device, py::object& element_type, const std::vector& shape, int64_t data_ptr) -> void { ORT_ENFORCE(data_ptr != 0, "Pointer to data memory is not valid"); InferenceSession* sess = io_binding->GetInferenceSession(); @@ -125,7 +126,7 @@ void addIoBindingMethods(pybind11::module& m) { OrtMemoryInfo info(GetDeviceName(device), OrtDeviceAllocator, device, device.Id()); auto ml_type = NumpyTypeToOnnxRuntimeType(type_num); OrtValue ml_value; - Tensor::InitOrtValue(ml_type, shape, reinterpret_cast(data_ptr), info, ml_value); + Tensor::InitOrtValue(ml_type, gsl::make_span(shape), reinterpret_cast(data_ptr), info, ml_value); auto status = io_binding->Get()->BindOutput(name, ml_value); if (!status.IsOK()) { diff --git a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc index f37539906f..2c52d468c6 100644 --- a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc @@ -108,7 +108,7 @@ void addOrtValueMethods(pybind11::module& m) { auto ml_value = std::make_unique(); auto ml_type = NumpyTypeToOnnxRuntimeType(type_num); - Tensor::InitOrtValue(ml_type, shape, std::move(allocator), *ml_value); + Tensor::InitOrtValue(ml_type, gsl::make_span(shape), std::move(allocator), *ml_value); return ml_value; }) diff --git a/onnxruntime/test/contrib_ops/unique_op_test.cc b/onnxruntime/test/contrib_ops/unique_op_test.cc index 32df72382a..501defbfc3 100644 --- a/onnxruntime/test/contrib_ops/unique_op_test.cc +++ b/onnxruntime/test/contrib_ops/unique_op_test.cc @@ -9,7 +9,7 @@ namespace test { TEST(UniqueOpTest, Unique_Spec_Example) { OpTester test("Unique", 1, onnxruntime::kMSDomain); - test.AddInput("x", {6}, {2.0, 1.0, 1.0, 3.0, 4.0, 3.0}); + test.AddInput("x", {6}, {2.0f, 1.0f, 1.0f, 3.0f, 4.0f, 3.0f}); test.AddOutput("uniques", {4}, {2.0f, 1.0f, 3.0f, 4.0f}); test.AddOutput("idx", {6}, {0, 1, 1, 2, 3, 2}); test.AddOutput("counts", {4}, {1, 2, 2, 1}); @@ -18,7 +18,7 @@ TEST(UniqueOpTest, Unique_Spec_Example) { TEST(UniqueOpTest, Unique_Complicated_Example) { OpTester test("Unique", 1, onnxruntime::kMSDomain); - test.AddInput("x", {12}, {2.0, 1.0, 1.0, 3.0, 4.0, 3.0, 2.0, 1.0, 1.0, 3.0, 4.0, 3.0}); + test.AddInput("x", {12}, {2.0f, 1.0f, 1.0f, 3.0f, 4.0f, 3.0f, 2.0f, 1.0f, 1.0f, 3.0f, 4.0f, 3.0f}); test.AddOutput("uniques", {4}, {2.0f, 1.0f, 3.0f, 4.0f}); test.AddOutput("idx", {12}, {0, 1, 1, 2, 3, 2, 0, 1, 1, 2, 3, 2}); test.AddOutput("counts", {4}, {2, 4, 4, 2}); @@ -27,8 +27,8 @@ TEST(UniqueOpTest, Unique_Complicated_Example) { TEST(UniqueOpTest, Unique_Example_SingleElement) { OpTester test("Unique", 1, onnxruntime::kMSDomain); - test.AddInput("x", {1}, {2.0}); - test.AddOutput("uniques", {1}, {2.0}); + test.AddInput("x", {1}, {2.0f}); + test.AddOutput("uniques", {1}, {2.0f}); test.AddOutput("idx", {1}, {0}); test.AddOutput("counts", {1}, {1}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); @@ -36,8 +36,8 @@ TEST(UniqueOpTest, Unique_Example_SingleElement) { TEST(UniqueOpTest, Unique_AllUniqueElements) { OpTester test("Unique", 1, onnxruntime::kMSDomain); - test.AddInput("x", {2}, {2.0, 3.0}); - test.AddOutput("uniques", {2}, {2.0, 3.0}); + test.AddInput("x", {2}, {2.0f, 3.0f}); + test.AddOutput("uniques", {2}, {2.0f, 3.0f}); test.AddOutput("idx", {2}, {0, 1}); test.AddOutput("counts", {2}, {1, 1}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); diff --git a/onnxruntime/test/eager/ort_invoker_test.cc b/onnxruntime/test/eager/ort_invoker_test.cc index ed2efccd76..ce437acc8a 100644 --- a/onnxruntime/test/eager/ort_invoker_test.cc +++ b/onnxruntime/test/eager/ort_invoker_test.cc @@ -39,7 +39,7 @@ TEST(InvokerTest, Basic) { ASSERT_STATUS_OK(kernel_invoker.Invoke("Add", {A, B}, result, nullptr)); const Tensor& C = result.back().Get(); auto& c_shape = C.Shape(); - EXPECT_EQ(c_shape.GetDimsAsVector(), dims_mul_x); + EXPECT_EQ(c_shape.GetDims(), gsl::make_span(dims_mul_x)); std::vector expected_result = {2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f}; auto* c_data = C.Data(); diff --git a/onnxruntime/test/framework/sparse_kernels_test.cc b/onnxruntime/test/framework/sparse_kernels_test.cc index 30d1232a7b..24c8097bae 100644 --- a/onnxruntime/test/framework/sparse_kernels_test.cc +++ b/onnxruntime/test/framework/sparse_kernels_test.cc @@ -1510,7 +1510,7 @@ TEST(SparseTensorConversionTests, CooConversion) { auto* cpu_provider = TestCPUExecutionProvider(); auto cpu_allocator = cpu_provider->GetAllocator(0, OrtMemTypeDefault); - const std::vector dense_shape{3, 3}; + const TensorShapeVector dense_shape{3, 3}; std::vector dense_data = { 0, 0, 1, 1, 0, 1, @@ -1638,7 +1638,7 @@ TEST(SparseTensorConversionTests, CooConversion) { gsl::make_span(expected_linear_indices))); ASSERT_EQ(str_cpu_src.Format(), SparseFormat::kCoo); ASSERT_TRUE(str_cpu_src.IsDataTypeString()); - ASSERT_EQ(str_cpu_src.DenseShape(), dense_shape); + ASSERT_EQ(str_cpu_src.DenseShape(), TensorShape(dense_shape)); ASSERT_EQ(str_cpu_src.NumValues(), expected_values_str.size()); auto values = str_cpu_src.Values().DataAsSpan(); ASSERT_TRUE(std::equal(expected_values_str.cbegin(), expected_values_str.cend(), values.cbegin(), values.cend())); diff --git a/onnxruntime/test/framework/tensor_shape_test.cc b/onnxruntime/test/framework/tensor_shape_test.cc index 7c799baf0f..86c64650ba 100644 --- a/onnxruntime/test/framework/tensor_shape_test.cc +++ b/onnxruntime/test/framework/tensor_shape_test.cc @@ -11,11 +11,11 @@ namespace onnxruntime { namespace utils { namespace test { -static void TestShapeWithVector(const std::vector& vector) { +static void TestShapeWithVector(const TensorShapeVector& vector) { // Test constructing from a vector TensorShape shape{vector}; - EXPECT_EQ(shape, vector); + EXPECT_EQ(shape, gsl::make_span(vector)); // Test copying to a new shape TensorShape shape_copy{shape}; @@ -38,8 +38,8 @@ TEST(TensorShapeTest, VariousSizes) { TestShapeWithVector({12, 23, 34, 45, 56, 67, 78, 89, 90}); // Test assigning a shape to a large then a small vector (causing it to switch from small block to large, then back to small) - std::vector small{1, 2, 3}; - std::vector large{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + TensorShapeVector small{1, 2, 3}; + TensorShapeVector large{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; TensorShape shape{small}; EXPECT_EQ(shape.GetDims(), gsl::make_span(small)); diff --git a/onnxruntime/test/onnx/microbenchmark/activation.cc b/onnxruntime/test/onnx/microbenchmark/activation.cc index d763e89219..976f484cce 100644 --- a/onnxruntime/test/onnx/microbenchmark/activation.cc +++ b/onnxruntime/test/onnx/microbenchmark/activation.cc @@ -147,12 +147,11 @@ static void RunSingleNode(const std::string& op_name, const std::string& domain, std::vector feeds(1); std::vector fetches(1); - std::vector shapes(static_cast(1), batch_size); - auto ml_tensor = DataTypeImpl::GetType(); + TensorShapeVector shapes(static_cast(1), batch_size); OrtMemoryInfo info("cpu", OrtDeviceAllocator); - feeds[0].Init(new Tensor(DataTypeImpl::GetType(), shapes, data, info), ml_tensor, ml_tensor->GetDeleteFunc()); - fetches[0].Init(new Tensor(DataTypeImpl::GetType(), shapes, output, info), ml_tensor, - ml_tensor->GetDeleteFunc()); + auto ml_float = DataTypeImpl::GetType(); + Tensor::InitOrtValue(ml_float, shapes, data, info, feeds[0]); + Tensor::InitOrtValue(ml_float, shapes, output, info, fetches[0]); GraphViewer v(k.model->MainGraph()); NodeIndexInfo node_index_info(v, *k.ort_value_idx_map); OrtThreadPoolParams tpo; diff --git a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc index ccc60266a4..d2a8196770 100644 --- a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc @@ -381,24 +381,25 @@ static void RunTest_v9(const std::string test_name, int64_t sequence_len, int64_ test.AddOutput("scan_loop_state_out_0", loop_state_shape, loop_state_out_0); - std::vector output_shape{sequence_len, 1}; + TensorShape output_shape{sequence_len, 1}; auto calculate_output_shape = [&](size_t output_index) { if (output_axes && output_axes->size() > output_index) { - auto axis = output_axes->at(output_index); - auto rank = gsl::narrow_cast(output_shape.size()); + const auto axis = output_axes->at(output_index); + const auto rank = gsl::narrow_cast(output_shape.NumDimensions()); // skip if this is an invalid input test and axis is out of the valid range if (axis >= -rank && axis < rank) { - std::vector permutations; - std::vector new_shape; - scan::detail::CalculateTransposedShapeForOutput(output_shape, HandleNegativeAxis(axis, output_shape.size()), + InlinedShapeVectorT permutations; + TensorShapeVector new_shape; + scan::detail::CalculateTransposedShapeForOutput(output_shape, HandleNegativeAxis(axis, rank), permutations, new_shape); - return new_shape; + return std::vector(new_shape.cbegin(), new_shape.cend()); } } - return output_shape; + const auto output_dims = output_shape.GetDims(); + return std::vector(output_dims.cbegin(), output_dims.cend()); }; test.AddOutput("scan_output_0", calculate_output_shape(0), output_0); diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index 4e7ae2faa6..f47d80f9ab 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -1097,7 +1097,7 @@ static void TestSumMultipleInputsNoBroadcasting(size_t num_inputs, const TensorS OpTester test{"Sum", 8}; - const auto dims = shape.GetDimsAsVector(); + const auto dims = GetShapeVector(shape); const std::vector input_data(shape.Size(), 1); for (size_t i = 0; i < num_inputs; ++i) { diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc index f7ffdf8b91..2c36b80ecc 100644 --- a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc +++ b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc @@ -2438,16 +2438,16 @@ TEST(ReductionOpTest, ArgMin_int32_neg_axis) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_ReduceDimWithZero1) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // R - keep_dims=1 - noop=false fast_kind = OptimizeShapeForFastReduce( std::vector{3, 0, 2}, std::vector(), fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{}; - expected_fast_axes = std::vector{}; - expected_fast_output_shape = std::vector{1, 0, 1}; + expected_fast_shape = {}; + expected_fast_axes = {}; + expected_fast_output_shape = {1, 0, 1}; ASSERT_EQ(fast_kind, FastReduceKind::kEmpty); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_shape, expected_fast_shape); @@ -2456,16 +2456,16 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_ReduceDimWithZero1) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_ReduceDimWithZero1b) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // R - keep_dims=1 - noop=false fast_kind = OptimizeShapeForFastReduce( std::vector{3, 0, 2}, std::vector{1}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{}; - expected_fast_axes = std::vector{}; - expected_fast_output_shape = std::vector{3, 0, 2}; + expected_fast_shape = {}; + expected_fast_axes = {}; + expected_fast_output_shape = {3, 0, 2}; ASSERT_EQ(fast_kind, FastReduceKind::kEmpty); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_shape, expected_fast_shape); @@ -2493,16 +2493,16 @@ TEST(ReductionOpTest, ReduceDimWithZero1) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_ReduceDimWithZero2) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // R - keep_dims=0 - noop=false fast_kind = OptimizeShapeForFastReduce( std::vector{3, 0, 2}, std::vector(), fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{}; - expected_fast_axes = std::vector{}; - expected_fast_output_shape = std::vector{}; + expected_fast_shape = {}; + expected_fast_axes = {}; + expected_fast_output_shape = {}; ASSERT_EQ(fast_kind, FastReduceKind::kEmpty); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_shape, expected_fast_shape); @@ -2531,16 +2531,16 @@ TEST(ReductionOpTest, ReduceDimWithZero2) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_ReduceDimWithZero3) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // R - keep_dims=0 - noop=false fast_kind = OptimizeShapeForFastReduce( std::vector{3, 0, 2}, std::vector{2}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{0, 2}; - expected_fast_axes = std::vector{1}; - expected_fast_output_shape = std::vector{3, 0}; + expected_fast_shape = {0, 2}; + expected_fast_axes = {1}; + expected_fast_output_shape = {3, 0}; ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_axes, expected_fast_axes); @@ -2696,16 +2696,16 @@ TEST(ReductionOpTest, ReduceInfLogSumExp_double) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_R_K) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // R - keep_dims=1 fast_kind = OptimizeShapeForFastReduce( std::vector{10}, std::vector{0}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{10}; - expected_fast_output_shape = std::vector{1}; - expected_fast_axes = std::vector{0}; + expected_fast_shape = {10}; + expected_fast_output_shape = {1}; + expected_fast_axes = {0}; ASSERT_EQ(fast_kind, FastReduceKind::kR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2714,9 +2714,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_R_K) { fast_kind = OptimizeShapeForFastReduce( std::vector{10, 11}, std::vector{0, 1}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{110}; - expected_fast_output_shape = std::vector{1, 1}; - expected_fast_axes = std::vector{0}; + expected_fast_shape = {110}; + expected_fast_output_shape = {1, 1}; + expected_fast_axes = {0}; ASSERT_EQ(fast_kind, FastReduceKind::kR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2726,9 +2726,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_R_K) { fast_kind = OptimizeShapeForFastReduce( std::vector{10}, std::vector{0}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{10}; - expected_fast_output_shape = std::vector(); - expected_fast_axes = std::vector{0}; + expected_fast_shape = {10}; + expected_fast_output_shape = {}; + expected_fast_axes = {0}; ASSERT_EQ(fast_kind, FastReduceKind::kR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2737,9 +2737,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_R_K) { fast_kind = OptimizeShapeForFastReduce( std::vector{10, 11}, std::vector{0, 1}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{110}; - expected_fast_output_shape = std::vector(); - expected_fast_axes = std::vector{0}; + expected_fast_shape = {110}; + expected_fast_output_shape = {}; + expected_fast_axes = {0}; ASSERT_EQ(fast_kind, FastReduceKind::kR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2748,25 +2748,27 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_R_K) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_R_empty) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // R - keep_dims=1 - noop=false fast_kind = OptimizeShapeForFastReduce( std::vector{10}, std::vector(), fast_shape, fast_output_shape, fast_axes, true); - expected_fast_axes = std::vector{0}; + expected_fast_axes = {0}; + expected_fast_shape = {10}; + expected_fast_output_shape = {1}; ASSERT_EQ(fast_kind, FastReduceKind::kR); - ASSERT_EQ(fast_shape, std::vector{10}); - ASSERT_EQ(fast_output_shape, std::vector{1}); + ASSERT_EQ(fast_shape, expected_fast_shape); + ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_axes, expected_fast_axes); fast_kind = OptimizeShapeForFastReduce( std::vector{10, 11}, std::vector(), fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{110}; - expected_fast_output_shape = std::vector{1, 1}; - expected_fast_axes = std::vector{0}; + expected_fast_shape = {110}; + expected_fast_output_shape = {1, 1}; + expected_fast_axes = {0}; ASSERT_EQ(fast_kind, FastReduceKind::kR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2776,9 +2778,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_R_empty) { fast_kind = OptimizeShapeForFastReduce( std::vector{10}, std::vector{}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{10}; - expected_fast_output_shape = std::vector{}; - expected_fast_axes = std::vector{0}; + expected_fast_shape = {10}; + expected_fast_output_shape = {}; + expected_fast_axes = {0}; ASSERT_EQ(fast_kind, FastReduceKind::kR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2787,9 +2789,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_R_empty) { fast_kind = OptimizeShapeForFastReduce( std::vector{10, 11}, std::vector{}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{110}; - expected_fast_output_shape = std::vector{}; - expected_fast_axes = std::vector{0}; + expected_fast_shape = {110}; + expected_fast_output_shape = {}; + expected_fast_axes = {0}; ASSERT_EQ(fast_kind, FastReduceKind::kR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2798,16 +2800,16 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_R_empty) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_K_empty) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // R - keep_dims=1 - noop=true fast_kind = OptimizeShapeForFastReduce( std::vector{10}, std::vector(), fast_shape, fast_output_shape, fast_axes, true, true); - expected_fast_shape = std::vector{10}; - expected_fast_output_shape = std::vector{10}; - expected_fast_axes = std::vector{}; + expected_fast_shape = {10}; + expected_fast_output_shape = {10}; + expected_fast_axes = {}; ASSERT_EQ(fast_kind, FastReduceKind::kK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2816,9 +2818,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_K_empty) { fast_kind = OptimizeShapeForFastReduce( std::vector{10, 11}, std::vector(), fast_shape, fast_output_shape, fast_axes, true, true); - expected_fast_shape = std::vector{110}; - expected_fast_output_shape = std::vector{10, 11}; - expected_fast_axes = std::vector{}; + expected_fast_shape = {110}; + expected_fast_output_shape = {10, 11}; + expected_fast_axes = {}; ASSERT_EQ(fast_kind, FastReduceKind::kK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2828,9 +2830,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_K_empty) { fast_kind = OptimizeShapeForFastReduce( std::vector{10}, std::vector{}, fast_shape, fast_output_shape, fast_axes, false, true); - expected_fast_shape = std::vector{10}; - expected_fast_output_shape = std::vector{10}; - expected_fast_axes = std::vector{}; + expected_fast_shape = {10}; + expected_fast_output_shape = {10}; + expected_fast_axes = {}; ASSERT_EQ(fast_kind, FastReduceKind::kK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2839,9 +2841,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_K_empty) { fast_kind = OptimizeShapeForFastReduce( std::vector{10, 11}, std::vector{}, fast_shape, fast_output_shape, fast_axes, false, true); - expected_fast_shape = std::vector{110}; - expected_fast_output_shape = std::vector{10, 11}; - expected_fast_axes = std::vector{}; + expected_fast_shape = {110}; + expected_fast_output_shape = {10, 11}; + expected_fast_axes = {}; ASSERT_EQ(fast_kind, FastReduceKind::kK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2850,36 +2852,36 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_K_empty) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_KR) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // KR - keep_dims=1 fast_kind = OptimizeShapeForFastReduce( - std::vector{10, 11}, std::vector{1}, + TensorShapeVector{10, 11}, TensorShapeVector{1}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{10, 11}; - expected_fast_output_shape = std::vector{10, 1}; - expected_fast_axes = std::vector{1}; + expected_fast_shape = {10, 11}; + expected_fast_output_shape = {10, 1}; + expected_fast_axes = {1}; ASSERT_EQ(fast_kind, FastReduceKind::kKR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_axes, expected_fast_axes); fast_kind = OptimizeShapeForFastReduce( - std::vector{9, 10, 11}, std::vector{1, 2}, + TensorShapeVector{9, 10, 11}, TensorShapeVector{1, 2}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{9, 110}; - expected_fast_output_shape = std::vector{9, 1, 1}; + expected_fast_shape = {9, 110}; + expected_fast_output_shape = {9, 1, 1}; ASSERT_EQ(fast_kind, FastReduceKind::kKR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_axes, expected_fast_axes); fast_kind = OptimizeShapeForFastReduce( - std::vector{9, 10, 11}, std::vector{2}, + TensorShapeVector{9, 10, 11}, TensorShapeVector{2}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{90, 11}; - expected_fast_output_shape = std::vector{9, 10, 1}; + expected_fast_shape = {90, 11}; + expected_fast_output_shape = {9, 10, 1}; ASSERT_EQ(fast_kind, FastReduceKind::kKR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2887,20 +2889,20 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_KR) { // KR - keep_dims=0 fast_kind = OptimizeShapeForFastReduce( - std::vector{10, 11}, std::vector{1}, + TensorShapeVector{10, 11}, TensorShapeVector{1}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{10, 11}; - expected_fast_output_shape = std::vector{10}; + expected_fast_shape = {10, 11}; + expected_fast_output_shape = {10}; ASSERT_EQ(fast_kind, FastReduceKind::kKR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_axes, expected_fast_axes); fast_kind = OptimizeShapeForFastReduce( - std::vector{9, 10, 11}, std::vector{1, 2}, + TensorShapeVector{9, 10, 11}, TensorShapeVector{1, 2}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{9, 110}; - expected_fast_output_shape = std::vector{9}; + expected_fast_shape = {9, 110}; + expected_fast_output_shape = {9}; ASSERT_EQ(fast_kind, FastReduceKind::kKR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2909,8 +2911,8 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_KR) { fast_kind = OptimizeShapeForFastReduce( std::vector{9, 10, 11}, std::vector{2}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{90, 11}; - expected_fast_output_shape = std::vector{9, 10}; + expected_fast_shape = {90, 11}; + expected_fast_output_shape = {9, 10}; ASSERT_EQ(fast_kind, FastReduceKind::kKR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2919,16 +2921,16 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_KR) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_KR_neg) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // KR - keep_dims=1 fast_kind = OptimizeShapeForFastReduce( - std::vector{10, 11}, std::vector{-1}, + TensorShapeVector{10, 11}, TensorShapeVector{-1}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{10, 11}; - expected_fast_output_shape = std::vector{10, 1}; - expected_fast_axes = std::vector{1}; + expected_fast_shape = TensorShapeVector{10, 11}; + expected_fast_output_shape = TensorShapeVector{10, 1}; + expected_fast_axes = TensorShapeVector{1}; ASSERT_EQ(fast_kind, FastReduceKind::kKR); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2937,36 +2939,36 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_KR_neg) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_RK) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // RK - keep_dims=1 fast_kind = OptimizeShapeForFastReduce( - std::vector{10, 11}, std::vector{0}, + TensorShapeVector{10, 11}, TensorShapeVector{0}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{10, 11}; - expected_fast_output_shape = std::vector{1, 11}; - expected_fast_axes = std::vector{0}; + expected_fast_shape = {10, 11}; + expected_fast_output_shape = {1, 11}; + expected_fast_axes = {0}; ASSERT_EQ(fast_kind, FastReduceKind::kRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_axes, expected_fast_axes); fast_kind = OptimizeShapeForFastReduce( - std::vector{9, 10, 11}, std::vector{0, 1}, + TensorShapeVector{9, 10, 11}, TensorShapeVector{0, 1}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{90, 11}; - expected_fast_output_shape = std::vector{1, 1, 11}; + expected_fast_shape = {90, 11}; + expected_fast_output_shape = {1, 1, 11}; ASSERT_EQ(fast_kind, FastReduceKind::kRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_axes, expected_fast_axes); fast_kind = OptimizeShapeForFastReduce( - std::vector{9, 10, 11}, std::vector{0}, + TensorShapeVector{9, 10, 11}, TensorShapeVector{0}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{9, 110}; - expected_fast_output_shape = std::vector{1, 10, 11}; + expected_fast_shape = {9, 110}; + expected_fast_output_shape = {1, 10, 11}; ASSERT_EQ(fast_kind, FastReduceKind::kRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2976,8 +2978,8 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_RK) { fast_kind = OptimizeShapeForFastReduce( std::vector{10, 11}, std::vector{0}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{10, 11}; - expected_fast_output_shape = std::vector{11}; + expected_fast_shape = {10, 11}; + expected_fast_output_shape = {11}; ASSERT_EQ(fast_kind, FastReduceKind::kRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2986,8 +2988,8 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_RK) { fast_kind = OptimizeShapeForFastReduce( std::vector{9, 10, 11}, std::vector{0, 1}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{90, 11}; - expected_fast_output_shape = std::vector{11}; + expected_fast_shape = {90, 11}; + expected_fast_output_shape = {11}; ASSERT_EQ(fast_kind, FastReduceKind::kRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -2995,8 +2997,8 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_RK) { fast_kind = OptimizeShapeForFastReduce( std::vector{9, 10, 11}, std::vector{0}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{9, 110}; - expected_fast_output_shape = std::vector{10, 11}; + expected_fast_shape = {9, 110}; + expected_fast_output_shape = {10, 11}; ASSERT_EQ(fast_kind, FastReduceKind::kRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -3005,46 +3007,46 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_RK) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_KRK) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // KRK - keep_dims=1 fast_kind = OptimizeShapeForFastReduce( std::vector{9, 10, 11}, std::vector{1}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{9, 10, 11}; - expected_fast_output_shape = std::vector{9, 1, 11}; - expected_fast_axes = std::vector{1}; + expected_fast_shape = {9, 10, 11}; + expected_fast_output_shape = {9, 1, 11}; + expected_fast_axes = {1}; ASSERT_EQ(fast_kind, FastReduceKind::kKRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_axes, expected_fast_axes); fast_kind = OptimizeShapeForFastReduce( - std::vector{7, 9, 10, 11}, std::vector{1, 2}, + TensorShapeVector{7, 9, 10, 11}, TensorShapeVector{1, 2}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{7, 90, 11}; - expected_fast_output_shape = std::vector{7, 1, 1, 11}; + expected_fast_shape = {7, 90, 11}; + expected_fast_output_shape = {7, 1, 1, 11}; ASSERT_EQ(fast_kind, FastReduceKind::kKRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_axes, expected_fast_axes); fast_kind = OptimizeShapeForFastReduce( - std::vector{7, 9, 10, 11}, std::vector{1}, + TensorShapeVector{7, 9, 10, 11}, TensorShapeVector{1}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{7, 9, 110}; - expected_fast_output_shape = std::vector{7, 1, 10, 11}; + expected_fast_shape = {7, 9, 110}; + expected_fast_output_shape = {7, 1, 10, 11}; ASSERT_EQ(fast_kind, FastReduceKind::kKRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); ASSERT_EQ(fast_axes, expected_fast_axes); fast_kind = OptimizeShapeForFastReduce( - std::vector{7, 9, 10, 11}, std::vector{2}, + TensorShapeVector{7, 9, 10, 11}, TensorShapeVector{2}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{63, 10, 11}; - expected_fast_output_shape = std::vector{7, 9, 1, 11}; + expected_fast_shape = {63, 10, 11}; + expected_fast_output_shape = {7, 9, 1, 11}; ASSERT_EQ(fast_kind, FastReduceKind::kKRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -3054,8 +3056,8 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_KRK) { fast_kind = OptimizeShapeForFastReduce( std::vector{9, 10, 11}, std::vector{1}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{9, 10, 11}; - expected_fast_output_shape = std::vector{9, 11}; + expected_fast_shape = {9, 10, 11}; + expected_fast_output_shape = {9, 11}; ASSERT_EQ(fast_kind, FastReduceKind::kKRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -3064,8 +3066,8 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_KRK) { fast_kind = OptimizeShapeForFastReduce( std::vector{7, 9, 10, 11}, std::vector{1, 2}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{7, 90, 11}; - expected_fast_output_shape = std::vector{7, 11}; + expected_fast_shape = {7, 90, 11}; + expected_fast_output_shape = {7, 11}; ASSERT_EQ(fast_kind, FastReduceKind::kKRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -3074,8 +3076,8 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_KRK) { fast_kind = OptimizeShapeForFastReduce( std::vector{7, 9, 10, 11}, std::vector{1}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{7, 9, 110}; - expected_fast_output_shape = std::vector{7, 10, 11}; + expected_fast_shape = {7, 9, 110}; + expected_fast_output_shape = {7, 10, 11}; ASSERT_EQ(fast_kind, FastReduceKind::kKRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -3084,8 +3086,8 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_KRK) { fast_kind = OptimizeShapeForFastReduce( std::vector{7, 9, 10, 11}, std::vector{2}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{63, 10, 11}; - expected_fast_output_shape = std::vector{7, 9, 11}; + expected_fast_shape = {63, 10, 11}; + expected_fast_output_shape = {7, 9, 11}; ASSERT_EQ(fast_kind, FastReduceKind::kKRK); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -3094,16 +3096,16 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_KRK) { TEST(ReductionOpTest, OptimizeShapeForFastReduce_NONE) { FastReduceKind fast_kind; - std::vector fast_shape, fast_output_shape, fast_axes; - std::vector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; + TensorShapeVector fast_shape, fast_output_shape, fast_axes; + TensorShapeVector expected_fast_shape, expected_fast_output_shape, expected_fast_axes; // RKRK fast_kind = OptimizeShapeForFastReduce( std::vector{7, 9, 10, 11}, std::vector{0, 2}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{7, 9, 10, 11}; - expected_fast_output_shape = std::vector{9, 11}; - expected_fast_axes = std::vector{0, 2}; + expected_fast_shape = {7, 9, 10, 11}; + expected_fast_output_shape = {9, 11}; + expected_fast_axes = {0, 2}; ASSERT_EQ(fast_kind, FastReduceKind::kNone); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -3112,9 +3114,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_NONE) { fast_kind = OptimizeShapeForFastReduce( std::vector{7, 9, 10, 11}, std::vector{1, 3}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{7, 9, 10, 11}; - expected_fast_output_shape = std::vector{7, 1, 10, 1}; - expected_fast_axes = std::vector{1, 3}; + expected_fast_shape = {7, 9, 10, 11}; + expected_fast_output_shape = {7, 1, 10, 1}; + expected_fast_axes = {1, 3}; ASSERT_EQ(fast_kind, FastReduceKind::kNone); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -3124,9 +3126,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_NONE) { fast_kind = OptimizeShapeForFastReduce( std::vector{7, 9, 10, 11, 2, 3, 4, 6}, std::vector{0, 1, 4, 5}, fast_shape, fast_output_shape, fast_axes, false); - expected_fast_shape = std::vector{63, 110, 6, 24}; - expected_fast_output_shape = std::vector{10, 11, 4, 6}; - expected_fast_axes = std::vector{0, 2}; + expected_fast_shape = {63, 110, 6, 24}; + expected_fast_output_shape = {10, 11, 4, 6}; + expected_fast_axes = {0, 2}; ASSERT_EQ(fast_kind, FastReduceKind::kNone); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); @@ -3135,9 +3137,9 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_NONE) { fast_kind = OptimizeShapeForFastReduce( std::vector{7, 9, 10, 11, 2, 3, 4, 6}, std::vector{0, 1, 4, 5}, fast_shape, fast_output_shape, fast_axes, true); - expected_fast_shape = std::vector{63, 110, 6, 24}; - expected_fast_output_shape = std::vector{1, 1, 10, 11, 1, 1, 4, 6}; - expected_fast_axes = std::vector{0, 2}; + expected_fast_shape = {63, 110, 6, 24}; + expected_fast_output_shape = {1, 1, 10, 11, 1, 1, 4, 6}; + expected_fast_axes = {0, 2}; ASSERT_EQ(fast_kind, FastReduceKind::kNone); ASSERT_EQ(fast_shape, expected_fast_shape); ASSERT_EQ(fast_output_shape, expected_fast_output_shape); diff --git a/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc b/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc index 56e509ebdd..af88c44d8c 100644 --- a/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc @@ -113,7 +113,7 @@ struct CastNonStringTester { auto output_span = gsl::make_span(output_buffer.get(), size); CastSpan(input_span, output_span); - TestCastOp(input_span, output_span, shape.GetDimsAsVector()); + TestCastOp(input_span, output_span, GetShapeVector(shape)); } }; diff --git a/onnxruntime/test/providers/cpu/tensor/copy_test.cc b/onnxruntime/test/providers/cpu/tensor/copy_test.cc index f8a72a5c8f..f121b2e568 100644 --- a/onnxruntime/test/providers/cpu/tensor/copy_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/copy_test.cc @@ -58,8 +58,8 @@ TEST_F(CopyTest, Transpose4D) { } std::unique_ptr dst = std::make_unique(numel); - std::vector dst_strides = {60, 5, 15, 1}; - std::vector src_strides = {60, 20, 5, 1}; + TensorShapeVector dst_strides = {60, 5, 15, 1}; + TensorShapeVector src_strides = {60, 20, 5, 1}; StridedCopy(tp.get(), dst.get(), dst_strides, {2, 3, 4, 5}, src.get(), src_strides); // stride to access the dst tensor as if it were contiguous @@ -90,8 +90,8 @@ TEST_F(CopyTest, Concat2D) { dst[i] = 0; } - std::vector dst_strides = {5, 1}; - std::vector src_strides = {2, 1}; + TensorShapeVector dst_strides = {5, 1}; + TensorShapeVector src_strides = {2, 1}; std::ptrdiff_t offset = 3; StridedCopy(tp.get(), dst.get() + offset, dst_strides, {6, 2}, src.get(), src_strides); @@ -110,9 +110,9 @@ TEST_F(CopyTest, Concat2D) { TEST_F(CopyTest, CoalesceTensorsTest) { { - std::vector strides_a{3, 1}; - std::vector strides_b{3, 1}; - std::vector shape{5, 3}; + TensorShapeVector strides_a{3, 1}; + TensorShapeVector strides_b{3, 1}; + TensorShapeVector shape{5, 3}; CoalesceDimensions({strides_a, strides_b}, shape); @@ -122,9 +122,9 @@ TEST_F(CopyTest, CoalesceTensorsTest) { } { - std::vector strides_a{3, 3, 1}; - std::vector strides_b{3, 3, 1}; - std::vector shape{5, 1, 3}; + TensorShapeVector strides_a{3, 3, 1}; + TensorShapeVector strides_b{3, 3, 1}; + TensorShapeVector shape{5, 1, 3}; CoalesceDimensions({strides_a, strides_b}, shape); @@ -133,9 +133,9 @@ TEST_F(CopyTest, CoalesceTensorsTest) { ASSERT_THAT(shape, testing::ElementsAre(15)); } { - std::vector strides_a{3, 3, 3, 1}; - std::vector strides_b{3, 3, 3, 1}; - std::vector shape{1, 5, 1, 3}; + TensorShapeVector strides_a{3, 3, 3, 1}; + TensorShapeVector strides_b{3, 3, 3, 1}; + TensorShapeVector shape{1, 5, 1, 3}; CoalesceDimensions({strides_a, strides_b}, shape); @@ -144,9 +144,9 @@ TEST_F(CopyTest, CoalesceTensorsTest) { ASSERT_THAT(shape, testing::ElementsAre(15)); } { - std::vector strides_a{3, 3, 3, 1}; - std::vector strides_b{3, 3, 3, 1}; - std::vector shape{1, 5, 1, 3}; + TensorShapeVector strides_a{3, 3, 3, 1}; + TensorShapeVector strides_b{3, 3, 3, 1}; + TensorShapeVector shape{1, 5, 1, 3}; CoalesceDimensions({strides_a, strides_b}, shape); @@ -155,9 +155,9 @@ TEST_F(CopyTest, CoalesceTensorsTest) { ASSERT_THAT(shape, testing::ElementsAre(15)); } { - std::vector strides_a{320, 1}; - std::vector strides_b{320, 1}; - std::vector shape{20, 10}; + TensorShapeVector strides_a{320, 1}; + TensorShapeVector strides_b{320, 1}; + TensorShapeVector shape{20, 10}; CoalesceDimensions({strides_a, strides_b}, shape); @@ -166,9 +166,9 @@ TEST_F(CopyTest, CoalesceTensorsTest) { ASSERT_THAT(shape, testing::ElementsAre(20, 10)); } { - std::vector strides_a{320, 20, 1}; - std::vector strides_b{320, 20, 1}; - std::vector shape{10, 2, 20}; + TensorShapeVector strides_a{320, 20, 1}; + TensorShapeVector strides_b{320, 20, 1}; + TensorShapeVector shape{10, 2, 20}; CoalesceDimensions({strides_a, strides_b}, shape); @@ -177,9 +177,9 @@ TEST_F(CopyTest, CoalesceTensorsTest) { ASSERT_THAT(shape, testing::ElementsAre(10, 40)); } { - std::vector strides_a{3, 1}; - std::vector strides_b{6, 1}; - std::vector shape{5, 3}; + TensorShapeVector strides_a{3, 1}; + TensorShapeVector strides_b{6, 1}; + TensorShapeVector shape{5, 3}; CoalesceDimensions({strides_a, strides_b}, shape); @@ -188,9 +188,9 @@ TEST_F(CopyTest, CoalesceTensorsTest) { ASSERT_THAT(shape, testing::ElementsAre(5, 3)); } { - std::vector strides_a{3, 1}; - std::vector strides_b{6, 1}; - std::vector shape{5, 3}; + TensorShapeVector strides_a{3, 1}; + TensorShapeVector strides_b{6, 1}; + TensorShapeVector shape{5, 3}; CoalesceDimensions({strides_a, strides_b}, shape); @@ -199,9 +199,9 @@ TEST_F(CopyTest, CoalesceTensorsTest) { ASSERT_THAT(shape, testing::ElementsAre(5, 3)); } { - std::vector strides_a{4, 1}; - std::vector strides_b{1, 1}; - std::vector shape{4, 1}; + TensorShapeVector strides_a{4, 1}; + TensorShapeVector strides_b{1, 1}; + TensorShapeVector shape{4, 1}; CoalesceDimensions({strides_a, strides_b}, shape); diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index f74768aee7..0bc17e79ef 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -550,10 +550,10 @@ void OpTester::AddShapeToTensorData(NodeArg& node_arg, gsl::span } #if !defined(DISABLE_SPARSE_TENSORS) -static std::unique_ptr MakeSparseTensor(MLDataType data_type, const std::vector& dims) { +static std::unique_ptr MakeSparseTensor(MLDataType data_type, const gsl::span& dims) { TensorShape shape{dims}; auto allocator = test::AllocatorManager::Instance().GetAllocator(CPU); - auto p_tensor = std::make_unique(data_type, shape, allocator); + auto p_tensor = std::make_unique(data_type, shape, std::move(allocator)); return p_tensor; } @@ -563,7 +563,7 @@ void OpTester::CopyDataToTensor(gsl::span data, Tensor& dst) { } NodeArg OpTester::MakeSparseNodeArg(int32_t dtype, const char* name, - const std::vector& dims, const std::vector* dim_params) { + const gsl::span& dims, const std::vector* dim_params) { std::vector dims_for_proto = GetDimsForProto(dims); TSparseTensorProto type_proto(dtype, add_shape_to_tensor_data_ ? &dims_for_proto : nullptr); NodeArg node_arg(name, &type_proto.proto); @@ -585,7 +585,7 @@ void OpTester::AddSparseTensorData(std::vector& data, NodeArg node_arg, void OpTester::AddSparseCooTensorData(std::vector& data, MLDataType data_type, const char* name, - const std::vector& dims, + gsl::span dims, gsl::span values, gsl::span indices, const CheckParams& check_params, @@ -606,7 +606,7 @@ void OpTester::AddSparseCooTensorData(std::vector& data, void OpTester::AddSparseCooTensorStrings(std::vector& data, const char* name, - const std::vector& dims, + gsl::span dims, gsl::span values, gsl::span indices, const std::vector* dim_params) { @@ -629,7 +629,7 @@ void OpTester::AddSparseCooTensorStrings(std::vector& data, void OpTester::AddSparseCsrTensorData(std::vector& data, MLDataType data_type, const char* name, - const std::vector& dims, + gsl::span dims, gsl::span values, gsl::span inner_indices, gsl::span outer_indices, @@ -653,7 +653,7 @@ void OpTester::AddSparseCsrTensorData(std::vector& data, void OpTester::AddSparseCsrTensorStrings(std::vector& data, const char* name, - const std::vector& dims, + gsl::span dims, gsl::span values, gsl::span inner_indices, gsl::span outer_indices, @@ -856,9 +856,9 @@ std::vector OpTester::ExecuteModel( if (add_shape_to_tensor_data_) { auto out_shape_proto = expected_data.def_.Shape(); EXPECT_TRUE(out_shape_proto != nullptr); - const auto& tensor_shape = + const auto tensor_shape = utils::GetTensorShapeFromTensorShapeProto(*out_shape_proto); - const auto& inferred_dims = tensor_shape.GetDims(); + const auto inferred_dims = tensor_shape.GetDims(); const auto& expected_shape = expected_data.data_.Get().Shape(); EXPECT_TRUE(inferred_dims.size() == diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h index 65eb7c97e7..6958d44a35 100644 --- a/onnxruntime/test/providers/provider_test_utils.h +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -27,6 +27,7 @@ #include "gtest/gtest.h" #include #include "core/util/math_cpuonly.h" +#include namespace onnxruntime { class InferenceSession; @@ -123,9 +124,8 @@ constexpr ONNX_NAMESPACE::TensorProto_DataType TypeToDataType() { template struct TTypeProto { - TTypeProto(const std::vector* shape = nullptr) { + explicit TTypeProto(const std::vector* shape = nullptr) { proto.mutable_tensor_type()->set_elem_type(TypeToDataType()); - if (shape) { auto mutable_shape = proto.mutable_tensor_type()->mutable_shape(); for (auto i : *shape) { @@ -155,13 +155,13 @@ struct TSparseTensorProto { proto.mutable_sparse_tensor_type()->set_elem_type(dtype); if (shape) { auto m_shape = proto.mutable_sparse_tensor_type()->mutable_shape(); - for_each(shape->cbegin(), shape->cend(), [m_shape](int64_t v) { + for (int64_t v : *shape) { auto* m_dim = m_shape->add_dim(); if (v != -1) m_dim->set_dim_value(v); else m_dim->set_dim_param("symbolic"); - }); + } } } ONNX_NAMESPACE::TypeProto proto; @@ -227,7 +227,6 @@ const SequenceTensorTypeProto SequenceTensorType::s_sequence #if !defined(DISABLE_OPTIONAL_TYPE) -template struct OptionalTypeProto { OptionalTypeProto(const ONNX_NAMESPACE::TypeProto& type_proto) { proto.mutable_optional_type()->mutable_elem_type()->CopyFrom(type_proto); @@ -290,20 +289,56 @@ class OpTester { // We have an initializer_list and vector version of the Add functions because std::vector is specialized for // bool and we can't get the raw data out. So those cases must use an initializer_list + + // Dims variant is needed to reduce the number of overloads + // MS compiler refuses to create a gsl::span from initializer_list especially + // if it contains a single element + using DimsVariant = std::variant, TensorShapeVector>; + template - void AddInput(const char* name, const std::vector& dims, const std::initializer_list& values, + void AddInput(const char* name, std::initializer_list dims, std::initializer_list values, + bool is_initializer = false, const std::vector* dim_params = nullptr) { + const DimsVariant dims_var = std::vector(dims); + AddData(input_data_, name, dims_var, values.begin(), values.size(), is_initializer, false, dim_params); + } + + template + void AddInput(const char* name, std::initializer_list dims, const std::vector& values, + bool is_initializer = false, const std::vector* dim_params = nullptr) { + const DimsVariant dims_var = std::vector(dims); + AddData(input_data_, name, dims_var, values.data(), values.size(), is_initializer, false, dim_params); + } + + template + void AddInput(const char* name, std::initializer_list dims, const T* p_values, + const size_t size, bool is_initializer = false, + const std::vector* dim_params = nullptr) { + const DimsVariant dims_var = std::vector(dims); + AddData(input_data_, name, dims_var, p_values, size, is_initializer, false, dim_params); + } + + + template + void AddInput(const char* name, std::initializer_list dims, const TensorShapeVector& values, + bool is_initializer = false, const std::vector* dim_params = nullptr) { + const DimsVariant dims_var = std::vector(dims); + AddData(input_data_, name, dims_var, values.data(), values.size(), is_initializer, false, dim_params); + } + + template + void AddInput(const char* name, const DimsVariant& dims, std::initializer_list values, bool is_initializer = false, const std::vector* dim_params = nullptr) { AddData(input_data_, name, dims, values.begin(), values.size(), is_initializer, false, dim_params); } template - void AddInput(const char* name, const std::vector& dims, const std::vector& values, + void AddInput(const char* name, const DimsVariant& dims, const std::vector& values, bool is_initializer = false, const std::vector* dim_params = nullptr) { AddData(input_data_, name, dims, values.data(), values.size(), is_initializer, false, dim_params); } template - void AddInput(const char* name, const std::vector& dims, const T* p_values, + void AddInput(const char* name, const DimsVariant& dims, const T* p_values, const size_t size, bool is_initializer = false, const std::vector* dim_params = nullptr) { AddData(input_data_, name, dims, p_values, size, is_initializer, false, dim_params); @@ -449,7 +484,7 @@ class OpTester { #if !defined(DISABLE_OPTIONAL_TYPE) template - void AddOptionalTypeTensorInput(const char* name, const std::vector& dims, + void AddOptionalTypeTensorInput(const char* name, const DimsVariant& dims, const std::initializer_list* values = nullptr, bool is_initializer = false, const std::vector* dim_params = nullptr) { AddData(input_data_, name, dims, values ? values->begin() : nullptr, @@ -457,7 +492,17 @@ class OpTester { } template - void AddOptionalTypeTensorOutput(const char* name, const std::vector& dims, + void AddOptionalTypeTensorInput(const char* name, std::initializer_list dims, + const std::initializer_list* values = nullptr, + bool is_initializer = false, const std::vector* dim_params = nullptr) { + DimsVariant dims_var = std::vector(dims); + AddData(input_data_, name, dims_var, values ? values->begin() : nullptr, + values ? values->size() : 0, is_initializer, false, dim_params, 0.0f, 0.0f, true); + } + + + template + void AddOptionalTypeTensorOutput(const char* name, const DimsVariant& dims, const std::initializer_list* expected_values = nullptr, bool sort_output = false, float rel_error = 0.0f, float abs_error = 0.0f) { AddData(output_data_, name, dims, expected_values ? expected_values->begin() : nullptr, @@ -465,6 +510,17 @@ class OpTester { sort_output, nullptr /* dim_params */, rel_error, abs_error, true); } + template + void AddOptionalTypeTensorOutput(const char* name, std::initializer_list dims, + const std::initializer_list* expected_values = nullptr, + bool sort_output = false, float rel_error = 0.0f, float abs_error = 0.0f) { + DimsVariant dims_var = std::vector(dims); + AddData(output_data_, name, dims_var, expected_values ? expected_values->begin() : nullptr, + expected_values ? expected_values->size() : 0, false, + sort_output, nullptr /* dim_params */, rel_error, abs_error, true); + } + + template void AddOptionalTypeSeqInput(const char* name, const SeqTensors* seq_tensors) { @@ -504,7 +560,32 @@ class OpTester { } template - void AddOutput(const char* name, const std::vector& dims, const std::initializer_list& expected_values, + void AddOutput(const char* name, std::initializer_list dims, std::initializer_list expected_values, + bool sort_output = false, float rel_error = 0.0f, float abs_error = 0.0f) { + const DimsVariant dims_var = std::vector(dims); + AddData(output_data_, name, dims_var, expected_values.begin(), expected_values.size(), false, + sort_output, nullptr /* dim_params */, rel_error, abs_error); + } + + template + void AddOutput(const char* name, std::initializer_list dims, const std::vector& expected_values, + bool sort_output = false, float rel_error = 0.0f, float abs_error = 0.0f) { + const DimsVariant dims_var = std::vector(dims); + AddData(output_data_, name, dims_var, expected_values.data(), expected_values.size(), false, + sort_output, nullptr /* dim_params */, rel_error, abs_error); + } + + template + void AddOutput(const char* name, std::initializer_list dims, const T* p_values, const size_t size, + bool sort_output = false, float rel_error = 0.0f, float abs_error = 0.0f) { + const DimsVariant dims_var = std::vector(dims); + AddData(output_data_, name, dims, p_values, size, false, + sort_output, nullptr /* dim_params */, rel_error, abs_error); + } + + + template + void AddOutput(const char* name, const DimsVariant& dims, std::initializer_list expected_values, bool sort_output = false, float rel_error = 0.0f, float abs_error = 0.0f) { AddData(output_data_, name, dims, expected_values.begin(), expected_values.size(), false, sort_output, nullptr /* dim_params */, rel_error, abs_error); @@ -512,14 +593,21 @@ class OpTester { // This function doesn't work for vector because const vector cannot invoke its data(). template - void AddOutput(const char* name, const std::vector& dims, const std::vector& expected_values, + void AddOutput(const char* name, const DimsVariant& dims, const std::vector& expected_values, bool sort_output = false, float rel_error = 0.0f, float abs_error = 0.0f) { AddData(output_data_, name, dims, expected_values.data(), expected_values.size(), false, sort_output, nullptr /* dim_params */, rel_error, abs_error); } template - void AddOutput(const char* name, const std::vector& dims, const T* p_values, const size_t size, + void AddOutput(const char* name, const DimsVariant& dims, const TensorShapeVector& expected_values, + bool sort_output = false, float rel_error = 0.0f, float abs_error = 0.0f) { + AddData(output_data_, name, dims, expected_values.data(), expected_values.size(), false, + sort_output, nullptr /* dim_params */, rel_error, abs_error); + } + + template + void AddOutput(const char* name, const DimsVariant& dims, const T* p_values, const size_t size, bool sort_output = false, float rel_error = 0.0f, float abs_error = 0.0f) { AddData(output_data_, name, dims, p_values, size, false, sort_output, nullptr /* dim_params */, rel_error, abs_error); @@ -829,11 +917,27 @@ class OpTester { #endif protected: + gsl::span ToDimsSpan(const DimsVariant& dims_var) { + gsl::span result; + switch (dims_var.index()) { + case 0: + result = std::get<0>(dims_var); + break; + case 1: + result = std::get<1>(dims_var); + break; + default: + ORT_THROW("Unhandled dims variant"); + } + return result; + } + template - void AddData(std::vector& data, const char* name, gsl::span dims, const T* values, + void AddData(std::vector& data, const char* name, const DimsVariant& dims_var, const T* values, int64_t values_count, bool is_initializer = false, bool sort_output = false, const std::vector* dim_params = nullptr, float rel_error = 0.0f, float abs_error = 0.0f, bool is_optional_type_tensor = false) { + auto dims = ToDimsSpan(dims_var); #if defined(DISABLE_OPTIONAL_TYPE) if (is_optional_type_tensor) { ORT_THROW("Optional type is not supported in this build"); @@ -877,7 +981,7 @@ class OpTester { TTypeProto tensor_type_proto(add_shape_to_tensor_data_ ? &dims_for_proto : nullptr); #if !defined(DISABLE_OPTIONAL_TYPE) - OptionalTypeProto optional_type_proto(tensor_type_proto.proto); + OptionalTypeProto optional_type_proto(tensor_type_proto.proto); auto node_arg = NodeArg(name, !is_optional_type_tensor ? &tensor_type_proto.proto : &optional_type_proto.proto); #else auto node_arg = NodeArg(name, &tensor_type_proto.proto); @@ -960,7 +1064,7 @@ class OpTester { SequenceTensorTypeProto sequence_tensor_proto; #if !defined(DISABLE_OPTIONAL_TYPE) - OptionalTypeProto optional_type_proto(sequence_tensor_proto.proto); + OptionalTypeProto optional_type_proto(sequence_tensor_proto.proto); auto node_arg = NodeArg(name, !is_optional_sequence_tensor_type ? &sequence_tensor_proto.proto : &optional_type_proto.proto); @@ -979,13 +1083,13 @@ class OpTester { #if !defined(DISABLE_SPARSE_TENSORS) NodeArg MakeSparseNodeArg(int32_t dtype, const char* name, - const std::vector& dims, + const gsl::span& dims, const std::vector* dim_params); void AddSparseCooTensorData(std::vector& data, MLDataType data_type, const char* name, - const std::vector& dims, + gsl::span dims, gsl::span values, gsl::span indices, const CheckParams& check_params, @@ -993,7 +1097,7 @@ class OpTester { void AddSparseCooTensorStrings(std::vector& data, const char* name, - const std::vector& dims, + gsl::span dims, gsl::span values, gsl::span indices, const std::vector* dim_params = nullptr); @@ -1001,7 +1105,7 @@ class OpTester { void AddSparseCsrTensorData(std::vector& data, MLDataType data_type, const char* name, - const std::vector& dims, + gsl::span dims, gsl::span values, gsl::span inner_indices, gsl::span outer_indices, @@ -1010,7 +1114,7 @@ class OpTester { void AddSparseCsrTensorStrings(std::vector& data, const char* name, - const std::vector& dims, + gsl::span dims, gsl::span values, gsl::span inner_indices, gsl::span outer_indices, @@ -1098,5 +1202,13 @@ inline CheckParams MakeCheckParams(const OpTester::Data& d) { return CheckParams{d.sort_output_, d.absolute_error_, d.relative_error_}; } +inline std::vector GetShapeVector(const TensorShape& shape) { + std::vector result; + const auto dims = shape.GetDims(); + result.resize(dims.size()); + result.assign(dims.cbegin(), dims.cend()); + return result; +} + } // namespace test } // namespace onnxruntime diff --git a/orttraining/orttraining/core/graph/optimizer/adam_optimizer_builder.cc b/orttraining/orttraining/core/graph/optimizer/adam_optimizer_builder.cc index dd68be9ba2..c7507b7a89 100644 --- a/orttraining/orttraining/core/graph/optimizer/adam_optimizer_builder.cc +++ b/orttraining/orttraining/core/graph/optimizer/adam_optimizer_builder.cc @@ -47,7 +47,7 @@ Status AdamOptimizerBuilder::Build( const auto uc_state_it = initial_states.find(ADAM_UC_PREFIX); if (uc_state_it != initial_states.end()) { const auto& init_tensor = uc_state_it->second.Get(); - ORT_THROW_IF_ERROR(IsMatchingTypeAndShape(init_tensor, ONNX_NAMESPACE::TensorProto_DataType_INT64, {1})); + ORT_THROW_IF_ERROR(IsMatchingTypeAndShape(init_tensor, ONNX_NAMESPACE::TensorProto_DataType_INT64, TensorShapeVector{1})); uc_tensor_proto = utils::TensorToTensorProto(init_tensor, update_count_string); } else { uc_tensor_proto = CreateTensorProto(update_count_string, 1); diff --git a/orttraining/orttraining/core/graph/optimizer/lamb_optimizer_builder.cc b/orttraining/orttraining/core/graph/optimizer/lamb_optimizer_builder.cc index b89933ce1d..ffdd042fd6 100644 --- a/orttraining/orttraining/core/graph/optimizer/lamb_optimizer_builder.cc +++ b/orttraining/orttraining/core/graph/optimizer/lamb_optimizer_builder.cc @@ -71,7 +71,7 @@ Status LambOptimizerBuilder::Build( const auto step_state_it = shared_optim_state.find(LAMB_STEP_TENSOR_NAME); if (step_state_it != shared_optim_state.end()) { const auto& init_tensor = step_state_it->second.Get(); - ORT_THROW_IF_ERROR(IsMatchingTypeAndShape(init_tensor, ONNX_NAMESPACE::TensorProto_DataType_INT64, {1})); + ORT_THROW_IF_ERROR(IsMatchingTypeAndShape(init_tensor, ONNX_NAMESPACE::TensorProto_DataType_INT64, TensorShapeVector{1})); step_tensor_proto = utils::TensorToTensorProto(init_tensor, LAMB_STEP_TENSOR_NAME); } else { step_tensor_proto = CreateTensorProto(LAMB_STEP_TENSOR_NAME, 1); diff --git a/orttraining/orttraining/core/graph/optimizer_builder.cc b/orttraining/orttraining/core/graph/optimizer_builder.cc index 7861d2e90a..e6982ef69f 100644 --- a/orttraining/orttraining/core/graph/optimizer_builder.cc +++ b/orttraining/orttraining/core/graph/optimizer_builder.cc @@ -9,6 +9,10 @@ namespace onnxruntime { namespace training { +namespace training_internal { +const int64_t single_span_element = 1; +} + // Register all optimizers here. void OptimizerBuilderRegistry::RegisterBuilders() { GetInstance().Register("AdamOptimizer"); @@ -19,7 +23,14 @@ void OptimizerBuilderRegistry::RegisterBuilders() { Status IsMatchingTypeAndShape( const onnxruntime::Tensor& tensor, const int32_t element_type, - const std::vector& expected_shape_dims) { + std::initializer_list expected_dims) { + return IsMatchingTypeAndShape(tensor, element_type, gsl::make_span(expected_dims.begin(), expected_dims.end())); +} + +Status IsMatchingTypeAndShape( + const onnxruntime::Tensor& tensor, + const int32_t element_type, + gsl::span expected_shape_dims) { ORT_RETURN_IF_NOT(tensor.GetElementType() == element_type, "Type mismatch"); const TensorShape& tensor_shape = tensor.Shape(); TensorShape expected_shape(expected_shape_dims); diff --git a/orttraining/orttraining/core/graph/optimizer_builder.h b/orttraining/orttraining/core/graph/optimizer_builder.h index f5f614f9ee..6bbd024182 100644 --- a/orttraining/orttraining/core/graph/optimizer_builder.h +++ b/orttraining/orttraining/core/graph/optimizer_builder.h @@ -18,36 +18,73 @@ const std::vector MOMENTS_PREFIXES({"Moment_1", "Moment_2"}); const std::string LAMB_STEP_TENSOR_NAME = "Step"; const std::string ADAM_UC_PREFIX = "Update_Count"; +namespace training_internal { +extern const int64_t single_span_element; +} + template -ONNX_NAMESPACE::TensorProto CreateTensorProto( +inline ONNX_NAMESPACE::TensorProto CreateTensorProto( const std::string& name, - T val, - const std::vector& dims = {1}) { - size_t count = static_cast(std::accumulate(dims.begin(), dims.end(), int64_t(1), std::multiplies{})); - std::vector values(count, val); + const std::vector& values, + std::initializer_list dims) { + return CreateTensorProto(name, values, gsl::make_span(dims.begin(), dims.end())); +} + +template +inline ONNX_NAMESPACE::TensorProto CreateTensorProto( + const std::string& name, + const std::vector& values, + gsl::span dims = gsl::span(training_internal::single_span_element)) { + const size_t count = static_cast(std::accumulate(dims.cbegin(), dims.cend(), int64_t(1), std::multiplies{})); + ORT_ENFORCE(values.size() == count); ONNX_NAMESPACE::TensorProto tensor_proto = ONNX_NAMESPACE::ToTensor(values); tensor_proto.set_name(name); - std::for_each(dims.begin(), dims.end(), [&](auto dim) { tensor_proto.add_dims(dim); }); + std::for_each(dims.cbegin(), dims.cend(), [&](auto dim) { tensor_proto.add_dims(dim); }); return tensor_proto; } template -ONNX_NAMESPACE::TensorProto CreateTensorProto( +inline ONNX_NAMESPACE::TensorProto CreateTensorProto( const std::string& name, - const std::vector& values, - const std::vector& dims = {1}) { - size_t count = static_cast(std::accumulate(dims.begin(), dims.end(), int64_t(1), std::multiplies{})); - ORT_ENFORCE(values.size() == count); + gsl::span values_span, + gsl::span dims = gsl::span(training_internal::single_span_element)) { + std::vector values; + values.reserve(values_span.size()); + values.assign(values_span.cbegin(), values_span.cend()); + return CreateTensorProto(name, values, dims); +} + + +template +inline ONNX_NAMESPACE::TensorProto CreateTensorProto( + const std::string& name, + T val, + gsl::span dims = gsl::span(training_internal::single_span_element)) { + size_t count = static_cast(std::accumulate(dims.cbegin(), dims.cend(), int64_t(1), std::multiplies{})); + std::vector values(count, val); ONNX_NAMESPACE::TensorProto tensor_proto = ONNX_NAMESPACE::ToTensor(values); tensor_proto.set_name(name); - std::for_each(dims.begin(), dims.end(), [&](auto dim) { tensor_proto.add_dims(dim); }); + std::for_each(dims.cbegin(), dims.cend(), [&](auto dim) { tensor_proto.add_dims(dim); }); return tensor_proto; } +template +inline ONNX_NAMESPACE::TensorProto CreateTensorProto( + const std::string& name, + T val, + std::initializer_list dims) { + return CreateTensorProto(name, val, gsl::make_span(dims.begin(), dims.end())); +} + Status IsMatchingTypeAndShape( const onnxruntime::Tensor& tensor, const int32_t element_type, - const std::vector& expected_shape); + gsl::span expected_shape); + +Status IsMatchingTypeAndShape( + const onnxruntime::Tensor& tensor, + const int32_t element_type, + std::initializer_list expected_dims); /** * The configuration for optimizer builder. diff --git a/orttraining/orttraining/core/graph/zero_optimizer_graph_builder.cc b/orttraining/orttraining/core/graph/zero_optimizer_graph_builder.cc index c7d4bb51c7..c44b593d78 100644 --- a/orttraining/orttraining/core/graph/zero_optimizer_graph_builder.cc +++ b/orttraining/orttraining/core/graph/zero_optimizer_graph_builder.cc @@ -168,7 +168,7 @@ static std::vector AddViewForParameter( ArgDef shape_argdef(argdef.name + "_view_shape_" + std::to_string(view_num), graph_defs.CreateTypeProto({dims}, ONNX_NAMESPACE::TensorProto_DataType_INT64)); - graph_defs.AddInitializers({CreateTensorProto(shape_argdef.name, shape.GetDimsAsVector(), {dims})}); + graph_defs.AddInitializers({CreateTensorProto(shape_argdef.name, shape.AsShapeVector(), {dims})}); auto dtype = static_cast(argdef.type_proto->tensor_type().elem_type()); ArgDef view_argdef(GetViewName(argdef.name, view_num), @@ -360,7 +360,7 @@ static Status ModifyParametersForOptimizerPartitioning( new_weight_argdefs.push_back(weight_argdef); new_gradient_argdefs.push_back(gradient_argdef); } else { - weight_partition_info[weight_argdef.name].original_dim = tensor_shape.GetDimsAsVector(); + weight_partition_info[weight_argdef.name].original_dim = tensor_shape.AsShapeVector(); if (offset < rank_start && offset + tensor_count <= rank_end) { int64_t size_for_previous_rank = rank_start - offset; int64_t size_for_current_rank = offset + tensor_count - rank_start; diff --git a/orttraining/orttraining/core/optimizer/megatron_transformer.cc b/orttraining/orttraining/core/optimizer/megatron_transformer.cc index 7e7ebe54a8..494ca5a381 100644 --- a/orttraining/orttraining/core/optimizer/megatron_transformer.cc +++ b/orttraining/orttraining/core/optimizer/megatron_transformer.cc @@ -150,11 +150,11 @@ bool MegatronTransformer::PartitionWeightByColumn(const Graph& graph, const Node if (rank == 2 && utils::HasDimValue(shape->dim(0)) && utils::HasDimValue(shape->dim(1))) { row_count = shape->dim(0).dim_value(); column_count = shape->dim(1).dim_value(); - weight_partition_info_[original_name].original_dim = std::vector{row_count, column_count}; + weight_partition_info_[original_name].original_dim = TensorShapeVector{row_count, column_count}; } else if (rank == 1) { row_count = 1; column_count = shape->dim(0).dim_value(); - weight_partition_info_[original_name].original_dim = std::vector{column_count}; + weight_partition_info_[original_name].original_dim = TensorShapeVector{column_count}; } else { LOGS_DEFAULT(WARNING) << "Initializer tensor's rank is " << rank << " (expected to be 1 or 2)."; return false; @@ -291,11 +291,11 @@ bool MegatronTransformer::PartitionWeightByRow(const Graph& graph, const NodeArg if (rank == 2 && utils::HasDimValue(shape->dim(0)) && utils::HasDimValue(shape->dim(1))) { row_count = shape->dim(0).dim_value(); column_count = shape->dim(1).dim_value(); - weight_partition_info_[original_name].original_dim = std::vector{row_count, column_count}; + weight_partition_info_[original_name].original_dim = {row_count, column_count}; } else if (rank == 1) { row_count = shape->dim(0).dim_value(); column_count = 1; - weight_partition_info_[original_name].original_dim = std::vector{row_count}; + weight_partition_info_[original_name].original_dim = {row_count}; } else { LOGS_DEFAULT(WARNING) << "Initializer tensor's rank is more than " << rank << " (expected to be 1 or 2)."; diff --git a/orttraining/orttraining/core/session/tensor_helper.cc b/orttraining/orttraining/core/session/tensor_helper.cc index 6957e1b552..0734c55c7c 100644 --- a/orttraining/orttraining/core/session/tensor_helper.cc +++ b/orttraining/orttraining/core/session/tensor_helper.cc @@ -11,7 +11,7 @@ void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const O namespace training { // Return the shape of a tensor slice. -std::vector GetSliceShape( +TensorShapeVector GetSliceShape( gsl::span shape, // before-slicing tensor shape const size_t slice_axis, // axis to slice along const size_t num_slices) { // number of slices along the slicing axis @@ -22,7 +22,7 @@ std::vector GetSliceShape( ORT_ENFORCE(shape.at(slice_axis) % num_slices == 0); // Shape of slice along slice_axis. - std::vector slice_shape(shape.size()); + TensorShapeVector slice_shape(shape.size()); // Compute original slice's shape. std::copy(shape.begin(), shape.end(), slice_shape.begin()); // Replace the sliced dimension. @@ -255,7 +255,7 @@ OrtValue ConcatenateTensors( // Concatenated tensors in CPU buffers. std::vector cpu_values; // Result tensor's shape. - std::vector new_shape = orig_values.front().Get().Shape().GetDimsAsVector(); + TensorShapeVector new_shape = orig_values.front().Get().Shape().AsShapeVector(); // Tensor elements' type. MLDataType elem_type = orig_values.front().Get().DataType(); int64_t new_dim = 0; diff --git a/orttraining/orttraining/core/session/training_session.h b/orttraining/orttraining/core/session/training_session.h index fb184051bc..1150241dae 100644 --- a/orttraining/orttraining/core/session/training_session.h +++ b/orttraining/orttraining/core/session/training_session.h @@ -34,7 +34,7 @@ class TrainingSession : public InferenceSession { */ struct PartitionInfo { // value of the original shape of the weight - std::vector original_dim; + TensorShapeVector original_dim; // indicates whether weight was megatron partitioned or not. // -1: not partitioned; 0: column partitioned; 1: row partitioned int megatron_row_partition = -1; diff --git a/orttraining/orttraining/eager/ort_tensor.cpp b/orttraining/orttraining/eager/ort_tensor.cpp index 72310817df..acdedbac68 100644 --- a/orttraining/orttraining/eager/ort_tensor.cpp +++ b/orttraining/orttraining/eager/ort_tensor.cpp @@ -80,12 +80,13 @@ int64_t ORTTensorImpl::size(int64_t d) const { void ORTTensorImpl::cacheSizeMetadata() { // TODO: wrap with change generation guard auto& tensor = tensor_.Get(); - auto shape = tensor.Shape(); - auto strides = GetStrides(shape.GetDims()); + const auto& shape = tensor.Shape(); + const auto dims = shape.GetDims(); + auto strides = GetStrides(dims); numel_ = shape.Size(); - sizes_and_strides_.set_sizes(shape.GetDimsAsVector()); + sizes_and_strides_.set_sizes(c10::IntArrayRef(dims.data(), dims.size())); for (std::size_t i = 0; i < strides.size(); i++) { sizes_and_strides_.stride_at_unchecked(i) = strides[i]; diff --git a/orttraining/orttraining/models/runner/training_util.cc b/orttraining/orttraining/models/runner/training_util.cc index 74f6c100f5..4141225751 100644 --- a/orttraining/orttraining/models/runner/training_util.cc +++ b/orttraining/orttraining/models/runner/training_util.cc @@ -99,7 +99,7 @@ std::vector DataSet::GetKthBatch(size_t batch_size, size_t k_th, Alloc const Tensor& first_tensor = data_[0]->at(input_index).Get(); MLDataType element_type = first_tensor.DataType(); - std::vector shape_vector = first_tensor.Shape().GetDimsAsVector(); + auto shape_vector = first_tensor.Shape().AsShapeVector(); if (first_tensor.Shape().Size() > 1) { shape_vector.insert(shape_vector.begin(), batch_size); } else { @@ -108,7 +108,7 @@ std::vector DataSet::GetKthBatch(size_t batch_size, size_t k_th, Alloc } AllocatorPtr alloc = allocator ? allocator : TrainingUtil::GetCpuAllocator(); - auto p_tensor = std::make_unique(element_type, shape_vector, alloc); + auto p_tensor = std::make_unique(element_type, shape_vector, std::move(alloc)); void* buffer = p_tensor->MutableDataRaw(); size_t memory_size_per_sample = first_tensor.SizeInBytes(); diff --git a/orttraining/orttraining/test/gradient/gradient_checker.cc b/orttraining/orttraining/test/gradient/gradient_checker.cc index 0bb1daf71f..3013f2686a 100644 --- a/orttraining/orttraining/test/gradient/gradient_checker.cc +++ b/orttraining/orttraining/test/gradient/gradient_checker.cc @@ -78,25 +78,25 @@ inline std::vector GradientChecker::EvaluateFunctionA if (x_infos[data_index].data_type == DataTypeImpl::GetTensorType()) { std::vector int64_data(data.size()); std::transform(data.begin(), data.end(), int64_data.begin(), [](X_T x) { return static_cast(x); }); - op_session.AddInput(name.c_str(), x_infos[data_index].shape.GetDimsAsVector(), int64_data); + op_session.AddInput(name.c_str(), x_infos[data_index].shape.AsShapeVector(), int64_data); } else if (x_infos[data_index].data_type == DataTypeImpl::GetTensorType()) { std::vector int32_data(data.size()); std::transform(data.begin(), data.end(), int32_data.begin(), [](X_T x) { return static_cast(x); }); - op_session.AddInput(name.c_str(), x_infos[data_index].shape.GetDimsAsVector(), int32_data); + op_session.AddInput(name.c_str(), x_infos[data_index].shape.AsShapeVector(), int32_data); } else if (x_infos[data_index].data_type == DataTypeImpl::GetTensorType()) { std::unique_ptr p_data(new bool[data.size()]); for (size_t i = 0; i < data.size(); ++i) { p_data[i] = static_cast(data[i]); } - op_session.AddInput(name.c_str(), x_infos[data_index].shape.GetDimsAsVector(), p_data.get(), data.size()); + op_session.AddInput(name.c_str(), x_infos[data_index].shape.AsShapeVector(), p_data.get(), data.size()); } else { - op_session.AddInput(name.c_str(), x_infos[data_index].shape.GetDimsAsVector(), data); + op_session.AddInput(name.c_str(), x_infos[data_index].shape.AsShapeVector(), data); } } for (size_t data_index = 0; data_index < y_infos.size(); data_index++) { std::string name = "output" + std::to_string(data_index); - op_session.AddOutput(name.c_str(), y_infos[data_index].shape.GetDimsAsVector(), (*y_datas)[data_index]); + op_session.AddOutput(name.c_str(), y_infos[data_index].shape.AsShapeVector(), (*y_datas)[data_index]); } op_session.Run(); return op_session.GetFetches(); @@ -142,25 +142,25 @@ inline Status GradientChecker::ComputeTheoreticalJacobianTransp if (x_infos[data_index].data_type == DataTypeImpl::GetTensorType()) { std::vector int64_data(data.size()); std::transform(data.begin(), data.end(), int64_data.begin(), [](X_T x) { return static_cast(x); }); - op_session.AddInput(name.c_str(), x_infos[data_index].shape.GetDimsAsVector(), int64_data); + op_session.AddInput(name.c_str(), x_infos[data_index].shape.AsShapeVector(), int64_data); } else if (x_infos[data_index].data_type == DataTypeImpl::GetTensorType()) { std::vector int32_data(data.size()); std::transform(data.begin(), data.end(), int32_data.begin(), [](X_T x) { return static_cast(x); }); - op_session.AddInput(name.c_str(), x_infos[data_index].shape.GetDimsAsVector(), int32_data); + op_session.AddInput(name.c_str(), x_infos[data_index].shape.AsShapeVector(), int32_data); } else if (x_infos[data_index].data_type == DataTypeImpl::GetTensorType()) { std::unique_ptr p_data(new bool[data.size()]); for (size_t i = 0; i < data.size(); ++i) { p_data[i] = static_cast(data[i]); } - op_session.AddInput(name.c_str(), x_infos[data_index].shape.GetDimsAsVector(), p_data.get(), data.size()); + op_session.AddInput(name.c_str(), x_infos[data_index].shape.AsShapeVector(), p_data.get(), data.size()); } else { - op_session.AddInput(name.c_str(), x_infos[data_index].shape.GetDimsAsVector(), data); + op_session.AddInput(name.c_str(), x_infos[data_index].shape.AsShapeVector(), data); } } for (size_t data_index = 0; data_index < y_num; data_index++) { std::string name = "output" + std::to_string(data_index); - op_session.AddOutput(name.c_str(), y_infos[data_index].shape.GetDimsAsVector(), (*y_datas)[data_index]); + op_session.AddOutput(name.c_str(), y_infos[data_index].shape.AsShapeVector(), (*y_datas)[data_index]); } // While calculating theoritical jacobian transpose we calculate the gradient by @@ -215,7 +215,7 @@ inline Status GradientChecker::InitOpTesterWithGraph( std::vector int64_data(data.size()); std::transform(data.begin(), data.end(), int64_data.begin(), [](X_T x) { return static_cast(x); }); op_session.AddInput(name.c_str(), - x_infos[data_index].shape.GetDimsAsVector(), + x_infos[data_index].shape.AsShapeVector(), int64_data, false, &x_infos[data_index].dim_params); @@ -223,7 +223,7 @@ inline Status GradientChecker::InitOpTesterWithGraph( std::vector int32_data(data.size()); std::transform(data.begin(), data.end(), int32_data.begin(), [](X_T x) { return static_cast(x); }); op_session.AddInput(name.c_str(), - x_infos[data_index].shape.GetDimsAsVector(), + x_infos[data_index].shape.AsShapeVector(), int32_data, false, &x_infos[data_index].dim_params); @@ -233,14 +233,14 @@ inline Status GradientChecker::InitOpTesterWithGraph( p_data[i] = static_cast(data[i]); } op_session.AddInput(name.c_str(), - x_infos[data_index].shape.GetDimsAsVector(), + x_infos[data_index].shape.AsShapeVector(), p_data.get(), data.size(), false, &x_infos[data_index].dim_params); } else { op_session.AddInput(name.c_str(), - x_infos[data_index].shape.GetDimsAsVector(), + x_infos[data_index].shape.AsShapeVector(), data, false, &x_infos[data_index].dim_params); @@ -255,10 +255,10 @@ inline Status GradientChecker::InitOpTesterWithGraph( std::vector int64_data(data.size()); std::transform(data.begin(), data.end(), int64_data.begin(), [](Y_T x) { return static_cast(x); }); op_session.AddOutput(name.c_str(), - y_infos[data_index].shape.GetDimsAsVector(), + y_infos[data_index].shape.AsShapeVector(), int64_data); } else { - op_session.AddOutput(name.c_str(), y_infos[data_index].shape.GetDimsAsVector(), data); + op_session.AddOutput(name.c_str(), y_infos[data_index].shape.AsShapeVector(), data); } } // Currently only allows setting int attributes to zero. TODO: Expand this diff --git a/orttraining/orttraining/test/gradient/gradient_checker.h b/orttraining/orttraining/test/gradient/gradient_checker.h index 5cd9c8dc05..6a857c298e 100644 --- a/orttraining/orttraining/test/gradient/gradient_checker.h +++ b/orttraining/orttraining/test/gradient/gradient_checker.h @@ -23,12 +23,12 @@ namespace onnxruntime { namespace test { struct TensorInfo { - TensorInfo(const std::initializer_list& shape, + TensorInfo(std::initializer_list shape_init, bool has_gradient = true, std::function* transformer = nullptr, MLDataType data_type = DataTypeImpl::GetTensorType(), const std::vector& dim_params = std::vector{}) - : shape(shape), + : shape(gsl::make_span(shape_init.begin(), shape_init.end())), has_gradient(has_gradient), transformer(transformer), data_type(data_type), diff --git a/orttraining/orttraining/test/gradient/gradient_op_test_utils.cc b/orttraining/orttraining/test/gradient/gradient_op_test_utils.cc index 809545cd2e..575bbe44a8 100644 --- a/orttraining/orttraining/test/gradient/gradient_op_test_utils.cc +++ b/orttraining/orttraining/test/gradient/gradient_op_test_utils.cc @@ -270,7 +270,7 @@ void GradientOpTester::FillFeedsAndOutputNames(std::unordered_map(i)) { values[data_index_of_output] = 1.0; //set only one value to one to construct jacobian matrix } - AddData(gradient_data, (output_data_[i].def_.Name() + "_grad").c_str(), shape.GetDims(), values.data(), values.size(), true); + AddData(gradient_data, (output_data_[i].def_.Name() + "_grad").c_str(), shape.AsShapeVector(), values.data(), values.size(), true); } for (size_t i = 0; i < gradient_data.size(); ++i) { diff --git a/orttraining/orttraining/test/gradient/gradient_ops_test.cc b/orttraining/orttraining/test/gradient/gradient_ops_test.cc index 4511336435..3b7818a252 100644 --- a/orttraining/orttraining/test/gradient/gradient_ops_test.cc +++ b/orttraining/orttraining/test/gradient/gradient_ops_test.cc @@ -90,8 +90,8 @@ static void RunReductionTests(const OpDef& op_def, float max_error; for (size_t i = 0; i < x_shapes.size(); i++) { max_error = 0; - TensorShape x_shape = x_shapes[i]; - TensorShape y_shape = y_shapes[i]; + TensorShape x_shape(gsl::make_span(x_shapes[i])); + TensorShape y_shape(gsl::make_span(y_shapes[i])); std::vector axes = axes_vec[i]; std::vector> x_datas; RandomValueGenerator random{}; @@ -101,7 +101,8 @@ static void RunReductionTests(const OpDef& op_def, if (keepdims_ip[i] != -1) attributes.push_back(MakeAttribute("keepdims", keepdims_ip[i])); if (axes_as_input) { std::vector axes_float; - std::transform(begin(axes), end(axes), std::back_inserter(axes_float), [](int64_t i) { return static_cast(i); }); + axes_float.reserve(axes.size()); + std::transform(std::begin(axes), std::end(axes), std::back_inserter(axes_float), [](int64_t i) { return static_cast(i); }); TensorInfo axes_info({static_cast(axes.size())}, false, nullptr, DataTypeImpl::GetTensorType()); input.push_back(axes_info); x_datas.push_back(axes_float); @@ -1327,8 +1328,8 @@ static void RunSqueezeUnsqueezeTests(const OpDef& op_def, float error_tolerance = 1e-3f; for (size_t i = 0; i < x_shapes.size(); i++) { - TensorShape x_shape = x_shapes[i]; - TensorShape y_shape = y_shapes[i]; + TensorShape x_shape(gsl::make_span(x_shapes[i])); + TensorShape y_shape(gsl::make_span(y_shapes[i])); std::vector axes = axes_ip[i]; std::vector> x_datas; RandomValueGenerator random{}; @@ -1650,7 +1651,7 @@ void TestSparseSoftmaxCrossEntropyGrad(const TensorShape& index_shape, const std // without weight { - std::vector logit_shape(index_shape.GetDimsAsVector()); + auto logit_shape(index_shape.AsShapeVector()); logit_shape.emplace_back(D); TensorInfo x_info(logit_shape); @@ -1664,7 +1665,7 @@ void TestSparseSoftmaxCrossEntropyGrad(const TensorShape& index_shape, const std // with weight { - std::vector logit_shape(index_shape.GetDimsAsVector()); + auto logit_shape(index_shape.AsShapeVector()); logit_shape.emplace_back(D); TensorInfo x_info(logit_shape); @@ -1712,7 +1713,7 @@ void TestSoftmaxCrossEntropyLossGrad(const TensorShape& index_shape, //label_sh // without weight and ignore_index { - std::vector logit_shape(index_shape.GetDimsAsVector()); + TensorShapeVector logit_shape(index_shape.AsShapeVector()); auto it = logit_shape.begin() + 1; logit_shape.insert(it, D); TensorInfo loss_info = {}; @@ -1732,7 +1733,7 @@ void TestSoftmaxCrossEntropyLossGrad(const TensorShape& index_shape, //label_sh // with weight and no ignore_index { - std::vector logit_shape(index_shape.GetDimsAsVector()); + TensorShapeVector logit_shape(index_shape.AsShapeVector()); auto it = logit_shape.begin() + 1; logit_shape.insert(it, D); TensorInfo loss_info = {}; @@ -1753,7 +1754,7 @@ void TestSoftmaxCrossEntropyLossGrad(const TensorShape& index_shape, //label_sh // without weight and ignore index { - std::vector logit_shape(index_shape.GetDimsAsVector()); + TensorShapeVector logit_shape(index_shape.AsShapeVector()); auto it = logit_shape.begin() + 1; logit_shape.insert(it, D); TensorInfo loss_info = {}; @@ -1773,7 +1774,7 @@ void TestSoftmaxCrossEntropyLossGrad(const TensorShape& index_shape, //label_sh // with weight and ignore_index { - std::vector logit_shape(index_shape.GetDimsAsVector()); + TensorShapeVector logit_shape(index_shape.AsShapeVector()); auto it = logit_shape.begin() + 1; logit_shape.insert(it, D); TensorInfo loss_info = {}; @@ -1930,11 +1931,11 @@ void TestDropoutOp(float ratio, TensorShape& x_shape, bool default_ratio = true) std::vector x_data(x_shape.Size(), input_constant); std::vector y_data(x_shape.Size(), 3.0f); - test.AddInput("x", x_shape.GetDimsAsVector(), x_data); + test.AddInput("x", x_shape.AsShapeVector(), x_data); if (!default_ratio) test.AddInput("ratio", {}, {ratio}); - test.AddOutput("y", x_shape.GetDimsAsVector(), y_data); - test.AddOutput("mask", x_shape.GetDimsAsVector(), {true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true}); + test.AddOutput("y", x_shape.AsShapeVector(), y_data); + test.AddOutput("mask", x_shape.AsShapeVector(), {true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true}); test.Run(); //Check output @@ -1975,9 +1976,9 @@ void TestDropoutGradOp(float ratio, TensorShape& x_shape, bool default_ratio = t output_constant, 0, output_constant, 0, output_constant, 0, output_constant, 0}); - test.AddInput("dy", x_shape.GetDimsAsVector(), dy_data); + test.AddInput("dy", x_shape.AsShapeVector(), dy_data); - test.AddInput("mask", x_shape.GetDimsAsVector(), {true, true, true, false, // + test.AddInput("mask", x_shape.AsShapeVector(), {true, true, true, false, // true, false, true, false, // true, false, true, false, // true, false, true, false}); @@ -1989,7 +1990,7 @@ void TestDropoutGradOp(float ratio, TensorShape& x_shape, bool default_ratio = t test.AddInput("training_mode", {}, {true}); - test.AddOutput("dx", x_shape.GetDimsAsVector(), dx_data); + test.AddOutput("dx", x_shape.AsShapeVector(), dx_data); test.Run(); } @@ -2215,7 +2216,7 @@ TEST(GradientCheckerTest, WhereGrad) { GradientChecker gradient_checker; OpDef op_def{"Where"}; - std::vector shape{4, 3, 2}; + TensorShapeVector shape{4, 3, 2}; TensorInfo x_info(shape), y_info(shape); std::function transformer = [](float x) { return static_cast(std::fmod(std::fabs(x), 1.0f) > 0.5f); diff --git a/orttraining/orttraining/test/training_ops/cpu/activation/activation_op_test.cc b/orttraining/orttraining/test/training_ops/cpu/activation/activation_op_test.cc index d4a52f6078..4ee8584737 100644 --- a/orttraining/orttraining/test/training_ops/cpu/activation/activation_op_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/activation/activation_op_test.cc @@ -215,16 +215,16 @@ void TestBiasGeluGradBroadcastBias(const std::string& op, int opset_version, con const std::vector dY(input_size, 1.0f); const std::vector B = ValueRange(bias_size, 1.0f); - test.AddInput("dY", input_shape.GetDimsAsVector(), dY); - test.AddInput("X", input_shape.GetDimsAsVector(), X); - test.AddInput("B", bias_shape.GetDimsAsVector(), B); + test.AddInput("dY", input_shape.AsShapeVector(), dY); + test.AddInput("X", input_shape.AsShapeVector(), X); + test.AddInput("B", bias_shape.AsShapeVector(), B); std::vector expected_dX{}; for (int64_t i = 0; i < input_size; ++i) { expected_dX.push_back(compute_gelu_grad_scalar_fn(dY[i], X[i] + B[i % bias_size])); } - test.AddOutput("dX", input_shape.GetDimsAsVector(), expected_dX); + test.AddOutput("dX", input_shape.AsShapeVector(), expected_dX); test.Run(); } diff --git a/orttraining/orttraining/test/training_ops/cpu/nn/dropout_op_test.cc b/orttraining/orttraining/test/training_ops/cpu/nn/dropout_op_test.cc index 226f82da49..e3bebf629a 100644 --- a/orttraining/orttraining/test/training_ops/cpu/nn/dropout_op_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/nn/dropout_op_test.cc @@ -189,8 +189,8 @@ void RunDropoutGradTest(float ratio, const std::vector& input_dims, boo mask_buffer.get(), mask_buffer.get() + input_shape.Size(), std::back_inserter(dx_data), [output_constant](bool mask_value) { return mask_value ? output_constant : 0.0f; }); - test.AddInput("dy", input_shape.GetDimsAsVector(), dy_data); - test.AddInput("mask", input_shape.GetDimsAsVector(), mask_buffer.get(), input_shape.Size()); + test.AddInput("dy", input_shape.AsShapeVector(), dy_data); + test.AddInput("mask", input_shape.AsShapeVector(), mask_buffer.get(), input_shape.Size()); if (!default_ratio) { test.AddInput("ratio", {1}, ratio_data); } else { @@ -198,7 +198,7 @@ void RunDropoutGradTest(float ratio, const std::vector& input_dims, boo } test.AddInput("training_mode", {}, {true}); - test.AddOutput("dx", input_shape.GetDimsAsVector(), dx_data); + test.AddOutput("dx", input_shape.AsShapeVector(), dx_data); test.Run(); } } // namespace diff --git a/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc b/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc index f955b7abb4..d7d652358e 100644 --- a/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc @@ -53,7 +53,7 @@ void ConfigureGatherGradRandomDataOpTester( ASSERT_LT(static_cast(axis), X_shape.NumDimensions()); const TensorShape dY_shape = [&]() { - std::vector dY_dims = X_shape.GetDimsAsVector(); + TensorShapeVector dY_dims = X_shape.AsShapeVector(); auto it = dY_dims.erase(dY_dims.begin() + axis); dY_dims.insert( it, indices_shape.GetDims().begin(), indices_shape.GetDims().end()); @@ -66,11 +66,13 @@ void ConfigureGatherGradRandomDataOpTester( const auto output = CalculateOutput(axis, X_shape, grad, indices); test.AddAttribute("axis", axis); + //auto shape_dims = X_shape.GetDims(); + //std::vector v_dims(shape_dims.cbegin(), shape_dims.cend()); test.AddInput( - "shape", {static_cast(X_shape.NumDimensions())}, X_shape.GetDimsAsVector()); - test.AddInput("indices", indices_shape.GetDimsAsVector(), indices); - test.AddInput("grad", dY_shape.GetDimsAsVector(), grad); - test.AddOutput("output", X_shape.GetDimsAsVector(), output); + "shape", {static_cast(X_shape.NumDimensions())}, X_shape.AsShapeVector()); + test.AddInput("indices", indices_shape.AsShapeVector(), indices); + test.AddInput("grad", dY_shape.AsShapeVector(), grad); + test.AddOutput("output", X_shape.AsShapeVector(), output); } template diff --git a/orttraining/orttraining/training_ops/cpu/aten_ops/aten_op.cc b/orttraining/orttraining/training_ops/cpu/aten_ops/aten_op.cc index 8be5b15317..81c54afa4b 100644 --- a/orttraining/orttraining/training_ops/cpu/aten_ops/aten_op.cc +++ b/orttraining/orttraining/training_ops/cpu/aten_ops/aten_op.cc @@ -39,7 +39,7 @@ bool IsATenOperatorExecutorInitialized() { return aten_ops::ATenOperatorExecutor::Instance().IsInitialized(); } -Status ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const std::vector& axes, bool keepdims) { +Status ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const gsl::span& axes, bool keepdims) { ORT_ENFORCE(aten_ops::ATenOperatorExecutor::Instance().IsInitialized() && !axes.empty()); std::vector dlpacks; auto* p_ctx_internal = static_cast(p_ctx); @@ -47,8 +47,8 @@ Status ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const std::vector dlpacks.emplace_back(dlpack::OrtValueToDlpack(ort_value)); OrtValue axes_tensor; OrtValue keepdims_tensor; - std::vector axes_tensor_shape(1, static_cast(axes.size())); - std::vector keepdims_tensor_shape(1, 1); + TensorShapeVector axes_tensor_shape(1, static_cast(axes.size())); + TensorShapeVector keepdims_tensor_shape(1, 1); auto ml_tensor = DataTypeImpl::GetType(); OrtMemoryInfo info("Cpu", OrtDeviceAllocator); axes_tensor.Init(new Tensor(DataTypeImpl::GetType(), axes_tensor_shape, diff --git a/orttraining/orttraining/training_ops/cpu/aten_ops/aten_op.h b/orttraining/orttraining/training_ops/cpu/aten_ops/aten_op.h index d7b802f77e..3ab071a271 100644 --- a/orttraining/orttraining/training_ops/cpu/aten_ops/aten_op.h +++ b/orttraining/orttraining/training_ops/cpu/aten_ops/aten_op.h @@ -23,7 +23,7 @@ class ATenOp : public OpKernel { }; bool IsATenOperatorExecutorInitialized(); -Status ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const std::vector& axes, bool keepdims); +Status ExecuteReduceSumATenOp(OpKernelContext* p_ctx, const gsl::span& axes, bool keepdims); } // namespace contrib } // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/cpu/loss/softmax_cross_entropy_loss.cc b/orttraining/orttraining/training_ops/cpu/loss/softmax_cross_entropy_loss.cc index 5fc3ae1b49..73fddefc33 100644 --- a/orttraining/orttraining/training_ops/cpu/loss/softmax_cross_entropy_loss.cc +++ b/orttraining/orttraining/training_ops/cpu/loss/softmax_cross_entropy_loss.cc @@ -72,7 +72,7 @@ void VerifyLogitWeightAndLabelShape(const TensorShape& logit_shape, } } -void GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, std::vector& new_shape, +void GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, TensorShapeVector& new_shape, std::vector& permutations) { if (ncd_to_ndc) { new_shape.emplace_back(tensor_shape[0]); @@ -118,7 +118,7 @@ Status SoftmaxCrossEntropyLoss::Compute(OpKernelContext* context) const GetNDCFromLogitAndLabelShape(logit_shape, label_shape, N_D, C); const T1* logit_data = logit.template Data(); OrtValue transpose_output; - std::vector new_shape; + TensorShapeVector new_shape; std::vector permutations; AllocatorPtr alloc; @@ -272,7 +272,7 @@ Status SoftmaxCrossEntropyLossGrad::Compute(OpKernelContext* context) co T1* d_logit_data = d_logit->template MutableData(); std::memset(d_logit_data, 0, sizeof(T1) * n_d); OrtValue transpose_output; - std::vector new_shape; + TensorShapeVector new_shape; std::vector permutations; AllocatorPtr alloc; diff --git a/orttraining/orttraining/training_ops/cpu/loss/softmax_cross_entropy_loss.h b/orttraining/orttraining/training_ops/cpu/loss/softmax_cross_entropy_loss.h index 64a355019c..04ff509ce7 100644 --- a/orttraining/orttraining/training_ops/cpu/loss/softmax_cross_entropy_loss.h +++ b/orttraining/orttraining/training_ops/cpu/loss/softmax_cross_entropy_loss.h @@ -44,7 +44,7 @@ void VerifyLogitWeightAndLabelShape(const TensorShape& logit_shape, const Tensor const TensorShape* weight_shape); void GetNDCFromLogitAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, int64_t& N_D, int64_t& C); -void GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, std::vector& new_shape, +void GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, TensorShapeVector& new_shape, std::vector& permutations); } // namespace contrib diff --git a/orttraining/orttraining/training_ops/cpu/nn/conv_grad.cc b/orttraining/orttraining/training_ops/cpu/nn/conv_grad.cc index f9d1652e89..a2e5a93eba 100644 --- a/orttraining/orttraining/training_ops/cpu/nn/conv_grad.cc +++ b/orttraining/orttraining/training_ops/cpu/nn/conv_grad.cc @@ -38,19 +38,19 @@ Status ConvGrad::Compute(OpKernelContext* context) const { ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X, W)); // Copied from conv_impl.h, maybe refactor - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape)); size_t kernel_rank = kernel_shape.size(); - std::vector pads(conv_attrs_.pads); + ConvAttributes::ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(kernel_rank * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(kernel_rank, 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(kernel_rank, 1); } diff --git a/orttraining/orttraining/training_ops/cpu/tensor/concat.cc b/orttraining/orttraining/training_ops/cpu/tensor/concat.cc index 54ed679657..4639adce67 100644 --- a/orttraining/orttraining/training_ops/cpu/tensor/concat.cc +++ b/orttraining/orttraining/training_ops/cpu/tensor/concat.cc @@ -23,7 +23,7 @@ Status ConcatTraining::Compute(OpKernelContext* ctx) const { auto input_count = Node().InputArgCount().front(); // Hold pointers to the input tensors to be used in the PrepareForCompute() step - std::vector input_tensors; + InlinedTensorsVector input_tensors; input_tensors.reserve(input_count); for (int i = 0; i < input_count; ++i) { input_tensors.push_back(ctx->Input(i)); diff --git a/orttraining/orttraining/training_ops/cpu/tensor/slice_grad.cc b/orttraining/orttraining/training_ops/cpu/tensor/slice_grad.cc index 0f2b1f5e88..00501fc08a 100644 --- a/orttraining/orttraining/training_ops/cpu/tensor/slice_grad.cc +++ b/orttraining/orttraining/training_ops/cpu/tensor/slice_grad.cc @@ -28,10 +28,10 @@ Status SliceGrad::Compute(OpKernelContext* context) const { Tensor& output = *context->Output(0, data_shape); memset(output.MutableDataRaw(), 0, output.SizeInBytes()); // Initialize the starts & ends to the actual tensor shape - std::vector input_starts; - std::vector input_ends; - std::vector input_axes; - std::vector input_steps; + TensorShapeVector input_starts; + TensorShapeVector input_ends; + TensorShapeVector input_axes; + TensorShapeVector input_steps; ORT_RETURN_IF_ERROR(FillVectorsFromInput(*context->Input(2), *context->Input(3), context->Input(4), context->Input(5), input_starts, input_ends, input_axes, input_steps)); @@ -56,10 +56,10 @@ Status SliceGrad::Compute(OpKernelContext* context) const { template Status SliceGrad::ComputeImpl(OpKernelContext* ctx, Tensor& output_grad_tensor, - const std::vector& output_dims, - std::vector* flattened_output_dims, - const std::vector& starts, - const std::vector& steps) const { + const gsl::span& output_dims, + TensorShapeVector* flattened_output_dims, + const gsl::span& starts, + const gsl::span& steps) const { TensorShape output_shape(output_dims); // output tensor's size is 0, nothing to fill - return if (output_shape.Size() == 0) @@ -86,7 +86,7 @@ Status SliceGrad::ComputeImpl(OpKernelContext* ctx, if (flattened_output_dims) { // if we have flattened output dims we need to also flatten the input dims. // as we're combining the innermost dims and keeping all values we can just copy the size of the last dim - std::vector flattened_input_dims(output_grad_tensor.Shape().GetDimsAsVector()); + TensorShapeVector flattened_input_dims(output_grad_tensor.Shape().AsShapeVector()); flattened_input_dims.resize(flattened_output_dims->size()); flattened_input_dims.back() = flattened_output_dims->back(); TensorShape input_shape(std::move(flattened_input_dims)); diff --git a/orttraining/orttraining/training_ops/cpu/tensor/slice_grad.h b/orttraining/orttraining/training_ops/cpu/tensor/slice_grad.h index 2e27847957..cd49f4b056 100644 --- a/orttraining/orttraining/training_ops/cpu/tensor/slice_grad.h +++ b/orttraining/orttraining/training_ops/cpu/tensor/slice_grad.h @@ -21,10 +21,10 @@ class SliceGrad final : public OpKernel, public SliceBase { template Status ComputeImpl(OpKernelContext* ctx, Tensor& output_grad_tensor, - const std::vector& output_dims, - std::vector* flattened_output_dims, - const std::vector& starts, - const std::vector& steps) const; + const gsl::span& output_dims, + TensorShapeVector* flattened_output_dims, + const gsl::span& starts, + const gsl::span& steps) const; }; } // namespace contrib diff --git a/orttraining/orttraining/training_ops/cpu/tensor/split.cc b/orttraining/orttraining/training_ops/cpu/tensor/split.cc index 73cb2db519..0b82a37ff5 100644 --- a/orttraining/orttraining/training_ops/cpu/tensor/split.cc +++ b/orttraining/orttraining/training_ops/cpu/tensor/split.cc @@ -118,7 +118,7 @@ Status SplitTraining::ComputeImpl(OpKernelContext& context, const Tensor& input) split_sizes)); // copy dimensions so we can update the selected axis in place - std::vector output_dimensions{input_shape.GetDimsAsVector()}; + auto output_dimensions{input_shape.AsShapeVector()}; int64_t input_offset = 0; const T* input_data = input.template Data(); diff --git a/orttraining/orttraining/training_ops/cuda/communication/recv.cc b/orttraining/orttraining/training_ops/cuda/communication/recv.cc index 3b6e78519a..4f230e2315 100644 --- a/orttraining/orttraining/training_ops/cuda/communication/recv.cc +++ b/orttraining/orttraining/training_ops/cuda/communication/recv.cc @@ -6,7 +6,7 @@ #include "orttraining/training_ops/cuda/communication/recv.h" #include "orttraining/training_ops/communication_common.h" #include "orttraining/training_ops/cuda/communication/nccl_service.h" -#include "core/providers/cuda/nvtx_profile.h" +#include "core/providers/cuda/nvtx_profile.h" #include "core/profile/context.h" #include "core/providers/cuda/cuda_check_memory.h" #include "core/providers/cuda/cuda_common.h" @@ -231,8 +231,8 @@ Status Recv::ComputeInternal(OpKernelContext* ctx) const { // we need to create outputs after receiving shapes. size_t begin = 0; for (int i = 0; i < num_tensors; ++i) { - std::vector tensor_shape(aggregated_tensor_shapes.begin() + begin, - aggregated_tensor_shapes.begin() + prefix_tensor_shape_sizes[i]); + TensorShapeVector tensor_shape(aggregated_tensor_shapes.begin() + begin, + aggregated_tensor_shapes.begin() + prefix_tensor_shape_sizes[i]); received_tensors[i] = ctx->Output(i + 1, tensor_shape); // Move the "begin" to the beginning dimension of the next received tensor. begin = prefix_tensor_shape_sizes[i]; diff --git a/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cc b/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cc index 6d8a522210..a7f86957bc 100644 --- a/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cc +++ b/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cc @@ -92,7 +92,7 @@ Status SoftmaxCrossEntropyLoss::ComputeInternal(OpKernelContext* ctx) co } OrtValue transpose_output; - std::vector new_shape; + TensorShapeVector new_shape; std::vector permutations; AllocatorPtr alloc; const OpKernelInfo& info = OpKernel::Info(); @@ -220,7 +220,7 @@ Status SoftmaxCrossEntropyLossGrad::ComputeInternal(OpKernelContext* ctx T* d_logit_data = d_logit->template MutableData(); const T* weight_data = nullptr; OrtValue transpose_output; - std::vector new_shape; + TensorShapeVector new_shape; std::vector permutations; AllocatorPtr alloc; const OpKernelInfo& info = OpKernel::Info(); diff --git a/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cc b/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cc index 6c51051d5e..c8f44c613c 100644 --- a/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cc +++ b/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cc @@ -71,7 +71,7 @@ Status SoftmaxCrossEntropy::ComputeInternal(OpKernelContext* ctx) const { temp_X.get(), // -(label * log(softmax)) N * D); - std::vector output_dims(2, 1); + TensorShapeVector output_dims(2, 1); Tensor* Y = ctx->Output(0, TensorShape({})); // Sum((label * log(softmax)) using Reduction return ReduceKernelShared( diff --git a/orttraining/orttraining/training_ops/cuda/math/div_grad.cc b/orttraining/orttraining/training_ops/cuda/math/div_grad.cc index 67b225f751..effd2df123 100644 --- a/orttraining/orttraining/training_ops/cuda/math/div_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/math/div_grad.cc @@ -23,12 +23,12 @@ DIVGRAD_REGISTER_KERNEL_TYPED(MLFloat16) DIVGRAD_REGISTER_KERNEL_TYPED(float) DIVGRAD_REGISTER_KERNEL_TYPED(double) -std::vector prepended_dimension_1(const TensorShape& shape, size_t total_rank) { +TensorShapeVector prepended_dimension_1(const TensorShape& shape, size_t total_rank) { size_t input_rank = shape.NumDimensions(); if (input_rank == total_rank) - return shape.GetDimsAsVector(); + return shape.AsShapeVector(); - std::vector dims(total_rank, 1); + TensorShapeVector dims(total_rank, 1); // https://github.com/onnx/onnx/blob/master/docs/Broadcasting.md // for property 3 of Multidirectional Broadcasting, we need to prepended with a dimension of length 1. @@ -95,7 +95,7 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { reinterpret_cast(db_data)); if (da_output_tensor) { - std::vector a_output_dims = prepended_dimension_1(a_shape, dy_shape.NumDimensions()); + auto a_output_dims = prepended_dimension_1(a_shape, dy_shape.NumDimensions()); ORT_RETURN_IF_ERROR((ReduceKernelShared( temp_da_data, dy_shape, @@ -124,7 +124,7 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { reinterpret_cast(temp_db_data)); if (db_output_tensor) { - std::vector b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); + auto b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); ORT_RETURN_IF_ERROR((ReduceKernelShared( temp_db_data, dy_shape, @@ -169,7 +169,7 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { } if (db_output_tensor) { - std::vector b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); + auto b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); ORT_RETURN_IF_ERROR((ReduceKernelShared( temp_db_data, dy_shape, @@ -216,7 +216,7 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { reinterpret_cast(db_data_ref)); if (need_reduce_da) { - std::vector a_output_dims = prepended_dimension_1(a_shape, dy_shape.NumDimensions()); + auto a_output_dims = prepended_dimension_1(a_shape, dy_shape.NumDimensions()); ORT_RETURN_IF_ERROR((ReduceKernelShared( da_data_ref, dy_shape, @@ -227,7 +227,7 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { } if (need_reduce_db) { - std::vector b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); + auto b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); ORT_RETURN_IF_ERROR((ReduceKernelShared( db_data_ref, dy_shape, diff --git a/orttraining/orttraining/training_ops/cuda/nn/conv_grad.cc b/orttraining/orttraining/training_ops/cuda/nn/conv_grad.cc index 1d64f14d27..df66635c6d 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/conv_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/nn/conv_grad.cc @@ -249,15 +249,15 @@ template Status ConvGrad::PrepareArgs(const Tensor& x, const Tensor& dY, const Tensor& w, Tensor* dB, Tensor* dX, Tensor* dW) const { const TensorShape& x_shape = x.Shape(); - std::vector x_dims = x_shape.GetDimsAsVector(); + auto x_dims = x_shape.AsShapeVector(); args_.x_data = reinterpret_cast(x.template Data()); const TensorShape& dy_shape = dY.Shape(); - std::vector dy_dims = dy_shape.GetDimsAsVector(); + auto dy_dims = dy_shape.AsShapeVector(); args_.dy_data = reinterpret_cast(dY.template Data()); const TensorShape& w_shape = w.Shape(); - std::vector w_dims = w_shape.GetDimsAsVector(); + auto w_dims = w_shape.AsShapeVector(); args_.w_data = reinterpret_cast(w.template Data()); args_.db_data = dB ? reinterpret_cast(dB->template MutableData()) : nullptr; @@ -273,21 +273,21 @@ Status ConvGrad::PrepareArgs(const Tensor& x, const Tensor& dY, const Tensor& // Update Attributes ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(&x, &w)); - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(w_shape, kernel_shape)); auto rank = kernel_shape.size(); - std::vector pads(conv_attrs_.pads); + ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(rank * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(rank, 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(rank, 1); } diff --git a/orttraining/orttraining/training_ops/cuda/nn/conv_grad.h b/orttraining/orttraining/training_ops/cuda/nn/conv_grad.h index c8eaf64e1c..6223359d35 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/conv_grad.h +++ b/orttraining/orttraining/training_ops/cuda/nn/conv_grad.h @@ -28,8 +28,8 @@ struct ConvParams { struct ConvArgs { // Update needed if x or w's dims changed. - std::vector last_x_dims; - std::vector last_w_dims; + TensorShapeVector last_x_dims; + TensorShapeVector last_w_dims; cudnnHandle_t handle; ConvParams params; diff --git a/orttraining/orttraining/training_ops/cuda/reduction/reduction_ops.cc b/orttraining/orttraining/training_ops/cuda/reduction/reduction_ops.cc index a01152140d..2aff14d8ba 100644 --- a/orttraining/orttraining/training_ops/cuda/reduction/reduction_ops.cc +++ b/orttraining/orttraining/training_ops/cuda/reduction/reduction_ops.cc @@ -94,8 +94,8 @@ Status ReduceKernel::ComputeImplEx& input_dims_cudnn = prepare_reduce_metadata.input_dims_cudnn; - std::vector& output_dims_cudnn = prepare_reduce_metadata.output_dims_cudnn; + auto& input_dims_cudnn = prepare_reduce_metadata.input_dims_cudnn; + auto& output_dims_cudnn = prepare_reduce_metadata.output_dims_cudnn; // special case when there is a dim value of 0 in the shape. if (input_count == 0) { diff --git a/orttraining/orttraining/training_ops/cuda/tensor/concat.cc b/orttraining/orttraining/training_ops/cuda/tensor/concat.cc index b1b8a95ae3..0352679ab5 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/concat.cc +++ b/orttraining/orttraining/training_ops/cuda/tensor/concat.cc @@ -20,7 +20,7 @@ Status ConcatTraining::ComputeInternal(OpKernelContext* ctx) const { auto input_count = Node().InputArgCount().front(); // Hold pointers to the input tensors to be used in the PrepareForCompute() step - std::vector input_tensors; + InlinedTensorsVector input_tensors; input_tensors.reserve(input_count); for (int i = 0; i < input_count; ++i) { input_tensors.push_back(ctx->Input(i)); @@ -34,11 +34,11 @@ Status ConcatTraining::ComputeInternal(OpKernelContext* ctx) const { return Status::OK(); } - std::vector concat_sizes(input_count); + TensorShapeVector concat_sizes(input_count); CudaAsyncBuffer input_ptr(this, input_count); gsl::span input_ptr_cpuspan = input_ptr.CpuSpan(); - std::vector axis_dimension_input_output_mapping(p.output_tensor->Shape()[p.axis]); + TensorShapeVector axis_dimension_input_output_mapping(p.output_tensor->Shape()[p.axis]); int index = 0; for (int i = 0; i < input_count; ++i) { auto input = p.inputs[i]; @@ -48,7 +48,7 @@ Status ConcatTraining::ComputeInternal(OpKernelContext* ctx) const { axis_dimension_input_output_mapping.at(index++) = i; } } - std::vector concat_sizes_range(concat_sizes); + TensorShapeVector concat_sizes_range(concat_sizes); for (size_t i = 1; i < concat_sizes_range.size(); ++i) { concat_sizes_range[i] += concat_sizes_range[i - 1]; } diff --git a/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.cc b/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.cc index d2205448b3..5c0c58d077 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.cc @@ -40,9 +40,9 @@ const Tensor* SliceGrad::GetSlicedOrUnslicedTensor(OpKernelContext* ctx) const { return GetOutputGradientTensor(ctx); } -Status SliceGrad::FillInputVectors(OpKernelContext* ctx, std::vector& input_starts, - std::vector& input_ends, std::vector& input_axes, - std::vector& input_steps) const { +Status SliceGrad::FillInputVectors(OpKernelContext* ctx, TensorShapeVector& input_starts, + TensorShapeVector& input_ends, TensorShapeVector& input_axes, + TensorShapeVector& input_steps) const { return FillVectorsFromInput(*ctx->Input(2), *ctx->Input(3), ctx->Input(4), ctx->Input(5), input_starts, input_ends, input_axes, input_steps); } diff --git a/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.h b/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.h index 03442fe3bb..cbf500ec23 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.h +++ b/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.h @@ -12,8 +12,8 @@ class SliceGrad final : public Slice { private: const Tensor* GetSlicedOrUnslicedTensor(OpKernelContext* ctx) const override; - Status FillInputVectors(OpKernelContext* ctx, std::vector& input_starts, std::vector& input_ends, - std::vector& input_axes, std::vector& input_steps) const override; + Status FillInputVectors(OpKernelContext* ctx, TensorShapeVector& input_starts, TensorShapeVector& input_ends, + TensorShapeVector& input_axes, TensorShapeVector& input_steps) const override; Status CallSliceImp(size_t element_size, size_t dimension_count, const TArray& starts_buffer, const TArray& steps_buffer, const TArray& input_strides, diff --git a/orttraining/orttraining/training_ops/cuda/tensor/split.cc b/orttraining/orttraining/training_ops/cuda/tensor/split.cc index 674596a69d..dd57e9032a 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/split.cc +++ b/orttraining/orttraining/training_ops/cuda/tensor/split.cc @@ -45,11 +45,11 @@ Status SplitTraining::ComputeInternal(OpKernelContext* ctx) const { auto input_data = input_tensor->DataRaw(); auto input_dims = input_shape.GetDims(); - std::vector output_dimensions{input_shape.GetDimsAsVector()}; + auto output_dimensions{input_shape.AsShapeVector()}; CudaAsyncBuffer output_ptr(this, num_outputs); gsl::span output_ptr_span = output_ptr.CpuSpan(); - std::vector axis_dimension_input_output_mapping(input_dims[axis]); + TensorShapeVector axis_dimension_input_output_mapping(input_dims[axis]); int index = 0; for (int i = 0; i < num_outputs; ++i) { // update size of dimension for axis we're splitting on diff --git a/orttraining/orttraining/training_ops/rocm/math/div_grad.cc b/orttraining/orttraining/training_ops/rocm/math/div_grad.cc index 301780db30..02e8e62af8 100644 --- a/orttraining/orttraining/training_ops/rocm/math/div_grad.cc +++ b/orttraining/orttraining/training_ops/rocm/math/div_grad.cc @@ -23,17 +23,17 @@ DIVGRAD_REGISTER_KERNEL_TYPED(MLFloat16) DIVGRAD_REGISTER_KERNEL_TYPED(float) // DIVGRAD_REGISTER_KERNEL_TYPED(double) -std::vector prepended_dimension_1(const TensorShape& shape, size_t total_rank) { +TensorShapeVector prepended_dimension_1(const TensorShape& shape, size_t total_rank) { size_t input_rank = shape.NumDimensions(); if (input_rank == total_rank) - return shape.GetDimsAsVector(); + return shape.AsShapeVector(); - std::vector dims(total_rank, 1); + TensorShapeVector dims(total_rank, 1); // https://github.com/onnx/onnx/blob/master/docs/Broadcasting.md // for property 3 of Multidirectional Broadcasting, we need to prepended with a dimension of length 1. if (input_rank > 0) - std::copy(shape.GetDims().begin(), shape.GetDims().end(), &dims[total_rank - input_rank]); + std::copy(shape.GetDims().cbegin(), shape.GetDims().cend(), &dims[total_rank - input_rank]); return dims; } @@ -95,7 +95,7 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { reinterpret_cast(db_data)); if (da_output_tensor) { - std::vector a_output_dims = prepended_dimension_1(a_shape, dy_shape.NumDimensions()); + auto a_output_dims = prepended_dimension_1(a_shape, dy_shape.NumDimensions()); ORT_RETURN_IF_ERROR((ReduceKernelShared( temp_da_data, dy_shape, @@ -124,7 +124,7 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { reinterpret_cast(temp_db_data)); if (db_output_tensor) { - std::vector b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); + auto b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); ORT_RETURN_IF_ERROR((ReduceKernelShared( temp_db_data, dy_shape, @@ -169,7 +169,7 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { } if (db_output_tensor) { - std::vector b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); + auto b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); ORT_RETURN_IF_ERROR((ReduceKernelShared( temp_db_data, dy_shape, @@ -216,7 +216,7 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { reinterpret_cast(db_data_ref)); if (need_reduce_da) { - std::vector a_output_dims = prepended_dimension_1(a_shape, dy_shape.NumDimensions()); + auto a_output_dims = prepended_dimension_1(a_shape, dy_shape.NumDimensions()); ORT_RETURN_IF_ERROR((ReduceKernelShared( da_data_ref, dy_shape, @@ -227,7 +227,7 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { } if (need_reduce_db) { - std::vector b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); + auto b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); ORT_RETURN_IF_ERROR((ReduceKernelShared( db_data_ref, dy_shape, diff --git a/orttraining/orttraining/training_ops/rocm/nn/conv_grad.cc b/orttraining/orttraining/training_ops/rocm/nn/conv_grad.cc index 30f5b68509..eecdf61e17 100644 --- a/orttraining/orttraining/training_ops/rocm/nn/conv_grad.cc +++ b/orttraining/orttraining/training_ops/rocm/nn/conv_grad.cc @@ -233,15 +233,15 @@ template Status ConvGrad::PrepareArgs(const Tensor& x, const Tensor& dY, const Tensor& w, Tensor* dB, Tensor* dX, Tensor* dW) const { const TensorShape& x_shape = x.Shape(); - std::vector x_dims = x_shape.GetDimsAsVector(); + auto x_dims = x_shape.AsShapeVector(); args_.x_data = reinterpret_cast(x.template Data()); const TensorShape& dy_shape = dY.Shape(); - std::vector dy_dims = dy_shape.GetDimsAsVector(); + auto dy_dims = dy_shape.AsShapeVector(); args_.dy_data = reinterpret_cast(dY.template Data()); const TensorShape& w_shape = w.Shape(); - std::vector w_dims = w_shape.GetDimsAsVector(); + auto w_dims = w_shape.AsShapeVector(); args_.w_data = reinterpret_cast(w.template Data()); args_.db_data = dB ? reinterpret_cast(dB->template MutableData()) : nullptr; @@ -257,21 +257,21 @@ Status ConvGrad::PrepareArgs(const Tensor& x, const Tensor& dY, const Tensor& // Update Attributes ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(&x, &w)); - std::vector kernel_shape; + TensorShapeVector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(w_shape, kernel_shape)); auto rank = kernel_shape.size(); - std::vector pads(conv_attrs_.pads); + ConvAttributes::ConvPadVector pads(conv_attrs_.pads); if (pads.empty()) { pads.resize(rank * 2, 0); } - std::vector dilations(conv_attrs_.dilations); + TensorShapeVector dilations(conv_attrs_.dilations); if (dilations.empty()) { dilations.resize(rank, 1); } - std::vector strides(conv_attrs_.strides); + TensorShapeVector strides(conv_attrs_.strides); if (strides.empty()) { strides.resize(rank, 1); } @@ -316,7 +316,7 @@ Status ConvGrad::PrepareArgs(const Tensor& x, const Tensor& dY, const Tensor& if (dB) { const TensorShape& db_shape = dB->Shape(); ORT_RETURN_IF_NOT(db_shape.NumDimensions() == 1, "bias should be 1D"); - std::vector db_dims(2 + kernel_shape.size(), 1); + TensorShapeVector db_dims(2 + kernel_shape.size(), 1); db_dims[1] = db_shape[0]; ORT_RETURN_IF_ERROR(args_.b_tensor.Set(db_dims, MiopenTensor::GetDataType())); } diff --git a/orttraining/orttraining/training_ops/rocm/nn/conv_grad.h b/orttraining/orttraining/training_ops/rocm/nn/conv_grad.h index 97428eab42..a56d88f396 100644 --- a/orttraining/orttraining/training_ops/rocm/nn/conv_grad.h +++ b/orttraining/orttraining/training_ops/rocm/nn/conv_grad.h @@ -26,8 +26,8 @@ struct ConvParams { struct ConvArgs { // Update needed if x or w's dims changed. - std::vector last_x_dims; - std::vector last_w_dims; + TensorShapeVector last_x_dims; + TensorShapeVector last_w_dims; miopenHandle_t handle; ConvParams params; diff --git a/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc b/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc index ffe475a7f3..6e1ec3cb0f 100644 --- a/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc +++ b/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc @@ -94,8 +94,8 @@ Status ReduceKernel::ComputeImplEx& input_dims_miopen = prepare_reduce_metadata.input_dims_miopen; - std::vector& output_dims_miopen = prepare_reduce_metadata.output_dims_miopen; + auto& input_dims_miopen = prepare_reduce_metadata.input_dims_miopen; + auto& output_dims_miopen = prepare_reduce_metadata.output_dims_miopen; // special case when there is a dim value of 0 in the shape. if (input_count == 0) { diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 59cdd99fc7..9e4ecc8512 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -48,6 +48,7 @@ jobs: script: | rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11 rm -f $(Build.BinariesDirectory)/Release/models + rm -rf $(Build.BinariesDirectory)/Release/_deps cd $(Build.BinariesDirectory)/Release find -executable -type f > $(Build.BinariesDirectory)/Release/perms.txt From 2afce4830c9ff48374ca11cf00b09afcc1ff2124 Mon Sep 17 00:00:00 2001 From: Chen Fu <1316708+chenfucn@users.noreply.github.com> Date: Mon, 24 Jan 2022 10:49:04 -0800 Subject: [PATCH 25/59] Symmetric QGEMM (#10289) Adding code for symmetric quantized matrix multiplication. Used in quantized convolution, achieving significant perf gain. TODO, use Symmetric Quantized GEMM in other operators! TODO address activation buffer overread in custom allocators and tensors supplied by users. DOT kernel perf test: Pixel 5a: Cartoongan 513.539 ms 471.786 ms Efficient 57.5169 ms 56.4174 ms Edgetpu 14.6673 ms 13.5959 ms NEON kernel perf test Pixel 3a Cartoongan 1423.53 ms 1069.92 ms Efficient 114.086 ms 107.968 ms Edgetpu 39.2632 ms 36.9839 ms Co-authored-by: Chen Fu --- cmake/onnxruntime_mlas.cmake | 4 + onnxruntime/core/framework/allocator.cc | 2 + onnxruntime/core/mlas/inc/mlas.h | 93 ++- .../mlas/lib/aarch64/SymQgemmS8KernelNeon.S | 536 +++++++++++++++++ .../mlas/lib/aarch64/SymQgemmS8KernelSdot.S | 388 +++++++++++++ .../mlas/lib/arm64/SymQgemmS8KernelNeon.asm | 538 ++++++++++++++++++ .../mlas/lib/arm64/SymQgemmS8KernelSdot.asm | 391 +++++++++++++ onnxruntime/core/mlas/lib/mlasi.h | 15 + onnxruntime/core/mlas/lib/platform.cpp | 2 + onnxruntime/core/mlas/lib/qgemm.cpp | 197 ++++++- onnxruntime/core/mlas/lib/qgemm.h | 114 ++++ .../core/mlas/lib/qgemm_kernel_neon.cpp | 40 ++ .../core/mlas/lib/qgemm_kernel_sdot.cpp | 275 +++++++++ .../providers/cpu/quantization/qlinearconv.cc | 222 +++++--- .../test/mlas/bench/bench_symm_qgemm.cpp | 103 ++++ .../test/mlas/unittest/test_symm_qgemm.cpp | 36 ++ .../test/mlas/unittest/test_symm_qgemm.h | 213 +++++++ .../mlas/unittest/test_symm_qgemm_fixture.h | 82 +++ 18 files changed, 3148 insertions(+), 103 deletions(-) create mode 100644 onnxruntime/core/mlas/lib/aarch64/SymQgemmS8KernelNeon.S create mode 100644 onnxruntime/core/mlas/lib/aarch64/SymQgemmS8KernelSdot.S create mode 100644 onnxruntime/core/mlas/lib/arm64/SymQgemmS8KernelNeon.asm create mode 100644 onnxruntime/core/mlas/lib/arm64/SymQgemmS8KernelSdot.asm create mode 100644 onnxruntime/test/mlas/bench/bench_symm_qgemm.cpp create mode 100644 onnxruntime/test/mlas/unittest/test_symm_qgemm.cpp create mode 100644 onnxruntime/test/mlas/unittest/test_symm_qgemm.h create mode 100644 onnxruntime/test/mlas/unittest/test_symm_qgemm_fixture.h diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 2fe9bd4e28..e552ff85b5 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -57,6 +57,8 @@ function(setup_mlas_source_for_windows) ${MLAS_SRC_DIR}/arm64/QgemmS8S8KernelSdot.asm ${MLAS_SRC_DIR}/arm64/SgemmKernelNeon.asm ${MLAS_SRC_DIR}/arm64/SgemvKernelNeon.asm + ${MLAS_SRC_DIR}/arm64/SymQgemmS8KernelNeon.asm + ${MLAS_SRC_DIR}/arm64/SymQgemmS8KernelSDot.asm ) else() target_sources(onnxruntime_mlas PRIVATE @@ -280,6 +282,8 @@ else() ${MLAS_SRC_DIR}/aarch64/QgemmS8S8KernelSdot.S ${MLAS_SRC_DIR}/aarch64/SgemmKernelNeon.S ${MLAS_SRC_DIR}/aarch64/SgemvKernelNeon.S + ${MLAS_SRC_DIR}/aarch64/SymQgemmS8KernelNeon.S + ${MLAS_SRC_DIR}/aarch64/SymQgemmS8KernelSdot.S ${MLAS_SRC_DIR}/qgemm_kernel_neon.cpp ${MLAS_SRC_DIR}/qgemm_kernel_udot.cpp ${MLAS_SRC_DIR}/qgemm_kernel_sdot.cpp diff --git a/onnxruntime/core/framework/allocator.cc b/onnxruntime/core/framework/allocator.cc index c1d3366fdd..99e950b8b6 100644 --- a/onnxruntime/core/framework/allocator.cc +++ b/onnxruntime/core/framework/allocator.cc @@ -43,6 +43,7 @@ bool IAllocator::CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, siz void* AllocatorDefaultAlloc(size_t size) { const size_t alignment = MlasGetPreferredBufferAlignment(); if (size <= 0) return nullptr; + size += MLAS_SYMM_QGEMM_BUF_OVERRUN; void* p; #if defined(_MSC_VER) p = mi_malloc_aligned(size, alignment); @@ -73,6 +74,7 @@ void AllocatorDefaultFree(void* p) { void* AllocatorDefaultAlloc(size_t size) { const size_t alignment = MlasGetPreferredBufferAlignment(); if (size <= 0) return nullptr; + size += MLAS_SYMM_QGEMM_BUF_OVERRUN; void* p; #if _MSC_VER p = _aligned_malloc(size, alignment); diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index 004b7640e3..66a7bd55b1 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -562,14 +562,6 @@ struct MLAS_GEMM_QUANT_DATA_PARAMS { const MLAS_QGEMM_OUTPUT_PROCESSOR* OutputProcessor = nullptr; }; -void -MLASCALL -MlasGemm( - const MLAS_GEMM_QUANT_SHAPE_PARAMS& Shape, - const MLAS_GEMM_QUANT_DATA_PARAMS& DataParams, - MLAS_THREADPOOL* ThreadPool - ); - /** * @brief Batched GEMM, for multiplying multiple pairs of matrices. * Note: We only support uniform batching, so shapes and types of the @@ -590,6 +582,62 @@ MlasGemmBatch( MLAS_THREADPOOL* ThreadPool ); +inline +void +MlasGemm( + const MLAS_GEMM_QUANT_SHAPE_PARAMS &Shape, + const MLAS_GEMM_QUANT_DATA_PARAMS &DataParams, + MLAS_THREADPOOL *ThreadPool) +{ + MlasGemmBatch(Shape, &DataParams, 1, ThreadPool); +} + +// +// Symmetric QGEMM has limited buffer overrun. +// Currently only supported in ARM64 +// +#if defined(MLAS_TARGET_ARM64) +constexpr size_t MLAS_SYMM_QGEMM_BUF_OVERRUN = 15; +#else +constexpr size_t MLAS_SYMM_QGEMM_BUF_OVERRUN = 0; +#endif + +/** + * @brief Supply data parameters for symmetric quantized GEMM. + * B matrix zero point must be zero, and it must be + * pre-packed, with column sums scaled by (-ZeroPointA) +*/ +struct MLAS_SYMM_QGEMM_DATA_PARAMS { + const void* A = nullptr; + size_t lda = 0; + const void* B = 0; + void* C = nullptr; + size_t ldc = 0; + // TODO!! add re-quantization parameters +}; + +/** + * @brief Batched QGEMM. Similar to MlasGemmBatch, but right hand side matrix + * must be symmetrically quantized and prepacked. + * + * @param [IN] Shape A single shape descriptor for all multiplicatons. + Currently A and B must be signed, and accumulation + mode not supported + * @param [IN] DataParams Array of data descriptors, one for each mutliplication + * B must be prepacked + * @param [IN] BatchN Number of multiplications + * @param [IN] ThreadPool +*/ +void +MLASCALL +MlasSymmQgemmBatch( + const MLAS_GEMM_QUANT_SHAPE_PARAMS& Shape, + const MLAS_SYMM_QGEMM_DATA_PARAMS* DataParams, + const size_t BatchN, + MLAS_THREADPOOL* ThreadPool + ); + + // // Buffer packing routines. // @@ -633,6 +681,35 @@ MlasGemmPackB( void* PackedB ); +/** + * @brief For symmetric quantized GEMM, returns size of the + * packing buffer needed for right hand side + * @param N Number of columns + * @param K Number of rows + * @param AIsSigned Whether left hand size is signed int8_t + * @return size of the packing buffer, + * 0 if operation not supported +*/ +size_t +MLASCALL +MlasSymmQgemmPackBSize( + size_t N, + size_t K, + bool AIsSigned + ); + +void +MLASCALL +MlasSymmQgemmPackB( + size_t N, + size_t K, + const int8_t* B, + size_t ldb, + bool AIsSigned, + int32_t ZeroPointA, + void* PackedB + ); + // // Convolution routines. // diff --git a/onnxruntime/core/mlas/lib/aarch64/SymQgemmS8KernelNeon.S b/onnxruntime/core/mlas/lib/aarch64/SymQgemmS8KernelNeon.S new file mode 100644 index 0000000000..f236ef4ed1 --- /dev/null +++ b/onnxruntime/core/mlas/lib/aarch64/SymQgemmS8KernelNeon.S @@ -0,0 +1,536 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SymQgemmS8KernelNeon.S + +Abstract: + + This module implements the kernels for the quantized integer matrix/matrix + multiply operation (QGEMM), where the right hand side is symmetrically quantized, + i.e. zero point being zero. + + This kernel only requires prepacking of the right hand side, which is usually + constant. When the packed right hand side is cached, we achieves higher performance + by avoid packing all together. + +--*/ + +#include "asmmacro.h" + +// +// Stack frame layout for the S8S8 kernel. +// d8-d15, x19-x30 need to be preserved if used +// + + .equ .LSQGemmS8Frame_SavedRegisters, (8 * 8) + .equ .LSQGemmS8Frame_ColumnSumBuffer, 0 + .LSQGemmS8Frame_SavedRegisters + + .text + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A (x0) - Supplies the address of matrix A. + + B (x1) - Supplies the address of matrix B. The matrix data has been packed + using MlasGemmQuantCopyPackB. + + C (x2) - Supplies the address of matrix C. + + PackedCountK (x3) - Supplies the number of packed columns from matrix A and + the number of packed rows from matrix B to iterate over. + + CountM (x4) - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN (x5) - Supplies the number of columns from matrix B and matrix C to + iterate over. + + ldc (x6) - Supplies the first dimension of matrix C. + + lda (x7) - Supplies the first dimension of matrix A. + + ColumnSumBuffer - Supplies the sum of each column from matrix B multiplied + by the zero point offset of matrix A. These values are accumulated into + every column of matrix C. + + +Return Value: + + Returns the number of rows handled. + +--*/ + + FUNCTION_ENTRY MlasSymQgemmS8KernelNeon + + stp d8,d9,[sp,#-.LSQGemmS8Frame_SavedRegisters]! + stp d10,d11,[sp,#16] + stp d12,d13,[sp,#32] + stp d14,d15,[sp,#48] + ldr x13,[sp,#.LSQGemmS8Frame_ColumnSumBuffer] + mov x14,x0 + mov x15,x3 + cmp x4,#1 // CountM == 1? + beq M1_ProcessLoop + cmp x4,#4 // CountM < 4? + blo M2_ProcessLoop + +// +// Process 4 rows of the matrices. +// B 16x4 +// ---------------------------------------- +// |v4.b[0] v5.b[0] v6.b[0] v7.b[0] | +// | ... ... ... ... | +// |v4.b[7] v5.b[7] v6.b[7] v7.b[7] | +// |v8.b[0] v9.b[0] v10.b[0] v11.b[0]| +// | ... ... ... ... | +// |v8.b[7] v9.b[7] v10.b[7] v11.b[7]| +// A 4x16 ---------------------------------------- +// ----------------------------------- ---------------------------------------- +// |v0.b[0]..v0.b[7] v2.b[0]..v2.b[7]| |v16.4s v17.4s v18.4s v19.4s | +// |v1.b[0]..v1.b[7] v3.b[0]..v3.b[7]| |v20.4s v21.4s v22.4s v23.4s | +// |v0.b[0]..v0.b[7] v2.b[0]..v2.b[7]| |v24.4s v25.4s v26.4s v27.4s | +// |v1.b[0]..v1.b[7] v3.b[0]..v3.b[7]| |v28.4s v29.4s v30.4s v31.4s | +// ----------------------------------- ---------------------------------------- +// +// Accumulators are horizontally aggregated to the left most register +// for each row. e.g. (v16.s[0], v16.s[1], v16.s[2], v16.s[3]) <- (v16, v17, v18, v19) +// + +M4_ProcessNextColumnLoop: + mov x0,x14 // reload matrix A0 + mov x3,x15 // reload PackedCountK + ldr d0,[x0],#8 // Load A0 + add x9,x14,x7 // A1 + ldr d2,[x0],#8 // Load A0 + movi v16.4s,#0 + movi v17.4s,#0 + ldp d4,d8,[x1],#64 // B + movi v18.4s,#0 + movi v19.4s,#0 + ldp d5,d9,[x1,#-48] + movi v20.4s,#0 + movi v21.4s,#0 + ldp d6,d10,[x1,#-32] + movi v22.4s,#0 + movi v23.4s,#0 + ldp d7,d11,[x1,#-16] + movi v24.4s,#0 + movi v25.4s,#0 + add x10,x9,x7 // A2 + ldp d1,d3,[x9],#16 // Load A1 + movi v26.4s,#0 + movi v27.4s,#0 + movi v28.4s,#0 + movi v29.4s,#0 + movi v30.4s,#0 + movi v31.4s,#0 + add x11,x10,x7 // A3 + +M4_ComputeBlockLoop: + smull v12.8h,v0.8b,v4.8b + smull v13.8h,v0.8b,v5.8b + smull v14.8h,v0.8b,v6.8b + smull v15.8h,v0.8b,v7.8b + smlal v12.8h,v2.8b,v8.8b + smlal v13.8h,v2.8b,v9.8b + smlal v14.8h,v2.8b,v10.8b + smlal v15.8h,v2.8b,v11.8b + ldp d0,d2,[x10],#16 // Load A2 + sadalp v16.4s,v12.8h + sadalp v17.4s,v13.8h + sadalp v18.4s,v14.8h + sadalp v19.4s,v15.8h + sub x3,x3,#1 + smull v12.8h,v1.8b,v4.8b + smull v13.8h,v1.8b,v5.8b + smull v14.8h,v1.8b,v6.8b + smull v15.8h,v1.8b,v7.8b + smlal v12.8h,v3.8b,v8.8b + smlal v13.8h,v3.8b,v9.8b + smlal v14.8h,v3.8b,v10.8b + smlal v15.8h,v3.8b,v11.8b + ldp d1,d3,[x11],#16 // Load A3 + sadalp v20.4s,v12.8h + sadalp v21.4s,v13.8h + sadalp v22.4s,v14.8h + sadalp v23.4s,v15.8h + cbz x3,M4_ComputeBlockLoopFinish + smull v12.8h,v0.8b,v4.8b + smull v13.8h,v0.8b,v5.8b + smull v14.8h,v0.8b,v6.8b + smull v15.8h,v0.8b,v7.8b + smlal v12.8h,v2.8b,v8.8b + smlal v13.8h,v2.8b,v9.8b + smlal v14.8h,v2.8b,v10.8b + smlal v15.8h,v2.8b,v11.8b + ldp d0,d2,[x0],#16 // Load A0 next iter + sadalp v24.4s,v12.8h + sadalp v25.4s,v13.8h + sadalp v26.4s,v14.8h + sadalp v27.4s,v15.8h + smull v12.8h,v1.8b,v4.8b + smull v13.8h,v1.8b,v5.8b + smull v14.8h,v1.8b,v6.8b + smull v15.8h,v1.8b,v7.8b + smlal v12.8h,v3.8b,v8.8b + ldp d4,d8,[x1],#64 // B + smlal v13.8h,v3.8b,v9.8b + ldp d5,d9,[x1,#-48] + smlal v14.8h,v3.8b,v10.8b + ldp d6,d10,[x1,#-32] + smlal v15.8h,v3.8b,v11.8b + ldp d7,d11,[x1,#-16] + sadalp v28.4s,v12.8h + ldp d1,d3,[x9],#16 // Load A1 next iter + sadalp v29.4s,v13.8h + sadalp v30.4s,v14.8h + sadalp v31.4s,v15.8h + b M4_ComputeBlockLoop + +M4_ComputeBlockLoopFinish: + smull v12.8h,v0.8b,v4.8b + smull v13.8h,v0.8b,v5.8b + smull v14.8h,v0.8b,v6.8b + smull v15.8h,v0.8b,v7.8b + smlal v12.8h,v2.8b,v8.8b + smlal v13.8h,v2.8b,v9.8b + smlal v14.8h,v2.8b,v10.8b + smlal v15.8h,v2.8b,v11.8b + ld1 {v2.4s},[x13],#16 // load ColumnSumBuffer[0] + sadalp v24.4s,v12.8h + sadalp v25.4s,v13.8h + sadalp v26.4s,v14.8h + sadalp v27.4s,v15.8h + smull v12.8h,v1.8b,v4.8b + smull v13.8h,v1.8b,v5.8b + smull v14.8h,v1.8b,v6.8b + smull v15.8h,v1.8b,v7.8b + smlal v12.8h,v3.8b,v8.8b + smlal v13.8h,v3.8b,v9.8b + smlal v14.8h,v3.8b,v10.8b + smlal v15.8h,v3.8b,v11.8b + sadalp v28.4s,v12.8h + sadalp v29.4s,v13.8h + sadalp v30.4s,v14.8h + sadalp v31.4s,v15.8h + addp v16.4s,v16.4s,v17.4s + addp v18.4s,v18.4s,v19.4s + addp v20.4s,v20.4s,v21.4s + addp v22.4s,v22.4s,v23.4s + addp v24.4s,v24.4s,v25.4s + addp v26.4s,v26.4s,v27.4s + addp v28.4s,v28.4s,v29.4s + addp v30.4s,v30.4s,v31.4s + addp v16.4s,v16.4s,v18.4s + addp v20.4s,v20.4s,v22.4s + addp v24.4s,v24.4s,v26.4s + addp v28.4s,v28.4s,v30.4s + + // accumulator += column sum B + add v16.4s,v16.4s,v2.4s + add v20.4s,v20.4s,v2.4s + add v24.4s,v24.4s,v2.4s + add v28.4s,v28.4s,v2.4s + +M4_StoreOutput: + add x10,x2,x6,lsl #2 + add x11,x10,x6,lsl #2 + add x12,x11,x6,lsl #2 + subs x5,x5,#4 // adjust CountN remaining + blo M4_StoreOutputPartial + st1 {v16.4s},[x2],#16 + st1 {v20.4s},[x10] + st1 {v24.4s},[x11] + st1 {v28.4s},[x12] + cbnz x5,M4_ProcessNextColumnLoop + +M4_ExitKernel: + mov x0,#4 // return number of rows handled + ldp d14,d15,[sp,#48] + ldp d12,d13,[sp,#32] + ldp d10,d11,[sp,#16] + ldp d8,d9,[sp],#.LSQGemmS8Frame_SavedRegisters + ret + +M4_StoreOutputPartial: + +M4_StoreOutputPartial_ZeroMode: + tbz x5,#1,M4_StoreOutputPartial1_ZeroMode + st1 {v16.2s},[x2],#8 + dup v16.4s,v16.s[2] // shift remaining elements down + st1 {v20.2s},[x10],#8 + dup v20.4s,v20.s[2] + st1 {v24.2s},[x11],#8 + dup v24.4s,v24.s[2] + st1 {v28.2s},[x12],#8 + dup v28.4s,v28.s[2] + +M4_StoreOutputPartial1_ZeroMode: + tbz x5,#0,M4_ExitKernel + st1 {v16.s}[0],[x2] + st1 {v20.s}[0],[x10] + st1 {v24.s}[0],[x11] + st1 {v28.s}[0],[x12] + b M4_ExitKernel + +// +// Process 2 rows of the matrices. +// +// Column Sum v2.s[0] v2.s[4] +// Each row sum replicated to all 4 elements of a vector register +// v30 v31 +// B 16x4 +// ---------------------------------------- +// |v4.b[0] v5.b[0] v6.b[0] v7.b[0] | +// | ... ... ... ... | +// |v4.b[7] v5.b[7] v6.b[7] v7.b[7] | +// |v24.b[0] v25.b[0] v26.b[0] v27.b[0]| +// | ... ... ... ... | +// |v24.b[7] v25.b[7] v26.b[7] v27.b[7]| +// A 2x16 ---------------------------------------- +// ----------------------------------- ---------------------------------------- +// |v0.b[0]..v0.b[7] v2.b[0]..v2.b[7]| |v16.4s v17.4s v18.4s v19.4s | +// |v1.b[0]..v1.b[7] v3.b[0]..v3.b[7]| |v20.4s v21.4s v22.4s v23.4s | +// ----------------------------------- ---------------------------------------- +// +// Accumulators are horizontally aggregated to the left most register +// for each row. e.g. (v16.s[0], v16.s[1], v16.s[2], v16.s[3]) <- (v16, v17, v18, v19) + +M2_ProcessLoop: + +M2_ProcessNextColumnLoop: + ldp d4,d24,[x1],#16 // B + mov x0,x14 // reload matrix A + mov x3,x15 // reload PackedCountK + ldp d0,d2,[x0],#16 // Load A0 + add x9,x14,x7 // A1 + movi v16.4s,#0 + movi v17.4s,#0 + ldp d5,d25,[x1],#16 + movi v18.4s,#0 + movi v19.4s,#0 + ldp d6,d26,[x1],#16 + movi v20.4s,#0 + movi v21.4s,#0 + ldp d7,d27,[x1],#16 + movi v22.4s,#0 + movi v23.4s,#0 + ldp d1,d3,[x9],#16 // Load A1 + +M2_ComputeBlockLoop: + sub x3,x3,#1 + smull v28.8h,v0.8b,v4.8b + smull v29.8h,v0.8b,v5.8b + smull v30.8h,v0.8b,v6.8b + smull v31.8h,v0.8b,v7.8b + cbz x3,M2_ComputeBlockLoopFinish + smlal v28.8h,v2.8b,v24.8b + smlal v29.8h,v2.8b,v25.8b + smlal v30.8h,v2.8b,v26.8b + smlal v31.8h,v2.8b,v27.8b + ldp d0,d2,[x0],#16 // Load A0 + sadalp v16.4s,v28.8h + sadalp v17.4s,v29.8h + sadalp v18.4s,v30.8h + sadalp v19.4s,v31.8h + smull v28.8h,v1.8b,v4.8b + smull v29.8h,v1.8b,v5.8b + smull v30.8h,v1.8b,v6.8b + smull v31.8h,v1.8b,v7.8b + smlal v28.8h,v3.8b,v24.8b + ldp d4,d24,[x1],#16 // B + smlal v29.8h,v3.8b,v25.8b + ldp d5,d25,[x1],#16 + smlal v30.8h,v3.8b,v26.8b + ldp d6,d26,[x1],#16 + smlal v31.8h,v3.8b,v27.8b + ldp d7,d27,[x1],#16 + sadalp v20.4s,v28.8h + ldp d1,d3,[x9],#16 // Load A1 + sadalp v21.4s,v29.8h + sadalp v22.4s,v30.8h + sadalp v23.4s,v31.8h + b M2_ComputeBlockLoop + +M2_ComputeBlockLoopFinish: + ld1 {v0.4s},[x13],#16 // load ColumnSumBuffer[0] + smlal v28.8h,v2.8b,v24.8b + smlal v29.8h,v2.8b,v25.8b + smlal v30.8h,v2.8b,v26.8b + smlal v31.8h,v2.8b,v27.8b + sadalp v16.4s,v28.8h + sadalp v17.4s,v29.8h + sadalp v18.4s,v30.8h + sadalp v19.4s,v31.8h + smull v28.8h,v1.8b,v4.8b + smull v29.8h,v1.8b,v5.8b + smull v30.8h,v1.8b,v6.8b + smull v31.8h,v1.8b,v7.8b + smlal v28.8h,v3.8b,v24.8b + smlal v29.8h,v3.8b,v25.8b + smlal v30.8h,v3.8b,v26.8b + smlal v31.8h,v3.8b,v27.8b + sadalp v20.4s,v28.8h + sadalp v21.4s,v29.8h + sadalp v22.4s,v30.8h + sadalp v23.4s,v31.8h + addp v16.4s,v16.4s,v17.4s + addp v18.4s,v18.4s,v19.4s + addp v20.4s,v20.4s,v21.4s + addp v22.4s,v22.4s,v23.4s + addp v16.4s,v16.4s,v18.4s + addp v20.4s,v20.4s,v22.4s + + // accumulator = column sum B + add v16.4s,v16.4s,v0.4s + add v20.4s,v20.4s,v0.4s + +M2_StoreOutput: + add x10,x2,x6,lsl #2 + subs x5,x5,#4 // adjust CountN remaining + blo M2_StoreOutputPartial + st1 {v16.4s},[x2],#16 + st1 {v20.4s},[x10] + cbnz x5,M2_ProcessNextColumnLoop + +M2_ExitKernel: + mov x0,#2 // return number of rows handled + ldp d14,d15,[sp,#48] + ldp d12,d13,[sp,#32] + ldp d10,d11,[sp,#16] + ldp d8,d9,[sp],#.LSQGemmS8Frame_SavedRegisters + ret + +M2_StoreOutputPartial: + +M2_StoreOutputPartial_ZeroMode: + tbz x5,#1,M2_StoreOutputPartial1_ZeroMode + st1 {v16.2s},[x2],#8 + dup v16.4s,v16.s[2] // shift remaining elements down + st1 {v20.2s},[x10],#8 + dup v20.4s,v20.s[2] + +M2_StoreOutputPartial1_ZeroMode: + tbz x5,#0,M2_ExitKernel + st1 {v16.s}[0],[x2] + st1 {v20.s}[0],[x10] + b M2_ExitKernel + +// +// Process 1 row of the matrices. +// +// Column Sum v2.s[0] v2.s[4] +// row sum replicated to all 4 elements of a vector register +// v31 +// B 16x4 +// ---------------------------------------- +// |v4.b[0] v5.b[0] v6.b[0] v7.b[0] | +// | ... ... ... ... | +// |v4.b[7] v5.b[7] v6.b[7] v7.b[7] | +// |v24.b[0] v25.b[0] v26.b[0] v27.b[0]| +// | ... ... ... ... | +// |v24.b[7] v25.b[7] v26.b[7] v27.b[7]| +// A 1x16 ---------------------------------------- +// ----------------------------------- ---------------------------------------- +// |v0.b[0]..v0.b[7] v2.b[0]..v2.b[7]| |v16.4s v17.4s v18.4s v19.4s | +// ----------------------------------- ---------------------------------------- +// +// Accumulators are horizontally aggregated to the left most register +// for each row. e.g. (v16.s[0], v16.s[1], v16.s[2], v16.s[3]) <- (v16, v17, v18, v19) +// +M1_ProcessLoop: + +M1_ProcessNextColumnLoop: + ldp d4,d24,[x1],#16 // B + ldp d5,d25,[x1],#16 + ldp d6,d26,[x1],#16 + ldp d7,d27,[x1],#16 + mov x0,x14 // reload matrix A + mov x3,x15 // reload PackedCountK + ldp d0,d2,[x0],#16 // A0 + movi v16.4s,#0 + movi v17.4s,#0 + movi v18.4s,#0 + movi v19.4s,#0 + +M1_ComputeBlockLoop: + sub x3,x3,#1 + smull v20.8h,v0.8b,v4.8b + smull v21.8h,v0.8b,v5.8b + cbz x3,M1_ComputeBlockLoopFinish + smull v22.8h,v0.8b,v6.8b + smull v23.8h,v0.8b,v7.8b + smlal v20.8h,v2.8b,v24.8b + ldp d4,d24,[x1],#16 // B + smlal v21.8h,v2.8b,v25.8b + ldp d5,d25,[x1],#16 + smlal v22.8h,v2.8b,v26.8b + ldp d6,d26,[x1],#16 + smlal v23.8h,v2.8b,v27.8b + ldp d0,d2,[x0],#16 // A0 + sadalp v16.4s,v20.8h + sadalp v17.4s,v21.8h + ldp d7,d27,[x1],#16 + sadalp v18.4s,v22.8h + sadalp v19.4s,v23.8h + b M1_ComputeBlockLoop + +M1_ComputeBlockLoopFinish: + ld1 {v4.4s},[x13],#16 // load ColumnSumBuffer[0] + smull v22.8h,v0.8b,v6.8b + smull v23.8h,v0.8b,v7.8b + smlal v20.8h,v2.8b,v24.8b + smlal v21.8h,v2.8b,v25.8b + smlal v22.8h,v2.8b,v26.8b + smlal v23.8h,v2.8b,v27.8b + sadalp v16.4s,v20.8h + sadalp v17.4s,v21.8h + sadalp v18.4s,v22.8h + sadalp v19.4s,v23.8h + addp v16.4s,v16.4s,v17.4s + addp v18.4s,v18.4s,v19.4s + addp v16.4s,v16.4s,v18.4s + + // accumulator += column sum B + add v16.4s,v16.4s,v4.4s + +M1_StoreOutput: + subs x5,x5,#4 // adjust CountN remaining + blo M1_StoreOutputPartial + st1 {v16.4s},[x2],#16 + cbnz x5,M1_ProcessNextColumnLoop + +M1_ExitKernel: + mov x0,#1 // return number of rows handled + ldp d14,d15,[sp,#48] + ldp d12,d13,[sp,#32] + ldp d10,d11,[sp,#16] + ldp d8,d9,[sp],#.LSQGemmS8Frame_SavedRegisters + ret + +M1_StoreOutputPartial: + +M1_StoreOutputPartial_ZeroMode: + tbz x5,#1,M1_StoreOutputPartial1_ZeroMode + st1 {v16.2s},[x2],#8 + dup v16.4s,v16.s[2] // shift remaining elements down + +M1_StoreOutputPartial1_ZeroMode: + tbz x5,#0,M1_ExitKernel + st1 {v16.s}[0],[x2] + b M1_ExitKernel + + .end diff --git a/onnxruntime/core/mlas/lib/aarch64/SymQgemmS8KernelSdot.S b/onnxruntime/core/mlas/lib/aarch64/SymQgemmS8KernelSdot.S new file mode 100644 index 0000000000..b364f00e5d --- /dev/null +++ b/onnxruntime/core/mlas/lib/aarch64/SymQgemmS8KernelSdot.S @@ -0,0 +1,388 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SymQgemmS8KernelSdot.S + +Abstract: + + This module implements the kernels for the quantized integer matrix/matrix + multiply operation (QGEMM), where the right hand side is symmetrically quantized, + i.e. zero point being zero. + + This kernel only requires prepacking of the right hand side, which is usually + constant. When the packed right hand side is cached, we achieves higher performance + by avoid packing all together. + +--*/ + +#include "asmmacro.h" +#include "AssembleDotProduct.h" + +// +// Stack frame layout for the symmetric convolution kernel. +// d8-d15, x19-x30 need to be preserved if used +// + .equ .LGemmS8S8KernelFrame_SavedRegisters, (4 * 8) + .equ .LGemmS8S8KernelFrame_ColumnSumBuffer, (0 + .LGemmS8S8KernelFrame_SavedRegisters) + + .text + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A (x0) - Supplies the address of matrix A. + + B (x1) - Supplies the address of matrix B. The matrix data has been packed + using MlasGemmQuantCopyPackB. + + C (x2) - Supplies the address of matrix C. + + PackedCountK (x3) - Supplies the number of packed columns from matrix A and + the number of packed rows from matrix B to iterate over. + Packed K should be 16x + + CountM (x4) - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN (x5) - Supplies the number of columns from matrix B and matrix C to + iterate over. + + ldc (x6) - Supplies the first dimension of matrix C. + + lda (x7) - Supplies the first dimension of matrix A. + + ColumnSumBuffer - Supplies the sum of each column from matrix B multiplied + by the zero point offset of matrix A. These values are accumulated into + every column of matrix C. + +Return Value: + + Returns the number of rows handled. + +--*/ + + FUNCTION_ENTRY MlasSymQgemmS8KernelSdot + + stp d8,d9,[sp,#-.LGemmS8S8KernelFrame_SavedRegisters]! + ldr x8,[sp,#.LGemmS8S8KernelFrame_ColumnSumBuffer] + stp d10,d11,[sp,#16] + + // compute C pointers: x2, x16, x17, x6 + cmp x4,#2 // M < 2 ? + add x16,x2,x6,lsl #2 // x16 -> C1 + add x17,x2,x6,lsl #3 // x17 -> C2 + csel x16,x2,x16,lo // if M < 2 x16/C1 -> C0 + csel x17,x16,x17,ls // if M <= 2 x17/C2 -> C1 + cmp x4,#4 // M < 4 ? + mov x12,#4 // set max M to 4 + add x6,x16,x6,lsl #3 // x6 -> C3 + mov x9,x0 // save A0 + mov x10,x3 // save K + csel x6,x17,x6,lo // if M < 4 x6/C3 -> C2 + csel x4,x12,x4,hi // if M > 4 M = 4; + +// Register Usage +// B (x1) -> 4x16 +// ---------------------------------------------------------------------------- +// |v4.b[0]..v4.b[12] v5.b[0]..v5.b[12] v6.b[0]..v6.b[12] v7.b[0]..v7.b[12]| +// | ... ... ... ... ... ... ... ... | +// |v4.b[3]..v4.b[15] v5.b[3]..v5.b[15] v6.b[3]..v6.b[15] v7.b[3]..v7.b[15]| +// A 4x4 ---------------------------------------------------------------------------- +// ------------------ ---------------------------------------------------------------------------- +// x0 |v0.b[0]..v0.b[3]| |v16.s[0]_v16.s[3] v20.s[0]_v20.s[3] v24.s[0]_v24.s[3] v28.s[0]_v28.s[3]| x2 +// x12 |v1.b[0]..v1.b[3]| |v17.s[0]_v17.s[3] v21.s[0]_v21.s[3] v25.s[0]_v25.s[3] v29.s[0]_v29.s[3]| x16 +// x13 |v2.b[0]..v2.b[3]| |v18.s[0]_v18.s[3] v22.s[0]_v22.s[3] v26.s[0]_v26.s[3] v30.s[0]_v30.s[3]| x17 +// x14 |v3.b[0]..v3.b[3]| |v19.s[0]_v19.s[3] v23.s[0]_v23.s[3] v27.s[0]_v27.s[3] v31.s[0]_v31.s[3]| x6 +// ------------------ ---------------------------------------------------------------------------- + +ProcessNextColumnLoop: + ldr q16,[x8],#16 // Init accumulators with column sums + ldr q20,[x8],#16 + ldr q24,[x8],#16 + ldr q28,[x8],#16 + mov x0,x9 // reload A0 + cmp x4,#2 // M < 2 ? + add x12,x9,x7 // x12 -> A1 + add x13,x0,x7,lsl #1 // x13 -> A2 + ldr q4,[x1],#16 // Load B + csel x12,x0,x12,lo // if M < 2 A1 -> A0 + csel x13,x12,x13,ls // if M <= 2 A2 -> A1 + cmp x4,4 // M < 4 ? + add x14,x12,x7,lsl #1 // x14 -> A3 + ldr q5,[x1],#16 + csel x14,x13,x14,lo // if M < 4 A3 -> A2 + ldr d0,[x0],#8 // Load A0 1st/2nd block of 4 + mov v17.16b,v16.16b + mov v18.16b,v16.16b + ldr d1,[x12],#8 // Load A1 + mov v19.16b,v16.16b + mov v21.16b,v20.16b + ldr d2,[x13],#8 // Load A2 + mov v22.16b,v20.16b + mov v23.16b,v20.16b + ldr d3,[x14],#8 // Load A3 + mov v25.16b,v24.16b + mov v26.16b,v24.16b + ldr q6,[x1],#16 + mov v27.16b,v24.16b + mov v29.16b,v28.16b + ldr q7,[x1],#16 + subs x3,x10,#2 // one loop iteration and epilogue consume k = 32 + mov v30.16b,v28.16b + mov v31.16b,v28.16b + b.lo BlockLoopEpilogue // Need 32 k for main loop + +BlockLoop: + SdotByElement 16, 4, 0,0 + SdotByElement 17, 4, 1,0 + ldr d8,[x0],#8 // Load A0 3rd/4th block of 4 + SdotByElement 18, 4, 2,0 + SdotByElement 19, 4, 3,0 + ldr q4,[x1],#16 + SdotByElement 20, 5, 0,0 + SdotByElement 21, 5, 1,0 + ldr d9,[x12],#8 + SdotByElement 22, 5, 2,0 + SdotByElement 23, 5, 3,0 + ldr q5,[x1],#16 + SdotByElement 24, 6, 0,0 + SdotByElement 25, 6, 1,0 + ldr d10,[x13],#8 + SdotByElement 26, 6, 2,0 + SdotByElement 27, 6, 3,0 + ldr q6,[x1],#16 + SdotByElement 28, 7, 0,0 + SdotByElement 29, 7, 1,0 + ldr d11,[x14],#8 + SdotByElement 30, 7, 2,0 + SdotByElement 31, 7, 3,0 + ldr q7,[x1],#16 + SdotByElement 16, 4, 0,1 + SdotByElement 17, 4, 1,1 + SdotByElement 18, 4, 2,1 + SdotByElement 19, 4, 3,1 + ldr q4,[x1],#16 + SdotByElement 20, 5, 0,1 + SdotByElement 21, 5, 1,1 + SdotByElement 22, 5, 2,1 + SdotByElement 23, 5, 3,1 + ldr q5,[x1],#16 + SdotByElement 24, 6, 0,1 + SdotByElement 25, 6, 1,1 + SdotByElement 26, 6, 2,1 + SdotByElement 27, 6, 3,1 + ldr q6,[x1],#16 + SdotByElement 28, 7, 0,1 + SdotByElement 29, 7, 1,1 + SdotByElement 30, 7, 2,1 + SdotByElement 31, 7, 3,1 + ldr q7,[x1],#16 + SdotByElement 16, 4, 8,0 + SdotByElement 17, 4, 9,0 + ldr d0,[x0],#8 + SdotByElement 18, 4,10,0 + SdotByElement 19, 4,11,0 + ldr q4,[x1],#16 + SdotByElement 20, 5, 8,0 + SdotByElement 21, 5, 9,0 + ldr d1,[x12],#8 + SdotByElement 22, 5,10,0 + SdotByElement 23, 5,11,0 + ldr q5,[x1],#16 + SdotByElement 24, 6, 8,0 + SdotByElement 25, 6, 9,0 + ldr d2,[x13],#8 + SdotByElement 26, 6,10,0 + SdotByElement 27, 6,11,0 + ldr q6,[x1],#16 + SdotByElement 28, 7, 8,0 + SdotByElement 29, 7, 9,0 + ldr d3,[x14],#8 + SdotByElement 30, 7,10,0 + SdotByElement 31, 7,11,0 + ldr q7,[x1],#16 + SdotByElement 16, 4, 8,1 + SdotByElement 17, 4, 9,1 + SdotByElement 18, 4,10,1 + SdotByElement 19, 4,11,1 + ldr q4,[x1],#16 + SdotByElement 20, 5, 8,1 + SdotByElement 21, 5, 9,1 + SdotByElement 22, 5,10,1 + SdotByElement 23, 5,11,1 + ldr q5,[x1],#16 + SdotByElement 24, 6, 8,1 + SdotByElement 25, 6, 9,1 + SdotByElement 26, 6,10,1 + SdotByElement 27, 6,11,1 + ldr q6,[x1],#16 + SdotByElement 28, 7, 8,1 + SdotByElement 29, 7, 9,1 + subs x3,x3,#1 // k -= 16 + SdotByElement 30, 7,10,1 + SdotByElement 31, 7,11,1 + ldr q7,[x1],#16 + b.hs BlockLoop + +BlockLoopEpilogue: + SdotByElement 16, 4, 0,0 + SdotByElement 17, 4, 1,0 + ldr d8,[x0],#8 + SdotByElement 18, 4, 2,0 + SdotByElement 19, 4, 3,0 + ldr q4,[x1],#16 + SdotByElement 20, 5, 0,0 + SdotByElement 21, 5, 1,0 + ldr d9,[x12],#8 + SdotByElement 22, 5, 2,0 + SdotByElement 23, 5, 3,0 + ldr q5,[x1],#16 + SdotByElement 24, 6, 0,0 + SdotByElement 25, 6, 1,0 + ldr d10,[x13],#8 + SdotByElement 26, 6, 2,0 + SdotByElement 27, 6, 3,0 + ldr q6,[x1],#16 + SdotByElement 28, 7, 0,0 + SdotByElement 29, 7, 1,0 + ldr d11,[x14],#8 + SdotByElement 30, 7, 2,0 + SdotByElement 31, 7, 3,0 + ldr q7,[x1],#16 + SdotByElement 16, 4, 0,1 + SdotByElement 17, 4, 1,1 + SdotByElement 18, 4, 2,1 + SdotByElement 19, 4, 3,1 + ldr q4,[x1],#16 + SdotByElement 20, 5, 0,1 + SdotByElement 21, 5, 1,1 + SdotByElement 22, 5, 2,1 + SdotByElement 23, 5, 3,1 + ldr q5,[x1],#16 + SdotByElement 24, 6, 0,1 + SdotByElement 25, 6, 1,1 + SdotByElement 26, 6, 2,1 + SdotByElement 27, 6, 3,1 + ldr q6,[x1],#16 + SdotByElement 28, 7, 0,1 + SdotByElement 29, 7, 1,1 + SdotByElement 30, 7, 2,1 + SdotByElement 31, 7, 3,1 + ldr q7,[x1],#16 + SdotByElement 16, 4, 8,0 + SdotByElement 17, 4, 9,0 + SdotByElement 18, 4,10,0 + SdotByElement 19, 4,11,0 + ldr q4,[x1],#16 + SdotByElement 20, 5, 8,0 + SdotByElement 21, 5, 9,0 + SdotByElement 22, 5,10,0 + SdotByElement 23, 5,11,0 + ldr q5,[x1],#16 + SdotByElement 24, 6, 8,0 + SdotByElement 25, 6, 9,0 + SdotByElement 26, 6,10,0 + SdotByElement 27, 6,11,0 + ldr q6,[x1],#16 + SdotByElement 28, 7, 8,0 + SdotByElement 29, 7, 9,0 + SdotByElement 30, 7,10,0 + SdotByElement 31, 7,11,0 + ldr q7,[x1],#16 + SdotByElement 16, 4, 8,1 + SdotByElement 17, 4, 9,1 + SdotByElement 18, 4,10,1 + SdotByElement 19, 4,11,1 + SdotByElement 20, 5, 8,1 + SdotByElement 21, 5, 9,1 + SdotByElement 22, 5,10,1 + SdotByElement 23, 5,11,1 + SdotByElement 24, 6, 8,1 + SdotByElement 25, 6, 9,1 + SdotByElement 26, 6,10,1 + SdotByElement 27, 6,11,1 + SdotByElement 28, 7, 8,1 + SdotByElement 29, 7, 9,1 + subs x5,x5,#16 // adjust CountN remaining + SdotByElement 30, 7,10,1 + SdotByElement 31, 7,11,1 + blo StoreOutputPartial + stp q16,q20,[x2],#32 + stp q24,q28,[x2],#32 + stp q17,q21,[x16],#32 + stp q25,q29,[x16],#32 + stp q18,q22,[x17],#32 + stp q26,q30,[x17],#32 + stp q19,q23,[x6],#32 + stp q27,q31,[x6],#32 + cbnz x5,ProcessNextColumnLoop + +ExitKernel: + mov x0,x4 // return number of rows handled + ldp d10,d11,[sp,#16] + ldp d8,d9,[sp],#.LGemmS8S8KernelFrame_SavedRegisters + ret + +// +// Store the partial 1 to 15 columns either overwriting the output matrix or +// accumulating into the existing contents of the output matrix. +// + +StoreOutputPartial: + tbz x5,#3,StoreOutputPartial4 + stp q16,q20,[x2],#32 + mov v16.16b,v24.16b // shift remaining elements down + mov v20.16b,v28.16b + stp q17,q21,[x16],#32 + mov v17.16b,v25.16b + mov v21.16b,v29.16b + stp q18,q22,[x17],#32 + mov v18.16b,v26.16b + mov v22.16b,v30.16b + stp q19,q23,[x6],#32 + mov v19.16b,v27.16b + mov v23.16b,v31.16b + +StoreOutputPartial4: + tbz x5,#2,StoreOutputPartial2 + st1 {v16.4s},[x2],#16 + mov v16.16b,v20.16b // shift remaining elements down + st1 {v17.4s},[x16],#16 + mov v17.16b,v21.16b + st1 {v18.4s},[x17],#16 + mov v18.16b,v22.16b + st1 {v19.4s},[x6],#16 + mov v19.16b,v23.16b + +StoreOutputPartial2: + tbz x5,#1,StoreOutputPartial1 + st1 {v16.2s},[x2],#8 + dup v16.4s,v16.s[2] // shift remaining elements down + st1 {v17.2s},[x16],#8 + dup v17.4s,v17.s[2] + st1 {v18.2s},[x17],#8 + dup v18.4s,v18.s[2] + st1 {v19.2s},[x6],#8 + dup v19.4s,v19.s[2] + +StoreOutputPartial1: + tbz x5,#0,ExitKernel + st1 {v16.s}[0],[x2] + st1 {v17.s}[0],[x16] + st1 {v18.s}[0],[x17] + st1 {v19.s}[0],[x6] + b ExitKernel + + .end diff --git a/onnxruntime/core/mlas/lib/arm64/SymQgemmS8KernelNeon.asm b/onnxruntime/core/mlas/lib/arm64/SymQgemmS8KernelNeon.asm new file mode 100644 index 0000000000..4770b071dd --- /dev/null +++ b/onnxruntime/core/mlas/lib/arm64/SymQgemmS8KernelNeon.asm @@ -0,0 +1,538 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SymQgemmS8KernelNeon.asm + +Abstract: + + This module implements the kernels for the quantized integer matrix/matrix + multiply operation (QGEMM), where the right hand side is symmetrically quantized, + i.e. zero point being zero. + + This kernel only requires prepacking of the right hand side, which is usually + constant. When the packed right hand side is cached, we achieves higher performance + by avoid packing all together. + +--*/ + +#include "kxarm64.h" + +// +// Stack frame layout for the S8S8 kernel. +// + +#define SQGemmS8Frame_SavedNeonRegisters (8 * 8) +#define SQGemmS8Frame_SavedRegisters SQGemmS8Frame_SavedNeonRegisters +#define SQGemmS8Frame_ColumnSumBuffer 0 + SQGemmS8Frame_SavedRegisters + + TEXTAREA + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A (x0) - Supplies the address of matrix A. + + B (x1) - Supplies the address of matrix B. The matrix data has been packed + using MlasGemmQuantCopyPackB. + + C (x2) - Supplies the address of matrix C. + + PackedCountK (x3) - Supplies the number of packed columns from matrix A and + the number of packed rows from matrix B to iterate over. + + CountM (x4) - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN (x5) - Supplies the number of columns from matrix B and matrix C to + iterate over. + + ldc (x6) - Supplies the first dimension of matrix C. + + lda (x7) - Supplies the first dimension of matrix A. + + ColumnSumBuffer - Supplies the sum of each column from matrix B multiplied + by the zero point offset of matrix A. These values are accumulated into + every column of matrix C. + + +Return Value: + + Returns the number of rows handled. + +--*/ + + NESTED_ENTRY MlasSymQgemmS8KernelNeon + + PROLOG_SAVE_REG_PAIR d8,d9,#-SQGemmS8Frame_SavedRegisters! + PROLOG_SAVE_REG_PAIR d10,d11,#16 + PROLOG_SAVE_REG_PAIR d12,d13,#32 + PROLOG_SAVE_REG_PAIR d14,d15,#48 + ldr x13,[sp,#SQGemmS8Frame_ColumnSumBuffer] + mov x14,x0 + mov x15,x3 + cmp x4,#1 // CountM == 1? + beq M1_ProcessLoop + cmp x4,#4 // CountM < 4? + blo M2_ProcessLoop + +// +// Process 4 rows of the matrices. +// B 16x4 +// ---------------------------------------- +// |v4.b[0] v5.b[0] v6.b[0] v7.b[0] | +// | ... ... ... ... | +// |v4.b[7] v5.b[7] v6.b[7] v7.b[7] | +// |v8.b[0] v9.b[0] v10.b[0] v11.b[0]| +// | ... ... ... ... | +// |v8.b[7] v9.b[7] v10.b[7] v11.b[7]| +// A 4x16 ---------------------------------------- +// ----------------------------------- ---------------------------------------- +// |v0.b[0]..v0.b[7] v2.b[0]..v2.b[7]| |v16.4s v17.4s v18.4s v19.4s | +// |v1.b[0]..v1.b[7] v3.b[0]..v3.b[7]| |v20.4s v21.4s v22.4s v23.4s | +// |v0.b[0]..v0.b[7] v2.b[0]..v2.b[7]| |v24.4s v25.4s v26.4s v27.4s | +// |v1.b[0]..v1.b[7] v3.b[0]..v3.b[7]| |v28.4s v29.4s v30.4s v31.4s | +// ----------------------------------- ---------------------------------------- +// +// Accumulators are horizontally aggregated to the left most register +// for each row. e.g. (v16.s[0], v16.s[1], v16.s[2], v16.s[3]) <- (v16, v17, v18, v19) +// + +M4_ProcessNextColumnLoop + mov x0,x14 // reload matrix A0 + mov x3,x15 // reload PackedCountK + ldr d0,[x0],#8 // Load A0 + add x9,x14,x7 // A1 + ldr d2,[x0],#8 // Load A0 + movi v16.4s,#0 + movi v17.4s,#0 + ldp d4,d8,[x1],#64 // B + movi v18.4s,#0 + movi v19.4s,#0 + ldp d5,d9,[x1,#-48] + movi v20.4s,#0 + movi v21.4s,#0 + ldp d6,d10,[x1,#-32] + movi v22.4s,#0 + movi v23.4s,#0 + ldp d7,d11,[x1,#-16] + movi v24.4s,#0 + movi v25.4s,#0 + add x10,x9,x7 // A2 + ldp d1,d3,[x9],#16 // Load A1 + movi v26.4s,#0 + movi v27.4s,#0 + movi v28.4s,#0 + movi v29.4s,#0 + movi v30.4s,#0 + movi v31.4s,#0 + add x11,x10,x7 // A3 + +M4_ComputeBlockLoop + smull v12.8h,v0.8b,v4.8b + smull v13.8h,v0.8b,v5.8b + smull v14.8h,v0.8b,v6.8b + smull v15.8h,v0.8b,v7.8b + smlal v12.8h,v2.8b,v8.8b + smlal v13.8h,v2.8b,v9.8b + smlal v14.8h,v2.8b,v10.8b + smlal v15.8h,v2.8b,v11.8b + ldp d0,d2,[x10],#16 // Load A2 + sadalp v16.4s,v12.8h + sadalp v17.4s,v13.8h + sadalp v18.4s,v14.8h + sadalp v19.4s,v15.8h + sub x3,x3,#1 + smull v12.8h,v1.8b,v4.8b + smull v13.8h,v1.8b,v5.8b + smull v14.8h,v1.8b,v6.8b + smull v15.8h,v1.8b,v7.8b + smlal v12.8h,v3.8b,v8.8b + smlal v13.8h,v3.8b,v9.8b + smlal v14.8h,v3.8b,v10.8b + smlal v15.8h,v3.8b,v11.8b + ldp d1,d3,[x11],#16 // Load A3 + sadalp v20.4s,v12.8h + sadalp v21.4s,v13.8h + sadalp v22.4s,v14.8h + sadalp v23.4s,v15.8h + cbz x3,M4_ComputeBlockLoopFinish + smull v12.8h,v0.8b,v4.8b + smull v13.8h,v0.8b,v5.8b + smull v14.8h,v0.8b,v6.8b + smull v15.8h,v0.8b,v7.8b + smlal v12.8h,v2.8b,v8.8b + smlal v13.8h,v2.8b,v9.8b + smlal v14.8h,v2.8b,v10.8b + smlal v15.8h,v2.8b,v11.8b + ldp d0,d2,[x0],#16 // Load A0 next iter + sadalp v24.4s,v12.8h + sadalp v25.4s,v13.8h + sadalp v26.4s,v14.8h + sadalp v27.4s,v15.8h + smull v12.8h,v1.8b,v4.8b + smull v13.8h,v1.8b,v5.8b + smull v14.8h,v1.8b,v6.8b + smull v15.8h,v1.8b,v7.8b + smlal v12.8h,v3.8b,v8.8b + ldp d4,d8,[x1],#64 // B + smlal v13.8h,v3.8b,v9.8b + ldp d5,d9,[x1,#-48] + smlal v14.8h,v3.8b,v10.8b + ldp d6,d10,[x1,#-32] + smlal v15.8h,v3.8b,v11.8b + ldp d7,d11,[x1,#-16] + sadalp v28.4s,v12.8h + ldp d1,d3,[x9],#16 // Load A1 next iter + sadalp v29.4s,v13.8h + sadalp v30.4s,v14.8h + sadalp v31.4s,v15.8h + b M4_ComputeBlockLoop + +M4_ComputeBlockLoopFinish + smull v12.8h,v0.8b,v4.8b + smull v13.8h,v0.8b,v5.8b + smull v14.8h,v0.8b,v6.8b + smull v15.8h,v0.8b,v7.8b + smlal v12.8h,v2.8b,v8.8b + smlal v13.8h,v2.8b,v9.8b + smlal v14.8h,v2.8b,v10.8b + smlal v15.8h,v2.8b,v11.8b + ld1 {v2.4s},[x13],#16 // load ColumnSumBuffer[0] + sadalp v24.4s,v12.8h + sadalp v25.4s,v13.8h + sadalp v26.4s,v14.8h + sadalp v27.4s,v15.8h + smull v12.8h,v1.8b,v4.8b + smull v13.8h,v1.8b,v5.8b + smull v14.8h,v1.8b,v6.8b + smull v15.8h,v1.8b,v7.8b + smlal v12.8h,v3.8b,v8.8b + smlal v13.8h,v3.8b,v9.8b + smlal v14.8h,v3.8b,v10.8b + smlal v15.8h,v3.8b,v11.8b + sadalp v28.4s,v12.8h + sadalp v29.4s,v13.8h + sadalp v30.4s,v14.8h + sadalp v31.4s,v15.8h + addp v16.4s,v16.4s,v17.4s + addp v18.4s,v18.4s,v19.4s + addp v20.4s,v20.4s,v21.4s + addp v22.4s,v22.4s,v23.4s + addp v24.4s,v24.4s,v25.4s + addp v26.4s,v26.4s,v27.4s + addp v28.4s,v28.4s,v29.4s + addp v30.4s,v30.4s,v31.4s + addp v16.4s,v16.4s,v18.4s + addp v20.4s,v20.4s,v22.4s + addp v24.4s,v24.4s,v26.4s + addp v28.4s,v28.4s,v30.4s + + // accumulator += column sum B + add v16.4s,v16.4s,v2.4s + add v20.4s,v20.4s,v2.4s + add v24.4s,v24.4s,v2.4s + add v28.4s,v28.4s,v2.4s + +M4_StoreOutput + add x10,x2,x6,lsl #2 + add x11,x10,x6,lsl #2 + add x12,x11,x6,lsl #2 + subs x5,x5,#4 // adjust CountN remaining + blo M4_StoreOutputPartial + st1 {v16.4s},[x2],#16 + st1 {v20.4s},[x10] + st1 {v24.4s},[x11] + st1 {v28.4s},[x12] + cbnz x5,M4_ProcessNextColumnLoop + +M4_ExitKernel + mov x0,#4 // return number of rows handled + EPILOG_RESTORE_REG_PAIR d14,d15,#48 + EPILOG_RESTORE_REG_PAIR d12,d13,#32 + EPILOG_RESTORE_REG_PAIR d10,d11,#16 + EPILOG_RESTORE_REG_PAIR d8,d9,#64! + EPILOG_RETURN + +M4_StoreOutputPartial + +M4_StoreOutputPartial_ZeroMode + tbz x5,#1,M4_StoreOutputPartial1_ZeroMode + st1 {v16.2s},[x2],#8 + dup v16.4s,v16.s[2] // shift remaining elements down + st1 {v20.2s},[x10],#8 + dup v20.4s,v20.s[2] + st1 {v24.2s},[x11],#8 + dup v24.4s,v24.s[2] + st1 {v28.2s},[x12],#8 + dup v28.4s,v28.s[2] + +M4_StoreOutputPartial1_ZeroMode + tbz x5,#0,M4_ExitKernel + st1 {v16.s}[0],[x2] + st1 {v20.s}[0],[x10] + st1 {v24.s}[0],[x11] + st1 {v28.s}[0],[x12] + b M4_ExitKernel + +// +// Process 2 rows of the matrices. +// +// Column Sum v2.s[0] v2.s[4] +// Each row sum replicated to all 4 elements of a vector register +// v30 v31 +// B 16x4 +// ---------------------------------------- +// |v4.b[0] v5.b[0] v6.b[0] v7.b[0] | +// | ... ... ... ... | +// |v4.b[7] v5.b[7] v6.b[7] v7.b[7] | +// |v24.b[0] v25.b[0] v26.b[0] v27.b[0]| +// | ... ... ... ... | +// |v24.b[7] v25.b[7] v26.b[7] v27.b[7]| +// A 2x16 ---------------------------------------- +// ----------------------------------- ---------------------------------------- +// |v0.b[0]..v0.b[7] v2.b[0]..v2.b[7]| |v16.4s v17.4s v18.4s v19.4s | +// |v1.b[0]..v1.b[7] v3.b[0]..v3.b[7]| |v20.4s v21.4s v22.4s v23.4s | +// ----------------------------------- ---------------------------------------- +// +// Accumulators are horizontally aggregated to the left most register +// for each row. e.g. (v16.s[0], v16.s[1], v16.s[2], v16.s[3]) <- (v16, v17, v18, v19) + +M2_ProcessLoop + +M2_ProcessNextColumnLoop + ldp d4,d24,[x1],#16 // B + mov x0,x14 // reload matrix A + mov x3,x15 // reload PackedCountK + ldp d0,d2,[x0],#16 // Load A0 + add x9,x14,x7 // A1 + movi v16.4s,#0 + movi v17.4s,#0 + ldp d5,d25,[x1],#16 + movi v18.4s,#0 + movi v19.4s,#0 + ldp d6,d26,[x1],#16 + movi v20.4s,#0 + movi v21.4s,#0 + ldp d7,d27,[x1],#16 + movi v22.4s,#0 + movi v23.4s,#0 + ldp d1,d3,[x9],#16 // Load A1 + +M2_ComputeBlockLoop + sub x3,x3,#1 + smull v28.8h,v0.8b,v4.8b + smull v29.8h,v0.8b,v5.8b + smull v30.8h,v0.8b,v6.8b + smull v31.8h,v0.8b,v7.8b + cbz x3,M2_ComputeBlockLoopFinish + smlal v28.8h,v2.8b,v24.8b + smlal v29.8h,v2.8b,v25.8b + smlal v30.8h,v2.8b,v26.8b + smlal v31.8h,v2.8b,v27.8b + ldp d0,d2,[x0],#16 // Load A0 + sadalp v16.4s,v28.8h + sadalp v17.4s,v29.8h + sadalp v18.4s,v30.8h + sadalp v19.4s,v31.8h + smull v28.8h,v1.8b,v4.8b + smull v29.8h,v1.8b,v5.8b + smull v30.8h,v1.8b,v6.8b + smull v31.8h,v1.8b,v7.8b + smlal v28.8h,v3.8b,v24.8b + ldp d4,d24,[x1],#16 // B + smlal v29.8h,v3.8b,v25.8b + ldp d5,d25,[x1],#16 + smlal v30.8h,v3.8b,v26.8b + ldp d6,d26,[x1],#16 + smlal v31.8h,v3.8b,v27.8b + ldp d7,d27,[x1],#16 + sadalp v20.4s,v28.8h + ldp d1,d3,[x9],#16 // Load A1 + sadalp v21.4s,v29.8h + sadalp v22.4s,v30.8h + sadalp v23.4s,v31.8h + b M2_ComputeBlockLoop + +M2_ComputeBlockLoopFinish + ld1 {v0.4s},[x13],#16 // load ColumnSumBuffer[0] + smlal v28.8h,v2.8b,v24.8b + smlal v29.8h,v2.8b,v25.8b + smlal v30.8h,v2.8b,v26.8b + smlal v31.8h,v2.8b,v27.8b + sadalp v16.4s,v28.8h + sadalp v17.4s,v29.8h + sadalp v18.4s,v30.8h + sadalp v19.4s,v31.8h + smull v28.8h,v1.8b,v4.8b + smull v29.8h,v1.8b,v5.8b + smull v30.8h,v1.8b,v6.8b + smull v31.8h,v1.8b,v7.8b + smlal v28.8h,v3.8b,v24.8b + smlal v29.8h,v3.8b,v25.8b + smlal v30.8h,v3.8b,v26.8b + smlal v31.8h,v3.8b,v27.8b + sadalp v20.4s,v28.8h + sadalp v21.4s,v29.8h + sadalp v22.4s,v30.8h + sadalp v23.4s,v31.8h + addp v16.4s,v16.4s,v17.4s + addp v18.4s,v18.4s,v19.4s + addp v20.4s,v20.4s,v21.4s + addp v22.4s,v22.4s,v23.4s + addp v16.4s,v16.4s,v18.4s + addp v20.4s,v20.4s,v22.4s + + // accumulator = column sum B + add v16.4s,v16.4s,v0.4s + add v20.4s,v20.4s,v0.4s + +M2_StoreOutput + add x10,x2,x6,lsl #2 + subs x5,x5,#4 // adjust CountN remaining + blo M2_StoreOutputPartial + st1 {v16.4s},[x2],#16 + st1 {v20.4s},[x10] + cbnz x5,M2_ProcessNextColumnLoop + +M2_ExitKernel + mov x0,#2 // return number of rows handled + EPILOG_RESTORE_REG_PAIR d14,d15,#48 + EPILOG_RESTORE_REG_PAIR d12,d13,#32 + EPILOG_RESTORE_REG_PAIR d10,d11,#16 + EPILOG_RESTORE_REG_PAIR d8,d9,#64! + EPILOG_RETURN + +M2_StoreOutputPartial + +M2_StoreOutputPartial_ZeroMode + tbz x5,#1,M2_StoreOutputPartial1_ZeroMode + st1 {v16.2s},[x2],#8 + dup v16.4s,v16.s[2] // shift remaining elements down + st1 {v20.2s},[x10],#8 + dup v20.4s,v20.s[2] + +M2_StoreOutputPartial1_ZeroMode + tbz x5,#0,M2_ExitKernel + st1 {v16.s}[0],[x2] + st1 {v20.s}[0],[x10] + b M2_ExitKernel + +// +// Process 1 row of the matrices. +// +// Column Sum v2.s[0] v2.s[4] +// row sum replicated to all 4 elements of a vector register +// v31 +// B 16x4 +// ---------------------------------------- +// |v4.b[0] v5.b[0] v6.b[0] v7.b[0] | +// | ... ... ... ... | +// |v4.b[7] v5.b[7] v6.b[7] v7.b[7] | +// |v24.b[0] v25.b[0] v26.b[0] v27.b[0]| +// | ... ... ... ... | +// |v24.b[7] v25.b[7] v26.b[7] v27.b[7]| +// A 1x16 ---------------------------------------- +// ----------------------------------- ---------------------------------------- +// |v0.b[0]..v0.b[7] v2.b[0]..v2.b[7]| |v16.4s v17.4s v18.4s v19.4s | +// ----------------------------------- ---------------------------------------- +// +// Accumulators are horizontally aggregated to the left most register +// for each row. e.g. (v16.s[0], v16.s[1], v16.s[2], v16.s[3]) <- (v16, v17, v18, v19) +// +M1_ProcessLoop + +M1_ProcessNextColumnLoop + ldp d4,d24,[x1],#16 // B + ldp d5,d25,[x1],#16 + ldp d6,d26,[x1],#16 + ldp d7,d27,[x1],#16 + mov x0,x14 // reload matrix A + mov x3,x15 // reload PackedCountK + ldp d0,d2,[x0],#16 // A0 + movi v16.4s,#0 + movi v17.4s,#0 + movi v18.4s,#0 + movi v19.4s,#0 + +M1_ComputeBlockLoop + sub x3,x3,#1 + smull v20.8h,v0.8b,v4.8b + smull v21.8h,v0.8b,v5.8b + cbz x3,M1_ComputeBlockLoopFinish + smull v22.8h,v0.8b,v6.8b + smull v23.8h,v0.8b,v7.8b + smlal v20.8h,v2.8b,v24.8b + ldp d4,d24,[x1],#16 // B + smlal v21.8h,v2.8b,v25.8b + ldp d5,d25,[x1],#16 + smlal v22.8h,v2.8b,v26.8b + ldp d6,d26,[x1],#16 + smlal v23.8h,v2.8b,v27.8b + ldp d0,d2,[x0],#16 // A0 + sadalp v16.4s,v20.8h + sadalp v17.4s,v21.8h + ldp d7,d27,[x1],#16 + sadalp v18.4s,v22.8h + sadalp v19.4s,v23.8h + b M1_ComputeBlockLoop + +M1_ComputeBlockLoopFinish + ld1 {v4.4s},[x13],#16 // load ColumnSumBuffer[0] + smull v22.8h,v0.8b,v6.8b + smull v23.8h,v0.8b,v7.8b + smlal v20.8h,v2.8b,v24.8b + smlal v21.8h,v2.8b,v25.8b + smlal v22.8h,v2.8b,v26.8b + smlal v23.8h,v2.8b,v27.8b + sadalp v16.4s,v20.8h + sadalp v17.4s,v21.8h + sadalp v18.4s,v22.8h + sadalp v19.4s,v23.8h + addp v16.4s,v16.4s,v17.4s + addp v18.4s,v18.4s,v19.4s + addp v16.4s,v16.4s,v18.4s + + // accumulator += column sum B + add v16.4s,v16.4s,v4.4s + +M1_StoreOutput + subs x5,x5,#4 // adjust CountN remaining + blo M1_StoreOutputPartial + st1 {v16.4s},[x2],#16 + cbnz x5,M1_ProcessNextColumnLoop + +M1_ExitKernel + mov x0,#1 // return number of rows handled + EPILOG_RESTORE_REG_PAIR d14,d15,#48 + EPILOG_RESTORE_REG_PAIR d12,d13,#32 + EPILOG_RESTORE_REG_PAIR d10,d11,#16 + EPILOG_RESTORE_REG_PAIR d8,d9,#64! + EPILOG_RETURN + +M1_StoreOutputPartial + +M1_StoreOutputPartial_ZeroMode + tbz x5,#1,M1_StoreOutputPartial1_ZeroMode + st1 {v16.2s},[x2],#8 + dup v16.4s,v16.s[2] // shift remaining elements down + +M1_StoreOutputPartial1_ZeroMode + tbz x5,#0,M1_ExitKernel + st1 {v16.s}[0],[x2] + b M1_ExitKernel + + NESTED_END MlasSymQgemmS8KernelNeon + + END diff --git a/onnxruntime/core/mlas/lib/arm64/SymQgemmS8KernelSdot.asm b/onnxruntime/core/mlas/lib/arm64/SymQgemmS8KernelSdot.asm new file mode 100644 index 0000000000..2d4f6ea52e --- /dev/null +++ b/onnxruntime/core/mlas/lib/arm64/SymQgemmS8KernelSdot.asm @@ -0,0 +1,391 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SymQgemmS8KernelSdot.asm + +Abstract: + + This module implements the kernels for the quantized integer matrix/matrix + multiply operation (QGEMM), where the right hand side is symmetrically quantized, + i.e. zero point being zero. + + This kernel only requires prepacking of the right hand side, which is usually + constant. When the packed right hand side is cached, we achieves higher performance + by avoid packing all together. + +--*/ + +#include "kxarm64.h" +#include "AssembleDotProduct.h" + +// +// Stack frame layout for the S8S8 kernel. +// + + +#define GemmS8S8KernelFrame_SavedRegisters (4 * 8) +#define GemmS8S8KernelFrame_ColumnSumBuffer (0 + GemmS8S8KernelFrame_SavedRegisters) + + TEXTAREA + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A (x0) - Supplies the address of matrix A. + + B (x1) - Supplies the address of matrix B. The matrix data has been packed + using MlasGemmQuantCopyPackB. + + C (x2) - Supplies the address of matrix C. + + PackedCountK (x3) - Supplies the number of packed columns from matrix A and + the number of packed rows from matrix B to iterate over. + Packed K should be 16x + + CountM (x4) - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN (x5) - Supplies the number of columns from matrix B and matrix C to + iterate over. + + ldc (x6) - Supplies the first dimension of matrix C. + + lda (x7) - Supplies the first dimension of matrix A. + + ColumnSumBuffer - Supplies the sum of each column from matrix B multiplied + by the zero point offset of matrix A. These values are accumulated into + every column of matrix C. + +Return Value: + + Returns the number of rows handled. + +--*/ + + NESTED_ENTRY MlasSymQgemmS8KernelSdot + + PROLOG_SAVE_REG_PAIR d8,d9,#-GemmS8S8KernelFrame_SavedRegisters! + PROLOG_NOP ldr x8,[sp,#GemmS8S8KernelFrame_ColumnSumBuffer] + PROLOG_SAVE_REG_PAIR d10,d11,#16 + + // compute C pointers: x2, x16, x17, x6 + cmp x4,#2 // M < 2 ? + add x16,x2,x6,lsl #2 // x16 -> C1 + add x17,x2,x6,lsl #3 // x17 -> C2 + csel x16,x2,x16,lo // if M < 2 x16/C1 -> C0 + csel x17,x16,x17,ls // if M <= 2 x17/C2 -> C1 + cmp x4,#4 // M < 4 ? + mov x12,#4 // set max M to 4 + add x6,x16,x6,lsl #3 // x6 -> C3 + mov x9,x0 // save A0 + mov x10,x3 // save K + csel x6,x17,x6,lo // if M < 4 x6/C3 -> C2 + csel x4,x12,x4,hi // if M > 4 M = 4; + +// Register Usage +// B (x1) -> 4x16 +// ---------------------------------------------------------------------------- +// |v4.b[0]..v4.b[12] v5.b[0]..v5.b[12] v6.b[0]..v6.b[12] v7.b[0]..v7.b[12]| +// | ... ... ... ... ... ... ... ... | +// |v4.b[3]..v4.b[15] v5.b[3]..v5.b[15] v6.b[3]..v6.b[15] v7.b[3]..v7.b[15]| +// A 4x4 ---------------------------------------------------------------------------- +// ------------------ ---------------------------------------------------------------------------- +// x0 |v0.b[0]..v0.b[3]| |v16.s[0]_v16.s[3] v20.s[0]_v20.s[3] v24.s[0]_v24.s[3] v28.s[0]_v28.s[3]| x2 +// x12 |v1.b[0]..v1.b[3]| |v17.s[0]_v17.s[3] v21.s[0]_v21.s[3] v25.s[0]_v25.s[3] v29.s[0]_v29.s[3]| x16 +// x13 |v2.b[0]..v2.b[3]| |v18.s[0]_v18.s[3] v22.s[0]_v22.s[3] v26.s[0]_v26.s[3] v30.s[0]_v30.s[3]| x17 +// x14 |v3.b[0]..v3.b[3]| |v19.s[0]_v19.s[3] v23.s[0]_v23.s[3] v27.s[0]_v27.s[3] v31.s[0]_v31.s[3]| x6 +// ------------------ ---------------------------------------------------------------------------- + +ProcessNextColumnLoop + ldr q16,[x8],#16 // Init accumulators with column sums + ldr q20,[x8],#16 + ldr q24,[x8],#16 + ldr q28,[x8],#16 + mov x0,x9 // reload A0 + cmp x4,#2 // M < 2 ? + add x12,x9,x7 // x12 -> A1 + add x13,x0,x7,lsl #1 // x13 -> A2 + ldr q4,[x1],#16 // Load B + csel x12,x0,x12,lo // if M < 2 A1 -> A0 + csel x13,x12,x13,ls // if M <= 2 A2 -> A1 + cmp x4,4 // M < 4 ? + add x14,x12,x7,lsl #1 // x14 -> A3 + ldr q5,[x1],#16 + csel x14,x13,x14,lo // if M < 4 A3 -> A2 + ldr d0,[x0],#8 // Load A0 1st/2nd block of 4 + mov v17.16b,v16.16b + mov v18.16b,v16.16b + ldr d1,[x12],#8 // Load A1 + mov v19.16b,v16.16b + mov v21.16b,v20.16b + ldr d2,[x13],#8 // Load A2 + mov v22.16b,v20.16b + mov v23.16b,v20.16b + ldr d3,[x14],#8 // Load A3 + mov v25.16b,v24.16b + mov v26.16b,v24.16b + ldr q6,[x1],#16 + mov v27.16b,v24.16b + mov v29.16b,v28.16b + ldr q7,[x1],#16 + subs x3,x10,#2 // one loop iteration and epilogue consume k = 32 + mov v30.16b,v28.16b + mov v31.16b,v28.16b + b.lo BlockLoopEpilogue // Need 32 k for main loop + +BlockLoop + sdot v16.4s,v4.16b,v0.4b[0] + sdot v17.4s,v4.16b,v1.4b[0] + ldr d8,[x0],#8 // Load A0 3rd/4th block of 4 + sdot v18.4s,v4.16b,v2.4b[0] + sdot v19.4s,v4.16b,v3.4b[0] + ldr q4,[x1],#16 + sdot v20.4s,v5.16b,v0.4b[0] + sdot v21.4s,v5.16b,v1.4b[0] + ldr d9,[x12],#8 + sdot v22.4s,v5.16b,v2.4b[0] + sdot v23.4s,v5.16b,v3.4b[0] + ldr q5,[x1],#16 + sdot v24.4s,v6.16b,v0.4b[0] + sdot v25.4s,v6.16b,v1.4b[0] + ldr d10,[x13],#8 + sdot v26.4s,v6.16b,v2.4b[0] + sdot v27.4s,v6.16b,v3.4b[0] + ldr q6,[x1],#16 + sdot v28.4s,v7.16b,v0.4b[0] + sdot v29.4s,v7.16b,v1.4b[0] + ldr d11,[x14],#8 + sdot v30.4s,v7.16b,v2.4b[0] + sdot v31.4s,v7.16b,v3.4b[0] + ldr q7,[x1],#16 + sdot v16.4s,v4.16b,v0.4b[1] + sdot v17.4s,v4.16b,v1.4b[1] + sdot v18.4s,v4.16b,v2.4b[1] + sdot v19.4s,v4.16b,v3.4b[1] + ldr q4,[x1],#16 + sdot v20.4s,v5.16b,v0.4b[1] + sdot v21.4s,v5.16b,v1.4b[1] + sdot v22.4s,v5.16b,v2.4b[1] + sdot v23.4s,v5.16b,v3.4b[1] + ldr q5,[x1],#16 + sdot v24.4s,v6.16b,v0.4b[1] + sdot v25.4s,v6.16b,v1.4b[1] + sdot v26.4s,v6.16b,v2.4b[1] + sdot v27.4s,v6.16b,v3.4b[1] + ldr q6,[x1],#16 + sdot v28.4s,v7.16b,v0.4b[1] + sdot v29.4s,v7.16b,v1.4b[1] + sdot v30.4s,v7.16b,v2.4b[1] + sdot v31.4s,v7.16b,v3.4b[1] + ldr q7,[x1],#16 + sdot v16.4s,v4.16b,v8.4b[0] + sdot v17.4s,v4.16b,v9.4b[0] + ldr d0,[x0],#8 + sdot v18.4s,v4.16b,v10.4b[0] + sdot v19.4s,v4.16b,v11.4b[0] + ldr q4,[x1],#16 + sdot v20.4s,v5.16b,v8.4b[0] + sdot v21.4s,v5.16b,v9.4b[0] + ldr d1,[x12],#8 + sdot v22.4s,v5.16b,v10.4b[0] + sdot v23.4s,v5.16b,v11.4b[0] + ldr q5,[x1],#16 + sdot v24.4s,v6.16b,v8.4b[0] + sdot v25.4s,v6.16b,v9.4b[0] + ldr d2,[x13],#8 + sdot v26.4s,v6.16b,v10.4b[0] + sdot v27.4s,v6.16b,v11.4b[0] + ldr q6,[x1],#16 + sdot v28.4s,v7.16b,v8.4b[0] + sdot v29.4s,v7.16b,v9.4b[0] + ldr d3,[x14],#8 + sdot v30.4s,v7.16b,v10.4b[0] + sdot v31.4s,v7.16b,v11.4b[0] + ldr q7,[x1],#16 + sdot v16.4s,v4.16b,v8.4b[1] + sdot v17.4s,v4.16b,v9.4b[1] + sdot v18.4s,v4.16b,v10.4b[1] + sdot v19.4s,v4.16b,v11.4b[1] + ldr q4,[x1],#16 + sdot v20.4s,v5.16b,v8.4b[1] + sdot v21.4s,v5.16b,v9.4b[1] + sdot v22.4s,v5.16b,v10.4b[1] + sdot v23.4s,v5.16b,v11.4b[1] + ldr q5,[x1],#16 + sdot v24.4s,v6.16b,v8.4b[1] + sdot v25.4s,v6.16b,v9.4b[1] + sdot v26.4s,v6.16b,v10.4b[1] + sdot v27.4s,v6.16b,v11.4b[1] + ldr q6,[x1],#16 + sdot v28.4s,v7.16b,v8.4b[1] + sdot v29.4s,v7.16b,v9.4b[1] + subs x3,x3,#1 // k -= 16 + sdot v30.4s,v7.16b,v10.4b[1] + sdot v31.4s,v7.16b,v11.4b[1] + ldr q7,[x1],#16 + b.hs BlockLoop + +BlockLoopEpilogue + sdot v16.4s,v4.16b,v0.4b[0] + sdot v17.4s,v4.16b,v1.4b[0] + ldr d8,[x0],#8 + sdot v18.4s,v4.16b,v2.4b[0] + sdot v19.4s,v4.16b,v3.4b[0] + ldr q4,[x1],#16 + sdot v20.4s,v5.16b,v0.4b[0] + sdot v21.4s,v5.16b,v1.4b[0] + ldr d9,[x12],#8 + sdot v22.4s,v5.16b,v2.4b[0] + sdot v23.4s,v5.16b,v3.4b[0] + ldr q5,[x1],#16 + sdot v24.4s,v6.16b,v0.4b[0] + sdot v25.4s,v6.16b,v1.4b[0] + ldr d10,[x13],#8 + sdot v26.4s,v6.16b,v2.4b[0] + sdot v27.4s,v6.16b,v3.4b[0] + ldr q6,[x1],#16 + sdot v28.4s,v7.16b,v0.4b[0] + sdot v29.4s,v7.16b,v1.4b[0] + ldr d11,[x14],#8 + sdot v30.4s,v7.16b,v2.4b[0] + sdot v31.4s,v7.16b,v3.4b[0] + ldr q7,[x1],#16 + sdot v16.4s,v4.16b,v0.4b[1] + sdot v17.4s,v4.16b,v1.4b[1] + sdot v18.4s,v4.16b,v2.4b[1] + sdot v19.4s,v4.16b,v3.4b[1] + ldr q4,[x1],#16 + sdot v20.4s,v5.16b,v0.4b[1] + sdot v21.4s,v5.16b,v1.4b[1] + sdot v22.4s,v5.16b,v2.4b[1] + sdot v23.4s,v5.16b,v3.4b[1] + ldr q5,[x1],#16 + sdot v24.4s,v6.16b,v0.4b[1] + sdot v25.4s,v6.16b,v1.4b[1] + sdot v26.4s,v6.16b,v2.4b[1] + sdot v27.4s,v6.16b,v3.4b[1] + ldr q6,[x1],#16 + sdot v28.4s,v7.16b,v0.4b[1] + sdot v29.4s,v7.16b,v1.4b[1] + sdot v30.4s,v7.16b,v2.4b[1] + sdot v31.4s,v7.16b,v3.4b[1] + ldr q7,[x1],#16 + sdot v16.4s,v4.16b,v8.4b[0] + sdot v17.4s,v4.16b,v9.4b[0] + sdot v18.4s,v4.16b,v10.4b[0] + sdot v19.4s,v4.16b,v11.4b[0] + ldr q4,[x1],#16 + sdot v20.4s,v5.16b,v8.4b[0] + sdot v21.4s,v5.16b,v9.4b[0] + sdot v22.4s,v5.16b,v10.4b[0] + sdot v23.4s,v5.16b,v11.4b[0] + ldr q5,[x1],#16 + sdot v24.4s,v6.16b,v8.4b[0] + sdot v25.4s,v6.16b,v9.4b[0] + sdot v26.4s,v6.16b,v10.4b[0] + sdot v27.4s,v6.16b,v11.4b[0] + ldr q6,[x1],#16 + sdot v28.4s,v7.16b,v8.4b[0] + sdot v29.4s,v7.16b,v9.4b[0] + sdot v30.4s,v7.16b,v10.4b[0] + sdot v31.4s,v7.16b,v11.4b[0] + ldr q7,[x1],#16 + sdot v16.4s,v4.16b,v8.4b[1] + sdot v17.4s,v4.16b,v9.4b[1] + sdot v18.4s,v4.16b,v10.4b[1] + sdot v19.4s,v4.16b,v11.4b[1] + sdot v20.4s,v5.16b,v8.4b[1] + sdot v21.4s,v5.16b,v9.4b[1] + sdot v22.4s,v5.16b,v10.4b[1] + sdot v23.4s,v5.16b,v11.4b[1] + sdot v24.4s,v6.16b,v8.4b[1] + sdot v25.4s,v6.16b,v9.4b[1] + sdot v26.4s,v6.16b,v10.4b[1] + sdot v27.4s,v6.16b,v11.4b[1] + sdot v28.4s,v7.16b,v8.4b[1] + sdot v29.4s,v7.16b,v9.4b[1] + subs x5,x5,#16 // adjust CountN remaining + sdot v30.4s,v7.16b,v10.4b[1] + sdot v31.4s,v7.16b,v11.4b[1] + blo StoreOutputPartial + stp q16,q20,[x2],#32 + stp q24,q28,[x2],#32 + stp q17,q21,[x16],#32 + stp q25,q29,[x16],#32 + stp q18,q22,[x17],#32 + stp q26,q30,[x17],#32 + stp q19,q23,[x6],#32 + stp q27,q31,[x6],#32 + cbnz x5,ProcessNextColumnLoop + +ExitKernel + mov x0,x4 // return number of rows handled + EPILOG_RESTORE_REG_PAIR d10,d11,#16 + EPILOG_RESTORE_REG_PAIR d8,d9,#GemmS8S8KernelFrame_SavedRegisters! + EPILOG_RETURN + +// +// Store the partial 1 to 15 columns either overwriting the output matrix or +// accumulating into the existing contents of the output matrix. +// + +StoreOutputPartial + tbz x5,#3,StoreOutputPartial4 + stp q16,q20,[x2],#32 + mov v16.16b,v24.16b // shift remaining elements down + mov v20.16b,v28.16b + stp q17,q21,[x16],#32 + mov v17.16b,v25.16b + mov v21.16b,v29.16b + stp q18,q22,[x17],#32 + mov v18.16b,v26.16b + mov v22.16b,v30.16b + stp q19,q23,[x6],#32 + mov v19.16b,v27.16b + mov v23.16b,v31.16b + +StoreOutputPartial4 + tbz x5,#2,StoreOutputPartial2 + st1 {v16.4s},[x2],#16 + mov v16.16b,v20.16b // shift remaining elements down + st1 {v17.4s},[x16],#16 + mov v17.16b,v21.16b + st1 {v18.4s},[x17],#16 + mov v18.16b,v22.16b + st1 {v19.4s},[x6],#16 + mov v19.16b,v23.16b + +StoreOutputPartial2 + tbz x5,#1,StoreOutputPartial1 + st1 {v16.2s},[x2],#8 + dup v16.4s,v16.s[2] // shift remaining elements down + st1 {v17.2s},[x16],#8 + dup v17.4s,v17.s[2] + st1 {v18.2s},[x17],#8 + dup v18.4s,v18.s[2] + st1 {v19.2s},[x6],#8 + dup v19.4s,v19.s[2] + +StoreOutputPartial1 + tbz x5,#0,ExitKernel + st1 {v16.s}[0],[x2] + st1 {v17.s}[0],[x16] + st1 {v18.s}[0],[x17] + st1 {v19.s}[0],[x6] + b ExitKernel + + NESTED_END MlasSymQgemmS8KernelSdot + + END diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index 77b5038786..d7e4fd55c7 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -697,6 +697,13 @@ extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmS8S8DispatchSdot; extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmU8X8DispatchWasmSimd; extern const MLAS_GEMM_QUANT_DISPATCH MlasGemmQuantDispatchDefault; +// +// Symmetric quantized qgemm dispatch structure +// +struct MLAS_SYMM_QGEMM_DISPATCH; +extern const MLAS_SYMM_QGEMM_DISPATCH MlasSymmQgemmS8DispatchNeon; +extern const MLAS_SYMM_QGEMM_DISPATCH MlasSymmQgemmS8DispatchSdot; + // // Symmetric quantized integer convolution dispatch structure. // @@ -792,6 +799,7 @@ struct MLAS_PLATFORM { #elif defined(MLAS_TARGET_ARM64) const MLAS_GEMM_QUANT_DISPATCH* GemmU8X8Dispatch; #endif + const MLAS_SYMM_QGEMM_DISPATCH* SymmQgemmDispatch{nullptr}; const MLAS_CONV_SYM_DISPATCH* ConvSymU8S8Dispatch{nullptr}; const MLAS_CONV_SYM_DISPATCH* ConvSymS8S8Dispatch{nullptr}; @@ -866,6 +874,13 @@ MlasExecuteThreaded( MLAS_THREADPOOL* ThreadPool ); +constexpr +size_t +MlasDivRoundup(size_t up, size_t down) +{ + return (up + down - 1) / down; +} + /** * @brief Distribute multiple iterations of work over a thread pool if supported * diff --git a/onnxruntime/core/mlas/lib/platform.cpp b/onnxruntime/core/mlas/lib/platform.cpp index 67e1e1675e..675e6625d5 100644 --- a/onnxruntime/core/mlas/lib/platform.cpp +++ b/onnxruntime/core/mlas/lib/platform.cpp @@ -353,6 +353,7 @@ Return Value: this->GemmU8X8Dispatch = &MlasGemmU8X8DispatchNeon; this->ConvSymU8S8Dispatch = &MlasConvSymDispatchNeon; + this->SymmQgemmDispatch = &MlasSymmQgemmS8DispatchNeon; // // Check if the processor supports ASIMD dot product instructions. @@ -371,6 +372,7 @@ Return Value: if (HasDotProductInstructions) { this->GemmU8X8Dispatch = &MlasGemmU8X8DispatchUdot; this->ConvSymU8S8Dispatch = &MlasConvSymDispatchDot; + this->SymmQgemmDispatch = &MlasSymmQgemmS8DispatchSdot; } #endif // MLAS_TARGET_ARM64 diff --git a/onnxruntime/core/mlas/lib/qgemm.cpp b/onnxruntime/core/mlas/lib/qgemm.cpp index dce27166fb..fb5a2f766c 100644 --- a/onnxruntime/core/mlas/lib/qgemm.cpp +++ b/onnxruntime/core/mlas/lib/qgemm.cpp @@ -109,41 +109,12 @@ Return Value: } -void -MLASCALL -MlasGemm( - const MLAS_GEMM_QUANT_SHAPE_PARAMS &Shape, - const MLAS_GEMM_QUANT_DATA_PARAMS &DataParams, - MLAS_THREADPOOL *ThreadPool) -/*++ - -Routine Description: - - This routine implements the quantized integer matrix/matrix multiply - operation (QGEMM). - -Arguments: - - Shape - Supplies the structure containing the GEMM input and output shapes. - - Data - Supplies the structure containing the GEMM input and output data layout - - ThreadPool - Supplies the thread pool object to use, else nullptr if the - base library threading support should be used. - -Return Value: - - None. - ---*/ -{ - MlasGemmBatch(Shape, &DataParams, 1, ThreadPool); -} #if defined(_MSC_VER) && !defined(__clang__) #pragma warning(push) // VC++ suggests we can attempt to make 'MlasBitsOfFp32' constexpr, but it is not valid. #pragma warning(disable : 26451) #endif + void MLASCALL MlasGemmBatch( @@ -220,6 +191,90 @@ MlasGemmBatch( MlasGemmQuantThreaded(&WorkBlock, &Shape, &DataParams[gemm_i], blk_i); }); } + +void +MLASCALL +MlasSymmQgemmBatch( + const MLAS_GEMM_QUANT_SHAPE_PARAMS& Shape, + const MLAS_SYMM_QGEMM_DATA_PARAMS* DataParams, + const size_t BatchN, + MLAS_THREADPOOL* ThreadPool + ) +{ + const size_t M = Shape.M; + const size_t N = Shape.N; + const size_t K = Shape.K; + const MLAS_SYMM_QGEMM_DISPATCH* dispatch = MlasPlatform.SymmQgemmDispatch; + MLAS_SYMM_QGEMM_OPERATION* operation = dispatch->Operation; + + if (ThreadPool == nullptr) { + // So our caller handles threaded job partition. + // Call single threaded operation directly + + for (size_t gemm_i = 0; gemm_i < BatchN; gemm_i++) { + auto Data = &DataParams[gemm_i]; + operation(&Shape, Data, 0, M, 0, N); + } + return; + } + + // + // Compute the number of target threads given the complexity of the SGEMM + // operation. Small requests should run using the single threaded path. + // + + const double Complexity = double(M) * double(N) * double(K) * double(BatchN); + + ptrdiff_t TargetThreadCount = ptrdiff_t(Complexity / double(MLAS_QGEMM_THREAD_COMPLEXITY)) + 1; + + ptrdiff_t MaximumThreadCount = MlasGetMaximumThreadCount(ThreadPool); + + if (TargetThreadCount >= MaximumThreadCount) { + TargetThreadCount = MaximumThreadCount; + } + + ptrdiff_t ThreadsPerGemm = TargetThreadCount / BatchN; + if (ThreadsPerGemm < 1) { + ThreadsPerGemm = 1; + } + + const size_t StrideM = dispatch->StrideM; + + size_t nc = N; + if ((size_t)MlasGetMaximumThreadCount(ThreadPool) > BatchN) { + // more than one thread per GEMM + + const size_t BlockedM = MlasDivRoundup(M, StrideM); + const size_t max_nc = MlasDivRoundup(N * BlockedM, ThreadsPerGemm); + if (max_nc < nc) { + nc = std::min(nc, MlasDivRoundup(nc, max_nc * MLAS_QGEMM_STRIDEN_THREAD_ALIGN) * + MLAS_QGEMM_STRIDEN_THREAD_ALIGN); + } + } + const size_t StrideN = nc; + + const size_t ThreadCountM = MlasDivRoundup(M, StrideM); + const size_t ThreadCountN = MlasDivRoundup(N, StrideN); + ThreadsPerGemm = ThreadCountM * ThreadCountN; + + MlasTrySimpleParallel(ThreadPool, ThreadsPerGemm * BatchN, [&](ptrdiff_t tid) { + const auto gemm_i = tid / ThreadsPerGemm; + const auto blk_i = tid % ThreadsPerGemm; + auto Data = &DataParams[gemm_i]; + + const ptrdiff_t ThreadIdN = blk_i / ThreadCountM; + const ptrdiff_t ThreadIdM = blk_i % ThreadCountM; + + const size_t RangeStartM = ThreadIdM * StrideM; + const size_t RangeCountM = std::min(Shape.M - RangeStartM, (size_t)StrideM); + + const size_t RangeStartN = ThreadIdN * StrideN; + const size_t RangeCountN = std::min(Shape.N - RangeStartN, (size_t)StrideN); + + operation(&Shape, Data, RangeStartM, RangeCountM, RangeStartN, RangeCountN); + }); +} + #if defined(_MSC_VER) && !defined(__clang__) #pragma warning(pop) #endif @@ -388,3 +443,85 @@ Return Value: B += ldb * CountK; } } + +#if defined(_MSC_VER) && !defined(__clang__) +#pragma warning(push) +// We can not make this function constexpr across different platforms +#pragma warning(disable : 26497) +#endif + +size_t +MLASCALL +MlasSymmQgemmPackBSize( + size_t N, + size_t K, + bool AIsSigned + ) +{ +#ifndef MLAS_TARGET_ARM64 + + // Only have arm64 impl for now + MLAS_UNREFERENCED_PARAMETER(N); + MLAS_UNREFERENCED_PARAMETER(K); + MLAS_UNREFERENCED_PARAMETER(AIsSigned); + return 0; +#else + + // Only support s8s8 for now + if (!AIsSigned) { + return 0; + } + + const auto* Dispatch = MlasPlatform.SymmQgemmDispatch; + + size_t PackedK = Dispatch->PackedK; + + // + // Compute the number of bytes required to hold the packed buffer. + // + + const size_t AlignedN = + (N + MLAS_QGEMM_STRIDEN_THREAD_ALIGN - 1) & ~(MLAS_QGEMM_STRIDEN_THREAD_ALIGN - 1); + const size_t AlignedK = (K + PackedK - 1) & ~(PackedK - 1); + + const size_t BytesRequired = + (AlignedN * sizeof(int32_t)) + (AlignedN * AlignedK * sizeof(uint8_t)); + const size_t BufferAlignment = MlasGetPreferredBufferAlignment(); + const size_t AlignedBytesRequired = (BytesRequired + BufferAlignment - 1) & + ~(BufferAlignment - 1); + + return AlignedBytesRequired; +#endif // !MLAS_TARGET_ARM64 +} +#if defined(_MSC_VER) && !defined(__clang__) +#pragma warning(pop) +#endif + + +void +MLASCALL +MlasSymmQgemmPackB( + size_t N, + size_t K, + const int8_t* B, + size_t ldb, + bool AIsSigned, + int32_t ZeroPointA, + void* PackedB + ) +{ + MLAS_UNREFERENCED_PARAMETER(AIsSigned); + + const MLAS_SYMM_QGEMM_DISPATCH* SymmQgemmDispatch = MlasPlatform.SymmQgemmDispatch; + + const size_t AlignedN = + (N + MLAS_QGEMM_STRIDEN_THREAD_ALIGN - 1) & ~(MLAS_QGEMM_STRIDEN_THREAD_ALIGN - 1); + int32_t* PackedColumnSumBuffer = (int32_t*)PackedB; + PackedB = PackedColumnSumBuffer + AlignedN; + + SymmQgemmDispatch->CopyPackBRoutine((uint8_t*)PackedB, (const uint8_t*)B, ldb, N, K, + PackedColumnSumBuffer, true); + for (size_t n = 0; n < AlignedN; n++) { + PackedColumnSumBuffer[n] *= -ZeroPointA; + } +} diff --git a/onnxruntime/core/mlas/lib/qgemm.h b/onnxruntime/core/mlas/lib/qgemm.h index 924cde75c6..ab566c5b50 100644 --- a/onnxruntime/core/mlas/lib/qgemm.h +++ b/onnxruntime/core/mlas/lib/qgemm.h @@ -167,6 +167,34 @@ MlasGemmQuantKernel( bool ZeroMode ); +/** + * @brief Usually a wrapper of assembly/intrinsic kernel + * of symmetric quant gemm + * @tparam KernelType + * @param A Left hand side matrix + * @param B Prepacked right hand side matrix + * @param C Result matrix + * @param PackedCountK Number of packed rows from B + * @param CountM Number of rows to process + * @param CountN Number of columns to process + * @param ldc Row stride of C + * @param lda Row stride of A + * @param ColumnSumVector Column sum of B scaled by zero point A + * @return Number of rows processed +*/ +template +size_t +MlasSymmQGemmKernel( + const int8_t* A, + const int8_t* B, + int32_t* C, + size_t PackedCountK, + size_t CountM, + size_t CountN, + size_t ldc, + size_t lda, + const int32_t* ColumnSumVector +); inline void @@ -658,6 +686,75 @@ Return Value: } } +/** + * @brief Operation for Quantized GEMM where B is symmetrically + * quantized and packed matrix + * @param Shape + * @param Data + * @param RangeStartM + * @param RangeCountM + * @param RangeStartN + * @param RangeCountN +*/ +template +void +MlasSymmQGemmPackedOperation( + const MLAS_GEMM_QUANT_SHAPE_PARAMS* Shape, + const MLAS_SYMM_QGEMM_DATA_PARAMS* Data, + const size_t RangeStartM, + const size_t RangeCountM, + const size_t RangeStartN, + const size_t RangeCountN + ) +{ + + const size_t K = Shape->K; + + const size_t lda = Data->lda; + const size_t ldc = Data->ldc; + + const int8_t* PanelA = (const int8_t*)(Data->A) + RangeStartM * lda; + const int8_t* PackedB = (const int8_t*)Data->B; + int32_t* C = (int32_t*)(Data->C) + RangeStartM * ldc + RangeStartN; + + // + // Extract the pointer to the column sum buffer from the packed matrix. + // + const size_t AlignedN = + (Shape->N + MLAS_QGEMM_STRIDEN_THREAD_ALIGN - 1) & ~(MLAS_QGEMM_STRIDEN_THREAD_ALIGN - 1); + const int32_t* PackedColumnSumBuffer = (const int32_t*)PackedB; + PackedB = (const int8_t*)(PackedColumnSumBuffer + AlignedN); + PackedColumnSumBuffer += RangeStartN; + + const size_t PackedCountK = (K + KernelType::PackedK - 1) / KernelType::PackedK; + + // + // Apply the global depth value constant without the ZeroPointB scaling from: + // + // (A[i] - ZeroPointA) * (B[i] - ZeroPointB) + // ==> + // A[i] * B[i] - A[i] * ZeroPointB - B[i] * ZeroPointA + ZeroPointA * ZeroPointB + // + // ZeroPointB is zero, which makes this much simpler + // + + const int8_t* b = PackedB + RangeStartN * KernelType::PackedK * PackedCountK; + int32_t* c = C; + + auto pa = PanelA; + size_t RowsRemaining = RangeCountM; + + while (RowsRemaining > 0) { + size_t RowsHandled = MlasSymmQGemmKernel( + pa, b, c, PackedCountK, RowsRemaining, RangeCountN, ldc, lda, PackedColumnSumBuffer); + + c += ldc * RowsHandled; + pa += lda * RowsHandled; + RowsRemaining -= RowsHandled; + } +} + + // // Quantized integer matrix/matrix dispatch structure. // @@ -673,6 +770,17 @@ void const size_t RangeCountN ); +typedef +void +(MLAS_SYMM_QGEMM_OPERATION)( + const MLAS_GEMM_QUANT_SHAPE_PARAMS* Shape, + const MLAS_SYMM_QGEMM_DATA_PARAMS* Data, + const size_t RangeStartM, + const size_t RangeCountM, + const size_t RangeStartN, + const size_t RangeCountN + ); + typedef void (MLAS_GEMM_QUANT_COPY_PACKB_ROUTINE)( @@ -693,6 +801,12 @@ struct MLAS_GEMM_QUANT_DISPATCH { size_t PackedStrideK; }; +struct MLAS_SYMM_QGEMM_DISPATCH { + MLAS_SYMM_QGEMM_OPERATION* Operation; + MLAS_GEMM_QUANT_COPY_PACKB_ROUTINE* CopyPackBRoutine; + size_t StrideM; /**< num of rows processed by kernel at a time */ + size_t PackedK; +}; MLAS_FORCEINLINE const MLAS_GEMM_QUANT_DISPATCH* diff --git a/onnxruntime/core/mlas/lib/qgemm_kernel_neon.cpp b/onnxruntime/core/mlas/lib/qgemm_kernel_neon.cpp index 46c5b2a4dd..ce9460be9b 100644 --- a/onnxruntime/core/mlas/lib/qgemm_kernel_neon.cpp +++ b/onnxruntime/core/mlas/lib/qgemm_kernel_neon.cpp @@ -608,6 +608,21 @@ extern "C" { const int32_t* ZeroPointB, bool ZeroMode ); + + size_t + MLASCALL + MlasSymQgemmS8KernelNeon( + const int8_t* A, + const int8_t* B, + int32_t* C, + size_t PackedCountK, + size_t CountM, + size_t CountN, + size_t ldc, + size_t lda, + const int32_t* ColumnSumVector + ); + } struct MLAS_GEMM_X8S8_KERNEL_NEON { @@ -1174,6 +1189,24 @@ MlasGemmQuantKernel( RowSumBuffer, ColumnSumBuffer, ZeroPointB, ZeroMode); } + +template<> +MLAS_FORCEINLINE +size_t MlasSymmQGemmKernel( + const int8_t* A, + const int8_t* B, + int32_t* C, + size_t PackedCountK, + size_t CountM, + size_t CountN, + size_t ldc, + size_t lda, + const int32_t* ColumnSumVector +) +{ + return MlasSymQgemmS8KernelNeon(A, B, C, PackedCountK, CountM, CountN, ldc, lda, + ColumnSumVector); +} const MLAS_GEMM_QUANT_DISPATCH MlasGemmX8S8DispatchNeon = { MlasGemmQuantOperation, @@ -1183,4 +1216,11 @@ const MLAS_GEMM_QUANT_DISPATCH MlasGemmX8S8DispatchNeon = { MLAS_GEMM_X8S8_KERNEL_NEON::PackedStrides.K, }; +const MLAS_SYMM_QGEMM_DISPATCH MlasSymmQgemmS8DispatchNeon = { + MlasSymmQGemmPackedOperation, + MlasGemmQuantCopyPackB, + 4, // StrideM + MLAS_GEMM_X8S8_KERNEL_NEON::PackedK +}; + #endif //defined(MLAS_TARGET_ARM64) diff --git a/onnxruntime/core/mlas/lib/qgemm_kernel_sdot.cpp b/onnxruntime/core/mlas/lib/qgemm_kernel_sdot.cpp index 7545570f7e..5b2d9a6e6c 100644 --- a/onnxruntime/core/mlas/lib/qgemm_kernel_sdot.cpp +++ b/onnxruntime/core/mlas/lib/qgemm_kernel_sdot.cpp @@ -757,3 +757,278 @@ const MLAS_GEMM_QUANT_DISPATCH MlasGemmS8S8DispatchSdot = { MLAS_GEMM_S8S8_KERNEL_SDOT::PackedK, MLAS_GEMM_S8S8_KERNEL_SDOT::PackedStrides.K, }; + + +/** + * @brief Type parameter for symmetric qgemm + */ +struct MLAS_SYMM_GEMM_S8S8_KERNEL_SDOT { + typedef uint8_t PackedAType; + typedef uint8_t PackedBType; + typedef int8_t OffsetAType; + typedef int8_t OffsetBType; + + static constexpr size_t PackedK = 16; +}; + +constexpr size_t MLAS_SYMM_GEMM_S8S8_KERNEL_SDOT::PackedK; + +template<> +void +MlasGemmQuantCopyPackB( + MLAS_SYMM_GEMM_S8S8_KERNEL_SDOT::PackedBType* Dst, + const uint8_t* B, + size_t ldb, + size_t CountN, + size_t CountK, + int32_t* ColumnSumBuffer, + bool BIsSigned + ) +{ + MLAS_UNREFERENCED_PARAMETER(BIsSigned); + int8_t* D = reinterpret_cast(Dst); + const int8x16_t ZeroVector = vmovq_n_s8(0); + int8x8_t BytesRow[4]; + + // Kernel MlasSymQgemmS8KernelSdot code loads 4x16 B block like: + // + // |v4.b[0]..v4.b[12] v5.b[0]..v5.b[12] v6.b[0]..v6.b[12] v7.b[0]..v7.b[12]| + // | ... ... ... ... ... ... ... ... | + // |v4.b[3]..v4.b[15] v5.b[3]..v5.b[15] v6.b[3]..v6.b[15] v7.b[3]..v7.b[15]| + // + // So we process B data similar with MlasGemmQuantCopyPackB + // But twice as wide and twice as deep: + // 16 CountN and 16 CountK + // + // [ A0 A1 A2 A3 B0 B1 B2 B3 C0 C1 C2 C3 D0 D1 D2 D3 ] + // [ E0 E1 E2 E3 F0 F1 F2 F3 G0 G1 G2 G3 H0 H1 H2 H3 ] + // [ I4 I5 I6 I7 J4 J5 J6 J7 K4 K5 K6 K7 L4 L5 L6 L7 ] + // [ M4 M5 M6 M7 N4 N5 N6 N7 O4 O5 O6 O7 P4 P5 P6 P7 ] + // .... repeat 4 times on K dimension .... + // + // If CountK is not aligned to a multiple of 16, then the packed buffer + // is padded with zero vectors. + // + // If CountN is not aligned to a multiple of 16, then the extra columns + // are padded with zeroes. + // + + while (CountN >= 16) { + + const int8_t* b = reinterpret_cast(B); + size_t k = CountK; + int32x4_t ColumnSums0[2]; + int32x4_t ColumnSums1[2]; + + ColumnSums0[0] = vmovq_n_s32(0); + ColumnSums0[1] = vmovq_n_s32(0); + ColumnSums1[0] = vmovq_n_s32(0); + ColumnSums1[1] = vmovq_n_s32(0); + + // + // Interleave rows of matrix B and write to the packed buffer. + // + + while (k >= 4) { + + BytesRow[0] = vld1_s8(&b[ldb * 0]); + BytesRow[1] = vld1_s8(&b[ldb * 1]); + BytesRow[2] = vld1_s8(&b[ldb * 2]); + BytesRow[3] = vld1_s8(&b[ldb * 3]); + MlasGemmS8S8CopyPackBProcessSDot(D, BytesRow, ColumnSums0); + D += 32; + + BytesRow[0] = vld1_s8(&b[ldb * 0 + 8]); + BytesRow[1] = vld1_s8(&b[ldb * 1 + 8]); + BytesRow[2] = vld1_s8(&b[ldb * 2 + 8]); + BytesRow[3] = vld1_s8(&b[ldb * 3 + 8]); + MlasGemmS8S8CopyPackBProcessSDot(D, BytesRow, ColumnSums1); + D += 32; + + b += ldb * 4; + k -= 4; + } + + if (k > 0) { + + BytesRow[0] = vld1_s8(&b[ldb * 0]); + BytesRow[1] = (k >= 2) ? vld1_s8(&b[ldb * 1]) : vget_low_s8(ZeroVector); + BytesRow[2] = (k > 2) ? vld1_s8(&b[ldb * 2]) : vget_low_s8(ZeroVector); + BytesRow[3] = vget_low_s8(ZeroVector); + MlasGemmS8S8CopyPackBProcessSDot(D, BytesRow, ColumnSums0); + D += 32; + + BytesRow[0] = vld1_s8(&b[ldb * 0 + 8]); + BytesRow[1] = (k >= 2) ? vld1_s8(&b[ldb * 1 + 8]) : vget_low_s8(ZeroVector); + BytesRow[2] = (k > 2) ? vld1_s8(&b[ldb * 2 + 8]) : vget_low_s8(ZeroVector); + BytesRow[3] = vget_low_s8(ZeroVector); + MlasGemmS8S8CopyPackBProcessSDot(D, BytesRow, ColumnSums1); + D += 32; + } + + // + // Zero pad the output buffer to a multiple of PackedK + // + constexpr size_t mask = MLAS_SYMM_GEMM_S8S8_KERNEL_SDOT::PackedK - 1; + size_t remain = (MLAS_SYMM_GEMM_S8S8_KERNEL_SDOT::PackedK - (CountK & mask)) & mask; + remain = remain >> 2; // divid by 4 + for (; remain > 0; remain--) { + vst1q_s8(&D[0], ZeroVector); + vst1q_s8(&D[16], ZeroVector); + vst1q_s8(&D[32], ZeroVector); + vst1q_s8(&D[48], ZeroVector); + D += 64; + } + + vst1q_s32(&ColumnSumBuffer[0], ColumnSums0[0]); + vst1q_s32(&ColumnSumBuffer[4], ColumnSums0[1]); + vst1q_s32(&ColumnSumBuffer[8], ColumnSums1[0]); + vst1q_s32(&ColumnSumBuffer[12], ColumnSums1[1]); + ColumnSumBuffer += 16; + + B += 16; + CountN -= 16; + } + + // + // Process the remaining columns of matrix B. + // + + if (CountN > 0) { + + const int8_t* b = reinterpret_cast(B); + size_t k = CountK; + int8_t PaddedMatrixBData[64]; + int32x4_t ColumnSums0[2]; + int32x4_t ColumnSums1[2]; + + vst1q_s8(&PaddedMatrixBData[0], ZeroVector); + vst1q_s8(&PaddedMatrixBData[16], ZeroVector); + vst1q_s8(&PaddedMatrixBData[32], ZeroVector); + vst1q_s8(&PaddedMatrixBData[48], ZeroVector); + + ColumnSums0[0] = vmovq_n_s32(0); + ColumnSums0[1] = vmovq_n_s32(0); + ColumnSums1[0] = vmovq_n_s32(0); + ColumnSums1[1] = vmovq_n_s32(0); + + // + // Interleave rows of matrix B using an intermediate zero padded stack + // buffer and write to the packed buffer. + // + + while (k > 0) { + + const int8_t* bcopy0 = &b[ldb * 0]; + const int8_t* bcopy1 = &b[ldb * 1]; + const int8_t* bcopy2 = &b[ldb * 2]; + const int8_t* bcopy3 = &b[ldb * 3]; + + if (k >= 4) { + + b += ldb * 4; + k -= 4; + + } else { + + vst1q_s8(&PaddedMatrixBData[0], ZeroVector); + vst1q_s8(&PaddedMatrixBData[16], ZeroVector); + vst1q_s8(&PaddedMatrixBData[32], ZeroVector); + vst1q_s8(&PaddedMatrixBData[48], ZeroVector); + + bcopy1 = (k >= 2) ? bcopy1 : &PaddedMatrixBData[48]; + bcopy2 = (k > 2) ? bcopy2 : &PaddedMatrixBData[48]; + bcopy3 = &PaddedMatrixBData[48]; + + k = 0; + } + + int8_t* padded = PaddedMatrixBData; + int8_t* padded_end = padded + CountN; + + do { + padded[0] = *bcopy0++; + padded[16] = *bcopy1++; + padded[32] = *bcopy2++; + padded[48] = *bcopy3++; + } while (++padded < padded_end); + + BytesRow[0] = vld1_s8(&PaddedMatrixBData[0]); + BytesRow[1] = vld1_s8(&PaddedMatrixBData[16]); + BytesRow[2] = vld1_s8(&PaddedMatrixBData[32]); + BytesRow[3] = vld1_s8(&PaddedMatrixBData[48]); + MlasGemmS8S8CopyPackBProcessSDot(D, BytesRow, ColumnSums0); + D += 32; + + BytesRow[0] = vld1_s8(&PaddedMatrixBData[8]); + BytesRow[1] = vld1_s8(&PaddedMatrixBData[24]); + BytesRow[2] = vld1_s8(&PaddedMatrixBData[40]); + BytesRow[3] = vld1_s8(&PaddedMatrixBData[56]); + MlasGemmS8S8CopyPackBProcessSDot(D, BytesRow, ColumnSums1); + D += 32; + } + + // + // Zero pad the output buffer to a multiple of PackedK + // + constexpr size_t mask = MLAS_SYMM_GEMM_S8S8_KERNEL_SDOT::PackedK - 1; + size_t remain = (MLAS_SYMM_GEMM_S8S8_KERNEL_SDOT::PackedK - (CountK & mask)) & mask; + remain = remain >> 2; // divid by 4 + for (; remain > 0; remain--) { + vst1q_s8(&D[0], ZeroVector); + vst1q_s8(&D[16], ZeroVector); + vst1q_s8(&D[32], ZeroVector); + vst1q_s8(&D[48], ZeroVector); + D += 64; + } + + vst1q_s32(&ColumnSumBuffer[0], ColumnSums0[0]); + vst1q_s32(&ColumnSumBuffer[4], ColumnSums0[1]); + vst1q_s32(&ColumnSumBuffer[8], ColumnSums1[0]); + vst1q_s32(&ColumnSumBuffer[12], ColumnSums1[1]); + } +} + +extern "C" { + // Prototype of SDOT symmetric qgemm kernel in assembly + + size_t + MLASCALL + MlasSymQgemmS8KernelSdot( + const int8_t* A, + const int8_t* B, + int32_t* C, + size_t PackedCountK, + size_t CountM, + size_t CountN, + size_t ldc, + size_t lda, + const int32_t* ColumnSumVector + ); + +} + +template<> +MLAS_FORCEINLINE +size_t MlasSymmQGemmKernel( + const int8_t* A, + const int8_t* B, + int32_t* C, + size_t PackedCountK, + size_t CountM, + size_t CountN, + size_t ldc, + size_t lda, + const int32_t* ColumnSumVector +) +{ + return MlasSymQgemmS8KernelSdot(A, B, C, PackedCountK, CountM, CountN, ldc, lda, + ColumnSumVector); +} + +const MLAS_SYMM_QGEMM_DISPATCH MlasSymmQgemmS8DispatchSdot = { + MlasSymmQGemmPackedOperation, + MlasGemmQuantCopyPackB, + 4, // StrideM + MLAS_SYMM_GEMM_S8S8_KERNEL_SDOT::PackedK +}; diff --git a/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc b/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc index 26a70a0033..68ccb69037 100644 --- a/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc +++ b/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc @@ -11,6 +11,22 @@ #include "core/util/qmath.h" #include "core/mlas/inc/mlas.h" +#if defined(_M_ARM64) || defined(__aarch64__) + +// +// TODO!! Hack! need to move this to MLAS +// +// We use a different job partition with mobile devices +// where the model tends to be smaller. When the entire +// weight matrix fits in the cache, we process a thin +// horizontal slice of the result matrix at a time. The +// thickness of this slice depend on micro-kernel M +// stride. +// +#define GEMM_KERNEL_STRIDE_M 4 + +#endif + namespace onnxruntime { using ConvPadVector = ConvAttributes::ConvPadVector; @@ -107,7 +123,7 @@ class QLinearConv : public OpKernel { return output_scales; } - static int32_t ComputeThreadCount(int64_t output_image_size, int64_t group_output_channels, int64_t kernel_dim) { + static int32_t ComputeTaskCount(int64_t output_image_size, int64_t group_output_channels, int64_t kernel_dim) { // Replicate the logic from MlasGemmU8X8Schedule to control the number of // worker threads used for the convolution. int32_t maximum_thread_count; @@ -122,16 +138,16 @@ class QLinearConv : public OpKernel { static_cast(group_output_channels) * static_cast(kernel_dim); - int32_t thread_count = maximum_thread_count; + int32_t task_count = maximum_thread_count; if (complexity < thread_complexity * maximum_thread_count) { - thread_count = static_cast(complexity / thread_complexity) + 1; + task_count = static_cast(complexity / thread_complexity) + 1; } - if (thread_count > output_image_size) { + if (task_count > output_image_size) { // Ensure that every thread produces at least one output. - thread_count = static_cast(output_image_size); + task_count = static_cast(output_image_size); } - return thread_count; + return task_count; } bool TryConvSymPrepack(const uint8_t* Wdata, @@ -143,46 +159,97 @@ class QLinearConv : public OpKernel { size_t kernel_size) { const Tensor* X_zero_point = nullptr; const Tensor* W_zero_point = nullptr; - if (Info().TryGetConstantInput(InputTensors::IN_X_ZERO_POINT, &X_zero_point) && IsScalarOr1ElementVector(X_zero_point) && - Info().TryGetConstantInput(InputTensors::IN_W_ZERO_POINT, &W_zero_point) && IsValidQuantParam(W_zero_point, static_cast(output_channels))) { - auto X_zero_point_value = *(X_zero_point->template Data()); - const size_t W_zero_point_size = static_cast(W_zero_point->Shape().Size()); - const auto* W_zero_point_data = W_zero_point->Data(); - if (std::all_of(W_zero_point_data, W_zero_point_data + W_zero_point_size, [](int8_t v) { return v == 0; })) { - size_t packed_size = MlasConvSymPackWSize(group_count, group_input_channels, group_output_channels, kernel_size, std::is_signed::value); - if (packed_size != 0) { - const Tensor* B = nullptr; - Info().TryGetConstantInput(8, &B); - const auto* Bdata = B != nullptr ? B->template Data() : nullptr; - column_sums_.resize(output_channels); - const int8_t* sdata = (const int8_t*)Wdata; - int32_t X_zero_point_fixup = MlasConvSymFixupInputZeroPoint(X_zero_point_value, std::is_signed::value); - for (size_t oc = 0; oc < output_channels; oc++) { - int32_t sum = 0; - for (size_t ks = 0; ks < kernel_size * group_input_channels; ks++) { - sum += *sdata++; - } - column_sums_[oc] = (Bdata != nullptr ? Bdata[oc] : 0) - sum * X_zero_point_fixup; - } + // We need activation and weight zero points for symmetric packing + if (!Info().TryGetConstantInput(InputTensors::IN_X_ZERO_POINT, &X_zero_point) || !IsScalarOr1ElementVector(X_zero_point) || + !Info().TryGetConstantInput(InputTensors::IN_W_ZERO_POINT, &W_zero_point) || !IsValidQuantParam(W_zero_point, static_cast(output_channels))) { + return false; + } - auto* packed_W = static_cast(alloc->Alloc(packed_size)); - packed_W_buffer_ = BufferUniquePtr(packed_W, BufferDeleter(alloc)); + auto X_zero_point_value = *(X_zero_point->template Data()); + const size_t W_zero_point_size = static_cast(W_zero_point->Shape().Size()); + const auto* W_zero_point_data = W_zero_point->Data(); + if (!std::all_of(W_zero_point_data, W_zero_point_data + W_zero_point_size, [](int8_t v) { return v == 0; })) { + // Symmetric means weight zero point must be zero + return false; + } - MlasConvSymPackW(group_count, - group_input_channels, - group_output_channels, - kernel_size, - reinterpret_cast(Wdata), - reinterpret_cast(packed_W), - packed_size, - std::is_signed::value); + // Try indirect conv packing + size_t packed_size = MlasConvSymPackWSize(group_count, group_input_channels, group_output_channels, kernel_size, std::is_signed::value); + if (packed_size != 0) { + const Tensor* B = nullptr; + Info().TryGetConstantInput(8, &B); + const auto* Bdata = B != nullptr ? B->template Data() : nullptr; - is_symmetric_conv_ = true; - - is_W_packed_ = true; - return true; + column_sums_.resize(output_channels); + const int8_t* sdata = (const int8_t*)Wdata; + int32_t X_zero_point_fixup = MlasConvSymFixupInputZeroPoint(X_zero_point_value, std::is_signed::value); + for (size_t oc = 0; oc < output_channels; oc++) { + int32_t sum = 0; + for (size_t ks = 0; ks < kernel_size * group_input_channels; ks++) { + sum += *sdata++; } + column_sums_[oc] = (Bdata != nullptr ? Bdata[oc] : 0) - sum * X_zero_point_fixup; + } + + auto* packed_W = static_cast(alloc->Alloc(packed_size)); + packed_W_buffer_ = BufferUniquePtr(packed_W, BufferDeleter(alloc)); + + MlasConvSymPackW(group_count, + group_input_channels, + group_output_channels, + kernel_size, + reinterpret_cast(Wdata), + reinterpret_cast(packed_W), + packed_size, + std::is_signed::value); + + is_symmetric_conv_ = true; + is_W_packed_ = true; + return true; + } + + // Try symmetric GEMM packing + // Don't pack the filter buffer if the MlasConvDepthwise path is used. + if (group_input_channels != 1 || group_output_channels != 1) { + const size_t kernel_dim = group_input_channels * kernel_size; + packed_W_size_ = MlasSymmQgemmPackBSize(group_output_channels, + kernel_dim, + std::is_same::value); + + if (packed_W_size_ != 0) { + size_t packed_W_data_size = SafeInt(group_count) * packed_W_size_; + auto* packed_W = static_cast(alloc->Alloc(packed_W_data_size)); + + memset(packed_W, 0, packed_W_data_size); + + packed_W_buffer_ = BufferUniquePtr(packed_W, BufferDeleter(alloc)); + + // Allocate a temporary buffer to hold the reordered oihw->hwio filter for + // a single group. + // + // Note: The size of this buffer is less than or equal to the size of the original + // weight tensor, so the allocation size is guaranteed to fit inside size_t. + auto* group_reordered_W = static_cast(alloc->Alloc(group_output_channels * group_input_channels * kernel_size)); + BufferUniquePtr group_reordered_W_buffer(group_reordered_W, BufferDeleter(alloc)); + + const size_t W_offset = group_output_channels * kernel_dim; + + for (int64_t group_id = 0; group_id < conv_attrs_.group; ++group_id) { + ReorderFilter(Wdata, (uint8_t*)(group_reordered_W), group_output_channels, group_input_channels, kernel_size); + MlasSymmQgemmPackB(group_output_channels, + kernel_dim, + group_reordered_W, + group_output_channels, + std::is_same::value, + X_zero_point_value, + packed_W); + packed_W += packed_W_size_; + Wdata += W_offset; + } + is_symmetric_gemm_ = true; + is_W_packed_ = true; + return true; } } @@ -213,6 +280,7 @@ class QLinearConv : public OpKernel { bool is_W_signed_{false}; bool is_W_packed_{false}; bool is_symmetric_conv_{false}; + bool is_symmetric_gemm_{false}; bool channels_last_{false}; std::vector column_sums_; }; @@ -324,7 +392,7 @@ Status QLinearConv::PrePack(const Tensor& tensor, int input_idx, Alloca } // Don't pack the filter buffer if the MlasConvDepthwise path is used. - if (group_input_channels != 1 && group_output_channels != 1) { + if (group_input_channels != 1 || group_output_channels != 1) { packed_W_size_ = MlasGemmPackBSize(group_output_channels, kernel_dim, std::is_same::value, @@ -547,7 +615,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { // Allocate temporary buffers for transposing to channels last format. if (!channels_last_) { - auto* transpose_input = alloc->Alloc(SafeInt(sizeof(ActType)) * X_offset); + auto* transpose_input = alloc->Alloc(SafeInt(sizeof(ActType)) * (X_offset + MLAS_SYMM_QGEMM_BUF_OVERRUN)); transpose_input_buffer = BufferUniquePtr(transpose_input, BufferDeleter(alloc)); auto* transpose_output = alloc->Alloc(SafeInt(sizeof(ActType)) * Y_offset); transpose_output_buffer = BufferUniquePtr(transpose_output, BufferDeleter(alloc)); @@ -567,6 +635,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { // Pointwise convolutions can use the original input tensor in place, // otherwise a temporary buffer is required for the im2col transform. int64_t group_col_buffer_size = (kernel_rank > 2) ? group_count * col_buffer_size : col_buffer_size; + group_col_buffer_size += MLAS_SYMM_QGEMM_BUF_OVERRUN; auto* col_data = alloc->Alloc(SafeInt(sizeof(ActType)) * group_col_buffer_size); col_buffer = BufferUniquePtr(col_data, BufferDeleter(alloc)); } @@ -579,9 +648,13 @@ Status QLinearConv::Compute(OpKernelContext* context) const { padding_data.resize(static_cast(C), X_zero_point_value); } - int32_t thread_count = ComputeThreadCount(output_image_size, group_output_channels, kernel_dim); concurrency::ThreadPool* thread_pool = context->GetOperatorThreadPool(); - thread_count = std::min(thread_count, concurrency::ThreadPool::DegreeOfParallelism(thread_pool)); +#if defined(_M_ARM64) || defined(__aarch64__) + int32_t task_count = (output_image_size + (GEMM_KERNEL_STRIDE_M - 1)) / GEMM_KERNEL_STRIDE_M; +#else + int32_t task_count = ComputeTaskCount(output_image_size, group_output_channels, kernel_dim); + task_count = std::min(task_count, concurrency::ThreadPool::DegreeOfParallelism(thread_pool)); +#endif for (int64_t image_id = 0; image_id < N; ++image_id) { const auto* input_data = Xdata; @@ -619,9 +692,14 @@ Status QLinearConv::Compute(OpKernelContext* context) const { } auto conv_worker = [&](ptrdiff_t batch) { - auto work = concurrency::ThreadPool::PartitionWork(batch, thread_count, static_cast(output_image_size)); +#if defined(_M_ARM64) || defined(__aarch64__) + int64_t output_start = batch * GEMM_KERNEL_STRIDE_M; + int64_t output_count = std::min((int64_t)GEMM_KERNEL_STRIDE_M, output_image_size - output_start); +#else + auto work = concurrency::ThreadPool::PartitionWork(batch, task_count, static_cast(output_image_size)); int64_t output_start = static_cast(work.start); int64_t output_count = static_cast(work.end) - work.start; +#endif ActType const** worker_indirection_buffer = nullptr; if (indirection_buffer) { @@ -687,22 +765,11 @@ Status QLinearConv::Compute(OpKernelContext* context) const { static_cast(kernel_size)); } else { for (int64_t group_id = 0; group_id < group_count; ++group_id) { - MLAS_GEMM_QUANT_DATA_PARAMS gemm_params; - gemm_params.ZeroPointA = static_cast(X_zero_point_value); - if (packed_W_buffer_) { - gemm_params.B = static_cast(packed_W_buffer_.get()) + group_id * packed_W_size_, - gemm_params.BIsPacked = true; - } else { - gemm_params.B = reordered_W + group_id * group_output_channels, - gemm_params.ldb = static_cast(M); - } - gemm_params.ZeroPointB = &W_zero_point_value; - gemm_params.C = worker_gemm_output + group_id * group_output_channels; - gemm_params.ldc = static_cast(M); - // Prepare the im2col transformation or use the input buffer directly for // pointwise convolutions. const auto* group_input_data = input_data + group_id * group_input_channels; + const uint8_t* AData; + size_t lda; if (col_buffer) { auto* worker_col_buffer = static_cast(col_buffer.get()) + output_start * kernel_dim; if (kernel_rank == 2) { @@ -749,11 +816,11 @@ Status QLinearConv::Compute(OpKernelContext* context) const { // Use the im2col buffer prepared outside the thread, indexed by group. worker_col_buffer += group_id * col_buffer_size; } - gemm_params.A = reinterpret_cast(worker_col_buffer); - gemm_params.lda = static_cast(kernel_dim); + AData = reinterpret_cast(worker_col_buffer); + lda = static_cast(kernel_dim); } else { - gemm_params.A = reinterpret_cast(group_input_data + output_start * C); - gemm_params.lda = static_cast(C); + AData = reinterpret_cast(group_input_data + output_start * C); + lda = static_cast(C); } MLAS_GEMM_QUANT_SHAPE_PARAMS gemm_shape; @@ -763,7 +830,32 @@ Status QLinearConv::Compute(OpKernelContext* context) const { gemm_shape.AIsSigned = std::is_signed::value; gemm_shape.BIsSigned = is_W_signed; - MlasGemm(gemm_shape, gemm_params, nullptr); + if (is_symmetric_gemm_) { + MLAS_SYMM_QGEMM_DATA_PARAMS symm_gemm; + symm_gemm.A = AData; + symm_gemm.lda = lda; + symm_gemm.C = worker_gemm_output + group_id * group_output_channels; + symm_gemm.ldc = static_cast(M); + symm_gemm.B = static_cast(packed_W_buffer_.get()) + group_id * packed_W_size_, + MlasSymmQgemmBatch(gemm_shape, &symm_gemm, 1, nullptr); + } else { + MLAS_GEMM_QUANT_DATA_PARAMS gemm_params; + gemm_params.ZeroPointA = static_cast(X_zero_point_value); + gemm_params.A = AData; + gemm_params.lda = lda; + if (packed_W_buffer_) { + gemm_params.B = static_cast(packed_W_buffer_.get()) + group_id * packed_W_size_, + gemm_params.BIsPacked = true; + } else { + gemm_params.B = reordered_W + group_id * group_output_channels, + gemm_params.ldb = static_cast(M); + } + gemm_params.ZeroPointB = &W_zero_point_value; + gemm_params.C = worker_gemm_output + group_id * group_output_channels; + gemm_params.ldc = static_cast(M); + + MlasGemm(gemm_shape, gemm_params, nullptr); + } } } @@ -782,7 +874,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { static_cast(M)); }; - concurrency::ThreadPool::TrySimpleParallelFor(thread_pool, thread_count, conv_worker); + concurrency::ThreadPool::TrySimpleParallelFor(thread_pool, task_count, conv_worker); if (!channels_last_) { // Transpose the output from channels last (NHWC) to channels first (NCHW). diff --git a/onnxruntime/test/mlas/bench/bench_symm_qgemm.cpp b/onnxruntime/test/mlas/bench/bench_symm_qgemm.cpp new file mode 100644 index 0000000000..bc0a5eb402 --- /dev/null +++ b/onnxruntime/test/mlas/bench/bench_symm_qgemm.cpp @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "mlas.h" +#include "bench_util.h" +#include "core/util/thread_utils.h" + +#include +#include +#include +#include + +static const std::vector qgemm_arg_names = {"M", "N", "K", "Batch", "Threads"}; + +void SYMMQGEMM(benchmark::State& state, bool a_signed) { + const int8_t a_zero_point = 29; + + if (state.range(0) <= 0) throw std::invalid_argument("M must greater than 0!"); + if (state.range(1) <= 0) throw std::invalid_argument("N must greater than 0!"); + if (state.range(2) <= 0) throw std::invalid_argument("K must greater than 0!"); + if (state.range(3) <= 0) throw std::invalid_argument("Batch must greater than 0!"); + if (state.range(4) <= 0) throw std::invalid_argument("Threads must greater than 0!"); + + const size_t M = static_cast(state.range(0)); + const size_t N = static_cast(state.range(1)); + const size_t K = static_cast(state.range(2)); + + const size_t batch = static_cast(state.range(3)); + const size_t threads = static_cast(state.range(4)); + + OrtThreadPoolParams tpo; + tpo.thread_pool_size = int(threads); + tpo.auto_set_affinity = true; + std::unique_ptr tp( + onnxruntime::concurrency::CreateThreadPool(&onnxruntime::Env::Default(), + tpo, onnxruntime::concurrency::ThreadPoolType::INTRA_OP)); + + auto A_holder = RandomVectorUniform(static_cast(M * K * batch) + 16, int8_t(-120), int8_t(120)); + auto B_holder = RandomVectorUniform(static_cast(N * K * batch), int8_t(-122), int8_t(122)); + std::vector C_holder(static_cast(M * N * batch)); + std::vector pack_b_holder; + + size_t packed_b_size = MlasGemmPackBSize(N, K, a_signed, true); + pack_b_holder.resize(packed_b_size * batch); + + MLAS_GEMM_QUANT_SHAPE_PARAMS gemm_shape; + + gemm_shape.M = static_cast(M); + gemm_shape.N = static_cast(N); + gemm_shape.K = static_cast(K); + gemm_shape.AIsSigned = true; + gemm_shape.BIsSigned = true; + + std::vector gemm_data_vec(batch); + for (size_t i = 0; i < batch; i++) { + auto& gemm_params = gemm_data_vec[i]; + gemm_params.lda = gemm_shape.K; + gemm_params.ldc = gemm_shape.N; + gemm_params.A = A_holder.data() + M * K * i; + gemm_params.C = C_holder.data() + M * N * i; + + MlasSymmQgemmPackB(N, K, (const int8_t*)gemm_params.B, N, a_signed, a_zero_point, (void*)(pack_b_holder.data() + packed_b_size * i)); + gemm_params.B = (void*)(pack_b_holder.data() + packed_b_size * i); + } + for (auto _ : state) { + MlasSymmQgemmBatch(gemm_shape, gemm_data_vec.data(), batch, tp.get()); + } +} + +static void SymmQGemmSize(benchmark::internal::Benchmark* b) { + b->ArgNames(qgemm_arg_names); + // Args for "M", "N", "K", "Batch", + + b->Args({512, 32128, 768, 1, 1}); + b->Args({512, 32128, 768, 1, 4}); + b->Args({512, 32128, 768, 1, 6}); + + b->Args({512, 3072, 768, 1, 1}); + b->Args({512, 3072, 768, 1, 4}); + b->Args({512, 3072, 768, 1, 6}); + + b->Args({512, 768, 3072, 1, 1}); + b->Args({512, 768, 3072, 1, 4}); + b->Args({512, 768, 3072, 1, 6}); + + b->Args({512, 768, 768, 1, 1}); + b->Args({512, 768, 768, 1, 4}); + b->Args({512, 768, 768, 1, 6}); + + b->Args({512, 64, 512, 1, 1}); + b->Args({512, 64, 512, 1, 4}); + b->Args({512, 64, 512, 1, 6}); + + b->Args({512, 512, 64, 12, 1}); + b->Args({512, 512, 64, 12, 4}); + b->Args({512, 512, 64, 12, 6}); + + b->Args({512, 64, 512, 12, 1}); + b->Args({512, 64, 512, 12, 4}); + b->Args({512, 64, 512, 12, 6}); +} + +BENCHMARK_CAPTURE(SYMMQGEMM, SignedActivation, true)->Apply(SymmQGemmSize)->UseRealTime(); diff --git a/onnxruntime/test/mlas/unittest/test_symm_qgemm.cpp b/onnxruntime/test/mlas/unittest/test_symm_qgemm.cpp new file mode 100644 index 0000000000..17fc378d38 --- /dev/null +++ b/onnxruntime/test/mlas/unittest/test_symm_qgemm.cpp @@ -0,0 +1,36 @@ +#include "test_symm_qgemm_fixture.h" + +template <> MlasSymmQgemmTest* MlasTestFixture>::mlas_tester(nullptr); +template <> MlasSymmQgemmTest* MlasTestFixture>::mlas_tester(nullptr); + +static size_t SymmQgemmRegistLongExecute() { + if (MlasSymmQgemmPackBSize(16, 16, true) == 0) { + return 0; + } + + size_t count = MlasLongExecuteTests>::RegisterLongExecute(); + + if (GetMlasThreadPool() != nullptr) { + count += MlasLongExecuteTests>::RegisterLongExecute(); + } + + return count; +} + +static size_t SymmQgemmRegistShortExecute() { + if (MlasSymmQgemmPackBSize(16, 16, true) == 0) { + return 0; + } + + size_t count = SymmQgemmShortExecuteTest::RegisterShortExecuteTests(); + + if (GetMlasThreadPool() != nullptr) { + count += SymmQgemmShortExecuteTest::RegisterShortExecuteTests(); + } + + return count; +} + +static UNUSED_VARIABLE bool added_to_main = AddTestRegister([](bool is_short_execute) { + return is_short_execute ? SymmQgemmRegistShortExecute() : SymmQgemmRegistLongExecute(); +}); \ No newline at end of file diff --git a/onnxruntime/test/mlas/unittest/test_symm_qgemm.h b/onnxruntime/test/mlas/unittest/test_symm_qgemm.h new file mode 100644 index 0000000000..7b4abb9078 --- /dev/null +++ b/onnxruntime/test/mlas/unittest/test_symm_qgemm.h @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "test_util.h" + +template +class MlasSymmQgemmTestBase : public MlasTestBase { + + protected: + MLAS_THREADPOOL* threadpool_; + + MlasSymmQgemmTestBase() : threadpool_(Threaded ? GetMlasThreadPool() : nullptr) {} + + void TestGemm(size_t M, + size_t N, + size_t K, + size_t BatchSize, + const uint8_t* A, + size_t lda, + int32_t offa, + bool AIsSigned, + const int8_t* B, + size_t ldb, + int32_t* C, + size_t ldc) { + MLAS_GEMM_QUANT_SHAPE_PARAMS GemmShape; + GemmShape.M = M; + GemmShape.N = N; + GemmShape.K = K; + GemmShape.AIsSigned = AIsSigned; + GemmShape.BIsSigned = true; + + size_t PackedBSize = MlasSymmQgemmPackBSize(N, K, AIsSigned); + int8_t* PackedB = (int8_t*)BufferBPacked.GetBuffer(PackedBSize * BatchSize); + + std::vector GemmParameters(BatchSize); + + for (size_t i = 0; i < GemmParameters.size(); i++) { + auto& params = GemmParameters[i]; + params.A = A + (M * K * i); + params.lda = lda; + params.C = C + (M * N * i); + params.ldc = ldc; + + MlasSymmQgemmPackB(N, K, B + (K * N * i), ldb, AIsSigned, offa, PackedB + PackedBSize * i); + params.B = PackedB + PackedBSize * i; + } + + MlasSymmQgemmBatch(GemmShape, GemmParameters.data(), BatchSize, threadpool_); + } + + private: + MatrixGuardBuffer BufferBPacked; +}; + +template +class MlasSymmQgemmTest; + +template +class MlasSymmQgemmTest : public MlasSymmQgemmTestBase { + public: + + void Test(size_t M, size_t N, size_t K, size_t BatchSize, int32_t offa) { + // Symmetric kernel will have limited buffer overrun when reading the input buffer + constexpr size_t OVERRUN = 15; + const uint8_t* A = BufferA.GetBuffer(K * M * BatchSize + OVERRUN); + const int8_t* B = BufferB.GetBuffer(N * K * BatchSize); + int32_t* C = BufferC.GetBuffer(N * M * BatchSize); + int32_t* CReference = BufferCReference.GetBuffer(N * M * BatchSize); + + Test(M, N, K, BatchSize, A, K, offa, B, N, C, CReference, N); + } + + void Test(size_t M, + size_t N, + size_t K, + size_t BatchSize, + const uint8_t* A, + size_t lda, + int32_t offa, + const int8_t* B, + size_t ldb, + int32_t* C, + int32_t* CReference, + size_t ldc) { + std::fill_n(C, M * N * BatchSize, -1); + std::fill_n(CReference, M * N * BatchSize, -1); + + this->TestGemm(M, N, K, BatchSize, A, lda, offa, std::is_signed::value, B, ldb, C, ldc); + ReferenceQgemm(M, N, K, BatchSize, (const AType*)A, lda, (AType)offa, B, ldb, (const int8_t)0, CReference, ldc); + + for (size_t batch = 0, f = 0; batch < BatchSize; batch++) { + for (size_t m = 0; m < M; m++) { + for (size_t n = 0; n < N; n++, f++) { + ASSERT_EQ(C[f], CReference[f]) << "@[" << batch << "x" << m << "x" << n << "], " + << "Batch=" << BatchSize << "M=" << M << ", N=" << N << ", K=" << K + << ", offa=" << offa << ", offb=--"; + } + } + } + } + + private: + void ReferenceQgemm(size_t M, + size_t N, + size_t K, + size_t BatchSize, + const AType* A, + size_t lda, + AType offa, + const int8_t* B, + size_t ldb, + int8_t offb, + int32_t* C, + size_t ldc) { + for (size_t batch = 0; batch < BatchSize; batch++) { + for (size_t m = 0; m < M; m++) { + for (size_t n = 0; n < N; n++) { + const AType* a = A + (M * K * batch) + (m * lda); + const int8_t* b = B + (K * N * batch) + n; + int32_t* c = C + (M * N * batch) + (m * ldc) + n; + int32_t sum = 0; + + for (size_t k = 0; k < K; k++) { + sum += ((int32_t(*b) - offb) * (int32_t(*a) - offa)); + b += ldb; + a += 1; + } + + *c = sum; + } + } + } + } + + MatrixGuardBuffer BufferA; + MatrixGuardBuffer BufferB; + MatrixGuardBuffer BufferC; + MatrixGuardBuffer BufferCReference; + + public: + static const char* GetTestSuiteName() { + static std::string suite_name = std::string("SymmQgemm") + + (std::is_signed::value ? "S8" : "U8") + + "_Int32" + + (Threaded ? "_Threaded" : "_SingleThread"); + return suite_name.c_str(); + } + + void ExecuteLong(void) override { + static const int32_t zero_points[] = {-18, 124}; + + for (size_t a = 0; a < _countof(zero_points); a++) { + int32_t offa = zero_points[a]; + + for (size_t M = 16; M < 160; M += 32) { + for (size_t N = 16; N < 160; N += 32) { + static const size_t ks[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16, 20, 32, 48, 64, 118, 119, 120, 121, 122, 160, 240, 320}; + for (size_t k = 0; k < _countof(ks); k++) { + size_t K = ks[k]; + + Test(M, N, K, 1, offa); + Test(M + 1, N, K, 1, offa); + Test(M, N + 1, K, 1, offa); + Test(M + 1, N + 1, K, 1, offa); + Test(M + 3, N + 2, K, 1, offa); + Test(M + 4, N, K, 1, offa); + Test(M, N + 4, K, 1, offa); + Test(M + 4, N + 4, K, 1, offa); + Test(M + 3, N + 7, K, 1, offa); + Test(M + 8, N, K, 1, offa); + Test(M, N + 8, K, 1, offa); + Test(M + 12, N + 12, K, 1, offa); + Test(M + 13, N, K, 1, offa); + Test(M, N + 15, K, 1, offa); + Test(M + 15, N + 15, K, 1, offa); + Test(M, N, K, 7 + a, offa); + Test(M + 3, N, K, 7 + a, offa); + Test(M, N + 1, K, 7 + a, offa); + Test(M + 12, N, K, 7 + a, offa); + Test(M, N + 15, K, 7 + a, offa); + Test(M + 15, N + 15, K, 7 + a, offa); + } + } + printf("a %zd/%zd b %zd M %zd\n", a, _countof(zero_points), _countof(zero_points), M); + } + } + + for (size_t M = 1; M < 160; M++) { + for (size_t N = 1; N < 160; N++) { + for (size_t K = 1; K < 160; K++) { + Test(M, N, K, 1, 18); + } + } + printf("M %zd\n", M); + } + + for (size_t M = 160; M < 320; M += 24) { + for (size_t N = 112; N < 320; N += 24) { + for (size_t K = 1; K < 16; K++) { + Test(M, N, K, 1, 1); + } + for (size_t K = 16; K < 160; K += 32) { + Test(M, N, K, 1, -5); + } + } + printf("M %zd\n", M); + } + } +}; + diff --git a/onnxruntime/test/mlas/unittest/test_symm_qgemm_fixture.h b/onnxruntime/test/mlas/unittest/test_symm_qgemm_fixture.h new file mode 100644 index 0000000000..749b47a368 --- /dev/null +++ b/onnxruntime/test/mlas/unittest/test_symm_qgemm_fixture.h @@ -0,0 +1,82 @@ + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "test_symm_qgemm.h" + +// +// Short Execute() test helper to register each test seperately by all parameters. +// +template +class SymmQgemmShortExecuteTest; + +template +class SymmQgemmShortExecuteTest : public MlasTestFixture> { + public: + explicit SymmQgemmShortExecuteTest(size_t M, size_t N, size_t K, size_t Batch, int32_t offa) + : M_(M), N_(N), K_(K), Batch_(Batch), offa_(offa) { + } + + void TestBody() override { + MlasTestFixture>::mlas_tester->Test(M_, N_, K_, Batch_, offa_); + } + + static size_t RegisterSingleTest(size_t M, size_t N, size_t K, size_t Batch, int32_t offa) { + std::stringstream ss; + ss << "Batch" << Batch << "/M" << M << "xN" << N << "xK" << K << "/" + << "offa" << offa; + auto test_name = ss.str(); + + testing::RegisterTest( + MlasSymmQgemmTest::GetTestSuiteName(), + test_name.c_str(), + nullptr, + test_name.c_str(), + __FILE__, + __LINE__, + // Important to use the fixture type as the return type here. + [=]() -> MlasTestFixture>* { + return new SymmQgemmShortExecuteTest( + M, N, K, Batch, offa); + }); + + return 1; + } + + static size_t RegisterShortExecuteTests() { + size_t test_registered = 0; + + for (size_t b = 1; b < 16; b++) { + test_registered += RegisterSingleTest(b, b, b, 1, 21); + test_registered += RegisterSingleTest(b, b, b, 2 + b / 4, -21); + } + for (size_t b = 1; b < 16; b++) { + test_registered += RegisterSingleTest(b, b, b, 1, 17); + } + for (size_t b = 16; b <= 256; b <<= 1) { + test_registered += RegisterSingleTest(b, b, b, 1, -1); + } + for (size_t b = 256; b < 320; b += 32) { + test_registered += RegisterSingleTest(b, b, b, 1, 85); + } + for (size_t b = 1; b < 96; b++) { + test_registered += RegisterSingleTest(1, b, 32, 1, 0); + test_registered += RegisterSingleTest(1, 32, b, 1, 0); + test_registered += RegisterSingleTest(1, b, b, 1, 0); + test_registered += RegisterSingleTest(1, b, 32, 3, 0); + test_registered += RegisterSingleTest(1, 32, b, 5, 0); + } + test_registered += RegisterSingleTest(43, 500, 401, 7, 113); + test_registered += RegisterSingleTest(2003, 212, 1020, 3, -5); + test_registered += RegisterSingleTest(202, 2003, 1023, 3, 15); + + return test_registered; + } + + private: + size_t M_, N_, K_, Batch_; + int32_t offa_; +}; + From df0c819850d8426706e853b629af3803ad5cc469 Mon Sep 17 00:00:00 2001 From: Chen Fu <1316708+chenfucn@users.noreply.github.com> Date: Mon, 24 Jan 2022 16:32:05 -0800 Subject: [PATCH 26/59] fix compilation error due to symantic conflict with another PR (#10370) Resolve PR conflicts between: #10289 and #10334 Co-authored-by: Chen Fu --- onnxruntime/core/mlas/lib/qgemm.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/mlas/lib/qgemm.cpp b/onnxruntime/core/mlas/lib/qgemm.cpp index fb5a2f766c..33ac5cfc27 100644 --- a/onnxruntime/core/mlas/lib/qgemm.cpp +++ b/onnxruntime/core/mlas/lib/qgemm.cpp @@ -204,7 +204,7 @@ MlasSymmQgemmBatch( const size_t M = Shape.M; const size_t N = Shape.N; const size_t K = Shape.K; - const MLAS_SYMM_QGEMM_DISPATCH* dispatch = MlasPlatform.SymmQgemmDispatch; + const MLAS_SYMM_QGEMM_DISPATCH* dispatch = GetMlasPlatform().SymmQgemmDispatch; MLAS_SYMM_QGEMM_OPERATION* operation = dispatch->Operation; if (ThreadPool == nullptr) { @@ -472,7 +472,7 @@ MlasSymmQgemmPackBSize( return 0; } - const auto* Dispatch = MlasPlatform.SymmQgemmDispatch; + const auto* Dispatch = GetMlasPlatform().SymmQgemmDispatch; size_t PackedK = Dispatch->PackedK; @@ -512,7 +512,7 @@ MlasSymmQgemmPackB( { MLAS_UNREFERENCED_PARAMETER(AIsSigned); - const MLAS_SYMM_QGEMM_DISPATCH* SymmQgemmDispatch = MlasPlatform.SymmQgemmDispatch; + const MLAS_SYMM_QGEMM_DISPATCH* SymmQgemmDispatch = GetMlasPlatform().SymmQgemmDispatch; const size_t AlignedN = (N + MLAS_QGEMM_STRIDEN_THREAD_ALIGN - 1) & ~(MLAS_QGEMM_STRIDEN_THREAD_ALIGN - 1); From 4b87d2c172bc77ca67a674009e48e3eb30cc155a Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Mon, 24 Jan 2022 19:06:09 -0800 Subject: [PATCH 27/59] Fix dockerfiles/Dockerfile.arm32v7 build. (#10360) Install CMake, ignore some Eigen warnings. --- dockerfiles/Dockerfile.arm32v7 | 6 +++--- dockerfiles/scripts/install_fedora_arm32.sh | 4 +++- include/onnxruntime/core/common/eigen_common_wrapper.h | 7 +++++++ .../core/platform/EigenNonBlockingThreadPool.h | 7 +++++++ onnxruntime/core/util/math_cpuonly.h | 7 +++++++ onnxruntime/test/perftest/performance_runner.cc | 8 ++++++++ onnxruntime/test/util/compare_ortvalue.cc | 7 +++++++ 7 files changed, 42 insertions(+), 4 deletions(-) diff --git a/dockerfiles/Dockerfile.arm32v7 b/dockerfiles/Dockerfile.arm32v7 index 9abfb47f65..54cc58c426 100644 --- a/dockerfiles/Dockerfile.arm32v7 +++ b/dockerfiles/Dockerfile.arm32v7 @@ -8,10 +8,10 @@ FROM arm32v7/fedora:34 MAINTAINER Changming Sun "chasun@microsoft.com" ADD . /code - -RUN /code/dockerfiles/scripts/install_fedora_arm32.sh && cd /code && ./build.sh --skip_submodule_sync --config Release --build_wheel --update --build --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) +RUN /code/dockerfiles/scripts/install_fedora_arm32.sh +RUN cd /code && ./build.sh --skip_submodule_sync --config Release --build_wheel --update --build --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) FROM arm64v8/centos:7 COPY --from=0 /code/build/Linux/Release/dist /root COPY --from=0 /code/dockerfiles/LICENSE-IMAGE.txt /code/LICENSE-IMAGE.txt -RUN yum install -y python3-wheel python3-pip && python3 -m pip install --upgrade pip && python3 -m pip install /root/*.whl && rm -rf /root/*.whl \ No newline at end of file +RUN yum install -y python3-wheel python3-pip && python3 -m pip install --upgrade pip && python3 -m pip install /root/*.whl && rm -rf /root/*.whl diff --git a/dockerfiles/scripts/install_fedora_arm32.sh b/dockerfiles/scripts/install_fedora_arm32.sh index 7f4b554213..c32859e696 100755 --- a/dockerfiles/scripts/install_fedora_arm32.sh +++ b/dockerfiles/scripts/install_fedora_arm32.sh @@ -1,3 +1,5 @@ -dnf install -y binutils gcc gcc-c++ aria2 python3-pip python3-wheel git python3-devel +#!/bin/bash +set -e +dnf install -y binutils gcc gcc-c++ aria2 python3-pip python3-wheel git python3-devel cmake python3 -m pip install --upgrade pip python3 -m pip install numpy diff --git a/include/onnxruntime/core/common/eigen_common_wrapper.h b/include/onnxruntime/core/common/eigen_common_wrapper.h index fa51a009a6..3ddb4e1e28 100644 --- a/include/onnxruntime/core/common/eigen_common_wrapper.h +++ b/include/onnxruntime/core/common/eigen_common_wrapper.h @@ -18,6 +18,13 @@ #ifdef HAS_DEPRECATED_COPY #pragma GCC diagnostic ignored "-Wdeprecated-copy" #endif +// cmake/external/eigen/unsupported/Eigen/CXX11/../../../Eigen/src/Core/arch/NEON/PacketMath.h:1633:9: +// error: ‘void* memcpy(void*, const void*, size_t)’ copying an object of non-trivial type ‘Eigen::internal::Packet4c’ +// {aka ‘struct Eigen::internal::eigen_packet_wrapper’} from an array of ‘const int8_t’ +// {aka ‘const signed char’} [-Werror=class-memaccess] +#ifdef HAS_CLASS_MEMACCESS +#pragma GCC diagnostic ignored "-Wclass-memaccess" +#endif #elif defined(_MSC_VER) // build\windows\debug\external\eigen3\unsupported\eigen\cxx11\src/Tensor/Tensor.h(76): // warning C4554: '&': check operator precedence for possible error; use parentheses to clarify precedence diff --git a/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h b/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h index 95bab63ba0..75eb86c9a0 100644 --- a/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h +++ b/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h @@ -20,6 +20,13 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wunused-result" +// cmake/external/eigen/unsupported/Eigen/CXX11/../../../Eigen/src/Core/arch/NEON/PacketMath.h:1633:9: +// error: ‘void* memcpy(void*, const void*, size_t)’ copying an object of non-trivial type ‘Eigen::internal::Packet4c’ +// {aka ‘struct Eigen::internal::eigen_packet_wrapper’} from an array of ‘const int8_t’ +// {aka ‘const signed char’} [-Werror=class-memaccess] +#ifdef HAS_CLASS_MEMACCESS +#pragma GCC diagnostic ignored "-Wclass-memaccess" +#endif #elif defined(_MSC_VER) #pragma warning(push) #pragma warning(disable : 4127) diff --git a/onnxruntime/core/util/math_cpuonly.h b/onnxruntime/core/util/math_cpuonly.h index ced9eb404e..d9214b16c0 100644 --- a/onnxruntime/core/util/math_cpuonly.h +++ b/onnxruntime/core/util/math_cpuonly.h @@ -35,6 +35,13 @@ #pragma GCC diagnostic ignored "-Wdeprecated-copy" #endif #endif +// cmake/external/eigen/Eigen/src/Core/arch/NEON/PacketMath.h:1633:9: +// error: ‘void* memcpy(void*, const void*, size_t)’ copying an object of non-trivial type ‘Eigen::internal::Packet4c’ +// {aka ‘struct Eigen::internal::eigen_packet_wrapper’} from an array of ‘const int8_t’ +// {aka ‘const signed char’} [-Werror=class-memaccess] +#ifdef HAS_CLASS_MEMACCESS +#pragma GCC diagnostic ignored "-Wclass-memaccess" +#endif #elif defined(_MSC_VER) // build\windows\debug\external\eigen3\unsupported\eigen\cxx11\src/Tensor/Tensor.h(76): // warning C4554: '&': check operator precedence for possible error; use parentheses to clarify precedence diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index f789ed5745..c59c5dc391 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -20,10 +20,18 @@ using onnxruntime::Status; // TODO: Temporary, while we bring up the threadpool impl... #include "core/platform/threadpool.h" +#include "onnxruntime_config.h" #if defined(__GNUC__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wunused-result" +// cmake/external/eigen/unsupported/Eigen/CXX11/../../../Eigen/src/Core/arch/NEON/PacketMath.h:1633:9: +// error: ‘void* memcpy(void*, const void*, size_t)’ copying an object of non-trivial type ‘Eigen::internal::Packet4c’ +// {aka ‘struct Eigen::internal::eigen_packet_wrapper’} from an array of ‘const int8_t’ +// {aka ‘const signed char’} [-Werror=class-memaccess] +#ifdef HAS_CLASS_MEMACCESS +#pragma GCC diagnostic ignored "-Wclass-memaccess" +#endif #endif #include #if defined(__GNUC__) diff --git a/onnxruntime/test/util/compare_ortvalue.cc b/onnxruntime/test/util/compare_ortvalue.cc index 53fccefbe4..c6bbae3465 100644 --- a/onnxruntime/test/util/compare_ortvalue.cc +++ b/onnxruntime/test/util/compare_ortvalue.cc @@ -10,6 +10,13 @@ #pragma GCC diagnostic ignored "-Wignored-qualifiers" #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wunused-result" +// cmake/external/eigen/Eigen/src/Core/arch/NEON/PacketMath.h:1633:9: +// error: ‘void* memcpy(void*, const void*, size_t)’ copying an object of non-trivial type ‘Eigen::internal::Packet4c’ +// {aka ‘struct Eigen::internal::eigen_packet_wrapper’} from an array of ‘const int8_t’ +// {aka ‘const signed char’} [-Werror=class-memaccess] +#ifdef HAS_CLASS_MEMACCESS +#pragma GCC diagnostic ignored "-Wclass-memaccess" +#endif #endif #include #include From 790c3be7e9117432df0f93128c5a2271fd6f1b40 Mon Sep 17 00:00:00 2001 From: pallavides Date: Mon, 24 Jan 2022 19:30:52 -0800 Subject: [PATCH 28/59] Fix Reshape issue when shape size is -1 (#10356) * Fix Reshape issue (in_place) when shape size is -1 --- orttraining/orttraining/eager/ort_ops.cpp | 10 ++++++---- orttraining/orttraining/eager/test/ort_tensor.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/orttraining/orttraining/eager/ort_ops.cpp b/orttraining/orttraining/eager/ort_ops.cpp index bebd3a82f7..3d5899d54f 100644 --- a/orttraining/orttraining/eager/ort_ops.cpp +++ b/orttraining/orttraining/eager/ort_ops.cpp @@ -4,14 +4,14 @@ #include "ort_ops.h" #include "ort_util.h" #include "ort_log.h" +#include "core/providers/cpu/tensor/reshape_helper.h" namespace torch_ort { namespace eager { -void copy(onnxruntime::ORTInvoker& invoker, +void copy(onnxruntime::ORTInvoker& invoker, const OrtValue& src, OrtValue& dst){ auto& ort_ep = invoker.GetCurrentExecutionProvider(); - const auto& src_tensor = src.Get(); auto* dst_tensor = dst.GetMutable(); if (!dst_tensor) @@ -25,16 +25,18 @@ void createInplaceOutputValue(OrtValue& input, V shape, OrtValue* p_mlv // the ort TensorShape class only accept std::vector, so have to conversion. std::vector new_shape; new_shape.assign(shape.begin(), shape.end()); + onnxruntime::ReshapeHelper helper(input.Get().Shape(), new_shape); CreateMLValue(input_ort_tensor->MutableDataRaw(), input_ort_tensor->DataType(), new_shape, p_mlvalue); } -template +template using Vector = std::vector>; template <> void createInplaceOutputValue(OrtValue& input, Vector shape, OrtValue* p_mlvalue){ auto* input_ort_tensor = input.GetMutable(); + onnxruntime::ReshapeHelper helper(input.Get().Shape(), shape); CreateMLValue(input_ort_tensor->MutableDataRaw(), input_ort_tensor->DataType(), shape, p_mlvalue); } @@ -42,4 +44,4 @@ void createInplaceOutputValue(OrtValue& input, Vector shape, Or template void createInplaceOutputValue(OrtValue& input, c10::ArrayRef shape, OrtValue* p_mlvalue); } // namespace eager -} // namespace torch_ort \ No newline at end of file +} // namespace torch_ort diff --git a/orttraining/orttraining/eager/test/ort_tensor.py b/orttraining/orttraining/eager/test/ort_tensor.py index b821a9179a..a1246db800 100644 --- a/orttraining/orttraining/eager/test/ort_tensor.py +++ b/orttraining/orttraining/eager/test/ort_tensor.py @@ -19,19 +19,25 @@ class OrtTensorTests(unittest.TestCase): ort_ones = cpu_ones.to('ort') assert ort_ones.is_ort assert torch.allclose(cpu_ones, ort_ones.cpu()) - + def test_reshape(self): cpu_ones = torch.ones(10, 10) ort_ones = cpu_ones.to('ort') y = ort_ones.reshape(-1) assert len(y.size()) == 1 assert y.size()[0] == 100 - + def test_view(self): cpu_ones = torch.ones(2048) ort_ones = cpu_ones.to('ort') y = ort_ones.view(4, 512) assert y.size() == (4, 512) + def test_view_neg1(self): + cpu_ones = torch.ones(784, 256) + ort_ones = cpu_ones.to('ort') + y = ort_ones.view(-1) + assert y.size()[0] == 200704 + if __name__ == '__main__': unittest.main() \ No newline at end of file From 6e95c0316d8375ac1926f77cde75d80bcdd453d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Tue, 25 Jan 2022 11:25:31 +0100 Subject: [PATCH 29/59] Builds onnxruntime + eager mode with the same value for _GLIBCXX_USE_CXX11_ABI as pytorch (#10114) * add _GLIBCXX_USE_CXX11_ABI * restrict to eager mode --- cmake/CMakeLists.txt | 4 ++++ tools/ci_build/build.py | 1 + 2 files changed, 5 insertions(+) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 07ce345345..185d5c0783 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -304,6 +304,10 @@ if (NOT MSVC AND NOT onnxruntime_ENABLE_BITCODE) string(APPEND CMAKE_C_FLAGS " -ffunction-sections -fdata-sections") endif() +if (onnxruntime_ENABLE_EAGER_MODE) + string(APPEND CMAKE_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=${_GLIBCXX_USE_CXX11_ABI}") +endif() + if (onnxruntime_BUILD_WEBASSEMBLY) # Enable LTO for release single-thread build if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT onnxruntime_ENABLE_WEBASSEMBLY_THREADS) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 56272d448a..e4d3375587 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1080,6 +1080,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home if args.build_eager_mode: import torch cmake_args += ["-Donnxruntime_PREBUILT_PYTORCH_PATH=%s" % os.path.dirname(torch.__file__)] + cmake_args += ['-D_GLIBCXX_USE_CXX11_ABI=' + str(int(torch._C._GLIBCXX_USE_CXX11_ABI))] cmake_args += ["-D{}".format(define) for define in cmake_extra_defines] From a0fe4a7c1c156d43840af42111a559dbbd9753ed Mon Sep 17 00:00:00 2001 From: Alexey Gladyshev Date: Tue, 25 Jan 2022 20:48:08 +0300 Subject: [PATCH 30/59] [TVM EP] Improved usability of TVM EP (#10241) * improved usability of TVM EP * moved technical import under a condition related to TVM EP only * Revert "moved technical import under a condition related to TVM EP only" * add conditional _ld_preload.py file extension for TVM EP * improve readability of inserted code --- setup.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/setup.py b/setup.py index 2a1ec7f25d..6130fcdbba 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ from shutil import copyfile import platform import subprocess import sys +import textwrap import datetime from pathlib import Path @@ -145,6 +146,33 @@ try: f.write(' import os\n') f.write(' os.environ["ORT_TENSORRT_UNAVAILABLE"] = "1"\n') + def _rewrite_ld_preload_tvm(self): + with open('onnxruntime/capi/_ld_preload.py', 'a') as f: + f.write(textwrap.dedent( + """ + import warnings + + try: + # This import is necessary in order to delegate the loading of libtvm.so to TVM. + import tvm + except ImportError as e: + warnings.warn( + f"WARNING: Failed to import TVM, libtvm.so was not loaded. More details: {e}" + ) + try: + # Working between the C++ and Python parts in TVM EP is done using the PackedFunc and + # Registry classes. In order to use a Python function in C++ code, it must be registered in + # the global table of functions. Registration is carried out through the JIT interface, + # so it is necessary to call special functions for registration. + # To do this, we need to make the following import. + import onnxruntime.providers.stvm + except ImportError as e: + warnings.warn( + f"WARNING: Failed to register python functions to work with TVM EP. More details: {e}" + ) + """ + )) + def run(self): if is_manylinux: source = 'onnxruntime/capi/onnxruntime_pybind11_state.so' @@ -207,6 +235,8 @@ try: self._rewrite_ld_preload(to_preload) self._rewrite_ld_preload_cuda(to_preload_cuda) self._rewrite_ld_preload_tensorrt(to_preload_tensorrt) + if package_name == 'onnxruntime-tvm': + self._rewrite_ld_preload_tvm() _bdist_wheel.run(self) if is_manylinux and not disable_auditwheel_repair: file = glob(path.join(self.dist_dir, '*linux*.whl'))[0] From df16c605e8aa287c7936e5fe80fdc337e744804b Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Tue, 25 Jan 2022 10:15:34 -0800 Subject: [PATCH 31/59] Add "available since" message for C API additions since v1.10.0. (#10348) --- .../core/session/onnxruntime_c_api.h | 61 ++++++++++++------- .../onnxruntime_session_options_config_keys.h | 7 ++- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 0ad3aec377..283a732faa 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -267,7 +267,7 @@ typedef OrtStatus* OrtStatusPtr; /** \brief Memory allocation interface * * Structure of function pointers that defines a memory allocator. This can be created and filled in by the user for custom allocators. -* +* * When an allocator is passed to any function, be sure that the allocator object is not destroyed until the last allocated object using it is freed. */ typedef struct OrtAllocator { @@ -376,7 +376,7 @@ typedef struct OrtCUDAProviderOptions { */ int arena_extend_strategy; - /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the CUDA EP + /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the CUDA EP * 0 = Use separate streams for copying and compute. * 1 = Use the same stream for copying and compute. * Defaults to 1. @@ -390,7 +390,7 @@ typedef struct OrtCUDAProviderOptions { */ int has_user_compute_stream; - /** \brief User provided compute stream. + /** \brief User provided compute stream. * If provided, please set `has_user_compute_stream` to 1. */ void* user_compute_stream; @@ -434,7 +434,7 @@ typedef struct OrtROCMProviderOptions { */ int arena_extend_strategy; - /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the ROCM EP + /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the ROCM EP * 0 = Use separate streams for copying and compute. * 1 = Use the same stream for copying and compute. * Defaults to 1. @@ -448,7 +448,7 @@ typedef struct OrtROCMProviderOptions { */ int has_user_compute_stream; - /** \brief User provided compute stream. + /** \brief User provided compute stream. * If provided, please set `has_user_compute_stream` to 1. */ void* user_compute_stream; @@ -3067,7 +3067,7 @@ struct OrtApi { * \brief Sets out to 1 iff an optional type OrtValue has an element, 0 otherwise (OrtValue is None) * Use this API to find if the optional type OrtValue is None or not. * If the optional type OrtValue is not None, use the OrtValue just like any other OrtValue. - * For example, if you get an OrtValue that corresponds to Optional(tensor) and + * For example, if you get an OrtValue that corresponds to Optional(tensor) and * if HasValue() returns true, use it as tensor and so on. * \param[in] value Input OrtValue. @@ -3079,7 +3079,7 @@ struct OrtApi { /// @} /// \name OrtKernelContext /// @{ - /** \brief Used for custom operators, gets the GPU compute stream to use to launch the custom a GPU kernel + /** \brief Used for custom operators, gets the GPU compute stream to use to launch the custom a GPU kernel * \see ::OrtCustomOp * \param[in] context OrtKernelContext instance * \param[out] out Returns pointer to a GPU compute stream that can be used to launch the custom GPU kernel. @@ -3088,7 +3088,7 @@ struct OrtApi { * Do not free or mutate the returned pointer as it refers to internal data owned by the underlying session. * Only use it for custom kernel launching. * - * \snippet{doc} snippets.dox OrtStatus Return Value + * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(KernelContext_GetGPUComputeStream, _In_ const OrtKernelContext* context, _Outptr_ void** out); @@ -3107,7 +3107,7 @@ struct OrtApi { /// \name GetExecutionProviderApi /// @{ /** \brief Get a pointer to the requested version of the Execution Provider specific - * API extensions to the OrtApi + * API extensions to the OrtApi * \param[in] provider_name The name of the execution provider name. Currently only the following * values are supported: "DML". * \param[in] version Must be ::ORT_API_VERSION. @@ -3126,16 +3126,16 @@ struct OrtApi { * * \param[in] options Session options * \param[in] ort_custom_create_thread_fn Custom thread creation function - * + * * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(SessionOptionsSetCustomCreateThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); - /** \brief Set creation options for custom thread + /** \brief Set creation options for custom thread * * \param[in] options Session options * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) - * + * * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(SessionOptionsSetCustomThreadCreationOptions, _Inout_ OrtSessionOptions* options, _In_ void* ort_custom_thread_creation_options); @@ -3144,7 +3144,7 @@ struct OrtApi { * * \param[in] options Session options * \param[in] ort_custom_join_thread_fn Custom join thread function, must not be nullptr when ort_custom_create_thread_fn is set - * + * * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(SessionOptionsSetCustomJoinThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); @@ -3156,7 +3156,7 @@ struct OrtApi { * * \param[inout] tp_options * \param[in] ort_custom_create_thread_fn Custom thread creation function - * + * * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(SetGlobalCustomCreateThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); @@ -3165,7 +3165,7 @@ struct OrtApi { * * \param[inout] tp_options * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) - * + * * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(SetGlobalCustomThreadCreationOptions, _Inout_ OrtThreadingOptions* tp_options, _In_ void* ort_custom_thread_creation_options); @@ -3174,7 +3174,7 @@ struct OrtApi { * * \param[inout] tp_options * \param[in] ort_custom_join_thread_fn Custom thread join function, must not be nullptr when global ort_custom_create_thread_fn is set - * + * * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(SetGlobalCustomJoinThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); @@ -3185,7 +3185,7 @@ struct OrtApi { * operation is provider specific and could be a no-op. * * \param[inout] binding_ptr - * + * * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(SynchronizeBoundInputs, _Inout_ OrtIoBinding* binding_ptr); @@ -3195,7 +3195,7 @@ struct OrtApi { * operation is provider specific and could be a no-op. * * \param[inout] binding_ptr - * + * * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(SynchronizeBoundOutputs, _Inout_ OrtIoBinding* binding_ptr); @@ -3219,6 +3219,8 @@ struct OrtApi { * \param[in] cuda_options * * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. */ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA_V2, _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptionsV2* cuda_options); @@ -3232,6 +3234,8 @@ struct OrtApi { * \param[out] out Newly created ::OrtCUDAProviderOptionsV2. Must be released with OrtApi::ReleaseCudaProviderOptions * * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. */ ORT_API2_STATUS(CreateCUDAProviderOptions, _Outptr_ OrtCUDAProviderOptionsV2** out); @@ -3249,6 +3253,8 @@ struct OrtApi { * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays * * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. */ ORT_API2_STATUS(UpdateCUDAProviderOptions, _Inout_ OrtCUDAProviderOptionsV2* cuda_options, _In_reads_(num_keys) const char* const* provider_options_keys, @@ -3258,21 +3264,29 @@ struct OrtApi { /** * Get serialized CUDA provider options string. * - * For example, "device_id=0;arena_extend_strategy=0;......" + * For example, "device_id=0;arena_extend_strategy=0;......" * - * \param cuda_options - OrtCUDAProviderOptionsV2 instance + * \param cuda_options - OrtCUDAProviderOptionsV2 instance * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() * the specified allocator will be used to allocate continuous buffers for output strings and lengths. * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. */ ORT_API2_STATUS(GetCUDAProviderOptionsAsString, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); /** \brief Release an ::OrtCUDAProviderOptionsV2 * * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does + * + * \since Version 1.11. */ void(ORT_API_CALL* ReleaseCUDAProviderOptions)(_Frees_ptr_opt_ OrtCUDAProviderOptionsV2* input); + /// @} + /** \brief Append MIGraphX provider to session options * * If MIGraphX is not available (due to a non MIGraphX enabled build, or if MIGraphX is not installed on the system), this function will return failure. @@ -3281,10 +3295,11 @@ struct OrtApi { * \param[in] migraphx_options * * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. */ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_MIGraphX, _In_ OrtSessionOptions* options, _In_ const OrtMIGraphXProviderOptions* migraphx_options); - /// @} }; /* @@ -3346,9 +3361,9 @@ struct OrtCustomOp { ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id); /* - * This is the old way to add the MIGraphX provider to the session, please use + * This is the old way to add the MIGraphX provider to the session, please use * SessionOptionsAppendExecutionProvider_MIGraphX above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with + * This function always exists, but will only succeed if Onnxruntime was built with * HIP support and the MIGraphX provider shared library exists * * \param device_id HIP device id, starts from zero. diff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h index 916ea586b4..55e328b1bb 100644 --- a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h +++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h @@ -100,8 +100,9 @@ static const char* const kOrtSessionOptionsConfigNnapiEpPartitioningStopOps = "e // Enabling dynamic block-sizing for multithreading. // With a positive value, thread pool will split a task of N iterations to blocks of size starting from: // N / (num_of_threads * dynamic_block_base) -// As execution progresses, the size will decrease according to the diminishing residual of N, +// As execution progresses, the size will decrease according to the diminishing residual of N, // meaning the task will be distributed in smaller granularity for better parallelism. -// For some models, it helps to reduce the variance of E2E inference latency and boost performane. +// For some models, it helps to reduce the variance of E2E inference latency and boost performance. // The feature will not function by default, specify any positive integer, e.g. "4", to enable it. -static const char* const kOrtSessionOptionsConfigDynamicBlockBase = "session.dynamic_block_base"; \ No newline at end of file +// Available since version 1.11. +static const char* const kOrtSessionOptionsConfigDynamicBlockBase = "session.dynamic_block_base"; From e1012a8662585b730ba1973450f1c7e0911735d5 Mon Sep 17 00:00:00 2001 From: sumitsays Date: Tue, 25 Jan 2022 13:00:44 -0800 Subject: [PATCH 32/59] Added OnRunEnd and Sync method in ExecutionProvider (#10362) Co-authored-by: Sumit Agarwal --- .../src/ExecutionProvider.cpp | 6 ++++++ .../src/ExecutionProvider.h | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.cpp index 32a6672593..7363bba4a3 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.cpp @@ -97,6 +97,12 @@ namespace Dml { m_context->Close(); } + + void ExecutionProviderImpl::WaitForOutstandingWork() + { + Flush(); + m_context->GetCurrentCompletionEvent().WaitForSignal(); + } HRESULT __stdcall ExecutionProviderImpl::AllocatePooledResource( size_t size, diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.h index 5c2db8bc70..257f481032 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.h @@ -130,6 +130,8 @@ namespace Dml // Waits for flushed work, discards unflushed work, and discards associated references to // prevent circular references. Must be the last call on the object before destruction. void Close() override; + + void WaitForOutstandingWork(); // Allocate a resource from pools. Releasing pooledResource returns it to the pool. STDMETHOD(AllocatePooledResource)( @@ -249,6 +251,22 @@ namespace Dml return m_impl->OnSessionInitializationEnd(); } + virtual onnxruntime::Status Sync() const final override + { + // Completely wait until the device has completed all preceding tasks. + // The application could have called SynchronizeBoundOutputs(). + m_impl->WaitForOutstandingWork(); + return Status::OK(); + } + + virtual onnxruntime::Status OnRunEnd(bool /*sync_stream*/) final override + { + // Flush any pending work to the GPU, but don't block for completion, permitting it + // to overlap other work. + m_impl->Flush(); + return Status::OK(); + } + void Flush() { return m_impl->Flush(); From 5eafbb50f9ad2b41885fb61a6278708b7264ed2a Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Tue, 25 Jan 2022 14:48:51 -0800 Subject: [PATCH 33/59] Fix possible null pointer dereference. (#10373) NodeInfo::p_node was used directly but it can be null from here: https://github.com/microsoft/onnxruntime/blob/2afce4830c9ff48374ca11cf00b09afcc1ff2124/onnxruntime/core/framework/session_state_utils.cc#L381-L382 Add an additional check that it is not null before use. --- onnxruntime/core/session/IOBinding.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/session/IOBinding.cc b/onnxruntime/core/session/IOBinding.cc index e0dd66d188..cf49da6860 100644 --- a/onnxruntime/core/session/IOBinding.cc +++ b/onnxruntime/core/session/IOBinding.cc @@ -52,7 +52,7 @@ static common::Status SyncProviders(const SessionState::NameNodeInfoMapType& nod std::set providers; for (auto& pair : node_info_map) { for (auto& node_info : pair.second) { - if (node_info.p_node->GetExecutionProviderType() != onnxruntime::kCpuExecutionProvider) { + if (node_info.p_node && node_info.p_node->GetExecutionProviderType() != onnxruntime::kCpuExecutionProvider) { providers.insert(node_info.p_node->GetExecutionProviderType()); } } From 9aa51379c93184a94171bc7902725d327e8722c5 Mon Sep 17 00:00:00 2001 From: "Tang, Cheng" Date: Tue, 25 Jan 2022 16:15:54 -0800 Subject: [PATCH 34/59] [eager mode]: add configuration for ort virtual device count (#10346) * add configuration for ort virtual device count * fix build break * fix ci build break Co-authored-by: Cheng Tang --- orttraining/orttraining/eager/ort_guard.cpp | 6 +++++- orttraining/orttraining/eager/ort_ops.cpp | 19 ++++--------------- orttraining/orttraining/eager/ort_util.cpp | 8 ++++++-- orttraining/orttraining/eager/ort_util.h | 1 + 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/orttraining/orttraining/eager/ort_guard.cpp b/orttraining/orttraining/eager/ort_guard.cpp index 65f75ff668..4626c3b915 100644 --- a/orttraining/orttraining/eager/ort_guard.cpp +++ b/orttraining/orttraining/eager/ort_guard.cpp @@ -9,6 +9,8 @@ namespace torch_ort { namespace eager { +constexpr const char* kORTVirtualDeviceCount="ORT_VIRTUAL_DEVICE_COUNT"; + struct ORTGuardImpl final : public c10::impl::DeviceGuardImplInterface { ORTGuardImpl() { } @@ -63,7 +65,9 @@ struct ORTGuardImpl final : public c10::impl::DeviceGuardImplInterface { at::DeviceIndex deviceCount() const noexcept override { ORT_LOG_FN(); - return 1; + const std::string ort_virtual_device_count = onnxruntime::Env::Default().GetEnvironmentVar(kORTVirtualDeviceCount); + // by default set device count to 1 + return ort_virtual_device_count.empty() ? 1 : std::stoi(ort_virtual_device_count); } // #pragma region events diff --git a/orttraining/orttraining/eager/ort_ops.cpp b/orttraining/orttraining/eager/ort_ops.cpp index 3d5899d54f..80fcf4432a 100644 --- a/orttraining/orttraining/eager/ort_ops.cpp +++ b/orttraining/orttraining/eager/ort_ops.cpp @@ -22,26 +22,15 @@ void copy(onnxruntime::ORTInvoker& invoker, template class V> void createInplaceOutputValue(OrtValue& input, V shape, OrtValue* p_mlvalue){ auto* input_ort_tensor = input.GetMutable(); - // the ort TensorShape class only accept std::vector, so have to conversion. - std::vector new_shape; - new_shape.assign(shape.begin(), shape.end()); - onnxruntime::ReshapeHelper helper(input.Get().Shape(), new_shape); + onnxruntime::TensorShapeVector target_shape{shape.begin(), shape.begin() + shape.size()}; + onnxruntime::ReshapeHelper helper(input.Get().Shape(), target_shape); + onnxruntime::TensorShape new_shape(target_shape); CreateMLValue(input_ort_tensor->MutableDataRaw(), input_ort_tensor->DataType(), new_shape, p_mlvalue); } -template -using Vector = std::vector>; - -template <> -void createInplaceOutputValue(OrtValue& input, Vector shape, OrtValue* p_mlvalue){ - auto* input_ort_tensor = input.GetMutable(); - onnxruntime::ReshapeHelper helper(input.Get().Shape(), shape); - CreateMLValue(input_ort_tensor->MutableDataRaw(), - input_ort_tensor->DataType(), shape, p_mlvalue); -} - template void createInplaceOutputValue(OrtValue& input, c10::ArrayRef shape, OrtValue* p_mlvalue); +template void createInplaceOutputValue(OrtValue& input, std::vector shape, OrtValue* p_mlvalue); } // namespace eager } // namespace torch_ort diff --git a/orttraining/orttraining/eager/ort_util.cpp b/orttraining/orttraining/eager/ort_util.cpp index 8a0cd87b89..d9ca7ee268 100644 --- a/orttraining/orttraining/eager/ort_util.cpp +++ b/orttraining/orttraining/eager/ort_util.cpp @@ -25,8 +25,7 @@ void CreateMLValue(onnxruntime::AllocatorPtr alloc, onnxruntime::DataTypeImpl::GetType()->GetDeleteFunc()); } -void CreateMLValue(void* data_ptr, onnxruntime::MLDataType element_type, const std::vector& dims, OrtValue* p_mlvalue) { - onnxruntime::TensorShape shape(dims); +void CreateMLValue(void* data_ptr, onnxruntime::MLDataType element_type, onnxruntime::TensorShape& shape, OrtValue* p_mlvalue){ OrtMemoryInfo *cpu_info; Ort::ThrowOnError(Ort::GetApi().CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &cpu_info)); std::unique_ptr p_tensor = std::make_unique(element_type, @@ -39,6 +38,11 @@ void CreateMLValue(void* data_ptr, onnxruntime::MLDataType element_type, const s onnxruntime::DataTypeImpl::GetType()->GetDeleteFunc()); } +void CreateMLValue(void* data_ptr, onnxruntime::MLDataType element_type, const std::vector& dims, OrtValue* p_mlvalue) { + onnxruntime::TensorShape shape(dims); + CreateMLValue(data_ptr, element_type, shape, p_mlvalue); +} + std::vector GetStrides(gsl::span shape) { std::vector strides(shape.size(), 1); for (auto i = shape.size(); i > 1; --i) { diff --git a/orttraining/orttraining/eager/ort_util.h b/orttraining/orttraining/eager/ort_util.h index cf82b47183..f34d86d2b5 100644 --- a/orttraining/orttraining/eager/ort_util.h +++ b/orttraining/orttraining/eager/ort_util.h @@ -16,6 +16,7 @@ void CreateMLValue(onnxruntime::AllocatorPtr alloc, OrtValue* p_mlvalue); void CreateMLValue(void* data_ptr, onnxruntime::MLDataType element_type, const std::vector& dims, OrtValue* p_mlvalue); +void CreateMLValue(void* data_ptr, onnxruntime::MLDataType element_type, onnxruntime::TensorShape& shape, OrtValue* p_mlvalue); template inline void CopyVectorToTensor(onnxruntime::ORTInvoker& invoker, From 4af116649c8f5f6e725ce8b314b7f8e38007f236 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Tue, 25 Jan 2022 17:13:46 -0800 Subject: [PATCH 35/59] [QDQ] Hookup NNAPI GetCapability/Compile with shared QDQ selectors (#10347) * add qdqgroup as input for NodeUnit * minor update * hookup nnapi_ep * minor update * update compiler setting * Add a simple UT * Pipeline change to add build minimal extended with NNAPI for Android * move GetAllNodeUnits to node_unit.h, add UT for NodeUnits, minor updates * minor updates * address CR comments Co-authored-by: gwang0000 <62914304+gwang0000@users.noreply.github.com> --- cmake/onnxruntime_optimizer.cmake | 15 +++ onnxruntime/core/graph/graph_utils.cc | 72 +++++----- onnxruntime/core/graph/graph_utils.h | 20 +-- .../selectors_actions/qdq_selectors.cc | 4 +- .../selectors_actions/qdq_selectors.h | 4 +- .../selectors_actions/shared/utils.cc | 9 +- .../selectors_actions/shared/utils.h | 14 +- .../selector_action_transformer.h | 8 +- onnxruntime/core/optimizer/utils.cc | 37 ++++-- onnxruntime/core/optimizer/utils.h | 16 ++- .../nnapi/nnapi_builtin/builders/helper.cc | 19 +-- .../nnapi_builtin/builders/model_builder.cc | 12 +- .../builders/op_support_checker.cc | 19 +++ .../nnapi_builtin/nnapi_execution_provider.cc | 44 +++++-- .../providers/shared/node_unit/node_unit.cc | 123 +++++++++++++++++- .../providers/shared/node_unit/node_unit.h | 21 ++- .../test/optimizer/qdq_transformer_test.cc | 108 ++++++++++----- .../test/providers/nnapi/nnapi_basic_test.cc | 15 +++ .../linux-cpu-minimal-build-ci-pipeline.yml | 34 +++++ 19 files changed, 447 insertions(+), 147 deletions(-) diff --git a/cmake/onnxruntime_optimizer.cmake b/cmake/onnxruntime_optimizer.cmake index 8b1e84acc4..1a3ceaa809 100644 --- a/cmake/onnxruntime_optimizer.cmake +++ b/cmake/onnxruntime_optimizer.cmake @@ -11,6 +11,21 @@ if (onnxruntime_MINIMAL_BUILD) "${ONNXRUNTIME_ROOT}/core/optimizer/graph_transformer.cc" ) + if (onnxruntime_EXTENDED_MINIMAL_BUILD AND onnxruntime_USE_NNAPI_BUILTIN) + list(APPEND onnxruntime_optimizer_src_patterns + "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/selectors_actions/shared/utils.h" + "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc" + "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h" + "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc" + "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/qdq_util.h" + "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/qdq_util.cc" + "${ONNXRUNTIME_ROOT}/core/optimizer/initializer.h" + "${ONNXRUNTIME_ROOT}/core/optimizer/initializer.cc" + "${ONNXRUNTIME_ROOT}/core/optimizer/utils.h" + "${ONNXRUNTIME_ROOT}/core/optimizer/utils.cc" + ) + endif() + if (onnxruntime_ENABLE_RUNTIME_OPTIMIZATION_REPLAY_IN_MINIMAL_BUILD) list(APPEND onnxruntime_optimizer_src_patterns "${ONNXRUNTIME_INCLUDE_DIR}/core/optimizer/graph_transformer_utils.h" diff --git a/onnxruntime/core/graph/graph_utils.cc b/onnxruntime/core/graph/graph_utils.cc index ad79b371f5..db3cdcf3c4 100644 --- a/onnxruntime/core/graph/graph_utils.cc +++ b/onnxruntime/core/graph/graph_utils.cc @@ -480,26 +480,6 @@ const Node* FirstChildByType(const Node& node, const std::string& child_type) { return nullptr; } -std::vector FindChildrenByType(const Node& node, const std::string& child_type) { - // find children and sort them by source argument index: - // Create a 2D vector to hold the result. - // 1st dimension index is output index, - // and the 2nd dimension stores the edges from the output. - std::vector> children(node.OutputDefs().size(), std::vector()); - for (auto it = node.OutputEdgesBegin(); it != node.OutputEdgesEnd(); it++) { - if (it->GetNode().OpType().compare(child_type) == 0) { - children[it->GetSrcArgIndex()].push_back(&(it->GetNode())); - } - } - - // aggregate children - std::vector agg_res; - for (size_t output_idx = 0; output_idx < children.size(); output_idx++) { - agg_res.insert(agg_res.end(), children[output_idx].begin(), children[output_idx].end()); - } - return agg_res; -} - const Node* FirstParentByType(const Node& node, const std::string& parent_type) { for (auto it = node.InputNodesBegin(); it != node.InputNodesEnd(); ++it) { if ((*it).OpType().compare(parent_type) == 0) { @@ -509,22 +489,6 @@ const Node* FirstParentByType(const Node& node, const std::string& parent_type) return nullptr; } -std::vector FindParentsByType(const Node& node, const std::string& parent_type) { - // find parents and sort them by destination argument index - // as there is at most one input edge for each input argument, - // there is no need of extra work like FindChildrenByType - std::vector parents(node.InputDefs().size(), nullptr); - for (auto it = node.InputEdgesBegin(); it != node.InputEdgesEnd(); it++) { - if (it->GetNode().OpType().compare(parent_type) == 0) { - parents[it->GetDstArgIndex()] = &(it->GetNode()); - } - } - - // remove unmatched nodes - parents.erase(std::remove(parents.begin(), parents.end(), nullptr), parents.end()); - return parents; -} - NodeArg& AddInitializer(Graph& graph, const ONNX_NAMESPACE::TensorProto& new_initializer) { // sanity check as AddInitializedTensor silently ignores attempts to add a duplicate initializer const ONNX_NAMESPACE::TensorProto* existing = nullptr; @@ -778,6 +742,42 @@ NodeArg& CreateNodeArg(Graph& graph, const NodeArg& base_arg) { #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) +std::vector FindParentsByType(const Node& node, const std::string& parent_type) { + // find parents and sort them by destination argument index + // as there is at most one input edge for each input argument, + // there is no need of extra work like FindChildrenByType + std::vector parents(node.InputDefs().size(), nullptr); + for (auto it = node.InputEdgesBegin(); it != node.InputEdgesEnd(); it++) { + if (it->GetNode().OpType().compare(parent_type) == 0) { + parents[it->GetDstArgIndex()] = &(it->GetNode()); + } + } + + // remove unmatched nodes + parents.erase(std::remove(parents.begin(), parents.end(), nullptr), parents.end()); + return parents; +} + +std::vector FindChildrenByType(const Node& node, const std::string& child_type) { + // find children and sort them by source argument index: + // Create a 2D vector to hold the result. + // 1st dimension index is output index, + // and the 2nd dimension stores the edges from the output. + std::vector> children(node.OutputDefs().size(), std::vector()); + for (auto it = node.OutputEdgesBegin(); it != node.OutputEdgesEnd(); it++) { + if (it->GetNode().OpType().compare(child_type) == 0) { + children[it->GetSrcArgIndex()].push_back(&(it->GetNode())); + } + } + + // aggregate children + std::vector agg_res; + for (size_t output_idx = 0; output_idx < children.size(); output_idx++) { + agg_res.insert(agg_res.end(), children[output_idx].begin(), children[output_idx].end()); + } + return agg_res; +} + const std::string& GetNodeInputName(const Node& node, int index) { const auto& inputs = node.InputDefs(); ORT_ENFORCE(index >= 0 && static_cast(index) < inputs.size(), diff --git a/onnxruntime/core/graph/graph_utils.h b/onnxruntime/core/graph/graph_utils.h index bbcce89c40..2b0302aa26 100644 --- a/onnxruntime/core/graph/graph_utils.h +++ b/onnxruntime/core/graph/graph_utils.h @@ -95,19 +95,9 @@ bool GetRepeatedNodeAttributeValues(const Node& node, /** Find the first child of the specified op type. */ const Node* FirstChildByType(const Node& node, const std::string& child_type); -/** Find node children by op types. - @returns The matched children are sorted by source argument index of their corresponding edge. -**/ -std::vector FindChildrenByType(const Node& node, const std::string& child_type); - /** Find the first parent of the specified op type. */ const Node* FirstParentByType(const Node& node, const std::string& parent_type); -/** Find node parents by op types. - @returns The matched parents are sorted by destination argument index of their corresponding edge. -**/ -std::vector FindParentsByType(const Node& node, const std::string& parent_type); - /** Tests if we can remove a node and merge its input edge (if any) with its output edges. Conditions: Input rules: @@ -290,6 +280,16 @@ NodeArg& CreateNodeArg(Graph& graph, const NodeArg& base_arg); #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) +/** Find node parents by op types. + @returns The matched parents are sorted by destination argument index of their corresponding edge. +**/ +std::vector FindParentsByType(const Node& node, const std::string& parent_type); + +/** Find node children by op types. + @returns The matched children are sorted by source argument index of their corresponding edge. +**/ +std::vector FindChildrenByType(const Node& node, const std::string& child_type); + /** Gets the name of the incoming NodeArg with the specified index for the given node. */ const std::string& GetNodeInputName(const Node& node, int index); diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc index 56f3015903..db0c1dfe58 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if !defined(ORT_MINIMAL_BUILD) +#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) #include "core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h" @@ -251,4 +251,4 @@ bool MatMulNodeGroupSelector::Check(const GraphViewer& graph_viewer, } // namespace QDQ } // namespace onnxruntime -#endif // !defined(ORT_MINIMAL_BUILD) +#endif // #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h index 398cfc1cce..7bef53e8c0 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h @@ -3,7 +3,7 @@ #pragma once -#if !defined(ORT_MINIMAL_BUILD) +#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) #include "core/optimizer/selectors_actions/selector_action_transformer.h" @@ -193,4 +193,4 @@ class MatMulSelector : public BaseSelector { } // namespace QDQ } // namespace onnxruntime -#endif // !defined(ORT_MINIMAL_BUILD) +#endif // !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc index 329ec66d94..b734278653 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) + #include "utils.h" #include @@ -10,7 +12,6 @@ #include #include -#include "core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.h" #include "core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h" namespace onnxruntime { @@ -101,7 +102,7 @@ void SelectorManager::InitializeSelectorsMap() { } } -void SelectorManager::Initialize() { +SelectorManager::SelectorManager() { CreateSelectors(); InitializeSelectorsMap(); } @@ -141,4 +142,6 @@ std::vector SelectorManager::GetQDQSelections(const GraphViewer& grap } } // namespace QDQ -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime + +#endif // !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.h b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.h index cb44ed2fa5..4ab89a5a2c 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.h +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.h @@ -4,9 +4,12 @@ #pragma once #include +#include "core/common/common.h" #include "core/graph/basic_types.h" -#include "core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h" -#include "core/optimizer/selectors_actions/helpers.h" + +#if !defined(ORT_MINIMAL_BUILD) +#include "onnx/defs/schema.h" +#endif namespace onnxruntime { @@ -15,6 +18,9 @@ class Node; namespace QDQ { +struct NodeGroup; +class NodeGroupSelector; + // struct that provides a join between selector and op versions supported struct OpVersionsAndSelector { using OpVersionsMap = std::unordered_map>; @@ -52,9 +58,7 @@ class Selectors { // class that manages qdq node group selections class SelectorManager { public: - SelectorManager() = default; - - void Initialize(); + SelectorManager(); // Methods that finds and returns a vector of QDQ::NodeGroup in a given graph // Can be used in QDQ support in different EPs diff --git a/onnxruntime/core/optimizer/selectors_actions/selector_action_transformer.h b/onnxruntime/core/optimizer/selectors_actions/selector_action_transformer.h index 3ac9c80551..2b8504c2c1 100644 --- a/onnxruntime/core/optimizer/selectors_actions/selector_action_transformer.h +++ b/onnxruntime/core/optimizer/selectors_actions/selector_action_transformer.h @@ -4,9 +4,9 @@ #pragma once #include -#if !defined(ORT_MINIMAL_BUILD) +#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) #include -#endif // !defined(ORT_MINIMAL_BUILD) +#endif // #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) #include "core/framework/kernel_registry_manager.h" #include "core/optimizer/graph_transformer.h" @@ -19,7 +19,7 @@ class Graph; class GraphViewer; class Node; -#if !defined(ORT_MINIMAL_BUILD) +#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) // Base class for a selector which checks for a match and returns the set of nodes involved. struct NodeSelector { @@ -33,7 +33,7 @@ struct NodeSelector { NodeSelector() = default; }; -#endif // !defined(ORT_MINIMAL_BUILD) +#endif // #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) // class to manage a set of selector and associated actions class SelectorActionRegistry { diff --git a/onnxruntime/core/optimizer/utils.cc b/onnxruntime/core/optimizer/utils.cc index b3852a7e83..7b84c392d8 100644 --- a/onnxruntime/core/optimizer/utils.cc +++ b/onnxruntime/core/optimizer/utils.cc @@ -1,5 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. + +#if !defined(ORT_MINIMAL_BUILD) #include "core/graph/constants.h" #include "core/graph/onnx_protobuf.h" #include "core/graph/graph_utils.h" @@ -13,29 +15,25 @@ #include #include #include +#endif // #if !defined(ORT_MINIMAL_BUILD) + +#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) +#include "core/graph/node_arg.h" +#endif // #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) using namespace onnxruntime; namespace onnxruntime { namespace optimizer_utils { +#if !defined(ORT_MINIMAL_BUILD) + bool IsFloatingPointDataType(const ONNX_NAMESPACE::TensorProto& tensor_proto) { return tensor_proto.data_type() == ONNX_NAMESPACE::TensorProto_DataType_FLOAT || tensor_proto.data_type() == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16 || tensor_proto.data_type() == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE; } -bool IsScalar(const NodeArg& input_arg) { - auto shape = input_arg.Shape(); - if (shape == nullptr) { - // shape inferencing wasn't able to populate shape information for this NodeArg - return false; - } - - auto dim_size = shape->dim_size(); - return dim_size == 0 || (dim_size == 1 && shape->dim(0).has_dim_value() && shape->dim(0).dim_value() == 1); -} - // Check whether input is a constant scalar with expected float value. bool IsInitializerWithExpectedValue(const Graph& graph, const NodeArg& input_arg, float expected_value, bool is_constant) { if (!IsScalar(input_arg)) { @@ -293,5 +291,22 @@ bool IsOperationDeterministic(const std::string& domain, const std::string& op) return itDomain->second.count(op) == 0; } +#endif // #if !defined(ORT_MINIMAL_BUILD) + +#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) + +bool IsScalar(const NodeArg& input_arg) { + auto shape = input_arg.Shape(); + if (shape == nullptr) { + // shape inferencing wasn't able to populate shape information for this NodeArg + return false; + } + + auto dim_size = shape->dim_size(); + return dim_size == 0 || (dim_size == 1 && shape->dim(0).has_dim_value() && shape->dim(0).dim_value() == 1); +} + +#endif // #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) + } // namespace optimizer_utils } // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/utils.h b/onnxruntime/core/optimizer/utils.h index d5b2f15dd4..9b87ee680d 100644 --- a/onnxruntime/core/optimizer/utils.h +++ b/onnxruntime/core/optimizer/utils.h @@ -3,8 +3,10 @@ #pragma once +#if !defined(ORT_MINIMAL_BUILD) #include "core/graph/onnx_protobuf.h" #include "core/graph/graph.h" +#endif // !#if !defined(ORT_MINIMAL_BUILD) namespace onnxruntime { class Graph; @@ -12,12 +14,11 @@ class NodeArg; namespace optimizer_utils { +#if !defined(ORT_MINIMAL_BUILD) + // Check if TensorProto contains a floating point type. bool IsFloatingPointDataType(const ONNX_NAMESPACE::TensorProto& tensor_proto); -// Check if NodeArg takes in a scalar tensor. -bool IsScalar(const NodeArg& input_arg); - /** Check whether a input is initializer with specified float value. @param expected_value is the expected value of the initializer. @param is_constant means whether the initializer is required to be constant. @@ -101,5 +102,14 @@ bool CheckOutputEdges(const Graph& graph, const Node& node, size_t expected_outp bool IsOperationDeterministic(const std::string& domain, const std::string& op); +#endif // !#if !defined(ORT_MINIMAL_BUILD) + +#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) + +// Check if NodeArg takes in a scalar tensor. +bool IsScalar(const NodeArg& input_arg); + +#endif // #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) + } // namespace optimizer_utils } // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index af93017649..339f1e1f65 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -434,7 +434,11 @@ bool IsValidSupportedNodeGroup(const std::vector& supported_node_pa return true; } -bool IsInternalQuantizedNode(const Node& node) { +static bool IsInternalQuantizedNodeUnit(const NodeUnit& node_unit) { + // First, ignore QDQ NodeUnit which is not internal quantized node + if (node_unit.UnitType() == NodeUnit::Type::QDQGroup) + return false; + // These operators can use uint8 input without specific QLinear version of it // However, the mode has to be internal to the graph/partition (they cannot consume graph inputs) static const std::unordered_set internal_quantized_op_types = @@ -445,6 +449,7 @@ bool IsInternalQuantizedNode(const Node& node) { "MaxPool", }; + const auto& node = node_unit.GetNode(); if (!Contains(internal_quantized_op_types, node.OpType())) return false; @@ -493,13 +498,11 @@ bool IsNodeSupportedInGroup(const NodeUnit& node_unit, const GraphViewer& graph_ if (!IsNodeSupported(node_unit, graph_viewer, params)) return false; - // TODO, ignore this step if the node_unit is qdq node_unit - // We also want to check if the node is supported as an internal quantized node - const auto& node = node_unit.GetNode(); - if (IsInternalQuantizedNode(node)) - return IsInternalQuantizationSupported(node, node_outputs_in_group); - else // This is not a internal quantized node, it is supported - return true; + // We also want to check if the node is supported as an internal quantized node_unit + if (IsInternalQuantizedNodeUnit(node_unit)) + return IsInternalQuantizationSupported(node_unit.GetNode(), node_outputs_in_group); + + return true; } bool IsInputSupported(const NodeArg& input, const std::string& parent_name) { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index fe6eade431..d599d57351 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -152,16 +152,7 @@ const NodeUnit& ModelBuilder::GetNodeUnit(const Node* node) const { } void ModelBuilder::PreprocessNodeUnits() { - // TODO, hookup shared QDQ selectors here to identify all the qdq NodeUnit in the graph - const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder(); - for (size_t i = 0; i < node_indices.size(); i++) { - const auto node_idx = node_indices[i]; - // TODO, check if the node is already part of a qdq group - const auto* node(graph_viewer_.GetNode(node_idx)); - auto node_unit = std::make_unique(*node); - node_unit_map_.insert({node, node_unit.get()}); - node_unit_holder_.push_back(std::move(node_unit)); - } + std::tie(node_unit_holder_, node_unit_map_) = GetAllNodeUnits(graph_viewer_); } // Help to get all quantized operators' input and the NodeUnit(s) using the input @@ -504,6 +495,7 @@ Status ModelBuilder::AddOperandFromPersistMemoryBuffer( Status ModelBuilder::AddOperations() { const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder(); std::unordered_set processed_node_units; + processed_node_units.reserve(node_unit_holder_.size()); for (size_t i = 0; i < node_indices.size(); i++) { const auto* node(graph_viewer_.GetNode(node_indices[i])); const NodeUnit& node_unit = GetNodeUnit(node); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc index 75eab4c837..1b1c6e7990 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc @@ -105,6 +105,11 @@ class BaseOpSupportChecker : public IOpSupportChecker { virtual int GetMinSupportedOpSet(const NodeUnit& /* node_unit */) const { return 1; } virtual int GetMaxSupportedOpSet(const NodeUnit& /* node_unit */) const { return 15; } + // Check if this node_unit's type is supported + // SingleNode type NodeUnit is supported + // QDQGroup type NodeUnit is by default unsupported, and this can be individually overwritten by inherited classes + virtual bool IsNodeUnitTypeSupported(const NodeUnit& node_unit) const; + private: bool HasSupportedOpSet(const NodeUnit& node_unit) const; bool HasSupportedInputs(const NodeUnit& node_unit) const; @@ -130,6 +135,9 @@ bool BaseOpSupportChecker::IsOpSupported(const InitializedTensorSet& initializer return false; } + if (!IsNodeUnitTypeSupported(node_unit)) + return false; + if (!HasSupportedInputs(node_unit)) return false; @@ -203,6 +211,17 @@ bool BaseOpSupportChecker::HasSupportedOpSet(const NodeUnit& node_unit) const { return true; } +bool BaseOpSupportChecker::IsNodeUnitTypeSupported(const NodeUnit& node_unit) const { + if (node_unit.UnitType() == NodeUnit::Type::QDQGroup) { + LOGS_DEFAULT(VERBOSE) << "QDQ NodeUnit [" << node_unit.OpType() + << "] is not supported for now"; + + return false; + } + + return true; +} + #pragma endregion op_base #pragma region op_binary diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 85a0cf3ad1..32fffec739 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -117,22 +117,46 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view } } + // Get all the NodeUnits in the graph_viewer + std::vector> node_unit_holder; + std::unordered_map node_unit_map; + + std::tie(node_unit_holder, node_unit_map) = GetAllNodeUnits(graph_viewer); + + // This holds the result of whether a NodeUnit is supported or not, + // to prevent nodes in a NodeUnit to be checked for multiple times + std::unordered_map node_unit_supported_result; + node_unit_supported_result.reserve(node_unit_holder.size()); + const auto excluded_nodes = utils::CreateExcludedNodeSet(graph_viewer, partitioning_stop_ops_); const bool check_excluded_nodes = !excluded_nodes.empty(); std::unordered_set node_outputs_in_current_group{}; const auto is_node_supported = [&](const Node& node) -> bool { - const bool excluded = check_excluded_nodes && Contains(excluded_nodes, &node); - // TODO, cache NodeUnit(s) and not generate new instance every time - const NodeUnit node_unit(node); - const bool supported = !excluded && - nnapi::IsNodeSupportedInGroup(node_unit, graph_viewer, params, - node_outputs_in_current_group); - LOGS_DEFAULT(VERBOSE) << "Operator type: [" << node_unit.OpType() - << "] index: [" << node_unit.Index() - << "] name: [" << node_unit.Name() - << "] supported: [" << supported + const NodeUnit* node_unit = node_unit_map.at(&node); + bool supported = false; + + // If we have visited one of the nodes in the node_unit, use the result directly + const auto it = node_unit_supported_result.find(node_unit); + if (it != node_unit_supported_result.cend()) { + supported = it->second; + } else { + // We only check the target node of the node unit for exclusion + const bool excluded = check_excluded_nodes && Contains(excluded_nodes, &node_unit->GetNode()); + supported = !excluded && + nnapi::IsNodeSupportedInGroup(*node_unit, graph_viewer, params, + node_outputs_in_current_group); + node_unit_supported_result[node_unit] = supported; + } + + LOGS_DEFAULT(VERBOSE) << "Node supported: [" << supported + << "] Operator type: [" << node.OpType() + << "] index: [" << node.Index() + << "] name: [" << node.Name() + << "] as part of the NodeUnit type: [" << node_unit->OpType() + << "] index: [" << node_unit->Index() + << "] name: [" << node_unit->Name() << "]"; if (supported) { diff --git a/onnxruntime/core/providers/shared/node_unit/node_unit.cc b/onnxruntime/core/providers/shared/node_unit/node_unit.cc index d443fe858f..a98336c6fe 100644 --- a/onnxruntime/core/providers/shared/node_unit/node_unit.cc +++ b/onnxruntime/core/providers/shared/node_unit/node_unit.cc @@ -2,7 +2,9 @@ // Licensed under the MIT License. #include "node_unit.h" -#include "core/graph/graph.h" +#include "core/graph/graph_viewer.h" +#include "core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h" +#include "core/optimizer/qdq_transformer/selectors_actions/shared/utils.h" namespace onnxruntime { @@ -79,13 +81,84 @@ bool IsVariadicQLinearOp(QLinearOpType type) { return type == QLinearOpType::QLinearConcat; } +const std::vector GetQDQOutputNodes(const GraphViewer& graph_viewer, const QDQ::NodeGroup& node_group) { + std::vector output_nodes; + output_nodes.reserve(node_group.q_nodes.size()); + for (const auto& node_idx : node_group.q_nodes) { + output_nodes.push_back(graph_viewer.GetNode(node_idx)); + } + return output_nodes; +} + +// Get the input or output NodeUnitIODef(s) for the given QDQ NodeGroup +std::vector GetQDQIODefs(const Node& target_node, const QDQ::NodeGroup& node_group, + bool is_input) { + const auto& dq_or_q_nodes = is_input ? node_group.dq_nodes : node_group.q_nodes; + const auto target_node_io_defs = is_input ? target_node.InputDefs() : target_node.OutputDefs(); + const size_t target_node_io_defs_size = target_node_io_defs.size(); + + // Find all the quantized IO defs and indices (for the input to the target node) + std::unordered_map quantized_io_defs; + quantized_io_defs.reserve(target_node_io_defs_size); + + auto cur = is_input ? target_node.InputEdgesBegin() : target_node.OutputEdgesBegin(); + auto end = is_input ? target_node.InputEdgesEnd() : target_node.OutputEdgesEnd(); + for (; cur != end; ++cur) { + const Node& node = cur->GetNode(); + + // If we can find the node index in the dq or q nodes, then this is a quantize node (can be DQ or Q depends on is_input) + if (std::find(dq_or_q_nodes.cbegin(), dq_or_q_nodes.cend(), node.Index()) != dq_or_q_nodes.cend()) { + const auto node_inputs = node.InputDefs(); + // quantization scale and zp are always the input[1, 2] + NodeUnitIODef::QuantParam quant_param{ + *node_inputs[1], + node_inputs.size() == 3 ? node_inputs[2] : nullptr}; + if (is_input) { + // DQ is input to the target node, use the DstArgIndex + auto idx = cur->GetDstArgIndex(); + // This is a DQ node, we are using x, x_scale, x_zp (input[0, 1, 2]) + quantized_io_defs.insert({idx, NodeUnitIODef{*node_inputs[0], quant_param}}); + } else { + // Q is output of the target node, use the SrcArgIndex + auto idx = cur->GetSrcArgIndex(); + // This is a Q node, we are using y (output[0]), y_scale, y_zp (input[1, 2]) + const auto node_outputs = node.OutputDefs(); + quantized_io_defs.insert({idx, NodeUnitIODef{*node_outputs[0], quant_param}}); + } + } + } + + // Construct the IODefs for this QDQ NodeGroup + std::vector io_defs; + io_defs.reserve(target_node_io_defs_size); + for (size_t i = 0; i < target_node_io_defs_size; i++) { + // If we can find the NodeUnitIODef for this index, this is a quantized input + if (quantized_io_defs.find(i) != quantized_io_defs.cend()) { + io_defs.push_back(std::move(quantized_io_defs.at(i))); + } else { + // This is a regular input + io_defs.push_back({*target_node_io_defs[i], std::nullopt}); + } + } + + return io_defs; +} + } // namespace NodeUnit::NodeUnit(const Node& node) : output_nodes_{&node}, target_node_(node), type_(Type::SingleNode) { - InitForNode(); + InitForSingleNode(); +} + +NodeUnit::NodeUnit(const GraphViewer& graph_viewer, const QDQ::NodeGroup& node_group) + : output_nodes_{GetQDQOutputNodes(graph_viewer, node_group)}, + target_node_(*graph_viewer.GetNode(node_group.target_node)), + type_(Type::QDQGroup), + inputs_{GetQDQIODefs(target_node_, node_group, true /* is_input */)}, + outputs_{GetQDQIODefs(target_node_, node_group, false /* is_input */)} { } const std::string& NodeUnit::Domain() const noexcept { return target_node_.Domain(); } @@ -96,7 +169,7 @@ NodeIndex NodeUnit::Index() const noexcept { return target_node_.Index(); } const Path& NodeUnit::ModelPath() const noexcept { return target_node_.ModelPath(); } ProviderType NodeUnit::GetExecutionProviderType() const noexcept { return target_node_.GetExecutionProviderType(); } -void NodeUnit::InitForNode() { +void NodeUnit::InitForSingleNode() { const auto& input_defs = target_node_.InputDefs(); const auto& output_defs = target_node_.OutputDefs(); auto qlinear_type = GetQLinearOpType(target_node_); @@ -172,4 +245,48 @@ void NodeUnit::InitForNode() { } } +std::pair>, std::unordered_map> +GetAllNodeUnits(const GraphViewer& graph_viewer) { + std::vector> node_unit_holder; + std::unordered_map node_unit_map; + + const auto add_node_unit_to_map = [&](const std::vector& node_indices, const NodeUnit* node_unit) { + for (const auto& node_idx : node_indices) { + const auto* node = graph_viewer.GetNode(node_idx); + node_unit_map.insert({node, node_unit}); + } + }; + + // Get QDQ NodeUnits first + QDQ::SelectorManager selector_mgr; + const auto qdq_selections = selector_mgr.GetQDQSelections(graph_viewer); + + for (const auto& qdq_selection : qdq_selections) { + auto qdq_unit = std::make_unique(graph_viewer, qdq_selection); + + // Fill the node to node_unit map for all nodes in the QDQ Group + add_node_unit_to_map(qdq_selection.dq_nodes, qdq_unit.get()); + add_node_unit_to_map(qdq_selection.q_nodes, qdq_unit.get()); + add_node_unit_to_map({qdq_selection.target_node}, qdq_unit.get()); + + node_unit_holder.push_back(std::move(qdq_unit)); + } + + // Get the left over SingleNode NodeUnits + const auto& node_indices = graph_viewer.GetNodesInTopologicalOrder(); + for (const auto node_idx : node_indices) { + const auto* node(graph_viewer.GetNode(node_idx)); + + // This is already part of a QDQ NodeUnit + if (node_unit_map.find(node) != node_unit_map.cend()) + continue; + + auto node_unit = std::make_unique(*node); + node_unit_map[node] = node_unit.get(); + node_unit_holder.push_back(std::move(node_unit)); + } + + return std::make_pair(std::move(node_unit_holder), std::move(node_unit_map)); +} + } // namespace onnxruntime diff --git a/onnxruntime/core/providers/shared/node_unit/node_unit.h b/onnxruntime/core/providers/shared/node_unit/node_unit.h index e109703c93..6157cff746 100644 --- a/onnxruntime/core/providers/shared/node_unit/node_unit.h +++ b/onnxruntime/core/providers/shared/node_unit/node_unit.h @@ -11,7 +11,6 @@ namespace onnxruntime { -class Graph; class GraphViewer; class Node; class NodeArg; @@ -49,6 +48,7 @@ class NodeUnit { public: explicit NodeUnit(const Node& node); + explicit NodeUnit(const GraphViewer& graph_viewer, const QDQ::NodeGroup& node_group); Type UnitType() const noexcept { return type_; } @@ -64,17 +64,24 @@ class NodeUnit { ProviderType GetExecutionProviderType() const noexcept; const Node& GetNode() const noexcept { return target_node_; } - const std::vector GetOutputNodes() const noexcept { return output_nodes_; } + const std::vector& GetOutputNodes() const noexcept { return output_nodes_; } private: + const std::vector output_nodes_; // all the nodes producing outputs for this NodeUnit + const Node& target_node_; + const Type type_; + std::vector inputs_; std::vector outputs_; - const std::vector output_nodes_; // all the nodes producing outputs for this NodeUnit - const Node& target_node_; - Type type_; - - void InitForNode(); // Initializing for single Node + // Initializing for a single Node + void InitForSingleNode(); }; +// Get all the nodes in the given graph_viewer as NodeUnits (SingleNode or QDQGroup) +// And return a map to quick query the NodeUnit which contains the given Node, +// Note, the value of the map is owned by the vector of std::unique_ptr +std::pair>, std::unordered_map> +GetAllNodeUnits(const GraphViewer& graph_viewer); + } // namespace onnxruntime diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index cbb716bfdd..9d74b8324b 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -23,7 +23,11 @@ #if defined(_MSC_VER) #pragma warning(disable : 4127) -#endif +#endif // #if defined(_MSC_VER) + +#ifdef USE_NNAPI +#include "core/providers/shared/node_unit/node_unit.h" +#endif // #ifdef USE_NNAPI namespace onnxruntime { namespace test { @@ -1831,6 +1835,9 @@ TEST(QDQTransformerTests, QDQ_Selector_Test) { onnxruntime::QDQ::ConvNodeGroupSelector conv_selector; + // Initialize SelectorManager + QDQ::SelectorManager selector_mgr; + // Create a GraphViewer covers the whole graph const GraphViewer whole_graph_viewer(graph); @@ -1844,6 +1851,63 @@ TEST(QDQTransformerTests, QDQ_Selector_Test) { ASSERT_EQ(std::vector({4}), qdq_group.q_nodes); } + // Check if SelectorManager get a conv qdq group selection as expected + { + const auto result = selector_mgr.GetQDQSelections(whole_graph_viewer); + ASSERT_FALSE(result.empty()); + const auto& qdq_group = result.at(0); + ASSERT_EQ(std::vector({0, 1, 2}), qdq_group.dq_nodes); + ASSERT_EQ(NodeIndex(3), qdq_group.target_node); + ASSERT_EQ(std::vector({4}), qdq_group.q_nodes); + } + +// The function GetAllNodeUnits is enabled for NNAPI EP only for now +#ifdef USE_NNAPI + { + // Get all the NodeUnits in the graph_viewer + std::vector> node_unit_holder; + std::unordered_map node_unit_map; + + std::tie(node_unit_holder, node_unit_map) = GetAllNodeUnits(whole_graph_viewer); + + // We should get a single QDQ Node unit in the result + ASSERT_EQ(1, node_unit_holder.size()); + ASSERT_EQ(5, node_unit_map.size()); + const auto& qdq_node_unit = *node_unit_holder[0]; + ASSERT_EQ(NodeUnit::Type::QDQGroup, qdq_node_unit.UnitType()); + + ASSERT_EQ(3, qdq_node_unit.Inputs().size()); + ASSERT_EQ(1, qdq_node_unit.Outputs().size()); + ASSERT_EQ(conv_node, &qdq_node_unit.GetNode()); + + const auto verify_io_def = [](const NodeUnitIODef& io_def, const Node& node) { + const auto& op_type = node.OpType(); + const bool is_dq = op_type == "DequantizeLinear"; + const bool is_q = op_type == "QuantizeLinear"; + ASSERT_TRUE(is_dq || is_q); + const auto input_defs = node.InputDefs(); + if (is_dq) { + ASSERT_EQ(&io_def.node_arg, input_defs[0]); + } else { // is_q + ASSERT_EQ(&io_def.node_arg, node.OutputDefs()[0]); + } + + ASSERT_EQ(&io_def.quant_param->scale, input_defs[1]); + + // [optional] zero point should be consistent between NodeUnitIODef and Input/OutputDefs + ASSERT_EQ(input_defs.size() == 3, !!io_def.quant_param->zero_point); + if (input_defs.size() == 3) // we have zero point + ASSERT_EQ(io_def.quant_param->zero_point, input_defs[2]); + }; + + // We know the graph has 5 nodes, DQ_input, DQ_weight, DQ_bias, Conv, Q_output (index 0-4) + verify_io_def(qdq_node_unit.Inputs()[0], *whole_graph_viewer.GetNode(0)); // DQ_input + verify_io_def(qdq_node_unit.Inputs()[1], *whole_graph_viewer.GetNode(1)); // DQ_weight + verify_io_def(qdq_node_unit.Inputs()[2], *whole_graph_viewer.GetNode(2)); // DQ_bias + verify_io_def(qdq_node_unit.Outputs()[0], *whole_graph_viewer.GetNode(4)); // Q_output + } +#endif // #ifdef USE_NNAPI + // Create a graph viewer covers part of the graph // Make sure the qdq conv selector will fail for the partial graph { @@ -1862,40 +1926,18 @@ TEST(QDQTransformerTests, QDQ_Selector_Test) { const GraphViewer partial_graph_viewer(graph, *compute_capability->sub_graph); ASSERT_EQ(3, partial_graph_viewer.NumberOfNodes()); - const auto result = conv_selector.GetQDQSelection(partial_graph_viewer, *conv_node); - ASSERT_FALSE(result.has_value()); - } -} -TEST(QDQTransformerTests, QDQ_Shared_GetSelectors_Test) { - const ORTCHAR_T* model_file_name = ORT_TSTR("testdata/transform/qdq_conv.onnx"); + // Check there is no qdq selection for the given nodes + { + const auto result = conv_selector.GetQDQSelection(partial_graph_viewer, *conv_node); + ASSERT_FALSE(result.has_value()); + } - SessionOptions so; - so.graph_optimization_level = TransformerLevel::Default; - InferenceSessionWrapper session_object{so, GetEnvironment()}; - ASSERT_STATUS_OK(session_object.Load(model_file_name)); - ASSERT_STATUS_OK(session_object.Initialize()); - const Graph& graph = session_object.GetGraph(); - const auto* conv_node = graph.GetNode(3); - - // Make sure node 3 is the conv node - ASSERT_TRUE(nullptr != conv_node); - ASSERT_EQ("Conv", conv_node->OpType()); - - const GraphViewer graph_viewer(graph); - - // Initialize SelectorManager - QDQ::SelectorManager selector_mgr; - selector_mgr.Initialize(); - - // Check if SelectorManager get a conv qdq group selection as expected - { - const auto result = selector_mgr.GetQDQSelections(graph_viewer); - ASSERT_EQ(false, result.empty()); - const auto& qdq_group = result.at(0); - ASSERT_EQ(std::vector({0, 1, 2}), qdq_group.dq_nodes); - ASSERT_EQ(NodeIndex(3), qdq_group.target_node); - ASSERT_EQ(std::vector({4}), qdq_group.q_nodes); + // Check SelectorManager will get empty result + { + const auto result = selector_mgr.GetQDQSelections(partial_graph_viewer); + ASSERT_TRUE(result.empty()); + } } } diff --git a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc index b8c4acd127..2cf6234f84 100644 --- a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc +++ b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc @@ -235,6 +235,21 @@ TEST(NnapiExecutionProviderTest, TestNoShapeInputModel) { << "No node should be taken by the NNAPI EP"; } +// For now since we don't support QDQ in NNAPI, even the infrastructure is there +// Need to verify a model with QDQ groups only will not be supported by NNAPI at all +// This may need to be changed when we gradually add support for different ops for QDQ +TEST(NnapiExecutionProviderTest, TestQDQConvModel) { + const ORTCHAR_T* model_file_name = ORT_TSTR("testdata/transform/qdq_conv.onnx"); + // test load only + SessionOptions so; + InferenceSessionWrapper session_object{so, GetEnvironment()}; + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(std::make_unique(0))); + ASSERT_STATUS_OK(session_object.Load(model_file_name)); + ASSERT_STATUS_OK(session_object.Initialize()); + ASSERT_EQ(CountAssignedNodes(session_object.GetGraph(), kNnapiExecutionProvider), 0) + << "No nodes should have been taken by the NNAPI EP"; +} + #endif // !(ORT_MINIMAL_BUILD TEST(NnapiExecutionProviderTest, NNAPIFlagsTest) { diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml index b0c86259d4..a51fe2d60b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml @@ -18,6 +18,8 @@ # minimal build. # 6c: extended minimal build with exceptions and python disabled checks that the exclusions don't break code paths # in an extended minimal build. +# 7. Build extended minimal ORT with NNAPI, with exceptions/RTTI/ml_ops disabled, for Android(arm64-v8a), +# this safe-guards the extended minimal build with NNAPI EP. jobs: - job: Linux_CPU_Minimal_Build_E2E @@ -261,6 +263,38 @@ jobs: onnxruntime_BUILD_UNIT_TESTS=OFF workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 + displayName: 7. Extended minimal build with NNAPI EP for Android(arm64-v8a) and skip tests. + inputs: + script: | + NDK_HOME=$(realpath $ANDROID_NDK_HOME) + docker run --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $ANDROID_HOME:/android_home \ + --volume $NDK_HOME:/ndk_home \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ + -e NIGHTLY_BUILD \ + onnxruntimecpubuild \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build/7 \ + --cmake_generator Ninja \ + --config MinSizeRel \ + --skip_submodule_sync \ + --parallel \ + --android \ + --android_sdk_path /android_home \ + --android_ndk_path /ndk_home \ + --android_abi=arm64-v8a \ + --android_api=29 \ + --use_nnapi \ + --minimal_build extended \ + --build_shared_lib \ + --disable_ml_ops \ + --disable_exceptions \ + --skip_tests + workingDirectory: $(Build.SourcesDirectory) + - task: PublishTestResults@2 displayName: 'Publish unit test results' inputs: From 5576e3553dd2a336389bf1b08a1f0cbebda547d7 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 26 Jan 2022 10:21:57 -0800 Subject: [PATCH 36/59] Remove python 3.6 from our python packaging pipeline (#10395) --- .../templates/py-packaging-stage.yml | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 208ffd63a1..adabdaa8f7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -62,8 +62,6 @@ stages: pool: Linux-CPU strategy: matrix: - Python36: - PythonVersion: '3.6' Python37: PythonVersion: '3.7' Python38: @@ -193,8 +191,6 @@ stages: pool: Onnxruntime-Linux-GPU strategy: matrix: - Python36: - PythonVersion: '3.6' Python37: PythonVersion: '3.7' Python38: @@ -333,10 +329,6 @@ stages: - ROCm_build_environment strategy: matrix: - Python36 Torch1100 Rocm42: - PythonVersion: '3.6' - TorchVersion: '1.10.0' - RocmVersion: '4.2' Python37 Torch1100 Rocm42: PythonVersion: '3.7' TorchVersion: '1.10.0' @@ -349,10 +341,6 @@ stages: PythonVersion: '3.9' TorchVersion: '1.10.0' RocmVersion: '4.2' - Python36 Torch1100 Rocm431: - PythonVersion: '3.6' - TorchVersion: '1.10.0' - RocmVersion: '4.3.1' Python37 Torch1100 Rocm431: PythonVersion: '3.7' TorchVersion: '1.10.0' @@ -602,10 +590,6 @@ stages: pool: 'Win-CPU-2021' strategy: matrix: - Python36_x64: - PythonVersion: '3.6' - MsbuildPlatform: x64 - buildArch: x64 Python37_x64: PythonVersion: '3.7' MsbuildPlatform: x64 @@ -618,10 +602,6 @@ stages: PythonVersion: '3.9' MsbuildPlatform: x64 buildArch: x64 - Python36_x86: - PythonVersion: '3.6' - MsbuildPlatform: Win32 - buildArch: x86 Python37_x86: PythonVersion: '3.7' MsbuildPlatform: Win32 @@ -810,11 +790,6 @@ stages: buildArch: x64 strategy: matrix: - Python36_GPU: - PythonVersion: '3.6' - EpBuildFlags: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.2.1.8.Windows10.x86_64.cuda-11.4.cudnn8.2" --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" - EnvSetupScript: setup_env_gpu.bat - EP_NAME: gpu Python37_GPU: PythonVersion: '3.7' EpBuildFlags: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.2.1.8.Windows10.x86_64.cuda-11.4.cudnn8.2" --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" @@ -830,11 +805,6 @@ stages: EpBuildFlags: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.2.1.8.Windows10.x86_64.cuda-11.4.cudnn8.2" --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" EnvSetupScript: setup_env_gpu.bat EP_NAME: gpu - Python36_dml: - PythonVersion: '3.6' - EpBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - EP_NAME: directml - EnvSetupScript: setup_env.bat Python37_dml: PythonVersion: '3.7' EpBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos @@ -1081,9 +1051,6 @@ stages: ARM64_Py37: PYTHON_EXE: '/opt/python/cp37-cp37m/bin/python3' Image: 'manylinux2014_aarch64' - ARM64_Py36: - PYTHON_EXE: '/opt/python/cp36-cp36m/bin/python3' - Image: 'manylinux2014_aarch64' # Uncomment the following lines to generate packages for PowerPC # POWERPC_Py39: # PYTHON_EXE: '/opt/python/cp39-cp39/bin/python3' @@ -1094,9 +1061,6 @@ stages: # POWERPC_Py37: # PYTHON_EXE: '/opt/python/cp37-cp37m/bin/python3' # Image: 'manylinux2014_ppc64le' - # POWERPC_Py36: - # PYTHON_EXE: '/opt/python/cp36-cp36m/bin/python3' - # Image: 'manylinux2014_ppc64le' steps: - checkout: self clean: true From a27503ebe4fd0ccb9d3454c78f2fe0eca9919c5e Mon Sep 17 00:00:00 2001 From: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com> Date: Wed, 26 Jan 2022 10:27:05 -0800 Subject: [PATCH 37/59] use strict mode (#10397) --- tools/perf_view/ort_perf_view.html | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tools/perf_view/ort_perf_view.html b/tools/perf_view/ort_perf_view.html index 6142161861..fc8becd109 100644 --- a/tools/perf_view/ort_perf_view.html +++ b/tools/perf_view/ort_perf_view.html @@ -42,19 +42,20 @@