diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 0447e4814d..e072f83076 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -78,17 +78,6 @@ if(HAS_DEPRECATED_COPY) set_source_files_properties("${ONNXRUNTIME_ROOT}/core/providers/cpu/tensor/where_op.cc" PROPERTIES COMPILE_FLAGS -Wno-deprecated-copy) endif() -if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT MSVC) - # For x86 platforms it is important to pass this flag to compiler. Without this gemmlowp will use slow reference code. - # These optimizations are not enabled on MSVC so excluding it. - message("enabling optimizations for gemmlowp") - set_source_files_properties("${ONNXRUNTIME_ROOT}/core/providers/cpu/math/matmul_integer.cc" PROPERTIES COMPILE_FLAGS "-msse4.1") - set_source_files_properties("${ONNXRUNTIME_ROOT}/core/providers/cpu/math/quantize_linear_matmul.cc" PROPERTIES COMPILE_FLAGS "-msse4.1") - set_source_files_properties("${ONNXRUNTIME_ROOT}/core/providers/cpu/nn/qlinearconv.cc" PROPERTIES COMPILE_FLAGS "-msse4.1") - set_source_files_properties("${ONNXRUNTIME_ROOT}/core/providers/cpu/nn/conv_integer.cc" PROPERTIES COMPILE_FLAGS "-msse4.1") -endif() - -set(gemmlowp_src ${PROJECT_SOURCE_DIR}/external/gemmlowp) set(re2_src ${ONNXRUNTIME_ROOT}/../cmake/external/re2) target_include_directories(onnxruntime_providers PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${gemmlowp_src} ${re2_src}) add_dependencies(onnxruntime_providers gsl onnx ${onnxruntime_EXTERNAL_DEPENDENCIES}) diff --git a/cmake/onnxruntime_util.cmake b/cmake/onnxruntime_util.cmake index a8b611d7c9..feea9f90ee 100644 --- a/cmake/onnxruntime_util.cmake +++ b/cmake/onnxruntime_util.cmake @@ -8,8 +8,17 @@ file(GLOB_RECURSE onnxruntime_util_srcs CONFIGURE_DEPENDS source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_util_srcs}) +if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT MSVC) + # For x86 platforms it is important to pass this flag to compiler. Without this gemmlowp will use slow reference code. + # These optimizations are not enabled on MSVC so excluding it. + message("enabling optimizations for gemmlowp") + set_source_files_properties("${ONNXRUNTIME_ROOT}/core/util/gemmlowp_common.cc" PROPERTIES COMPILE_FLAGS "-msse4.1") +endif() + +set(gemmlowp_src ${PROJECT_SOURCE_DIR}/external/gemmlowp) + add_library(onnxruntime_util ${onnxruntime_util_srcs}) -target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${MKLML_INCLUDE_DIR} PUBLIC ${eigen_INCLUDE_DIRS}) +target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${MKLML_INCLUDE_DIR} ${gemmlowp_src} PUBLIC ${eigen_INCLUDE_DIRS}) onnxruntime_add_include_to_target(onnxruntime_util onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf) if(UNIX) target_compile_options(onnxruntime_util PUBLIC "-Wno-error=comment") diff --git a/onnxruntime/core/providers/common.h b/onnxruntime/core/providers/common.h index 16a065cb8c..5bd8f69c5b 100644 --- a/onnxruntime/core/providers/common.h +++ b/onnxruntime/core/providers/common.h @@ -4,6 +4,7 @@ #pragma once #include "core/common/common.h" +#include "core/framework/tensor.h" namespace onnxruntime { @@ -20,4 +21,16 @@ inline int64_t HandleNegativeAxis(int64_t axis, int64_t tensor_rank) { return axis = axis < 0 ? axis + tensor_rank : axis; } +/** +Returns true if given tensor is a scalar or 1D tensor of size 1 +**/ +inline bool IsScalarOr1ElementVector(const Tensor* input) { + if (input->Shape().NumDimensions() == 0 || + (input->Shape().NumDimensions() == 1 && input->Shape().GetDims().size() == 1)) { + return true; + } else { + return false; + } +} + } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/matmul_integer.cc b/onnxruntime/core/providers/cpu/math/matmul_integer.cc index 9a64e3fe42..6e8464d4d5 100644 --- a/onnxruntime/core/providers/cpu/math/matmul_integer.cc +++ b/onnxruntime/core/providers/cpu/math/matmul_integer.cc @@ -1,14 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifdef _MSC_VER -#pragma warning(disable : 4244) -#pragma warning(disable : 4267) -#endif - #include "core/providers/cpu/math/matmul_integer.h" #include "core/providers/cpu/math/matmul_helper.h" -#include "core/util/gemmlowp_common_wrapper.h" +#include "core/util/gemmlowp_common.h" +#include "core/providers/common.h" namespace onnxruntime { @@ -24,25 +20,7 @@ ONNX_OPERATOR_KERNEL_EX( .TypeConstraint("T3", DataTypeImpl::GetTensorType()), MatMulInteger); -Status GemmlowpMultiply(const uint8_t* lhs_data, const uint8_t* rhs_data, - int32_t* result_data, const int lhs_offset, const int rhs_offset, - int m, int n, int k) { - const std::tuple<> empty_pipeline = {}; - // TODO exp ColMajor order for rhs and result. That may be faster - const auto matOrder = gemmlowp::MapOrder::RowMajor; - gemmlowp::MatrixMap lhs(lhs_data, m, k); - gemmlowp::MatrixMap rhs(rhs_data, k, n); - gemmlowp::MatrixMap result(result_data, m, n); - - gemmlowp::GemmContext gemm_context; - gemmlowp::GemmWithOutputPipeline( - &gemm_context, lhs, rhs, &result, -lhs_offset, -rhs_offset, empty_pipeline); - - return Status::OK(); -} - -template<> +template <> Status MatMulInteger::Compute(OpKernelContext* ctx) const { auto a = ctx->Input(0); auto b = ctx->Input(1); @@ -57,28 +35,26 @@ Status MatMulInteger::Compute(OpKernelContext* ctx) c int32_t b_offset = 0; if (has_a_zero_point_) { auto a_zero_point = ctx->Input(2); - ORT_ENFORCE(a_zero_point->Shape().NumDimensions() == 0 || - (a_zero_point->Shape().NumDimensions() == 1 && a_zero_point->Shape().GetDims().size() == 1), - "Currently only scalar zero_point is supported. TODO: add per channel zero point support."); + ORT_ENFORCE(IsScalarOr1ElementVector(a_zero_point), + "MatmulInteger : input1 zero point must be a scalar or 1D tensor of size 1"); a_offset = static_cast(*a_zero_point->template Data()); } if (has_b_zero_point_) { auto b_zero_point = ctx->Input(3); - ORT_ENFORCE(b_zero_point->Shape().NumDimensions() == 0 || - (b_zero_point->Shape().NumDimensions() == 1 && b_zero_point->Shape().GetDims().size() == 1), - "Currently only scalar zero_point is supported. TODO: add per channel zero point support."); + ORT_ENFORCE(IsScalarOr1ElementVector(b_zero_point), + "MatmulInteger : input2 zero point must be a scalar or 1D tensor of size 1"); b_offset = static_cast(*b_zero_point->template Data()); } for (size_t i = 0; i < helper.OutputOffsets().size(); i++) { - GemmlowpMultiply(a->template Data() + helper.LeftOffsets()[i], - b->template Data() + helper.RightOffsets()[i], - y->template MutableData() + helper.OutputOffsets()[i], - a_offset, - b_offset, - static_cast(helper.M()), - static_cast(helper.N()), - static_cast(helper.K())); + GemmlowpMultiplyu8u8_s32(a->template Data() + helper.LeftOffsets()[i], + b->template Data() + helper.RightOffsets()[i], + y->template MutableData() + helper.OutputOffsets()[i], + a_offset, + b_offset, + static_cast(helper.M()), + static_cast(helper.N()), + static_cast(helper.K())); } return Status::OK(); diff --git a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc index b3cf0dcbe7..164b58f208 100644 --- a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc +++ b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc @@ -1,14 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifdef _MSC_VER -#pragma warning(disable : 4244) -#pragma warning(disable : 4267) -#endif - #include "core/providers/cpu/math/quantize_linear_matmul.h" #include "core/providers/cpu/math/matmul_helper.h" -#include "core/util/gemmlowp_common_wrapper.h" +#include "core/providers/common.h" namespace onnxruntime { @@ -24,55 +19,7 @@ ONNX_OPERATOR_KERNEL_EX( .TypeConstraint("T3", DataTypeImpl::GetTensorType()), QLinearMatMul); -Status GemmlowpMultiply(const uint8_t* lhs_data, const uint8_t* rhs_data, uint8_t* result_data, - const int lhs_offset, const int rhs_offset, const int result_offset, - int m, int n, int k, int32_t int_multiplier, int32_t right_shift) { - gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage; - quantize_down_stage.result_offset_after_shift = result_offset; - quantize_down_stage.result_fixedpoint_multiplier = int_multiplier; - quantize_down_stage.result_shift = right_shift; - gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage; - const auto& output_pipeline = std::make_tuple(quantize_down_stage, saturating_cast_stage); - - // TODO exp ColMajor order for rhs and result. That may be faster - const auto matOrder = gemmlowp::MapOrder::RowMajor; - gemmlowp::MatrixMap lhs(lhs_data, m, k); - gemmlowp::MatrixMap rhs(rhs_data, k, n); - gemmlowp::MatrixMap result(result_data, m, n); - - gemmlowp::GemmContext gemm_context; - gemmlowp::GemmWithOutputPipeline( - &gemm_context, lhs, rhs, &result, -lhs_offset, -rhs_offset, output_pipeline); - - return Status::OK(); -} - -void QuantizeMultiplier(float fp_multiplier, std::int32_t* integer_multiplier, int* right_shift) { - auto* fp_as_bits = reinterpret_cast(&fp_multiplier); - auto current_exponent = (*fp_as_bits >> 23); - // bring multiplier in [.5,1) range and calculate the shift - auto bumped_multiplier_as_bits = - (*fp_as_bits & UINT32_C(0x007fffff)) | UINT32_C(0x3f000000); - auto* bumped_multiplier = reinterpret_cast(&bumped_multiplier_as_bits); - auto shift = 126 - current_exponent; - // convert to fixed point number - auto int_multiplier = static_cast(std::round(*bumped_multiplier * (1ll << 31))); - - *integer_multiplier = static_cast(int_multiplier); - *right_shift = shift; -} - -void ScaleAndZeropointPairValidationHelper(const Tensor* scale, const Tensor* zeropoint) { - ORT_ENFORCE(scale->Shape().NumDimensions() == 0 || - (scale->Shape().NumDimensions() == 1 && scale->Shape().GetDims().size() == 1), - "scale must be a scalar"); - ORT_ENFORCE(zeropoint->Shape().NumDimensions() == 0 || - (zeropoint->Shape().NumDimensions() == 1 && zeropoint->Shape().GetDims().size() == 1), - "zeropoint must be a scalar"); -} - -template<> +template <> Status QLinearMatMul::Compute(OpKernelContext* ctx) const { auto a = ctx->Input(0); auto b = ctx->Input(3); @@ -82,16 +29,27 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) c ORT_RETURN_IF_ERROR(helper.Compute(a->Shape(), b->Shape())); Tensor* y = ctx->Output(0, helper.OutputShape()); - // validate scale and zero points + // validate offsets + auto a_offset = ctx->Input(2); + auto b_offset = ctx->Input(5); + auto y_offset = ctx->Input(7); + ORT_ENFORCE(IsScalarOr1ElementVector(a_offset), + "QLinearMatmul : input zero point must be a scalar or 1D tensor of size 1"); + ORT_ENFORCE(IsScalarOr1ElementVector(b_offset), + "QLinearMatmul : weight zero point must be a scalar or 1D tensor of size 1"); + ORT_ENFORCE(IsScalarOr1ElementVector(y_offset), + "QLinearMatmul : result zero point must be a scalar or 1D tensor of size 1"); + + // validate scale auto a_scale = ctx->Input(1); - auto a_zero_point = ctx->Input(2); - ScaleAndZeropointPairValidationHelper(a_scale, a_zero_point); auto b_scale = ctx->Input(4); - auto b_zero_point = ctx->Input(5); - ScaleAndZeropointPairValidationHelper(b_scale, b_zero_point); auto y_scale = ctx->Input(6); - auto y_zero_point = ctx->Input(7); - ScaleAndZeropointPairValidationHelper(y_scale, y_zero_point); + ORT_ENFORCE(IsScalarOr1ElementVector(a_scale), + "QLinearMatmul : input scale must be a scalar or 1D tensor of size 1"); + ORT_ENFORCE(IsScalarOr1ElementVector(b_scale), + "QLinearMatmul : weight scale must be a scalar or 1D tensor of size 1"); + ORT_ENFORCE(IsScalarOr1ElementVector(y_scale), + "QLinearMatmul : result scale must be a scalar or 1D tensor of size 1"); auto a_scale_data = *(a_scale->template Data()); auto b_scale_data = *(b_scale->template Data()); @@ -103,17 +61,17 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) c QuantizeMultiplier(real_multiplier, &integer_multiplier, &right_shift); for (size_t i = 0; i < helper.OutputOffsets().size(); i++) { - GemmlowpMultiply(a->template Data() + helper.LeftOffsets()[i], - b->template Data() + helper.RightOffsets()[i], - y->template MutableData() + helper.OutputOffsets()[i], - *a_zero_point->template Data(), - *b_zero_point->template Data(), - *y_zero_point->template Data(), - static_cast(helper.M()), - static_cast(helper.N()), - static_cast(helper.K()), - integer_multiplier, - right_shift); + GemmlowpMultiplyu8u8_u8(a->template Data() + helper.LeftOffsets()[i], + b->template Data() + helper.RightOffsets()[i], + y->template MutableData() + helper.OutputOffsets()[i], + *a_offset->template Data(), + *b_offset->template Data(), + *y_offset->template Data(), + static_cast(helper.M()), + static_cast(helper.N()), + static_cast(helper.K()), + integer_multiplier, + right_shift); } return Status::OK(); diff --git a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.h b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.h index 778bb03ca0..aada308756 100644 --- a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.h +++ b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.h @@ -6,6 +6,7 @@ #include "core/common/common.h" #include "core/framework/op_kernel.h" #include "core/util/math_cpuonly.h" +#include "core/util/gemmlowp_common.h" namespace onnxruntime { @@ -16,6 +17,6 @@ class QLinearMatMul final : public OpKernel { } Status Compute(OpKernelContext* context) const override; - + }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/conv_integer.cc b/onnxruntime/core/providers/cpu/nn/conv_integer.cc index fbd182312f..49d2a3c575 100644 --- a/onnxruntime/core/providers/cpu/nn/conv_integer.cc +++ b/onnxruntime/core/providers/cpu/nn/conv_integer.cc @@ -1,15 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifdef _MSC_VER -#pragma warning(disable : 4244) -#pragma warning(disable : 4267) -#endif #include "core/providers/cpu/nn/conv_integer.h" #include "core/util/math.h" #include "core/util/math_cpuonly.h" -#include "core/util/gemmlowp_common_wrapper.h" +#include "core/util/gemmlowp_common.h" +#include "core/providers/common.h" namespace onnxruntime { @@ -28,27 +25,17 @@ Status ConvInteger::Compute(OpKernelContext* context) const { size_t num_inputs = OpKernel::Node().InputDefs().size(); const auto* X = context->Input(0); const auto* W = context->Input(1); - int32_t input_offset = 0; - int32_t filter_offset = 0; + uint8_t input_offset = 0; + uint8_t filter_offset = 0; if (num_inputs >= 3) { const auto* X_Zero_Point = context->Input(2); - if (X_Zero_Point->Shape().NumDimensions() == 0 || - (X_Zero_Point->Shape().NumDimensions() == 1 && X_Zero_Point->Shape().GetDims().size() == 1)) { - input_offset = static_cast(*(X_Zero_Point->Data())); - } else { - //TODO: Add support for per-channel quantization. - return Status(common::ONNXRUNTIME, common::FAIL, "Non per-tensor quantization is not supported now."); - } + ORT_ENFORCE(IsScalarOr1ElementVector(X_Zero_Point), "Must be a scalar or 1D tensor or size 1."); + input_offset = *(X_Zero_Point->Data()); } if (num_inputs >= 4) { const auto* W_Zero_Point = context->Input(3); - if (W_Zero_Point->Shape().NumDimensions() == 0 || - (W_Zero_Point->Shape().NumDimensions() == 1 && W_Zero_Point->Shape().GetDims().size() == 1)) { - filter_offset = static_cast(*(W_Zero_Point->Data())); - } else { - //TODO: Add support for per-channel quantization. - return Status(common::ONNXRUNTIME, common::FAIL, "Non per-tensor quantization is not supported now."); - } + ORT_ENFORCE(IsScalarOr1ElementVector(W_Zero_Point), "Non per-tensor quantization is not supported now."); + filter_offset = *(W_Zero_Point->Data()); } const int64_t N = X->Shape()[0]; @@ -118,27 +105,17 @@ Status ConvInteger::Compute(OpKernelContext* context) const { static_cast(kernel_shape.size()), col_buffer_data, &CPUMathUtil::Instance(), - false, - input_offset); + false, + input_offset); - const uint8_t* filter_data_as_uint8 = W->template Data() + group_id * W_offset; - static const gemmlowp::MapOrder ResultOrder = gemmlowp::MapOrder::RowMajor; - static const gemmlowp::MapOrder LhsOrder = gemmlowp::MapOrder::RowMajor; - static const gemmlowp::MapOrder RhsOrder = gemmlowp::MapOrder::RowMajor; - gemmlowp::MatrixMap lhs( - filter_data_as_uint8, static_cast(M / group_), static_cast(kernel_dim)); - gemmlowp::MatrixMap rhs( - col_buffer_data, static_cast(kernel_dim), static_cast(output_image_size)); - gemmlowp::MatrixMap result( - Ydata + group_id * Y_offset, static_cast(M / group_), static_cast(output_image_size)); - const std::tuple<> empty_pipeline = {}; - - gemmlowp::GemmContext gemm_context; - // TODO: worker thread pool needs to be handled. - gemmlowp::GemmWithOutputPipeline( - &gemm_context, lhs, rhs, &result, -filter_offset, -input_offset, - empty_pipeline); + GemmlowpMultiplyu8u8_s32(W->template Data() + group_id * W_offset, + col_buffer_data, + Ydata + group_id * Y_offset, + filter_offset, + input_offset, + static_cast(M / group_), + static_cast(output_image_size), + static_cast(kernel_dim)); } Xdata += X_offset * group_; diff --git a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc index 1cf064f7ea..d6dce056ea 100644 --- a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc +++ b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc @@ -1,14 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifdef _MSC_VER -#pragma warning(disable : 4244) -#pragma warning(disable : 4267) -#endif - #include "core/providers/cpu/nn/qlinearconv.h" #include "core/util/math.h" #include "core/util/math_cpuonly.h" +#include "core/providers/common.h" namespace onnxruntime { ONNX_OPERATOR_KERNEL_EX( @@ -26,25 +22,32 @@ Status QLinearConv::Compute(OpKernelContext* context) const { const auto* X = context->Input(0); const auto* W = context->Input(3); - // validate scale and zero points - auto input_scale = context->Input(1); + // validate offsets auto input_offset = context->Input(2); - ScaleAndZeropointPairValidationHelper(input_scale, input_offset); - auto filter_scale = context->Input(4); auto filter_offset = context->Input(5); - ScaleAndZeropointPairValidationHelper(filter_scale, filter_offset); - auto result_scale = context->Input(6); auto result_offset = context->Input(7); - ScaleAndZeropointPairValidationHelper(result_scale, result_offset); + ORT_ENFORCE(IsScalarOr1ElementVector(input_offset), + "QLinearConv : input zero point must be a scalar or 1D tensor of size 1"); + ORT_ENFORCE(IsScalarOr1ElementVector(filter_offset), + "QLinearConv : filter zero point must be a scalar or 1D tensor of size 1"); + ORT_ENFORCE(IsScalarOr1ElementVector(result_offset), + "QLinearConv : result zero point must be a scalar or 1D tensor of size 1"); + + // validate scale + auto input_scale = context->Input(1); + auto filter_scale = context->Input(4); + auto result_scale = context->Input(6); + ORT_ENFORCE(IsScalarOr1ElementVector(input_scale), + "QLinearConv : input scale must be a scalar or 1D tensor of size 1"); + ORT_ENFORCE(IsScalarOr1ElementVector(filter_scale), + "QLinearConv : filter scale must be a scalar or 1D tensor of size 1"); + ORT_ENFORCE(IsScalarOr1ElementVector(result_scale), + "QLinearConv : result scale must be a scalar or 1D tensor of size 1"); auto input_scale_data = *(input_scale->template Data()); auto filter_scale_data = *(filter_scale->template Data()); auto result_scale_data = *(result_scale->template Data()); - auto input_offset_data = *(input_offset->template Data()); - auto filter_offset_data = *(filter_offset->template Data()); - auto result_offset_data = *(result_offset->template Data()); - const float real_multiplier = (input_scale_data * filter_scale_data) / result_scale_data; int32_t integer_multiplier; int right_shift; @@ -54,7 +57,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { const Tensor* bias = nullptr; if (num_inputs == 9) { bias = context->Input(8); - } + } const int64_t N = X->Shape()[0]; const int64_t C = X->Shape()[1]; @@ -95,7 +98,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { const int64_t kernel_size = TensorShape(kernel_shape).Size(); const int64_t X_offset = C / group_ * input_image_size; const int64_t Y_offset = Y->Shape().Size() / Y->Shape()[0] / group_; - const int64_t W_offset = W->Shape().Size() / group_; + const int64_t W_offset = W->Shape().Size() / group_; const int64_t kernel_dim = C / group_ * kernel_size; const int64_t col_buffer_size = kernel_dim * output_image_size; const int bias_offset = static_cast(M / group_); @@ -124,35 +127,21 @@ Status QLinearConv::Compute(OpKernelContext* context) const { static_cast(kernel_shape.size()), col_buffer_data, &CPUMathUtil::Instance(), - false, - input_offset_data); + false, + *input_offset->template Data()); - const uint8_t* filter_data_as_uint8 = W->template Data() + group_id * W_offset; - static const gemmlowp::MapOrder MatOrder = gemmlowp::MapOrder::RowMajor; - gemmlowp::MatrixMap lhs( - filter_data_as_uint8, static_cast(M / group_), static_cast(kernel_dim)); - gemmlowp::MatrixMap rhs( - col_buffer_data, static_cast(kernel_dim), static_cast(output_image_size)); - gemmlowp::MatrixMap result( - Ydata + group_id * Y_offset, static_cast(M / group_), static_cast(output_image_size)); - - // TODO: worker thread pool needs to be handled. - gemmlowp::GemmContext gemm_context; - if (bias == nullptr) { - auto output_pipeline = MakeOutputPipelineWithOutBias(result_offset_data, - integer_multiplier, right_shift); - gemmlowp::GemmWithOutputPipeline( - &gemm_context, lhs, rhs, &result, -filter_offset_data, -input_offset_data, - output_pipeline); - } else { - auto output_pipeline = MakeOutputPipelineWithBias(bias->template Data() + group_id * bias_offset, - static_cast(M / group_), result_offset_data, integer_multiplier, right_shift); - gemmlowp::GemmWithOutputPipeline( - &gemm_context, lhs, rhs, &result, -filter_offset_data, -input_offset_data, - output_pipeline); - } + GemmlowpMultiplyu8u8_u8(W->template Data() + group_id * W_offset, + col_buffer_data, + Ydata + group_id * Y_offset, + *filter_offset->template Data(), + *input_offset->template Data(), + *result_offset->template Data(), + static_cast(M / group_), + static_cast(output_image_size), + static_cast(kernel_dim), + integer_multiplier, + right_shift, + bias == nullptr ? nullptr : bias->template Data() + group_id * bias_offset); } Xdata += X_offset * group_; @@ -161,28 +150,4 @@ Status QLinearConv::Compute(OpKernelContext* context) const { return Status::OK(); } - -void QLinearConv::QuantizeMultiplier(float fp_multiplier, std::int32_t* integer_multiplier, int* right_shift) const { - auto* fp_as_bits = reinterpret_cast(&fp_multiplier); - auto current_exponent = (*fp_as_bits >> 23); - // bring multiplier in [.5,1) range and calculate the shift - auto bumped_multiplier_as_bits = - (*fp_as_bits & UINT32_C(0x007fffff)) | UINT32_C(0x3f000000); - auto* bumped_multiplier = reinterpret_cast(&bumped_multiplier_as_bits); - auto shift = 126 - current_exponent; - // convert to fixed point number - auto int_multiplier = static_cast(std::round(*bumped_multiplier * (1ll << 31))); - - *integer_multiplier = static_cast(int_multiplier); - *right_shift = shift; -} - -void QLinearConv::ScaleAndZeropointPairValidationHelper(const Tensor* scale, const Tensor* zeropoint) const { - ORT_ENFORCE(scale->Shape().NumDimensions() == 0 || - (scale->Shape().NumDimensions() == 1 && scale->Shape().GetDims().size() == 1), - "scale must be a scalar"); - ORT_ENFORCE(zeropoint->Shape().NumDimensions() == 0 || - (zeropoint->Shape().NumDimensions() == 1 && zeropoint->Shape().GetDims().size() == 1), - "zeropoint must be a scalar"); -} } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/qlinearconv.h b/onnxruntime/core/providers/cpu/nn/qlinearconv.h index c5e7919371..9179da587c 100644 --- a/onnxruntime/core/providers/cpu/nn/qlinearconv.h +++ b/onnxruntime/core/providers/cpu/nn/qlinearconv.h @@ -4,7 +4,7 @@ #pragma once #include "core/providers/cpu/nn/conv_base.h" -#include "core/util/gemmlowp_common_wrapper.h" +#include "core/util/gemmlowp_common.h" namespace onnxruntime { class QLinearConv : public OpKernel, public ConvBase { @@ -12,44 +12,6 @@ class QLinearConv : public OpKernel, public ConvBase { explicit QLinearConv(const OpKernelInfo& info) : OpKernel(info), ConvBase(info) { } - Status Compute(OpKernelContext* context) const override; - - void QuantizeMultiplier(float fp_multiplier, std::int32_t* integer_multiplier, int* right_shift) const; - - void ScaleAndZeropointPairValidationHelper(const Tensor* scale, const Tensor* zeropoint) const; + Status Compute(OpKernelContext* context) const override; }; - -typedef gemmlowp::VectorMap ColVectorMap; - -inline std::tuple, - gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint, - gemmlowp::OutputStageSaturatingCastToUint8> -MakeOutputPipelineWithBias(const int32_t* bias, - int rows, - std::int32_t result_offset, - std::int32_t result_mult_int, - std::int32_t result_shift) { - ColVectorMap bias_vector(bias, rows); - gemmlowp::OutputStageBiasAddition bias_addition_stage; - bias_addition_stage.bias_vector = bias_vector; - gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage; - quantize_down_stage.result_offset_after_shift = result_offset; - quantize_down_stage.result_fixedpoint_multiplier = result_mult_int; - quantize_down_stage.result_shift = result_shift; - gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage; - return std::make_tuple(bias_addition_stage, quantize_down_stage, saturating_cast_stage); -} - -inline std::tuple -MakeOutputPipelineWithOutBias(std::int32_t result_offset, - std::int32_t result_mult_int, - std::int32_t result_shift) { - gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage; - quantize_down_stage.result_offset_after_shift = result_offset; - quantize_down_stage.result_fixedpoint_multiplier = result_mult_int; - quantize_down_stage.result_shift = result_shift; - gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage; - return std::make_tuple(quantize_down_stage, saturating_cast_stage); -} } // namespace onnxruntime diff --git a/onnxruntime/core/util/gemmlowp_common.cc b/onnxruntime/core/util/gemmlowp_common.cc new file mode 100644 index 0000000000..7754e717a7 --- /dev/null +++ b/onnxruntime/core/util/gemmlowp_common.cc @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/util/gemmlowp_common_wrapper.h" +#include "core/util/gemmlowp_common.h" + +namespace onnxruntime { + +Status GemmlowpMultiplyu8u8_u8(const uint8_t* lhs_data, const uint8_t* rhs_data, uint8_t* result_data, + const int lhs_offset, const int rhs_offset, const int result_offset, + int m, int n, int k, int32_t int_multiplier, int32_t right_shift, const int32_t* bias) { + // TODO exp ColMajor order for rhs and result. That may be faster + const auto matOrder = gemmlowp::MapOrder::RowMajor; + gemmlowp::MatrixMap lhs(lhs_data, m, k); + gemmlowp::MatrixMap rhs(rhs_data, k, n); + gemmlowp::MatrixMap result(result_data, m, n); + + gemmlowp::GemmContext gemm_context; + + if (bias == nullptr) { + auto output_pipeline = MakeOutputPipelineWithOutBias(result_offset, + int_multiplier, right_shift); + gemmlowp::GemmWithOutputPipeline( + &gemm_context, lhs, rhs, &result, -lhs_offset, -rhs_offset, output_pipeline); + } else { + auto output_pipeline = MakeOutputPipelineWithBias(bias, m, result_offset, int_multiplier, right_shift); + gemmlowp::GemmWithOutputPipeline( + &gemm_context, lhs, rhs, &result, -lhs_offset, -rhs_offset, output_pipeline); + } + + return Status::OK(); +} + +Status GemmlowpMultiplyu8u8_s32(const uint8_t* lhs_data, const uint8_t* rhs_data, int32_t* result_data, + const int lhs_offset, const int rhs_offset, int m, int n, int k) { + // TODO exp ColMajor order for rhs and result. That may be faster + const auto matOrder = gemmlowp::MapOrder::RowMajor; + gemmlowp::MatrixMap lhs(lhs_data, m, k); + gemmlowp::MatrixMap rhs(rhs_data, k, n); + gemmlowp::MatrixMap result(result_data, m, n); + + gemmlowp::GemmContext gemm_context; + + const std::tuple<> empty_pipeline = {}; + + gemmlowp::GemmWithOutputPipeline( + &gemm_context, lhs, rhs, &result, -lhs_offset, -rhs_offset, empty_pipeline); + + return Status::OK(); +} +} \ No newline at end of file diff --git a/onnxruntime/core/util/gemmlowp_common.h b/onnxruntime/core/util/gemmlowp_common.h new file mode 100644 index 0000000000..63fb187bdd --- /dev/null +++ b/onnxruntime/core/util/gemmlowp_common.h @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once +#include "core/util/gemmlowp_common_wrapper.h" +#include "core/util/math.h" + + +namespace onnxruntime { + +void inline QuantizeMultiplier(float fp_multiplier, std::int32_t* integer_multiplier, int* right_shift) { + auto* fp_as_bits = reinterpret_cast(&fp_multiplier); + auto current_exponent = (*fp_as_bits >> 23); + // bring multiplier in [.5,1) range and calculate the shift + auto bumped_multiplier_as_bits = + (*fp_as_bits & UINT32_C(0x007fffff)) | UINT32_C(0x3f000000); + auto* bumped_multiplier = reinterpret_cast(&bumped_multiplier_as_bits); + auto shift = 126 - current_exponent; + // convert to fixed point number + auto int_multiplier = static_cast(std::round(*bumped_multiplier * (1ll << 31))); + + *integer_multiplier = static_cast(int_multiplier); + *right_shift = shift; +} + +typedef gemmlowp::VectorMap ColVectorMap; + +inline std::tuple, + gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint, + gemmlowp::OutputStageSaturatingCastToUint8> +MakeOutputPipelineWithBias(const int32_t* bias, + int rows, + std::int32_t result_offset, + std::int32_t result_mult_int, + std::int32_t result_shift) { + ColVectorMap bias_vector(bias, rows); + gemmlowp::OutputStageBiasAddition bias_addition_stage; + bias_addition_stage.bias_vector = bias_vector; + gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage; + quantize_down_stage.result_offset_after_shift = result_offset; + quantize_down_stage.result_fixedpoint_multiplier = result_mult_int; + quantize_down_stage.result_shift = result_shift; + gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage; + return std::make_tuple(bias_addition_stage, quantize_down_stage, saturating_cast_stage); +} + +inline std::tuple +MakeOutputPipelineWithOutBias(std::int32_t result_offset, + std::int32_t result_mult_int, + std::int32_t result_shift) { + gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage; + quantize_down_stage.result_offset_after_shift = result_offset; + quantize_down_stage.result_fixedpoint_multiplier = result_mult_int; + quantize_down_stage.result_shift = result_shift; + gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage; + return std::make_tuple(quantize_down_stage, saturating_cast_stage); +} + +Status GemmlowpMultiplyu8u8_u8(const uint8_t* lhs_data, const uint8_t* rhs_data, uint8_t* result_data, + const int lhs_offset, const int rhs_offset, const int result_offset, + int m, int n, int k, int32_t int_multiplier, int32_t right_shift, const int32_t* bias = nullptr); + +Status GemmlowpMultiplyu8u8_s32(const uint8_t* lhs_data, const uint8_t* rhs_data, int32_t* result_data, + const int lhs_offset, const int rhs_offset, int m, int n, int k); + +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/util/gemmlowp_common_wrapper.h b/onnxruntime/core/util/gemmlowp_common_wrapper.h index 48f3afdb26..9a2386e659 100644 --- a/onnxruntime/core/util/gemmlowp_common_wrapper.h +++ b/onnxruntime/core/util/gemmlowp_common_wrapper.h @@ -13,6 +13,8 @@ #else #pragma warning(push) #pragma warning(disable : 4100) //'identifier' : unreferenced formal parameter +#pragma warning(disable : 4244) +#pragma warning(disable : 4267) #endif #include "public/gemmlowp.h"