diff --git a/cmake/onnxruntime_util.cmake b/cmake/onnxruntime_util.cmake index 3cab36ac14..34b098276a 100644 --- a/cmake/onnxruntime_util.cmake +++ b/cmake/onnxruntime_util.cmake @@ -8,19 +8,12 @@ file(GLOB_RECURSE onnxruntime_util_srcs CONFIGURE_DEPENDS source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_util_srcs}) -if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT MSVC) - # For x86 platforms it is important to pass this flag to compiler. Without this gemmlowp will use slow reference code. - # These optimizations are not enabled on MSVC so excluding it. - message("enabling optimizations for gemmlowp") - set_source_files_properties("${ONNXRUNTIME_ROOT}/core/util/gemmlowp_common.cc" PROPERTIES COMPILE_FLAGS "-msse4.1") -endif() - set(gemmlowp_src ${PROJECT_SOURCE_DIR}/external/gemmlowp) add_library(onnxruntime_util ${onnxruntime_util_srcs}) if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8) #TODO: fix the warnings, they are dangerous - target_compile_options(onnxruntime_util PRIVATE "/wd4244") + target_compile_options(onnxruntime_util PRIVATE "/wd4244") endif() target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${MKLML_INCLUDE_DIR} ${gemmlowp_src} PUBLIC ${eigen_INCLUDE_DIRS}) onnxruntime_add_include_to_target(onnxruntime_util onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf) diff --git a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc index 164b58f208..f64e90026d 100644 --- a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc +++ b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc @@ -3,7 +3,12 @@ #include "core/providers/cpu/math/quantize_linear_matmul.h" #include "core/providers/cpu/math/matmul_helper.h" +#include "core/common/safeint.h" #include "core/providers/common.h" +#include "core/util/math_cpuonly.h" +#include "core/util/qmath.h" +#include "core/util/gemmlowp_common.h" +#include "core/mlas/inc/mlas.h" namespace onnxruntime { @@ -56,11 +61,44 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) c auto y_scale_data = *(y_scale->template Data()); const float real_multiplier = (a_scale_data * b_scale_data) / y_scale_data; + +#ifdef MLAS_SUPPORTS_GEMM_U8X8 + AllocatorPtr alloc; + ORT_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&alloc)); + auto gemm_output_data = alloc->Alloc(SafeInt(sizeof(int32_t)) * + static_cast(helper.M()) * static_cast(helper.N())); + BufferUniquePtr gemm_output_buffer(gemm_output_data, BufferDeleter(alloc)); + auto* gemm_output = static_cast(gemm_output_buffer.get()); +#else + // Compute the fixed point multiplier and shift for requantizing with GEMMLOWP. int32_t integer_multiplier; int right_shift; QuantizeMultiplier(real_multiplier, &integer_multiplier, &right_shift); +#endif for (size_t i = 0; i < helper.OutputOffsets().size(); i++) { +#ifdef MLAS_SUPPORTS_GEMM_U8X8 + QGemmu8u8_s32(static_cast(helper.M()), + static_cast(helper.N()), + static_cast(helper.K()), + a->template Data() + helper.LeftOffsets()[i], + static_cast(helper.K()), + *a_offset->template Data(), + b->template Data() + helper.RightOffsets()[i], + static_cast(helper.N()), + *b_offset->template Data(), + gemm_output, + static_cast(helper.N()), + ctx->GetOperatorThreadPool()); + + MlasRequantizeOutput(gemm_output, + y->template MutableData() + helper.OutputOffsets()[i], + nullptr, + static_cast(helper.M()), + static_cast(helper.N()), + real_multiplier, + *y_offset->template Data()); +#else GemmlowpMultiplyu8u8_u8(a->template Data() + helper.LeftOffsets()[i], b->template Data() + helper.RightOffsets()[i], y->template MutableData() + helper.OutputOffsets()[i], @@ -72,6 +110,7 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) c static_cast(helper.K()), integer_multiplier, right_shift); +#endif } return Status::OK(); diff --git a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.h b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.h index aada308756..dfc7a77729 100644 --- a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.h +++ b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.h @@ -3,10 +3,7 @@ #pragma once -#include "core/common/common.h" #include "core/framework/op_kernel.h" -#include "core/util/math_cpuonly.h" -#include "core/util/gemmlowp_common.h" namespace onnxruntime { diff --git a/onnxruntime/core/util/gemmlowp_common.cc b/onnxruntime/core/util/gemmlowp_common.cc index 7b14a09b51..d4db43d715 100644 --- a/onnxruntime/core/util/gemmlowp_common.cc +++ b/onnxruntime/core/util/gemmlowp_common.cc @@ -6,6 +6,40 @@ namespace onnxruntime { +typedef gemmlowp::VectorMap ColVectorMap; + +inline std::tuple, + gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint, + gemmlowp::OutputStageSaturatingCastToUint8> +MakeOutputPipelineWithBias(const int32_t* bias, + int rows, + std::int32_t result_offset, + std::int32_t result_mult_int, + std::int32_t result_shift) { + ColVectorMap bias_vector(bias, rows); + gemmlowp::OutputStageBiasAddition bias_addition_stage; + bias_addition_stage.bias_vector = bias_vector; + gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage; + quantize_down_stage.result_offset_after_shift = result_offset; + quantize_down_stage.result_fixedpoint_multiplier = result_mult_int; + quantize_down_stage.result_shift = result_shift; + gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage; + return std::make_tuple(bias_addition_stage, quantize_down_stage, saturating_cast_stage); +} + +inline std::tuple +MakeOutputPipelineWithOutBias(std::int32_t result_offset, + std::int32_t result_mult_int, + std::int32_t result_shift) { + gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage; + quantize_down_stage.result_offset_after_shift = result_offset; + quantize_down_stage.result_fixedpoint_multiplier = result_mult_int; + quantize_down_stage.result_shift = result_shift; + gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage; + return std::make_tuple(quantize_down_stage, saturating_cast_stage); +} + void GemmlowpMultiplyu8u8_u8(const uint8_t* lhs_data, const uint8_t* rhs_data, uint8_t* result_data, const int lhs_offset, const int rhs_offset, const int result_offset, int m, int n, int k, int32_t int_multiplier, int32_t right_shift, const int32_t* bias) { @@ -44,6 +78,6 @@ void GemmlowpMultiplyu8u8_s32(const uint8_t* lhs_data, const uint8_t* rhs_data, gemmlowp::GemmWithOutputPipeline( &gemm_context, lhs, rhs, &result, -lhs_offset, -rhs_offset, empty_pipeline); +} } -} \ No newline at end of file diff --git a/onnxruntime/core/util/gemmlowp_common.h b/onnxruntime/core/util/gemmlowp_common.h index 99ea341fd4..01d7632bcd 100644 --- a/onnxruntime/core/util/gemmlowp_common.h +++ b/onnxruntime/core/util/gemmlowp_common.h @@ -28,40 +28,6 @@ void inline QuantizeMultiplier(float fp_multiplier, std::int32_t* integer_multip *right_shift = shift; } -typedef gemmlowp::VectorMap ColVectorMap; - -inline std::tuple, - gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint, - gemmlowp::OutputStageSaturatingCastToUint8> -MakeOutputPipelineWithBias(const int32_t* bias, - int rows, - std::int32_t result_offset, - std::int32_t result_mult_int, - std::int32_t result_shift) { - ColVectorMap bias_vector(bias, rows); - gemmlowp::OutputStageBiasAddition bias_addition_stage; - bias_addition_stage.bias_vector = bias_vector; - gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage; - quantize_down_stage.result_offset_after_shift = result_offset; - quantize_down_stage.result_fixedpoint_multiplier = result_mult_int; - quantize_down_stage.result_shift = result_shift; - gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage; - return std::make_tuple(bias_addition_stage, quantize_down_stage, saturating_cast_stage); -} - -inline std::tuple -MakeOutputPipelineWithOutBias(std::int32_t result_offset, - std::int32_t result_mult_int, - std::int32_t result_shift) { - gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage; - quantize_down_stage.result_offset_after_shift = result_offset; - quantize_down_stage.result_fixedpoint_multiplier = result_mult_int; - quantize_down_stage.result_shift = result_shift; - gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage; - return std::make_tuple(quantize_down_stage, saturating_cast_stage); -} - void GemmlowpMultiplyu8u8_u8(const uint8_t* lhs_data, const uint8_t* rhs_data, uint8_t* result_data, const int lhs_offset, const int rhs_offset, const int result_offset, int m, int n, int k, int32_t int_multiplier, int32_t right_shift, const int32_t* bias = nullptr); @@ -73,4 +39,4 @@ void GemmlowpMultiplyu8u8_s32(const uint8_t* lhs_data, const uint8_t* rhs_data, #ifdef __GNUC__ #pragma GCC diagnostic pop -#endif \ No newline at end of file +#endif