mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
QLinearMatMul speed up (#3283)
The equivalent of PR#3196 but done for QLinearMatMul. Use MLAS to do a u8u8=s32 GEMM and then requantize this intermediate buffer.
This commit is contained in:
Tracy Sharpe
GitHub
parent
9c3b6d2e4b
commit
57468c651c
5 changed files with 76 additions and 47 deletions
|
|
@ -8,19 +8,12 @@ file(GLOB_RECURSE onnxruntime_util_srcs CONFIGURE_DEPENDS
|
|||
|
||||
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_util_srcs})
|
||||
|
||||
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT MSVC)
|
||||
# For x86 platforms it is important to pass this flag to compiler. Without this gemmlowp will use slow reference code.
|
||||
# These optimizations are not enabled on MSVC so excluding it.
|
||||
message("enabling optimizations for gemmlowp")
|
||||
set_source_files_properties("${ONNXRUNTIME_ROOT}/core/util/gemmlowp_common.cc" PROPERTIES COMPILE_FLAGS "-msse4.1")
|
||||
endif()
|
||||
|
||||
set(gemmlowp_src ${PROJECT_SOURCE_DIR}/external/gemmlowp)
|
||||
|
||||
add_library(onnxruntime_util ${onnxruntime_util_srcs})
|
||||
if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
|
||||
#TODO: fix the warnings, they are dangerous
|
||||
target_compile_options(onnxruntime_util PRIVATE "/wd4244")
|
||||
target_compile_options(onnxruntime_util PRIVATE "/wd4244")
|
||||
endif()
|
||||
target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${MKLML_INCLUDE_DIR} ${gemmlowp_src} PUBLIC ${eigen_INCLUDE_DIRS})
|
||||
onnxruntime_add_include_to_target(onnxruntime_util onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,12 @@
|
|||
|
||||
#include "core/providers/cpu/math/quantize_linear_matmul.h"
|
||||
#include "core/providers/cpu/math/matmul_helper.h"
|
||||
#include "core/common/safeint.h"
|
||||
#include "core/providers/common.h"
|
||||
#include "core/util/math_cpuonly.h"
|
||||
#include "core/util/qmath.h"
|
||||
#include "core/util/gemmlowp_common.h"
|
||||
#include "core/mlas/inc/mlas.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
@ -56,11 +61,44 @@ Status QLinearMatMul<uint8_t, uint8_t, uint8_t>::Compute(OpKernelContext* ctx) c
|
|||
auto y_scale_data = *(y_scale->template Data<float>());
|
||||
|
||||
const float real_multiplier = (a_scale_data * b_scale_data) / y_scale_data;
|
||||
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8
|
||||
AllocatorPtr alloc;
|
||||
ORT_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&alloc));
|
||||
auto gemm_output_data = alloc->Alloc(SafeInt<size_t>(sizeof(int32_t)) *
|
||||
static_cast<size_t>(helper.M()) * static_cast<size_t>(helper.N()));
|
||||
BufferUniquePtr gemm_output_buffer(gemm_output_data, BufferDeleter(alloc));
|
||||
auto* gemm_output = static_cast<int32_t*>(gemm_output_buffer.get());
|
||||
#else
|
||||
// Compute the fixed point multiplier and shift for requantizing with GEMMLOWP.
|
||||
int32_t integer_multiplier;
|
||||
int right_shift;
|
||||
QuantizeMultiplier(real_multiplier, &integer_multiplier, &right_shift);
|
||||
#endif
|
||||
|
||||
for (size_t i = 0; i < helper.OutputOffsets().size(); i++) {
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8
|
||||
QGemmu8u8_s32(static_cast<int>(helper.M()),
|
||||
static_cast<int>(helper.N()),
|
||||
static_cast<int>(helper.K()),
|
||||
a->template Data<uint8_t>() + helper.LeftOffsets()[i],
|
||||
static_cast<int>(helper.K()),
|
||||
*a_offset->template Data<uint8_t>(),
|
||||
b->template Data<uint8_t>() + helper.RightOffsets()[i],
|
||||
static_cast<int>(helper.N()),
|
||||
*b_offset->template Data<uint8_t>(),
|
||||
gemm_output,
|
||||
static_cast<int>(helper.N()),
|
||||
ctx->GetOperatorThreadPool());
|
||||
|
||||
MlasRequantizeOutput(gemm_output,
|
||||
y->template MutableData<uint8_t>() + helper.OutputOffsets()[i],
|
||||
nullptr,
|
||||
static_cast<size_t>(helper.M()),
|
||||
static_cast<size_t>(helper.N()),
|
||||
real_multiplier,
|
||||
*y_offset->template Data<uint8_t>());
|
||||
#else
|
||||
GemmlowpMultiplyu8u8_u8(a->template Data<uint8_t>() + helper.LeftOffsets()[i],
|
||||
b->template Data<uint8_t>() + helper.RightOffsets()[i],
|
||||
y->template MutableData<uint8_t>() + helper.OutputOffsets()[i],
|
||||
|
|
@ -72,6 +110,7 @@ Status QLinearMatMul<uint8_t, uint8_t, uint8_t>::Compute(OpKernelContext* ctx) c
|
|||
static_cast<int>(helper.K()),
|
||||
integer_multiplier,
|
||||
right_shift);
|
||||
#endif
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
|
|
|
|||
|
|
@ -3,10 +3,7 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#include "core/util/math_cpuonly.h"
|
||||
#include "core/util/gemmlowp_common.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,40 @@
|
|||
|
||||
namespace onnxruntime {
|
||||
|
||||
typedef gemmlowp::VectorMap<const std::int32_t, gemmlowp::VectorShape::Col> ColVectorMap;
|
||||
|
||||
inline std::tuple<gemmlowp::OutputStageBiasAddition<ColVectorMap>,
|
||||
gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint,
|
||||
gemmlowp::OutputStageSaturatingCastToUint8>
|
||||
MakeOutputPipelineWithBias(const int32_t* bias,
|
||||
int rows,
|
||||
std::int32_t result_offset,
|
||||
std::int32_t result_mult_int,
|
||||
std::int32_t result_shift) {
|
||||
ColVectorMap bias_vector(bias, rows);
|
||||
gemmlowp::OutputStageBiasAddition<ColVectorMap> bias_addition_stage;
|
||||
bias_addition_stage.bias_vector = bias_vector;
|
||||
gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage;
|
||||
quantize_down_stage.result_offset_after_shift = result_offset;
|
||||
quantize_down_stage.result_fixedpoint_multiplier = result_mult_int;
|
||||
quantize_down_stage.result_shift = result_shift;
|
||||
gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage;
|
||||
return std::make_tuple(bias_addition_stage, quantize_down_stage, saturating_cast_stage);
|
||||
}
|
||||
|
||||
inline std::tuple<gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint,
|
||||
gemmlowp::OutputStageSaturatingCastToUint8>
|
||||
MakeOutputPipelineWithOutBias(std::int32_t result_offset,
|
||||
std::int32_t result_mult_int,
|
||||
std::int32_t result_shift) {
|
||||
gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage;
|
||||
quantize_down_stage.result_offset_after_shift = result_offset;
|
||||
quantize_down_stage.result_fixedpoint_multiplier = result_mult_int;
|
||||
quantize_down_stage.result_shift = result_shift;
|
||||
gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage;
|
||||
return std::make_tuple(quantize_down_stage, saturating_cast_stage);
|
||||
}
|
||||
|
||||
void GemmlowpMultiplyu8u8_u8(const uint8_t* lhs_data, const uint8_t* rhs_data, uint8_t* result_data,
|
||||
const int lhs_offset, const int rhs_offset, const int result_offset,
|
||||
int m, int n, int k, int32_t int_multiplier, int32_t right_shift, const int32_t* bias) {
|
||||
|
|
@ -44,6 +78,6 @@ void GemmlowpMultiplyu8u8_s32(const uint8_t* lhs_data, const uint8_t* rhs_data,
|
|||
|
||||
gemmlowp::GemmWithOutputPipeline<std::uint8_t, std::int32_t, gemmlowp::DefaultL8R8BitDepthParams>(
|
||||
&gemm_context, lhs, rhs, &result, -lhs_offset, -rhs_offset, empty_pipeline);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -28,40 +28,6 @@ void inline QuantizeMultiplier(float fp_multiplier, std::int32_t* integer_multip
|
|||
*right_shift = shift;
|
||||
}
|
||||
|
||||
typedef gemmlowp::VectorMap<const std::int32_t, gemmlowp::VectorShape::Col> ColVectorMap;
|
||||
|
||||
inline std::tuple<gemmlowp::OutputStageBiasAddition<ColVectorMap>,
|
||||
gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint,
|
||||
gemmlowp::OutputStageSaturatingCastToUint8>
|
||||
MakeOutputPipelineWithBias(const int32_t* bias,
|
||||
int rows,
|
||||
std::int32_t result_offset,
|
||||
std::int32_t result_mult_int,
|
||||
std::int32_t result_shift) {
|
||||
ColVectorMap bias_vector(bias, rows);
|
||||
gemmlowp::OutputStageBiasAddition<ColVectorMap> bias_addition_stage;
|
||||
bias_addition_stage.bias_vector = bias_vector;
|
||||
gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage;
|
||||
quantize_down_stage.result_offset_after_shift = result_offset;
|
||||
quantize_down_stage.result_fixedpoint_multiplier = result_mult_int;
|
||||
quantize_down_stage.result_shift = result_shift;
|
||||
gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage;
|
||||
return std::make_tuple(bias_addition_stage, quantize_down_stage, saturating_cast_stage);
|
||||
}
|
||||
|
||||
inline std::tuple<gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint,
|
||||
gemmlowp::OutputStageSaturatingCastToUint8>
|
||||
MakeOutputPipelineWithOutBias(std::int32_t result_offset,
|
||||
std::int32_t result_mult_int,
|
||||
std::int32_t result_shift) {
|
||||
gemmlowp::OutputStageQuantizeDownInt32ByFixedPoint quantize_down_stage;
|
||||
quantize_down_stage.result_offset_after_shift = result_offset;
|
||||
quantize_down_stage.result_fixedpoint_multiplier = result_mult_int;
|
||||
quantize_down_stage.result_shift = result_shift;
|
||||
gemmlowp::OutputStageSaturatingCastToUint8 saturating_cast_stage;
|
||||
return std::make_tuple(quantize_down_stage, saturating_cast_stage);
|
||||
}
|
||||
|
||||
void GemmlowpMultiplyu8u8_u8(const uint8_t* lhs_data, const uint8_t* rhs_data, uint8_t* result_data,
|
||||
const int lhs_offset, const int rhs_offset, const int result_offset,
|
||||
int m, int n, int k, int32_t int_multiplier, int32_t right_shift, const int32_t* bias = nullptr);
|
||||
|
|
@ -73,4 +39,4 @@ void GemmlowpMultiplyu8u8_s32(const uint8_t* lhs_data, const uint8_t* rhs_data,
|
|||
|
||||
#ifdef __GNUC__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in a new issue