From 45483dcf1f63bf21767cc18468b2e51cb1e1f661 Mon Sep 17 00:00:00 2001 From: Tracy Sharpe <42477615+tracysh@users.noreply.github.com> Date: Tue, 20 Oct 2020 08:45:13 -0700 Subject: [PATCH] Add QLinearConv for activations=u8, weights=s8 (#5510) --- cmake/onnxruntime_mlas.cmake | 1 + onnxruntime/core/mlas/inc/mlas.h | 46 ++ onnxruntime/core/mlas/lib/quantize.cpp | 212 ++++++++ onnxruntime/core/mlas/lib/transpose.cpp | 174 ++++++ .../providers/cpu/cpu_execution_provider.cc | 11 +- .../core/providers/cpu/nn/conv_attributes.h | 21 +- .../core/providers/cpu/nn/qlinearconv.cc | 497 ++++++++++++++++-- onnxruntime/core/util/math.h | 18 + onnxruntime/core/util/math_cpu.cc | 74 ++- .../providers/cpu/nn/qlinearconv_op_test.cc | 333 ++++++++++++ 10 files changed, 1326 insertions(+), 61 deletions(-) create mode 100644 onnxruntime/core/mlas/lib/transpose.cpp diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 99c69070a6..9aa7c8adef 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -9,6 +9,7 @@ set(mlas_common_srcs ${ONNXRUNTIME_ROOT}/core/mlas/lib/qgemm.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/convolve.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/pooling.cpp + ${ONNXRUNTIME_ROOT}/core/mlas/lib/transpose.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/reorder.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/snchwc.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/activate.cpp diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index feb979b6a2..64abcd47a0 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -479,6 +479,28 @@ MlasConvertHalfToFloatBuffer( size_t Count ); +// +// Transpose routines. +// + +void +MLASCALL +MlasTranspose( + const uint8_t* Input, + uint8_t* Output, + size_t M, + size_t N + ); + +void +MLASCALL +MlasTranspose( + const uint32_t* Input, + uint32_t* Output, + size_t M, + size_t N + ); + // // Buffer reordering routines. // @@ -603,6 +625,30 @@ MlasRequantizeOutput( uint8_t ZeroPoint ); +void +MLASCALL +MlasRequantizeOutputColumn( + const int32_t* Input, + uint8_t* Output, + const int32_t* Bias, + size_t M, + size_t N, + const float Scale, + uint8_t ZeroPoint + ); + +void +MLASCALL +MlasRequantizeOutputColumn( + const int32_t* Input, + uint8_t* Output, + const int32_t* Bias, + size_t M, + size_t N, + const float* Scale, + uint8_t ZeroPoint + ); + void MLASCALL MlasFindMinMaxElement( diff --git a/onnxruntime/core/mlas/lib/quantize.cpp b/onnxruntime/core/mlas/lib/quantize.cpp index 8a3f7c9263..522a25bad2 100644 --- a/onnxruntime/core/mlas/lib/quantize.cpp +++ b/onnxruntime/core/mlas/lib/quantize.cpp @@ -421,6 +421,218 @@ Return Value: } } +void +MLASCALL +MlasRequantizeOutputColumn( + const int32_t* Input, + uint8_t* Output, + const int32_t* Bias, + size_t M, + size_t N, + float Scale, + uint8_t ZeroPoint + ) +/*++ + +Routine Description: + + This routine requantizes the intermediate buffer to the output buffer + optionally adding the supplied bias. + +Arguments: + + Input - Supplies the input matrix. + + Output - Supplies the output matrix. + + Bias - Supplies the optional bias vector to be added to the input buffer + before requantization. + + Buffer - Supplies the output matrix. + + M - Supplies the number of elements of the bias vector and the number of + rows in the output matrix. + + N - Supplies the number of columns of the output matrix. + + Scale - Supplies the quantization scale. + + ZeroPoint - Supplies the quantization zero point value. + +Return Value: + + None. + +--*/ +{ + MLAS_FLOAT32X4 ScaleVector = MlasBroadcastFloat32x4(Scale); + MLAS_FLOAT32X4 MinimumValueVector = MlasBroadcastFloat32x4(float(0 - ZeroPoint)); + MLAS_FLOAT32X4 MaximumValueVector = MlasBroadcastFloat32x4(float(255 - ZeroPoint)); + MLAS_INT32X4 ZeroPointVector = MlasBroadcastInt32x4(ZeroPoint); + MLAS_INT32X4 BiasVector = _mm_setzero_si128(); + + // + // Step through each row of the output matrix. + // + + while (M-- > 0) { + + const int32_t* bias = Bias; + + size_t n = N; + + while (n >= 4) { + + MLAS_INT32X4 IntegerVector = _mm_loadu_si128((const __m128i *)Input); + + if (bias != nullptr) { + BiasVector = _mm_loadu_si128((const __m128i*)bias); + bias += 4; + } + + IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector, + ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector); + + IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); + IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); + + *((int32_t*)Output) = _mm_cvtsi128_si32(IntegerVector); + + Input += 4; + Output += 4; + n -= 4; + } + + while (n > 0) { + + MLAS_INT32X4 IntegerVector = _mm_cvtsi32_si128(*Input); + + if (bias != nullptr) { + BiasVector = _mm_cvtsi32_si128(*bias); + bias += 1; + } + + IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector, + ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector); + + *Output = (uint8_t)_mm_cvtsi128_si32(IntegerVector); + + Input += 1; + Output += 1; + n -= 1; + } + } +} + +void +MLASCALL +MlasRequantizeOutputColumn( + const int32_t* Input, + uint8_t* Output, + const int32_t* Bias, + size_t M, + size_t N, + const float* Scale, + uint8_t ZeroPoint + ) +/*++ + +Routine Description: + + This routine requantizes the intermediate buffer to the output buffer + optionally adding the supplied bias. + +Arguments: + + Input - Supplies the input matrix. + + Output - Supplies the output matrix. + + Bias - Supplies the optional bias vector to be added to the input buffer + before requantization. + + Buffer - Supplies the output matrix. + + M - Supplies the number of elements of the bias vector and the number of + rows in the output matrix. + + N - Supplies the number of columns of the output matrix. + + Scale - Supplies the quantization scale vector. + + ZeroPoint - Supplies the quantization zero point value. + +Return Value: + + None. + +--*/ +{ + MLAS_FLOAT32X4 MinimumValueVector = MlasBroadcastFloat32x4(float(0 - ZeroPoint)); + MLAS_FLOAT32X4 MaximumValueVector = MlasBroadcastFloat32x4(float(255 - ZeroPoint)); + MLAS_INT32X4 ZeroPointVector = MlasBroadcastInt32x4(ZeroPoint); + MLAS_INT32X4 BiasVector = _mm_setzero_si128(); + + // + // Step through each row of the output matrix. + // + + while (M-- > 0) { + + const int32_t* bias = Bias; + const float* scale = Scale; + + size_t n = N; + + while (n >= 4) { + + MLAS_INT32X4 IntegerVector = _mm_loadu_si128((const __m128i *)Input); + + if (bias != nullptr) { + BiasVector = _mm_loadu_si128((const __m128i*)bias); + bias += 4; + } + + MLAS_FLOAT32X4 ScaleVector = MlasLoadFloat32x4(scale); + scale += 4; + + IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector, + ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector); + + IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); + IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); + + *((int32_t*)Output) = _mm_cvtsi128_si32(IntegerVector); + + Input += 4; + Output += 4; + n -= 4; + } + + while (n > 0) { + + MLAS_INT32X4 IntegerVector = _mm_cvtsi32_si128(*Input); + + if (bias != nullptr) { + BiasVector = _mm_cvtsi32_si128(*bias); + bias += 1; + } + + MLAS_FLOAT32X4 ScaleVector = _mm_load_ss(scale); + scale += 1; + + IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector, + ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector); + + *Output = (uint8_t)_mm_cvtsi128_si32(IntegerVector); + + Input += 1; + Output += 1; + n -= 1; + } + } +} + #endif void diff --git a/onnxruntime/core/mlas/lib/transpose.cpp b/onnxruntime/core/mlas/lib/transpose.cpp new file mode 100644 index 0000000000..f52b514410 --- /dev/null +++ b/onnxruntime/core/mlas/lib/transpose.cpp @@ -0,0 +1,174 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + transpose.cpp + +Abstract: + + This module implements the transpose operation. + +--*/ + +#include "mlasi.h" + +#ifdef MLAS_TARGET_AMD64_IX86 + +void +MLASCALL +MlasTranspose( + const uint8_t* Input, + uint8_t* Output, + size_t M, + size_t N + ) +/*++ + +Routine Description: + + This routine transposes the input matrix (M rows by N columns) to the + output matrix (N rows by M columns). + +Arguments: + + Input - Supplies the input buffer. + + Output - Supplies the output buffer. + + M - Supplies the number of rows for the input matrix and the number of + columns for the output matrix. + + N - Supplies the number of columns for the input matrix and the number of + rows for the output matrix. + +Return Value: + + None. + +--*/ +{ + size_t n = N; + + // + // Transpose elements from the input matrix to the output matrix 8 columns + // at a time. + // + + while (n >= 8) { + + const uint8_t* s = Input; + uint8_t* d = Output; + size_t m = M; + + while (m >= 8) { + + __m128i a0 = _mm_loadl_epi64((const __m128i*)&s[N * 0]); + __m128i a1 = _mm_loadl_epi64((const __m128i*)&s[N * 1]); + __m128i b0 = _mm_unpacklo_epi8(a0, a1); + + __m128i a2 = _mm_loadl_epi64((const __m128i*)&s[N * 2]); + __m128i a3 = _mm_loadl_epi64((const __m128i*)&s[N * 3]); + __m128i b1 = _mm_unpacklo_epi8(a2, a3); + + __m128i a4 = _mm_loadl_epi64((const __m128i*)&s[N * 4]); + __m128i a5 = _mm_loadl_epi64((const __m128i*)&s[N * 5]); + __m128i b2 = _mm_unpacklo_epi8(a4, a5); + + __m128i a6 = _mm_loadl_epi64((const __m128i*)&s[N * 6]); + __m128i a7 = _mm_loadl_epi64((const __m128i*)&s[N * 7]); + __m128i b3 = _mm_unpacklo_epi8(a6, a7); + + __m128i c0 = _mm_unpacklo_epi16(b0, b1); + __m128i c1 = _mm_unpackhi_epi16(b0, b1); + __m128i c2 = _mm_unpacklo_epi16(b2, b3); + __m128i c3 = _mm_unpackhi_epi16(b2, b3); + + __m128 d0 = _mm_castsi128_ps(_mm_unpacklo_epi32(c0, c2)); + _mm_storel_pi((__m64*)&d[M * 0], d0); + _mm_storeh_pi((__m64*)&d[M * 1], d0); + + __m128 d1 = _mm_castsi128_ps(_mm_unpackhi_epi32(c0, c2)); + _mm_storel_pi((__m64*)&d[M * 2], d1); + _mm_storeh_pi((__m64*)&d[M * 3], d1); + + __m128 d2 = _mm_castsi128_ps(_mm_unpacklo_epi32(c1, c3)); + _mm_storel_pi((__m64*)&d[M * 4], d2); + _mm_storeh_pi((__m64*)&d[M * 5], d2); + + __m128 d3 = _mm_castsi128_ps(_mm_unpackhi_epi32(c1, c3)); + _mm_storel_pi((__m64*)&d[M * 6], d3); + _mm_storeh_pi((__m64*)&d[M * 7], d3); + + s += N * 8; + d += 8; + m -= 8; + } + + while (m > 0) { + + d[M * 0] = s[0]; + d[M * 1] = s[1]; + d[M * 2] = s[2]; + d[M * 3] = s[3]; + d[M * 4] = s[4]; + d[M * 5] = s[5]; + d[M * 6] = s[6]; + d[M * 7] = s[7]; + + s += N; + d += 1; + m -= 1; + } + + Input += 8; + Output += M * 8; + n -= 8; + } + + // + // Transpose elements from the input matrix to the output matrix for the + // remaining columns. + // + + while (n > 0) { + + const uint8_t* s = Input; + uint8_t* d = Output; + size_t m = M; + + while (m >= 8) { + + d[0] = s[N * 0]; + d[1] = s[N * 1]; + d[2] = s[N * 2]; + d[3] = s[N * 3]; + d[4] = s[N * 4]; + d[5] = s[N * 5]; + d[6] = s[N * 6]; + d[7] = s[N * 7]; + + s += N * 8; + d += 8; + m -= 8; + } + + while (m > 0) { + + d[0] = s[0]; + + s += N; + d += 1; + m -= 1; + } + + Input += 1; + Output += M; + n -= 1; + } +} + +#endif diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 873110b3d4..32bae0d967 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -4,6 +4,7 @@ #include "core/providers/cpu/cpu_execution_provider.h" #include "core/framework/op_kernel.h" #include "core/framework/kernel_registry.h" +#include "core/mlas/inc/mlas.h" #ifndef DISABLE_CONTRIB_OPS #include "contrib_ops/cpu/cpu_contrib_kernels.h" @@ -283,7 +284,8 @@ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOn class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, QLinearMatMul); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t, MatMulInteger); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, ConvInteger); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, QLinearConv); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t, QLinearConv); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, int8_t, QLinearConv); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, Slice); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 11, Dropout); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, NonMaxSuppression); @@ -959,7 +961,12 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, +#if defined(MLAS_TARGET_AMD64_IX86) + BuildKernelCreateInfo, +#endif BuildKernelCreateInfo, BuildKernelCreateInfoShape()[1]; - const int64_t M = W->Shape()[0]; + Status ValidateInputShape(const TensorShape& input_shape, const TensorShape& weight_shape) const { + const int64_t C = input_shape[1]; + const int64_t M = weight_shape[0]; - if (X->Shape().NumDimensions() != W->Shape().NumDimensions()) { + if (input_shape.NumDimensions() != weight_shape.NumDimensions()) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "X num_dims does not match W num_dims.", - " X: ", X->Shape().ToString().c_str(), - " W: ", W->Shape().ToString().c_str()); + " X: ", input_shape.ToString().c_str(), + " W: ", weight_shape.ToString().c_str()); } - if (C != W->Shape()[1] * group) { + if (C != weight_shape[1] * group) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input channels C is not equal to kernel channels * group.", " C: ", C, - " kernel channels: ", W->Shape()[1], + " kernel channels: ", weight_shape[1], " group: ", group); } @@ -104,6 +103,10 @@ struct ConvAttributes { return Status::OK(); } + Status ValidateInputShape(const Tensor* input, const Tensor* weight) const { + return ValidateInputShape(input->Shape(), weight->Shape()); + } + Status InferOutputShape(const TensorShape& input_shape, const std::vector& kernel_shape, const std::vector& strides_p, diff --git a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc index 1941ffae2d..1d28da229c 100644 --- a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc +++ b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc @@ -13,35 +13,39 @@ namespace onnxruntime { -class QLinearConv : public OpKernel { +template +class QLinearConv; + +template <> +class QLinearConv : public OpKernel { public: - explicit QLinearConv(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info) {} + explicit QLinearConv(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info) {} Status Compute(OpKernelContext* context) const override; + protected: ConvAttributes conv_attrs_; }; -ONNX_OPERATOR_KERNEL_EX( +ONNX_CPU_OPERATOR_TYPED_KERNEL( QLinearConv, - kOnnxDomain, 10, - kCpuExecutionProvider, + uint8_t, KernelDefBuilder() .TypeConstraint("T1", DataTypeImpl::GetTensorType()) .TypeConstraint("T2", DataTypeImpl::GetTensorType()) .TypeConstraint("T3", DataTypeImpl::GetTensorType()) .TypeConstraint("T4", DataTypeImpl::GetTensorType()), - QLinearConv); + QLinearConv); -Status QLinearConv::Compute(OpKernelContext* context) const { - const auto* X = context->Input(0); - const auto* W = context->Input(3); +Status QLinearConv::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + const Tensor* W = context->Input(3); // validate offsets - auto X_zero_point = context->Input(2); - auto W_zero_point = context->Input(5); - auto Y_zero_point = context->Input(7); + const Tensor* X_zero_point = context->Input(2); + const Tensor* W_zero_point = context->Input(5); + const Tensor* Y_zero_point = context->Input(7); ORT_ENFORCE(IsScalarOr1ElementVector(X_zero_point), "QLinearConv : input zero point must be a scalar or 1D tensor of size 1"); ORT_ENFORCE(IsScalarOr1ElementVector(W_zero_point), @@ -54,9 +58,9 @@ Status QLinearConv::Compute(OpKernelContext* context) const { auto Y_zero_point_value = *(Y_zero_point->template Data()); // validate scale - auto X_scale = context->Input(1); - auto W_scale = context->Input(4); - auto Y_scale = context->Input(6); + const Tensor* X_scale = context->Input(1); + const Tensor* W_scale = context->Input(4); + const Tensor* Y_scale = context->Input(6); ORT_ENFORCE(IsScalarOr1ElementVector(X_scale), "QLinearConv : input scale must be a scalar or 1D tensor of size 1"); ORT_ENFORCE(IsScalarOr1ElementVector(W_scale), @@ -68,11 +72,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { auto W_scale_value = *(W_scale->template Data()); auto Y_scale_value = *(Y_scale->template Data()); - size_t num_inputs = OpKernel::Node().InputDefs().size(); - const Tensor* B = nullptr; - if (num_inputs == 9) { - B = context->Input(8); - } + const Tensor* B = context->Input(8); const int64_t N = X->Shape()[0]; const int64_t C = X->Shape()[1]; @@ -82,17 +82,19 @@ Status QLinearConv::Compute(OpKernelContext* context) const { std::vector kernel_shape; ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape)); + const size_t kernel_rank = kernel_shape.size(); + std::vector pads(conv_attrs_.pads); if (pads.empty()) { - pads.resize(kernel_shape.size() * 2, 0); + pads.resize(kernel_rank * 2, 0); } std::vector dilations(conv_attrs_.dilations); if (dilations.empty()) { - dilations.resize(kernel_shape.size(), 1); + dilations.resize(kernel_rank, 1); } std::vector strides(conv_attrs_.strides); if (strides.empty()) { - strides.resize(kernel_shape.size(), 1); + strides.resize(kernel_rank, 1); } std::vector Y_dims({N, M}); @@ -109,14 +111,15 @@ Status QLinearConv::Compute(OpKernelContext* context) const { const int64_t input_image_size = input_shape.Size(); const int64_t output_image_size = output_shape.Size(); const int64_t kernel_size = TensorShape(kernel_shape).Size(); - const int64_t X_offset = C / conv_attrs_.group * input_image_size; - const int64_t Y_offset = Y->Shape().Size() / Y->Shape()[0] / conv_attrs_.group; - const int64_t W_offset = W->Shape().Size() / conv_attrs_.group; - const int64_t B_offset = M / conv_attrs_.group; - const int64_t kernel_dim = C / conv_attrs_.group * kernel_size; - const int64_t col_buffer_size = kernel_dim * output_image_size; - const size_t kernel_rank = kernel_shape.size(); + const int64_t group_input_channels = W->Shape()[1]; + const int64_t group_output_channels = M / conv_attrs_.group; + + const int64_t X_offset = group_input_channels * input_image_size; + const int64_t Y_offset = Y->Shape().Size() / Y->Shape()[0] / conv_attrs_.group; + const int64_t kernel_dim = group_input_channels * kernel_size; + const int64_t W_offset = W->Shape().Size() / conv_attrs_.group; + const int64_t col_buffer_size = kernel_dim * output_image_size; AllocatorPtr alloc; ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); @@ -160,13 +163,13 @@ Status QLinearConv::Compute(OpKernelContext* context) const { const auto* Bdata = B != nullptr ? B->template Data() : nullptr; auto* Ydata = Y->template MutableData(); - for (int image_id = 0; image_id < N; ++image_id) { - for (int group_id = 0; group_id < conv_attrs_.group; ++group_id) { + for (int64_t image_id = 0; image_id < N; ++image_id) { + for (int64_t group_id = 0; group_id < conv_attrs_.group; ++group_id) { if (col_buffer_data != nullptr) { if (kernel_rank == 2) { math::Im2col()( Xdata, - C / conv_attrs_.group, + group_input_channels, input_shape[0], input_shape[1], kernel_shape[0], @@ -200,7 +203,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { } #ifdef MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT - QGemm(static_cast(M / conv_attrs_.group), + QGemm(static_cast(group_output_channels), static_cast(output_image_size), static_cast(kernel_dim), Wdata + group_id * W_offset, @@ -216,8 +219,8 @@ Status QLinearConv::Compute(OpKernelContext* context) const { MlasRequantizeOutput(gemm_output, Ydata, - Bdata != nullptr ? Bdata + group_id * B_offset : nullptr, - static_cast(M / conv_attrs_.group), + Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr, + static_cast(group_output_channels), static_cast(output_image_size), real_multiplier, Y_zero_point_value); @@ -228,12 +231,12 @@ Status QLinearConv::Compute(OpKernelContext* context) const { W_zero_point_value, X_zero_point_value, Y_zero_point_value, - static_cast(M / conv_attrs_.group), + static_cast(group_output_channels), static_cast(output_image_size), static_cast(kernel_dim), integer_multiplier, right_shift, - Bdata != nullptr ? Bdata + group_id * B_offset : nullptr); + Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr); #endif Xdata += X_offset; @@ -244,4 +247,422 @@ Status QLinearConv::Compute(OpKernelContext* context) const { return Status::OK(); } +#if defined(MLAS_TARGET_AMD64_IX86) + +template <> +class QLinearConv : public OpKernel { + public: + explicit QLinearConv(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info), is_W_packed_(false) {} + + Status Compute(OpKernelContext* context) const override; + Status PrePack(const Tensor& tensor, int input_idx, bool& is_packed) override; + + private: + static void ReorderFilter(const uint8_t* input, + uint8_t* output, + size_t output_channels, + size_t input_channels, + size_t kernel_size); + + ConvAttributes conv_attrs_; + TensorShape W_shape_; +#ifdef MLAS_SUPPORTS_PACKED_GEMM_U8X8 + BufferUniquePtr packed_W_buffer_; + size_t packed_W_size_; +#endif + BufferUniquePtr reordered_W_buffer_; + bool is_W_packed_; +}; + +ONNX_CPU_OPERATOR_TYPED_KERNEL( + QLinearConv, + 10, + int8_t, + KernelDefBuilder() + .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("T2", DataTypeImpl::GetTensorType()) + .TypeConstraint("T3", DataTypeImpl::GetTensorType()) + .TypeConstraint("T4", DataTypeImpl::GetTensorType()), + QLinearConv); + +void QLinearConv::ReorderFilter(const uint8_t* input, + uint8_t* output, + size_t output_channels, + size_t input_channels, + size_t kernel_size) { + for (size_t k = 0; k < kernel_size; k++) { + for (size_t ic = 0; ic < input_channels; ic++) { + for (size_t oc = 0; oc < output_channels; oc++) { + size_t index = (oc * input_channels * kernel_size) + (ic * kernel_size) + k; + *output++ = input[index]; + } + } + } +} + +Status QLinearConv::PrePack(const Tensor& tensor, int input_idx, bool& is_packed) { + is_packed = false; + + // Support packing the weight matrix. + if (input_idx != 3) { + return Status::OK(); + } + + const auto& shape = tensor.Shape(); + size_t rank = shape.NumDimensions(); + if (rank != 4) { + return Status::OK(); + } + + if (shape[0] % conv_attrs_.group != 0) { + return Status::OK(); + } + + // Note: The tensor has already been allocated with this tensor shape, so all + // shape indices are guaranteed to fit inside size_t. + const size_t output_channels = static_cast(shape[0]); + const size_t group_input_channels = static_cast(shape[1]); + const size_t kernel_size = static_cast(shape[2] * shape[3]); + + const size_t group_count = static_cast(conv_attrs_.group); + const size_t group_output_channels = output_channels / group_count; + const size_t kernel_dim = group_input_channels * kernel_size; + + const auto* Wdata = static_cast(tensor.DataRaw()); + W_shape_ = shape; + + auto alloc = Info().GetAllocator(0, OrtMemTypeDefault); + +#ifdef MLAS_SUPPORTS_PACKED_GEMM_U8X8 + packed_W_size_ = MlasGemmPackBSize(group_output_channels, kernel_dim, true); + + if (packed_W_size_ != 0) { + auto* packed_W = static_cast(alloc->Alloc(SafeInt(group_count) * packed_W_size_)); + packed_W_buffer_ = BufferUniquePtr(packed_W, BufferDeleter(alloc)); + + // Allocate a temporary buffer to hold the reordered oihw->ohwi filter for + // a single group. + // + // Note: The size of this buffer is less than or equal to the size of the original + // weight tensor, so the allocation size is guaranteed to fit inside size_t. + auto* group_reordered_W = static_cast(alloc->Alloc(group_output_channels * group_input_channels * kernel_size)); + BufferUniquePtr group_reordered_W_buffer(group_reordered_W, BufferDeleter(alloc)); + + const size_t W_offset = group_output_channels * kernel_dim; + + for (int64_t group_id = 0; group_id < conv_attrs_.group; ++group_id) { + ReorderFilter(Wdata, group_reordered_W, group_output_channels, group_input_channels, kernel_size); + MlasGemmPackB(group_output_channels, kernel_dim, group_reordered_W, group_output_channels, true, packed_W); + packed_W += packed_W_size_; + Wdata += W_offset; + } + + is_W_packed_ = true; + is_packed = true; + return Status::OK(); + } +#endif + + auto* reordered_W = static_cast(alloc->Alloc(SafeInt(sizeof(uint8_t)) * shape.Size())); + reordered_W_buffer_ = BufferUniquePtr(reordered_W, BufferDeleter(alloc)); + + ReorderFilter(Wdata, reordered_W, output_channels, group_input_channels, kernel_size); + + is_W_packed_ = true; + is_packed = true; + return Status::OK(); +} + +Status QLinearConv::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + const Tensor* W = is_W_packed_ ? nullptr : context->Input(3); + const auto& W_shape = is_W_packed_ ? W_shape_ : W->Shape(); + + const int64_t N = X->Shape()[0]; + const int64_t M = W_shape[0]; + + // validate offsets + const Tensor* X_zero_point = context->Input(2); + const Tensor* W_zero_point = context->Input(5); + const Tensor* Y_zero_point = context->Input(7); + ORT_ENFORCE(IsScalarOr1ElementVector(X_zero_point), + "QLinearConv : input zero point must be a scalar or 1D tensor of size 1"); + ORT_ENFORCE(IsScalarOr1ElementVector(Y_zero_point), + "QLinearConv : result zero point must be a scalar or 1D tensor of size 1"); + + auto X_zero_point_value = *(X_zero_point->template Data()); + auto Y_zero_point_value = *(Y_zero_point->template Data()); + + const auto& W_zero_point_shape = W_zero_point->Shape(); + if (W_zero_point_shape.NumDimensions() == 0 || + (W_zero_point_shape.NumDimensions() == 1 && (W_zero_point_shape[0] == 1 || W_zero_point_shape[0] == M))) { + const int64_t W_zero_point_size = W_zero_point_shape.Size(); + const auto* W_zero_point_data = W_zero_point->template Data(); + for (int64_t i = 0; i < W_zero_point_size; i++) { + if (W_zero_point_data[i] != 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "QLinearConv : filter zero point must be zero"); + } + } + } else { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "QLinearConv : filter zero point shape invalid"); + } + + // validate scale + const Tensor* X_scale = context->Input(1); + const Tensor* W_scale = context->Input(4); + const Tensor* Y_scale = context->Input(6); + ORT_ENFORCE(IsScalarOr1ElementVector(X_scale), + "QLinearConv : input scale must be a scalar or 1D tensor of size 1"); + ORT_ENFORCE(IsScalarOr1ElementVector(Y_scale), + "QLinearConv : result scale must be a scalar or 1D tensor of size 1"); + + auto X_scale_value = *(X_scale->template Data()); + auto Y_scale_value = *(Y_scale->template Data()); + + std::vector output_scales; + const auto& W_scale_shape = W_scale->Shape(); + if (W_scale_shape.NumDimensions() == 0 || + (W_scale_shape.NumDimensions() == 1 && (W_scale_shape[0] == 1 || W_scale_shape[0] == M))) { + const int64_t W_scale_size = W_scale_shape.Size(); + const auto* W_scale_data = W_scale->template Data(); + output_scales.resize(static_cast(W_scale_size)); + for (int64_t i = 0; i < W_scale_size; i++) { + output_scales[i] = (X_scale_value * W_scale_data[i] / Y_scale_value); + } + } else { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "QLinearConv : filter scale shape invalid"); + } + + const Tensor* B = context->Input(8); + + ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X->Shape(), W_shape)); + + std::vector kernel_shape; + ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W_shape, kernel_shape)); + + const size_t kernel_rank = kernel_shape.size(); + ORT_ENFORCE(kernel_rank == 2, "QLinearConv : must be 2D convolution"); + + std::vector pads(conv_attrs_.pads); + if (pads.empty()) { + pads.resize(kernel_rank * 2, 0); + } + std::vector dilations(conv_attrs_.dilations); + if (dilations.empty()) { + dilations.resize(kernel_rank, 1); + } + std::vector strides(conv_attrs_.strides); + if (strides.empty()) { + strides.resize(kernel_rank, 1); + } + + std::vector Y_dims({N, M}); + TensorShape input_shape = X->Shape().Slice(2); + ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShape(input_shape, kernel_shape, strides, dilations, pads, Y_dims)); + Tensor* Y = context->Output(0, TensorShape(Y_dims)); + TensorShape output_shape = Y->Shape().Slice(2); + + // Bail out early if one of the dimensions is zero. + if (Y->Shape().Size() == 0) { + return Status::OK(); + } + + const int64_t input_image_size = input_shape.Size(); + const int64_t output_image_size = output_shape.Size(); + const int64_t kernel_size = TensorShape(kernel_shape).Size(); + + const int64_t group_count = conv_attrs_.group; + const int64_t group_input_channels = W_shape[1]; + const int64_t group_output_channels = M / group_count; + + const int64_t X_offset = group_input_channels * input_image_size; + const int64_t Y_offset = group_output_channels * output_image_size; + const int64_t kernel_dim = group_input_channels * kernel_size; + const int64_t col_buffer_size = kernel_dim * output_image_size; + + AllocatorPtr alloc; + ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + + // Use an intermediate int32_t buffer for the GEMM computation before + // requantizing to the output type. + auto gemm_output_data = alloc->Alloc(SafeInt(sizeof(int32_t)) * Y_offset); + BufferUniquePtr gemm_output_buffer(gemm_output_data, BufferDeleter(alloc)); + auto* gemm_output = static_cast(gemm_output_buffer.get()); + + const auto* Xdata = X->template Data(); + const auto* Bdata = B != nullptr ? B->template Data() : nullptr; + auto* Ydata = Y->template MutableData(); + + auto* transpose_input = static_cast(alloc->Alloc(SafeInt(sizeof(uint8_t)) * X_offset)); + BufferUniquePtr transpose_input_buffer(transpose_input, BufferDeleter(alloc)); + + auto* transpose_output = static_cast(alloc->Alloc(SafeInt(sizeof(uint8_t)) * Y_offset)); + BufferUniquePtr transpose_output_buffer(transpose_output, BufferDeleter(alloc)); + + // Handle the case of a dynamic weight filter. + BufferUniquePtr reordered_W_buffer; + uint8_t* reordered_W = nullptr; + bool use_reordered_W = true; +#ifdef MLAS_SUPPORTS_PACKED_GEMM_U8X8 + if (packed_W_buffer_) { + use_reordered_W = false; + } +#endif + if (use_reordered_W) { + if (reordered_W_buffer_) { + reordered_W = static_cast(reordered_W_buffer_.get()); + } else { + // Weight tensor was not constant or prepacking is disabled. + reordered_W = static_cast(alloc->Alloc(SafeInt(sizeof(uint8_t)) * W_shape.Size())); + reordered_W_buffer = BufferUniquePtr(reordered_W, BufferDeleter(alloc)); + ReorderFilter(static_cast(W->DataRaw()), + reordered_W, + static_cast(M), + static_cast(group_input_channels), + static_cast(kernel_size)); + } + } + + // Pointwise convolutions can use the original input tensor in place, + // otherwise a temporary buffer is required for the im2col transform. + BufferUniquePtr col_buffer; + if (kernel_size != 1 || !conv_attrs_.HasStridesOneAndNoPadding()) { + auto* col_data = alloc->Alloc(SafeInt(sizeof(uint8_t)) * col_buffer_size); + col_buffer = BufferUniquePtr(col_data, BufferDeleter(alloc)); + } + auto* col_buffer_data = static_cast(col_buffer.get()); + + // Replicate the logic from MlasGemmU8X8Schedule to control the number of + // worker threads used for the convolution. + constexpr int32_t maximum_thread_count = 16; + constexpr double thread_complexity = static_cast(64 * 1024); + + const double complexity = static_cast(output_image_size) * + static_cast(group_output_channels) * + static_cast(kernel_dim); + + int32_t thread_count = maximum_thread_count; + if (complexity < thread_complexity * maximum_thread_count) { + thread_count = static_cast(complexity / thread_complexity) + 1; + } + if (thread_count > output_image_size) { + // Ensure that every thread produces at least one output. + thread_count = static_cast(output_image_size); + } + + concurrency::ThreadPool* thread_pool = context->GetOperatorThreadPool(); + thread_count = std::min(thread_count, concurrency::ThreadPool::DegreeOfParallelism(thread_pool)); + + for (int64_t image_id = 0; image_id < N; ++image_id) { + for (int64_t group_id = 0; group_id < group_count; ++group_id) { + // Transpose the input from channels first (NCHW) to channels last (NHWC). + MlasTranspose(Xdata, + transpose_input, + static_cast(group_input_channels), + static_cast(input_image_size)); + + auto conv_worker = [&](ptrdiff_t batch) { + auto work = concurrency::ThreadPool::PartitionWork(batch, thread_count, static_cast(output_image_size)); + int64_t output_start = static_cast(work.start); + int64_t output_count = static_cast(work.end - work.start); + + // Prepare the im2col transformation or use the input buffer directly for + // pointwise convolutions. + uint8_t* worker_gemm_input; + if (col_buffer_data != nullptr) { + worker_gemm_input = col_buffer_data + output_start * kernel_dim; + math::Im2col()( + transpose_input, + group_input_channels, + input_shape[0], + input_shape[1], + kernel_shape[0], + kernel_shape[1], + dilations[0], + dilations[1], + pads[0], + pads[1], + strides[0], + strides[1], + output_shape[1], + output_start, + output_count, + worker_gemm_input, + X_zero_point_value); + } else { + worker_gemm_input = transpose_input + output_start * kernel_dim; + } + + auto* worker_gemm_output = gemm_output + output_start * group_output_channels; + auto* worker_transpose_output = transpose_output + output_start * group_output_channels; + +#ifdef MLAS_SUPPORTS_PACKED_GEMM_U8X8 + if (packed_W_buffer_) { + MlasGemm(static_cast(output_count), + static_cast(group_output_channels), + static_cast(kernel_dim), + worker_gemm_input, + static_cast(kernel_dim), + X_zero_point_value, + static_cast(packed_W_buffer_.get()) + group_id * packed_W_size_, + 0, + true, + worker_gemm_output, + static_cast(group_output_channels), + nullptr); + } else +#endif + { + MlasGemm(static_cast(output_count), + static_cast(group_output_channels), + static_cast(kernel_dim), + worker_gemm_input, + static_cast(kernel_dim), + X_zero_point_value, + reordered_W + group_id * group_output_channels, + static_cast(M), + 0, + true, + worker_gemm_output, + static_cast(group_output_channels), + nullptr); + } + + if (output_scales.size() == 1) { + MlasRequantizeOutputColumn(worker_gemm_output, + worker_transpose_output, + Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr, + static_cast(output_count), + static_cast(group_output_channels), + output_scales[0], + Y_zero_point_value); + } else { + MlasRequantizeOutputColumn(worker_gemm_output, + worker_transpose_output, + Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr, + static_cast(output_count), + static_cast(group_output_channels), + output_scales.data() + group_id * group_output_channels, + Y_zero_point_value); + } + }; + + concurrency::ThreadPool::TrySimpleParallelFor(thread_pool, thread_count, conv_worker); + + // Transpose the output from channels last (NHWC) to channels first (NCHW). + MlasTranspose(transpose_output, + Ydata, + static_cast(output_image_size), + static_cast(group_output_channels)); + + Xdata += X_offset; + Ydata += Y_offset; + } + } + + return Status::OK(); +} + +#endif + } // namespace onnxruntime diff --git a/onnxruntime/core/util/math.h b/onnxruntime/core/util/math.h index 88f0407386..b5253a5946 100644 --- a/onnxruntime/core/util/math.h +++ b/onnxruntime/core/util/math.h @@ -234,6 +234,24 @@ struct Im2col { int64_t stride_w, T* data_col, T padding_value = 0); + void operator()( + const T* data_im, + int64_t channels, + int64_t input_h, + int64_t input_w, + int64_t kernel_h, + int64_t kernel_w, + int64_t dilation_h, + int64_t dilation_w, + int64_t pad_t, + int64_t pad_l, + int64_t stride_h, + int64_t stride_w, + int64_t output_w, + int64_t output_start, + int64_t output_count, + T* data_col, + T padding_value = 0); }; template diff --git a/onnxruntime/core/util/math_cpu.cc b/onnxruntime/core/util/math_cpu.cc index 10b8807741..42ccc24b3c 100644 --- a/onnxruntime/core/util/math_cpu.cc +++ b/onnxruntime/core/util/math_cpu.cc @@ -297,7 +297,6 @@ DELEGATE_SIMPLE_UNARY_FUNCTION(float, Log, log) DELEGATE_SIMPLE_UNARY_FUNCTION(float, Sqr, square) #undef DELEGATE_SIMPLE_UNARY_FUNCTION - #define EIGEN_SIMPLE_BINARY_FUNCTION(T, Funcname, expr) \ template <> \ void Funcname(int N, const T* a, const T* b, T* y, CPUMathUtil*) { \ @@ -412,12 +411,12 @@ void Im2col::operator()(const T* data_im, int64_t channel template struct Im2col; template struct Im2col; -template <> -void Im2col::operator()(const float* data_im, int64_t channels, int64_t height, - int64_t width, int64_t kernel_h, int64_t kernel_w, - int64_t dilation_h, int64_t dilation_w, int64_t pad_t, - int64_t pad_l, int64_t pad_b, int64_t pad_r, int64_t stride_h, - int64_t stride_w, float* data_col, float padding_value) { +template +void Im2col::operator()(const T* data_im, int64_t channels, int64_t height, + int64_t width, int64_t kernel_h, int64_t kernel_w, + int64_t dilation_h, int64_t dilation_w, int64_t pad_t, + int64_t pad_l, int64_t pad_b, int64_t pad_r, int64_t stride_h, + int64_t stride_w, T* data_col, T padding_value) { const int64_t dkernel_h = dilation_h * (kernel_h - 1) + 1; const int64_t dkernel_w = dilation_w * (kernel_w - 1) + 1; @@ -430,13 +429,11 @@ void Im2col::operator()(const float* data_im, int64_t for (int64_t w = 0; w < width_col; ++w) { for (int64_t ih = h_pad; ih < h_pad + dkernel_h; ih += dilation_h) { for (int64_t iw = w_pad; iw < w_pad + dkernel_w; iw += dilation_w) { - if (ih >= 0 && ih < height && iw >= 0 && iw < width) { - memcpy(data_col, data_im + (ih * width + iw) * channels, - sizeof(float) * channels); + if (is_a_ge_zero_and_a_lt_b(ih, height) && is_a_ge_zero_and_a_lt_b(iw, width)) { + data_col = std::copy_n(data_im + (ih * width + iw) * channels, channels, data_col); } else { - std::fill_n(data_col, channels, padding_value); + data_col = std::fill_n(data_col, channels, padding_value); } - data_col += channels; } } w_pad += stride_w; @@ -445,6 +442,59 @@ void Im2col::operator()(const float* data_im, int64_t } } +template +void Im2col::operator()(const T* data_im, int64_t channels, int64_t input_h, + int64_t input_w, int64_t kernel_h, int64_t kernel_w, + int64_t dilation_h, int64_t dilation_w, int64_t pad_t, + int64_t pad_l, int64_t stride_h, int64_t stride_w, + int64_t output_w, int64_t output_start, int64_t output_count, + T* data_col, T padding_value) { + for (int64_t m = output_start; m < output_start + output_count; m++) { + int64_t mh = m / output_w; + int64_t mw = m % output_w; + + int64_t oh = mh * stride_h; + int64_t ow = mw * stride_w; + + for (int64_t kh = 0; kh < kernel_h; kh++) { + int64_t ih = kh * dilation_h + oh - pad_t; + + if (is_a_ge_zero_and_a_lt_b(ih, input_h)) { + if (dilation_w == 1) { + int64_t kw = kernel_w; + int64_t iw = ow - pad_l; + while (kw > 0) { + if (is_a_ge_zero_and_a_lt_b(iw, input_w)) { + // Increase the copy count size to reduce the number of copy calls. + int64_t batch_w = std::min(kw, input_w - iw); + data_col = std::copy_n(data_im + (ih * input_w + iw) * channels, batch_w * channels, data_col); + iw += batch_w; + kw -= batch_w; + } else { + data_col = std::fill_n(data_col, channels, padding_value); + iw++; + kw--; + } + } + } else { + for (int64_t kw = 0; kw < kernel_w; kw++) { + int64_t iw = kw * dilation_w + ow - pad_l; + if (is_a_ge_zero_and_a_lt_b(iw, input_w)) { + data_col = std::copy_n(data_im + (ih * input_w + iw) * channels, channels, data_col); + } else { + data_col = std::fill_n(data_col, channels, padding_value); + } + } + } + } else { + data_col = std::fill_n(data_col, kernel_w * channels, padding_value); + } + } + } +} + +template struct Im2col; + template <> void Col2im(const float* data_col, int64_t channels, int64_t height, int64_t width, int64_t kernel_h, int64_t kernel_w, diff --git a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc index 3f6678f2f4..a3341b0978 100644 --- a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc @@ -3,6 +3,8 @@ #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" +#include "core/mlas/inc/mlas.h" +#include namespace onnxruntime { namespace test { @@ -258,6 +260,337 @@ TEST(QLinearConvTest, WithGroup_2D) { {kNGraphExecutionProvider}); } +#if defined(MLAS_TARGET_AMD64_IX86) + +template +class QLinearConvOpTester { + private: + template + struct QuantizedTensor { + std::vector data_; + std::vector shape_; + std::vector scale_; + T zero_point_{0}; + }; + + std::default_random_engine generator_{1234}; + QuantizedTensor X_; + QuantizedTensor W_; + std::vector B_; + std::vector pads_; + std::vector strides_; + std::vector dilations_; + int64_t groups_{0}; + float output_scale_{1.0f}; + T1 output_zero_point_{0}; + + static size_t ShapeSize(const std::vector& shape) { + return static_cast(std::accumulate(shape.cbegin(), shape.cend(), 1LL, std::multiplies())); + } + + template + void GenerateRandom(QuantizedTensor& tensor, + const std::vector& shape, + float scale, + T zero_point, + int32_t min_value, + int32_t max_value) { + std::uniform_int_distribution distribution(min_value, max_value); + size_t shape_size = ShapeSize(shape); + tensor.data_.resize(shape_size); + for (size_t n = 0; n < shape_size; n++) { + tensor.data_[n] = static_cast(distribution(generator_)); + } + tensor.shape_ = shape; + tensor.scale_ = {scale}; + tensor.zero_point_ = {zero_point}; + } + + template + struct RequantizeValues { + RequantizeValues(int32_t zero_point) { + min_value_ = static_cast(static_cast(std::numeric_limits::min()) - zero_point); + max_value_ = static_cast(static_cast(std::numeric_limits::max()) - zero_point); + zero_point_ = static_cast(zero_point); + } + float min_value_; + float max_value_; + float zero_point_; + }; + + inline float RoundHalfToEven(float input) { + if (!std::isfinite(input)) { + return input; + } + // std::remainder returns x - n, where n is the integral value nearest to x. When |x - n| = 0.5, n is chosen to be even + return input - std::remainderf(input, 1.f); + } + + template + T RequantizeOutput(int32_t sum, float scale, RequantizeValues& requantize_values) { + float f = static_cast(sum) * scale; + f = std::min(f, requantize_values.max_value_); + f = std::max(f, requantize_values.min_value_); + return static_cast(RoundHalfToEven(f) + requantize_values.zero_point_); + } + + void ComputeExpectedOutput(std::vector& Y_data, std::vector& Y_shape) { + ORT_ENFORCE(W_.shape_.size() > 2); + ORT_ENFORCE(X_.shape_.size() == W_.shape_.size()); + + const size_t kernel_rank = W_.shape_.size() - 2; + + const int64_t batch_count = X_.shape_[0]; + const int64_t input_channels = X_.shape_[1]; + const int64_t output_channels = W_.shape_[0]; + const int64_t group_count = std::max(groups_, 1LL); + const int64_t group_input_channels = W_.shape_[1]; + const int64_t group_output_channels = output_channels / group_count; + + ORT_ENFORCE(input_channels == group_input_channels * group_count); + ORT_ENFORCE(output_channels == group_output_channels * group_count); + + const int64_t* input_shape = X_.shape_.data() + 2; + const int64_t* kernel_shape = W_.shape_.data() + 2; + + std::vector pads(pads_); + if (pads.empty()) { + pads.resize(kernel_rank * 2, 0); + } + std::vector dilations(dilations_); + if (dilations.empty()) { + dilations.resize(kernel_rank, 1); + } + std::vector strides(strides_); + if (strides.empty()) { + strides.resize(kernel_rank, 1); + } + + // Compute the expected shape of the output. + Y_shape.reserve(kernel_rank + 2); + Y_shape.push_back(batch_count); + Y_shape.push_back(output_channels); + for (size_t n = 0; n < kernel_rank; n++) { + Y_shape.push_back(((input_shape[n] + pads[n] + pads[kernel_rank + n]) - + (dilations[n] * (kernel_shape[n] - 1) + 1)) / strides[n] + 1); + } + const int64_t* output_shape = Y_shape.data() + 2; + Y_data.resize(ShapeSize(Y_shape)); + + const int64_t input_h = input_shape[0]; + const int64_t input_w = input_shape[1]; + const int64_t input_image_size = input_h * input_w; + const int64_t kernel_h = kernel_shape[0]; + const int64_t kernel_w = kernel_shape[1]; + const int64_t kernel_size = kernel_h * kernel_w; + const int64_t output_h = output_shape[0]; + const int64_t output_w = output_shape[1]; + const int64_t pad_t = pads[0]; + const int64_t pad_l = pads[1]; + const int64_t dilation_h = dilations[0]; + const int64_t dilation_w = dilations[1]; + const int64_t stride_h = strides[0]; + const int64_t stride_w = strides[1]; + const int32_t X_zero_point = X_.zero_point_; + + const T1* Xdata = X_.data_.data(); + T1* Ydata = Y_data.data(); + + RequantizeValues requantize_values(output_zero_point_); + + for (int64_t batch = 0; batch < batch_count; batch++) { + const T2* weight_group = W_.data_.data(); + for (int64_t group = 0; group < group_count; group++) { + const T2* weight_row = weight_group; + + for (int64_t oc = 0; oc < group_output_channels; oc++) { + int64_t channel_index = group * group_output_channels + oc; + int32_t bias = B_.empty() ? 0 : B_[channel_index]; + float weight_scale = W_.scale_[(W_.scale_.size() == 1) ? 0 : channel_index]; + float requantize_scale = (X_.scale_[0] * weight_scale) / output_scale_; + + for (int64_t oh = 0; oh < output_h; oh++) { + for (int64_t ow = 0; ow < output_w; ow++) { + int32_t sum = bias; + const T1* input_image = Xdata; + const T2* weight_data = weight_row; + for (int64_t ic = 0; ic < group_input_channels; ic++) { + for (int64_t kh = 0; kh < kernel_h; kh++) { + int64_t ih = kh * dilation_h + oh * stride_h - pad_t; + for (int64_t kw = 0; kw < kernel_w; kw++) { + int64_t iw = kw * dilation_w + ow * stride_w - pad_l; + int32_t w_value = static_cast(*weight_data++); + if (static_cast(ih) < static_cast(input_h) && + static_cast(iw) < static_cast(input_w)) { + int32_t x_value = static_cast(input_image[ih * input_w + iw]) - X_zero_point; + sum += x_value * w_value; + } + } + } + input_image += input_image_size; + } + *Ydata++ = RequantizeOutput(sum, requantize_scale, requantize_values); + } + } + + weight_row += group_input_channels * kernel_size; + } + + Xdata += group_input_channels * input_image_size; + weight_group += group_output_channels * group_input_channels * kernel_size; + } + } + } + + void Run(bool all_input_initializer_except_x) { + OpTester test("QLinearConv", 10); + + std::vector Y_data; + std::vector Y_shape; + ComputeExpectedOutput(Y_data, Y_shape); + + test.AddInput("x", X_.shape_, X_.data_); + test.AddInput("x_scale", {}, X_.scale_, all_input_initializer_except_x); + test.AddInput("x_zero_point", {}, {X_.zero_point_}); + + const std::vector W_scale_shape{static_cast(W_.scale_.size())}; + test.AddInput("w", W_.shape_, W_.data_, all_input_initializer_except_x); + test.AddInput("w_scale", W_scale_shape, W_.scale_, all_input_initializer_except_x); + test.AddInput("w_zero_point", {}, {W_.zero_point_}); + + test.AddInput("y_scale", {}, {output_scale_}, all_input_initializer_except_x); + test.AddInput("y_zero_point", {}, {output_zero_point_}); + + if (!B_.empty()) { + const std::vector B_shape{static_cast(B_.size())}; + test.AddInput("b", B_shape, B_); + } + + test.AddOutput("y", Y_shape, Y_data); + + if (!pads_.empty()) { + test.AddAttribute("pads", pads_); + } + if (!strides_.empty()) { + test.AddAttribute("strides", strides_); + } + if (!dilations_.empty()) { + test.AddAttribute("dilations", dilations_); + } + if (groups_ > 0) { + test.AddAttribute("group", groups_); + } + + test.Run(OpTester::ExpectResult::kExpectSuccess, ""); + } + + public: + QLinearConvOpTester() { + } + + void GenerateRandomInput(const std::vector& shape, float scale, T1 zero_point) { + GenerateRandom(X_, shape, scale, zero_point, 0, 63); + } + + void GenerateRandomWeights(const std::vector& shape, float scale, T2 zero_point) { + GenerateRandom(W_, shape, scale, zero_point, -63, 63); + } + + void SetWeightScales(const std::vector& scales) { + W_.scale_ = scales; + } + + void GenerateRandomBias() { + ORT_ENFORCE(W_.shape_.size() >= 1); + const size_t output_channels = static_cast(W_.shape_[0]); + B_.resize(output_channels); + std::uniform_int_distribution distribution(-423, 423); + for (size_t n = 0; n < output_channels; n++) { + B_[n] = distribution(generator_); + } + } + + void SetPads(const std::vector& pads) { + pads_ = pads; + } + + void SetStrides(const std::vector& strides) { + strides_ = strides; + } + + void SetDilations(const std::vector& dilations) { + dilations_ = dilations; + } + + void SetGroups(int64_t groups) { + groups_ = groups; + } + + void SetOutputScaleAndZeroPoint(float output_scale, T1 output_zero_point) { + output_scale_ = output_scale; + output_zero_point_ = output_zero_point; + } + + void Run() { + for (bool all_input_initializer_except_x : std::initializer_list{false, true}) { + Run(all_input_initializer_except_x); + } + } +}; + +TEST(QLinearConvTest, Conv2D_U8S8) { + QLinearConvOpTester test; + test.GenerateRandomInput({3, 24, 15, 11}, .05f, 4); + test.GenerateRandomWeights({32, 24, 3, 3}, .125f, 0); + test.GenerateRandomBias(); + test.SetPads({1, 1, 1, 1}); + test.SetOutputScaleAndZeroPoint(.55f, 54); + test.Run(); +} + +TEST(QLinearConvTest, Conv2D_U8S8_Dilations) { + QLinearConvOpTester test; + test.GenerateRandomInput({1, 4, 19, 16}, .02f, 20); + test.GenerateRandomWeights({6, 4, 3, 2}, .11f, 0); + test.SetDilations({2, 2}); + test.SetOutputScaleAndZeroPoint(.24f, 15); + test.Run(); +} + +TEST(QLinearConvTest, Conv2D_U8S8_Strides) { + QLinearConvOpTester test; + test.GenerateRandomInput({1, 7, 18, 24}, .04f, 16); + test.GenerateRandomWeights({5, 7, 2, 3}, .14f, 0); + test.SetStrides({2, 2}); + test.SetOutputScaleAndZeroPoint(.31f, 30); + test.Run(); +} + +TEST(QLinearConvTest, Conv2D_U8S8_Groups) { + QLinearConvOpTester test; + test.GenerateRandomInput({1, 8, 13, 17}, .03f, 7); + test.GenerateRandomWeights({12, 4, 3, 3}, .10f, 0); + test.GenerateRandomBias(); + test.SetPads({1, 1, 1, 1}); + test.SetGroups(2); + test.SetOutputScaleAndZeroPoint(.76f, 88); + test.Run(); +} + +TEST(QLinearConvTest, Conv2D_U8S8_Groups_PerChannel) { + QLinearConvOpTester test; + test.GenerateRandomInput({1, 8, 13, 17}, .03f, 7); + test.GenerateRandomWeights({10, 4, 3, 3}, .10f, 0); + test.SetWeightScales({.15f, .14f, .11f, .13f, .15f, .09f, .12f, .16f, .17f, .07f}); + test.GenerateRandomBias(); + test.SetPads({1, 1, 1, 1}); + test.SetGroups(2); + test.SetOutputScaleAndZeroPoint(.76f, 88); + test.Run(); +} + +#endif + } // namespace } // namespace test } // namespace onnxruntime