mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
Add QLinearConv for activations=u8, weights=s8 (#5510)
This commit is contained in:
Tracy Sharpe
GitHub
parent
280cdf31f5
commit
45483dcf1f
10 changed files with 1326 additions and 61 deletions
|
|
@ -9,6 +9,7 @@ set(mlas_common_srcs
|
|||
${ONNXRUNTIME_ROOT}/core/mlas/lib/qgemm.cpp
|
||||
${ONNXRUNTIME_ROOT}/core/mlas/lib/convolve.cpp
|
||||
${ONNXRUNTIME_ROOT}/core/mlas/lib/pooling.cpp
|
||||
${ONNXRUNTIME_ROOT}/core/mlas/lib/transpose.cpp
|
||||
${ONNXRUNTIME_ROOT}/core/mlas/lib/reorder.cpp
|
||||
${ONNXRUNTIME_ROOT}/core/mlas/lib/snchwc.cpp
|
||||
${ONNXRUNTIME_ROOT}/core/mlas/lib/activate.cpp
|
||||
|
|
|
|||
|
|
@ -479,6 +479,28 @@ MlasConvertHalfToFloatBuffer(
|
|||
size_t Count
|
||||
);
|
||||
|
||||
//
|
||||
// Transpose routines.
|
||||
//
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasTranspose(
|
||||
const uint8_t* Input,
|
||||
uint8_t* Output,
|
||||
size_t M,
|
||||
size_t N
|
||||
);
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasTranspose(
|
||||
const uint32_t* Input,
|
||||
uint32_t* Output,
|
||||
size_t M,
|
||||
size_t N
|
||||
);
|
||||
|
||||
//
|
||||
// Buffer reordering routines.
|
||||
//
|
||||
|
|
@ -603,6 +625,30 @@ MlasRequantizeOutput(
|
|||
uint8_t ZeroPoint
|
||||
);
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutputColumn(
|
||||
const int32_t* Input,
|
||||
uint8_t* Output,
|
||||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
const float Scale,
|
||||
uint8_t ZeroPoint
|
||||
);
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutputColumn(
|
||||
const int32_t* Input,
|
||||
uint8_t* Output,
|
||||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
const float* Scale,
|
||||
uint8_t ZeroPoint
|
||||
);
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasFindMinMaxElement(
|
||||
|
|
|
|||
|
|
@ -421,6 +421,218 @@ Return Value:
|
|||
}
|
||||
}
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutputColumn(
|
||||
const int32_t* Input,
|
||||
uint8_t* Output,
|
||||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
float Scale,
|
||||
uint8_t ZeroPoint
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine requantizes the intermediate buffer to the output buffer
|
||||
optionally adding the supplied bias.
|
||||
|
||||
Arguments:
|
||||
|
||||
Input - Supplies the input matrix.
|
||||
|
||||
Output - Supplies the output matrix.
|
||||
|
||||
Bias - Supplies the optional bias vector to be added to the input buffer
|
||||
before requantization.
|
||||
|
||||
Buffer - Supplies the output matrix.
|
||||
|
||||
M - Supplies the number of elements of the bias vector and the number of
|
||||
rows in the output matrix.
|
||||
|
||||
N - Supplies the number of columns of the output matrix.
|
||||
|
||||
Scale - Supplies the quantization scale.
|
||||
|
||||
ZeroPoint - Supplies the quantization zero point value.
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
{
|
||||
MLAS_FLOAT32X4 ScaleVector = MlasBroadcastFloat32x4(Scale);
|
||||
MLAS_FLOAT32X4 MinimumValueVector = MlasBroadcastFloat32x4(float(0 - ZeroPoint));
|
||||
MLAS_FLOAT32X4 MaximumValueVector = MlasBroadcastFloat32x4(float(255 - ZeroPoint));
|
||||
MLAS_INT32X4 ZeroPointVector = MlasBroadcastInt32x4(ZeroPoint);
|
||||
MLAS_INT32X4 BiasVector = _mm_setzero_si128();
|
||||
|
||||
//
|
||||
// Step through each row of the output matrix.
|
||||
//
|
||||
|
||||
while (M-- > 0) {
|
||||
|
||||
const int32_t* bias = Bias;
|
||||
|
||||
size_t n = N;
|
||||
|
||||
while (n >= 4) {
|
||||
|
||||
MLAS_INT32X4 IntegerVector = _mm_loadu_si128((const __m128i *)Input);
|
||||
|
||||
if (bias != nullptr) {
|
||||
BiasVector = _mm_loadu_si128((const __m128i*)bias);
|
||||
bias += 4;
|
||||
}
|
||||
|
||||
IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector,
|
||||
ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector);
|
||||
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
|
||||
*((int32_t*)Output) = _mm_cvtsi128_si32(IntegerVector);
|
||||
|
||||
Input += 4;
|
||||
Output += 4;
|
||||
n -= 4;
|
||||
}
|
||||
|
||||
while (n > 0) {
|
||||
|
||||
MLAS_INT32X4 IntegerVector = _mm_cvtsi32_si128(*Input);
|
||||
|
||||
if (bias != nullptr) {
|
||||
BiasVector = _mm_cvtsi32_si128(*bias);
|
||||
bias += 1;
|
||||
}
|
||||
|
||||
IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector,
|
||||
ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector);
|
||||
|
||||
*Output = (uint8_t)_mm_cvtsi128_si32(IntegerVector);
|
||||
|
||||
Input += 1;
|
||||
Output += 1;
|
||||
n -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutputColumn(
|
||||
const int32_t* Input,
|
||||
uint8_t* Output,
|
||||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
const float* Scale,
|
||||
uint8_t ZeroPoint
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine requantizes the intermediate buffer to the output buffer
|
||||
optionally adding the supplied bias.
|
||||
|
||||
Arguments:
|
||||
|
||||
Input - Supplies the input matrix.
|
||||
|
||||
Output - Supplies the output matrix.
|
||||
|
||||
Bias - Supplies the optional bias vector to be added to the input buffer
|
||||
before requantization.
|
||||
|
||||
Buffer - Supplies the output matrix.
|
||||
|
||||
M - Supplies the number of elements of the bias vector and the number of
|
||||
rows in the output matrix.
|
||||
|
||||
N - Supplies the number of columns of the output matrix.
|
||||
|
||||
Scale - Supplies the quantization scale vector.
|
||||
|
||||
ZeroPoint - Supplies the quantization zero point value.
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
{
|
||||
MLAS_FLOAT32X4 MinimumValueVector = MlasBroadcastFloat32x4(float(0 - ZeroPoint));
|
||||
MLAS_FLOAT32X4 MaximumValueVector = MlasBroadcastFloat32x4(float(255 - ZeroPoint));
|
||||
MLAS_INT32X4 ZeroPointVector = MlasBroadcastInt32x4(ZeroPoint);
|
||||
MLAS_INT32X4 BiasVector = _mm_setzero_si128();
|
||||
|
||||
//
|
||||
// Step through each row of the output matrix.
|
||||
//
|
||||
|
||||
while (M-- > 0) {
|
||||
|
||||
const int32_t* bias = Bias;
|
||||
const float* scale = Scale;
|
||||
|
||||
size_t n = N;
|
||||
|
||||
while (n >= 4) {
|
||||
|
||||
MLAS_INT32X4 IntegerVector = _mm_loadu_si128((const __m128i *)Input);
|
||||
|
||||
if (bias != nullptr) {
|
||||
BiasVector = _mm_loadu_si128((const __m128i*)bias);
|
||||
bias += 4;
|
||||
}
|
||||
|
||||
MLAS_FLOAT32X4 ScaleVector = MlasLoadFloat32x4(scale);
|
||||
scale += 4;
|
||||
|
||||
IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector,
|
||||
ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector);
|
||||
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
|
||||
*((int32_t*)Output) = _mm_cvtsi128_si32(IntegerVector);
|
||||
|
||||
Input += 4;
|
||||
Output += 4;
|
||||
n -= 4;
|
||||
}
|
||||
|
||||
while (n > 0) {
|
||||
|
||||
MLAS_INT32X4 IntegerVector = _mm_cvtsi32_si128(*Input);
|
||||
|
||||
if (bias != nullptr) {
|
||||
BiasVector = _mm_cvtsi32_si128(*bias);
|
||||
bias += 1;
|
||||
}
|
||||
|
||||
MLAS_FLOAT32X4 ScaleVector = _mm_load_ss(scale);
|
||||
scale += 1;
|
||||
|
||||
IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector,
|
||||
ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector);
|
||||
|
||||
*Output = (uint8_t)_mm_cvtsi128_si32(IntegerVector);
|
||||
|
||||
Input += 1;
|
||||
Output += 1;
|
||||
n -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
void
|
||||
|
|
|
|||
174
onnxruntime/core/mlas/lib/transpose.cpp
Normal file
174
onnxruntime/core/mlas/lib/transpose.cpp
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
transpose.cpp
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the transpose operation.
|
||||
|
||||
--*/
|
||||
|
||||
#include "mlasi.h"
|
||||
|
||||
#ifdef MLAS_TARGET_AMD64_IX86
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasTranspose(
|
||||
const uint8_t* Input,
|
||||
uint8_t* Output,
|
||||
size_t M,
|
||||
size_t N
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine transposes the input matrix (M rows by N columns) to the
|
||||
output matrix (N rows by M columns).
|
||||
|
||||
Arguments:
|
||||
|
||||
Input - Supplies the input buffer.
|
||||
|
||||
Output - Supplies the output buffer.
|
||||
|
||||
M - Supplies the number of rows for the input matrix and the number of
|
||||
columns for the output matrix.
|
||||
|
||||
N - Supplies the number of columns for the input matrix and the number of
|
||||
rows for the output matrix.
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
{
|
||||
size_t n = N;
|
||||
|
||||
//
|
||||
// Transpose elements from the input matrix to the output matrix 8 columns
|
||||
// at a time.
|
||||
//
|
||||
|
||||
while (n >= 8) {
|
||||
|
||||
const uint8_t* s = Input;
|
||||
uint8_t* d = Output;
|
||||
size_t m = M;
|
||||
|
||||
while (m >= 8) {
|
||||
|
||||
__m128i a0 = _mm_loadl_epi64((const __m128i*)&s[N * 0]);
|
||||
__m128i a1 = _mm_loadl_epi64((const __m128i*)&s[N * 1]);
|
||||
__m128i b0 = _mm_unpacklo_epi8(a0, a1);
|
||||
|
||||
__m128i a2 = _mm_loadl_epi64((const __m128i*)&s[N * 2]);
|
||||
__m128i a3 = _mm_loadl_epi64((const __m128i*)&s[N * 3]);
|
||||
__m128i b1 = _mm_unpacklo_epi8(a2, a3);
|
||||
|
||||
__m128i a4 = _mm_loadl_epi64((const __m128i*)&s[N * 4]);
|
||||
__m128i a5 = _mm_loadl_epi64((const __m128i*)&s[N * 5]);
|
||||
__m128i b2 = _mm_unpacklo_epi8(a4, a5);
|
||||
|
||||
__m128i a6 = _mm_loadl_epi64((const __m128i*)&s[N * 6]);
|
||||
__m128i a7 = _mm_loadl_epi64((const __m128i*)&s[N * 7]);
|
||||
__m128i b3 = _mm_unpacklo_epi8(a6, a7);
|
||||
|
||||
__m128i c0 = _mm_unpacklo_epi16(b0, b1);
|
||||
__m128i c1 = _mm_unpackhi_epi16(b0, b1);
|
||||
__m128i c2 = _mm_unpacklo_epi16(b2, b3);
|
||||
__m128i c3 = _mm_unpackhi_epi16(b2, b3);
|
||||
|
||||
__m128 d0 = _mm_castsi128_ps(_mm_unpacklo_epi32(c0, c2));
|
||||
_mm_storel_pi((__m64*)&d[M * 0], d0);
|
||||
_mm_storeh_pi((__m64*)&d[M * 1], d0);
|
||||
|
||||
__m128 d1 = _mm_castsi128_ps(_mm_unpackhi_epi32(c0, c2));
|
||||
_mm_storel_pi((__m64*)&d[M * 2], d1);
|
||||
_mm_storeh_pi((__m64*)&d[M * 3], d1);
|
||||
|
||||
__m128 d2 = _mm_castsi128_ps(_mm_unpacklo_epi32(c1, c3));
|
||||
_mm_storel_pi((__m64*)&d[M * 4], d2);
|
||||
_mm_storeh_pi((__m64*)&d[M * 5], d2);
|
||||
|
||||
__m128 d3 = _mm_castsi128_ps(_mm_unpackhi_epi32(c1, c3));
|
||||
_mm_storel_pi((__m64*)&d[M * 6], d3);
|
||||
_mm_storeh_pi((__m64*)&d[M * 7], d3);
|
||||
|
||||
s += N * 8;
|
||||
d += 8;
|
||||
m -= 8;
|
||||
}
|
||||
|
||||
while (m > 0) {
|
||||
|
||||
d[M * 0] = s[0];
|
||||
d[M * 1] = s[1];
|
||||
d[M * 2] = s[2];
|
||||
d[M * 3] = s[3];
|
||||
d[M * 4] = s[4];
|
||||
d[M * 5] = s[5];
|
||||
d[M * 6] = s[6];
|
||||
d[M * 7] = s[7];
|
||||
|
||||
s += N;
|
||||
d += 1;
|
||||
m -= 1;
|
||||
}
|
||||
|
||||
Input += 8;
|
||||
Output += M * 8;
|
||||
n -= 8;
|
||||
}
|
||||
|
||||
//
|
||||
// Transpose elements from the input matrix to the output matrix for the
|
||||
// remaining columns.
|
||||
//
|
||||
|
||||
while (n > 0) {
|
||||
|
||||
const uint8_t* s = Input;
|
||||
uint8_t* d = Output;
|
||||
size_t m = M;
|
||||
|
||||
while (m >= 8) {
|
||||
|
||||
d[0] = s[N * 0];
|
||||
d[1] = s[N * 1];
|
||||
d[2] = s[N * 2];
|
||||
d[3] = s[N * 3];
|
||||
d[4] = s[N * 4];
|
||||
d[5] = s[N * 5];
|
||||
d[6] = s[N * 6];
|
||||
d[7] = s[N * 7];
|
||||
|
||||
s += N * 8;
|
||||
d += 8;
|
||||
m -= 8;
|
||||
}
|
||||
|
||||
while (m > 0) {
|
||||
|
||||
d[0] = s[0];
|
||||
|
||||
s += N;
|
||||
d += 1;
|
||||
m -= 1;
|
||||
}
|
||||
|
||||
Input += 1;
|
||||
Output += M;
|
||||
n -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
#include "core/providers/cpu/cpu_execution_provider.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
#include "core/framework/kernel_registry.h"
|
||||
#include "core/mlas/inc/mlas.h"
|
||||
|
||||
#ifndef DISABLE_CONTRIB_OPS
|
||||
#include "contrib_ops/cpu/cpu_contrib_kernels.h"
|
||||
|
|
@ -283,7 +284,8 @@ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOn
|
|||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, QLinearMatMul);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t, MatMulInteger);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, ConvInteger);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, QLinearConv);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t, QLinearConv);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, int8_t, QLinearConv);
|
||||
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, Slice);
|
||||
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 11, Dropout);
|
||||
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, NonMaxSuppression);
|
||||
|
|
@ -959,7 +961,12 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
|
|||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t,
|
||||
MatMulInteger)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, ConvInteger)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, QLinearConv)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t,
|
||||
QLinearConv)>,
|
||||
#if defined(MLAS_TARGET_AMD64_IX86)
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, int8_t,
|
||||
QLinearConv)>,
|
||||
#endif
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10,
|
||||
Slice)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 11,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
#pragma once
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/common/exceptions.h"
|
||||
#include "core/framework/op_node_proto_helper.h"
|
||||
#include "core/providers/common.h"
|
||||
#include "core/util/math.h"
|
||||
|
|
@ -79,20 +78,20 @@ struct ConvAttributes {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ValidateInputShape(const Tensor* X, const Tensor* W) const {
|
||||
const int64_t C = X->Shape()[1];
|
||||
const int64_t M = W->Shape()[0];
|
||||
Status ValidateInputShape(const TensorShape& input_shape, const TensorShape& weight_shape) const {
|
||||
const int64_t C = input_shape[1];
|
||||
const int64_t M = weight_shape[0];
|
||||
|
||||
if (X->Shape().NumDimensions() != W->Shape().NumDimensions()) {
|
||||
if (input_shape.NumDimensions() != weight_shape.NumDimensions()) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "X num_dims does not match W num_dims.",
|
||||
" X: ", X->Shape().ToString().c_str(),
|
||||
" W: ", W->Shape().ToString().c_str());
|
||||
" X: ", input_shape.ToString().c_str(),
|
||||
" W: ", weight_shape.ToString().c_str());
|
||||
}
|
||||
|
||||
if (C != W->Shape()[1] * group) {
|
||||
if (C != weight_shape[1] * group) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input channels C is not equal to kernel channels * group.",
|
||||
" C: ", C,
|
||||
" kernel channels: ", W->Shape()[1],
|
||||
" kernel channels: ", weight_shape[1],
|
||||
" group: ", group);
|
||||
}
|
||||
|
||||
|
|
@ -104,6 +103,10 @@ struct ConvAttributes {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ValidateInputShape(const Tensor* input, const Tensor* weight) const {
|
||||
return ValidateInputShape(input->Shape(), weight->Shape());
|
||||
}
|
||||
|
||||
Status InferOutputShape(const TensorShape& input_shape,
|
||||
const std::vector<int64_t>& kernel_shape,
|
||||
const std::vector<int64_t>& strides_p,
|
||||
|
|
|
|||
|
|
@ -13,35 +13,39 @@
|
|||
|
||||
namespace onnxruntime {
|
||||
|
||||
class QLinearConv : public OpKernel {
|
||||
template <typename T>
|
||||
class QLinearConv;
|
||||
|
||||
template <>
|
||||
class QLinearConv<uint8_t> : public OpKernel {
|
||||
public:
|
||||
explicit QLinearConv(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info) {}
|
||||
explicit QLinearConv<uint8_t>(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info) {}
|
||||
|
||||
Status Compute(OpKernelContext* context) const override;
|
||||
|
||||
protected:
|
||||
ConvAttributes conv_attrs_;
|
||||
};
|
||||
|
||||
ONNX_OPERATOR_KERNEL_EX(
|
||||
ONNX_CPU_OPERATOR_TYPED_KERNEL(
|
||||
QLinearConv,
|
||||
kOnnxDomain,
|
||||
10,
|
||||
kCpuExecutionProvider,
|
||||
uint8_t,
|
||||
KernelDefBuilder()
|
||||
.TypeConstraint("T1", DataTypeImpl::GetTensorType<uint8_t>())
|
||||
.TypeConstraint("T2", DataTypeImpl::GetTensorType<uint8_t>())
|
||||
.TypeConstraint("T3", DataTypeImpl::GetTensorType<uint8_t>())
|
||||
.TypeConstraint("T4", DataTypeImpl::GetTensorType<int32_t>()),
|
||||
QLinearConv);
|
||||
QLinearConv<uint8_t>);
|
||||
|
||||
Status QLinearConv::Compute(OpKernelContext* context) const {
|
||||
const auto* X = context->Input<Tensor>(0);
|
||||
const auto* W = context->Input<Tensor>(3);
|
||||
Status QLinearConv<uint8_t>::Compute(OpKernelContext* context) const {
|
||||
const Tensor* X = context->Input<Tensor>(0);
|
||||
const Tensor* W = context->Input<Tensor>(3);
|
||||
|
||||
// validate offsets
|
||||
auto X_zero_point = context->Input<Tensor>(2);
|
||||
auto W_zero_point = context->Input<Tensor>(5);
|
||||
auto Y_zero_point = context->Input<Tensor>(7);
|
||||
const Tensor* X_zero_point = context->Input<Tensor>(2);
|
||||
const Tensor* W_zero_point = context->Input<Tensor>(5);
|
||||
const Tensor* Y_zero_point = context->Input<Tensor>(7);
|
||||
ORT_ENFORCE(IsScalarOr1ElementVector(X_zero_point),
|
||||
"QLinearConv : input zero point must be a scalar or 1D tensor of size 1");
|
||||
ORT_ENFORCE(IsScalarOr1ElementVector(W_zero_point),
|
||||
|
|
@ -54,9 +58,9 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
auto Y_zero_point_value = *(Y_zero_point->template Data<uint8_t>());
|
||||
|
||||
// validate scale
|
||||
auto X_scale = context->Input<Tensor>(1);
|
||||
auto W_scale = context->Input<Tensor>(4);
|
||||
auto Y_scale = context->Input<Tensor>(6);
|
||||
const Tensor* X_scale = context->Input<Tensor>(1);
|
||||
const Tensor* W_scale = context->Input<Tensor>(4);
|
||||
const Tensor* Y_scale = context->Input<Tensor>(6);
|
||||
ORT_ENFORCE(IsScalarOr1ElementVector(X_scale),
|
||||
"QLinearConv : input scale must be a scalar or 1D tensor of size 1");
|
||||
ORT_ENFORCE(IsScalarOr1ElementVector(W_scale),
|
||||
|
|
@ -68,11 +72,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
auto W_scale_value = *(W_scale->template Data<float>());
|
||||
auto Y_scale_value = *(Y_scale->template Data<float>());
|
||||
|
||||
size_t num_inputs = OpKernel::Node().InputDefs().size();
|
||||
const Tensor* B = nullptr;
|
||||
if (num_inputs == 9) {
|
||||
B = context->Input<Tensor>(8);
|
||||
}
|
||||
const Tensor* B = context->Input<Tensor>(8);
|
||||
|
||||
const int64_t N = X->Shape()[0];
|
||||
const int64_t C = X->Shape()[1];
|
||||
|
|
@ -82,17 +82,19 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
std::vector<int64_t> kernel_shape;
|
||||
ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape));
|
||||
|
||||
const size_t kernel_rank = kernel_shape.size();
|
||||
|
||||
std::vector<int64_t> pads(conv_attrs_.pads);
|
||||
if (pads.empty()) {
|
||||
pads.resize(kernel_shape.size() * 2, 0);
|
||||
pads.resize(kernel_rank * 2, 0);
|
||||
}
|
||||
std::vector<int64_t> dilations(conv_attrs_.dilations);
|
||||
if (dilations.empty()) {
|
||||
dilations.resize(kernel_shape.size(), 1);
|
||||
dilations.resize(kernel_rank, 1);
|
||||
}
|
||||
std::vector<int64_t> strides(conv_attrs_.strides);
|
||||
if (strides.empty()) {
|
||||
strides.resize(kernel_shape.size(), 1);
|
||||
strides.resize(kernel_rank, 1);
|
||||
}
|
||||
|
||||
std::vector<int64_t> Y_dims({N, M});
|
||||
|
|
@ -109,14 +111,15 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
const int64_t input_image_size = input_shape.Size();
|
||||
const int64_t output_image_size = output_shape.Size();
|
||||
const int64_t kernel_size = TensorShape(kernel_shape).Size();
|
||||
const int64_t X_offset = C / conv_attrs_.group * input_image_size;
|
||||
const int64_t Y_offset = Y->Shape().Size() / Y->Shape()[0] / conv_attrs_.group;
|
||||
const int64_t W_offset = W->Shape().Size() / conv_attrs_.group;
|
||||
const int64_t B_offset = M / conv_attrs_.group;
|
||||
const int64_t kernel_dim = C / conv_attrs_.group * kernel_size;
|
||||
const int64_t col_buffer_size = kernel_dim * output_image_size;
|
||||
|
||||
const size_t kernel_rank = kernel_shape.size();
|
||||
const int64_t group_input_channels = W->Shape()[1];
|
||||
const int64_t group_output_channels = M / conv_attrs_.group;
|
||||
|
||||
const int64_t X_offset = group_input_channels * input_image_size;
|
||||
const int64_t Y_offset = Y->Shape().Size() / Y->Shape()[0] / conv_attrs_.group;
|
||||
const int64_t kernel_dim = group_input_channels * kernel_size;
|
||||
const int64_t W_offset = W->Shape().Size() / conv_attrs_.group;
|
||||
const int64_t col_buffer_size = kernel_dim * output_image_size;
|
||||
|
||||
AllocatorPtr alloc;
|
||||
ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc));
|
||||
|
|
@ -160,13 +163,13 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
const auto* Bdata = B != nullptr ? B->template Data<int32_t>() : nullptr;
|
||||
auto* Ydata = Y->template MutableData<uint8_t>();
|
||||
|
||||
for (int image_id = 0; image_id < N; ++image_id) {
|
||||
for (int group_id = 0; group_id < conv_attrs_.group; ++group_id) {
|
||||
for (int64_t image_id = 0; image_id < N; ++image_id) {
|
||||
for (int64_t group_id = 0; group_id < conv_attrs_.group; ++group_id) {
|
||||
if (col_buffer_data != nullptr) {
|
||||
if (kernel_rank == 2) {
|
||||
math::Im2col<uint8_t, StorageOrder::NCHW>()(
|
||||
Xdata,
|
||||
C / conv_attrs_.group,
|
||||
group_input_channels,
|
||||
input_shape[0],
|
||||
input_shape[1],
|
||||
kernel_shape[0],
|
||||
|
|
@ -200,7 +203,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
}
|
||||
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT
|
||||
QGemm(static_cast<int>(M / conv_attrs_.group),
|
||||
QGemm(static_cast<int>(group_output_channels),
|
||||
static_cast<int>(output_image_size),
|
||||
static_cast<int>(kernel_dim),
|
||||
Wdata + group_id * W_offset,
|
||||
|
|
@ -216,8 +219,8 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
|
||||
MlasRequantizeOutput(gemm_output,
|
||||
Ydata,
|
||||
Bdata != nullptr ? Bdata + group_id * B_offset : nullptr,
|
||||
static_cast<size_t>(M / conv_attrs_.group),
|
||||
Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr,
|
||||
static_cast<size_t>(group_output_channels),
|
||||
static_cast<size_t>(output_image_size),
|
||||
real_multiplier,
|
||||
Y_zero_point_value);
|
||||
|
|
@ -228,12 +231,12 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
W_zero_point_value,
|
||||
X_zero_point_value,
|
||||
Y_zero_point_value,
|
||||
static_cast<int>(M / conv_attrs_.group),
|
||||
static_cast<int>(group_output_channels),
|
||||
static_cast<int>(output_image_size),
|
||||
static_cast<int>(kernel_dim),
|
||||
integer_multiplier,
|
||||
right_shift,
|
||||
Bdata != nullptr ? Bdata + group_id * B_offset : nullptr);
|
||||
Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr);
|
||||
#endif
|
||||
|
||||
Xdata += X_offset;
|
||||
|
|
@ -244,4 +247,422 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
#if defined(MLAS_TARGET_AMD64_IX86)
|
||||
|
||||
template <>
|
||||
class QLinearConv<int8_t> : public OpKernel {
|
||||
public:
|
||||
explicit QLinearConv<int8_t>(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info), is_W_packed_(false) {}
|
||||
|
||||
Status Compute(OpKernelContext* context) const override;
|
||||
Status PrePack(const Tensor& tensor, int input_idx, bool& is_packed) override;
|
||||
|
||||
private:
|
||||
static void ReorderFilter(const uint8_t* input,
|
||||
uint8_t* output,
|
||||
size_t output_channels,
|
||||
size_t input_channels,
|
||||
size_t kernel_size);
|
||||
|
||||
ConvAttributes conv_attrs_;
|
||||
TensorShape W_shape_;
|
||||
#ifdef MLAS_SUPPORTS_PACKED_GEMM_U8X8
|
||||
BufferUniquePtr packed_W_buffer_;
|
||||
size_t packed_W_size_;
|
||||
#endif
|
||||
BufferUniquePtr reordered_W_buffer_;
|
||||
bool is_W_packed_;
|
||||
};
|
||||
|
||||
ONNX_CPU_OPERATOR_TYPED_KERNEL(
|
||||
QLinearConv,
|
||||
10,
|
||||
int8_t,
|
||||
KernelDefBuilder()
|
||||
.TypeConstraint("T1", DataTypeImpl::GetTensorType<uint8_t>())
|
||||
.TypeConstraint("T2", DataTypeImpl::GetTensorType<int8_t>())
|
||||
.TypeConstraint("T3", DataTypeImpl::GetTensorType<uint8_t>())
|
||||
.TypeConstraint("T4", DataTypeImpl::GetTensorType<int32_t>()),
|
||||
QLinearConv<int8_t>);
|
||||
|
||||
void QLinearConv<int8_t>::ReorderFilter(const uint8_t* input,
|
||||
uint8_t* output,
|
||||
size_t output_channels,
|
||||
size_t input_channels,
|
||||
size_t kernel_size) {
|
||||
for (size_t k = 0; k < kernel_size; k++) {
|
||||
for (size_t ic = 0; ic < input_channels; ic++) {
|
||||
for (size_t oc = 0; oc < output_channels; oc++) {
|
||||
size_t index = (oc * input_channels * kernel_size) + (ic * kernel_size) + k;
|
||||
*output++ = input[index];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Status QLinearConv<int8_t>::PrePack(const Tensor& tensor, int input_idx, bool& is_packed) {
|
||||
is_packed = false;
|
||||
|
||||
// Support packing the weight matrix.
|
||||
if (input_idx != 3) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
const auto& shape = tensor.Shape();
|
||||
size_t rank = shape.NumDimensions();
|
||||
if (rank != 4) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
if (shape[0] % conv_attrs_.group != 0) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// Note: The tensor has already been allocated with this tensor shape, so all
|
||||
// shape indices are guaranteed to fit inside size_t.
|
||||
const size_t output_channels = static_cast<size_t>(shape[0]);
|
||||
const size_t group_input_channels = static_cast<size_t>(shape[1]);
|
||||
const size_t kernel_size = static_cast<size_t>(shape[2] * shape[3]);
|
||||
|
||||
const size_t group_count = static_cast<size_t>(conv_attrs_.group);
|
||||
const size_t group_output_channels = output_channels / group_count;
|
||||
const size_t kernel_dim = group_input_channels * kernel_size;
|
||||
|
||||
const auto* Wdata = static_cast<const uint8_t*>(tensor.DataRaw());
|
||||
W_shape_ = shape;
|
||||
|
||||
auto alloc = Info().GetAllocator(0, OrtMemTypeDefault);
|
||||
|
||||
#ifdef MLAS_SUPPORTS_PACKED_GEMM_U8X8
|
||||
packed_W_size_ = MlasGemmPackBSize(group_output_channels, kernel_dim, true);
|
||||
|
||||
if (packed_W_size_ != 0) {
|
||||
auto* packed_W = static_cast<uint8_t*>(alloc->Alloc(SafeInt<size_t>(group_count) * packed_W_size_));
|
||||
packed_W_buffer_ = BufferUniquePtr(packed_W, BufferDeleter(alloc));
|
||||
|
||||
// Allocate a temporary buffer to hold the reordered oihw->ohwi filter for
|
||||
// a single group.
|
||||
//
|
||||
// Note: The size of this buffer is less than or equal to the size of the original
|
||||
// weight tensor, so the allocation size is guaranteed to fit inside size_t.
|
||||
auto* group_reordered_W = static_cast<uint8_t*>(alloc->Alloc(group_output_channels * group_input_channels * kernel_size));
|
||||
BufferUniquePtr group_reordered_W_buffer(group_reordered_W, BufferDeleter(alloc));
|
||||
|
||||
const size_t W_offset = group_output_channels * kernel_dim;
|
||||
|
||||
for (int64_t group_id = 0; group_id < conv_attrs_.group; ++group_id) {
|
||||
ReorderFilter(Wdata, group_reordered_W, group_output_channels, group_input_channels, kernel_size);
|
||||
MlasGemmPackB(group_output_channels, kernel_dim, group_reordered_W, group_output_channels, true, packed_W);
|
||||
packed_W += packed_W_size_;
|
||||
Wdata += W_offset;
|
||||
}
|
||||
|
||||
is_W_packed_ = true;
|
||||
is_packed = true;
|
||||
return Status::OK();
|
||||
}
|
||||
#endif
|
||||
|
||||
auto* reordered_W = static_cast<uint8_t*>(alloc->Alloc(SafeInt<size_t>(sizeof(uint8_t)) * shape.Size()));
|
||||
reordered_W_buffer_ = BufferUniquePtr(reordered_W, BufferDeleter(alloc));
|
||||
|
||||
ReorderFilter(Wdata, reordered_W, output_channels, group_input_channels, kernel_size);
|
||||
|
||||
is_W_packed_ = true;
|
||||
is_packed = true;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status QLinearConv<int8_t>::Compute(OpKernelContext* context) const {
|
||||
const Tensor* X = context->Input<Tensor>(0);
|
||||
const Tensor* W = is_W_packed_ ? nullptr : context->Input<Tensor>(3);
|
||||
const auto& W_shape = is_W_packed_ ? W_shape_ : W->Shape();
|
||||
|
||||
const int64_t N = X->Shape()[0];
|
||||
const int64_t M = W_shape[0];
|
||||
|
||||
// validate offsets
|
||||
const Tensor* X_zero_point = context->Input<Tensor>(2);
|
||||
const Tensor* W_zero_point = context->Input<Tensor>(5);
|
||||
const Tensor* Y_zero_point = context->Input<Tensor>(7);
|
||||
ORT_ENFORCE(IsScalarOr1ElementVector(X_zero_point),
|
||||
"QLinearConv : input zero point must be a scalar or 1D tensor of size 1");
|
||||
ORT_ENFORCE(IsScalarOr1ElementVector(Y_zero_point),
|
||||
"QLinearConv : result zero point must be a scalar or 1D tensor of size 1");
|
||||
|
||||
auto X_zero_point_value = *(X_zero_point->template Data<uint8_t>());
|
||||
auto Y_zero_point_value = *(Y_zero_point->template Data<uint8_t>());
|
||||
|
||||
const auto& W_zero_point_shape = W_zero_point->Shape();
|
||||
if (W_zero_point_shape.NumDimensions() == 0 ||
|
||||
(W_zero_point_shape.NumDimensions() == 1 && (W_zero_point_shape[0] == 1 || W_zero_point_shape[0] == M))) {
|
||||
const int64_t W_zero_point_size = W_zero_point_shape.Size();
|
||||
const auto* W_zero_point_data = W_zero_point->template Data<int8_t>();
|
||||
for (int64_t i = 0; i < W_zero_point_size; i++) {
|
||||
if (W_zero_point_data[i] != 0) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "QLinearConv : filter zero point must be zero");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "QLinearConv : filter zero point shape invalid");
|
||||
}
|
||||
|
||||
// validate scale
|
||||
const Tensor* X_scale = context->Input<Tensor>(1);
|
||||
const Tensor* W_scale = context->Input<Tensor>(4);
|
||||
const Tensor* Y_scale = context->Input<Tensor>(6);
|
||||
ORT_ENFORCE(IsScalarOr1ElementVector(X_scale),
|
||||
"QLinearConv : input scale must be a scalar or 1D tensor of size 1");
|
||||
ORT_ENFORCE(IsScalarOr1ElementVector(Y_scale),
|
||||
"QLinearConv : result scale must be a scalar or 1D tensor of size 1");
|
||||
|
||||
auto X_scale_value = *(X_scale->template Data<float>());
|
||||
auto Y_scale_value = *(Y_scale->template Data<float>());
|
||||
|
||||
std::vector<float> output_scales;
|
||||
const auto& W_scale_shape = W_scale->Shape();
|
||||
if (W_scale_shape.NumDimensions() == 0 ||
|
||||
(W_scale_shape.NumDimensions() == 1 && (W_scale_shape[0] == 1 || W_scale_shape[0] == M))) {
|
||||
const int64_t W_scale_size = W_scale_shape.Size();
|
||||
const auto* W_scale_data = W_scale->template Data<float>();
|
||||
output_scales.resize(static_cast<size_t>(W_scale_size));
|
||||
for (int64_t i = 0; i < W_scale_size; i++) {
|
||||
output_scales[i] = (X_scale_value * W_scale_data[i] / Y_scale_value);
|
||||
}
|
||||
} else {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "QLinearConv : filter scale shape invalid");
|
||||
}
|
||||
|
||||
const Tensor* B = context->Input<Tensor>(8);
|
||||
|
||||
ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X->Shape(), W_shape));
|
||||
|
||||
std::vector<int64_t> kernel_shape;
|
||||
ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W_shape, kernel_shape));
|
||||
|
||||
const size_t kernel_rank = kernel_shape.size();
|
||||
ORT_ENFORCE(kernel_rank == 2, "QLinearConv : must be 2D convolution");
|
||||
|
||||
std::vector<int64_t> pads(conv_attrs_.pads);
|
||||
if (pads.empty()) {
|
||||
pads.resize(kernel_rank * 2, 0);
|
||||
}
|
||||
std::vector<int64_t> dilations(conv_attrs_.dilations);
|
||||
if (dilations.empty()) {
|
||||
dilations.resize(kernel_rank, 1);
|
||||
}
|
||||
std::vector<int64_t> strides(conv_attrs_.strides);
|
||||
if (strides.empty()) {
|
||||
strides.resize(kernel_rank, 1);
|
||||
}
|
||||
|
||||
std::vector<int64_t> Y_dims({N, M});
|
||||
TensorShape input_shape = X->Shape().Slice(2);
|
||||
ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShape(input_shape, kernel_shape, strides, dilations, pads, Y_dims));
|
||||
Tensor* Y = context->Output(0, TensorShape(Y_dims));
|
||||
TensorShape output_shape = Y->Shape().Slice(2);
|
||||
|
||||
// Bail out early if one of the dimensions is zero.
|
||||
if (Y->Shape().Size() == 0) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
const int64_t input_image_size = input_shape.Size();
|
||||
const int64_t output_image_size = output_shape.Size();
|
||||
const int64_t kernel_size = TensorShape(kernel_shape).Size();
|
||||
|
||||
const int64_t group_count = conv_attrs_.group;
|
||||
const int64_t group_input_channels = W_shape[1];
|
||||
const int64_t group_output_channels = M / group_count;
|
||||
|
||||
const int64_t X_offset = group_input_channels * input_image_size;
|
||||
const int64_t Y_offset = group_output_channels * output_image_size;
|
||||
const int64_t kernel_dim = group_input_channels * kernel_size;
|
||||
const int64_t col_buffer_size = kernel_dim * output_image_size;
|
||||
|
||||
AllocatorPtr alloc;
|
||||
ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc));
|
||||
|
||||
// Use an intermediate int32_t buffer for the GEMM computation before
|
||||
// requantizing to the output type.
|
||||
auto gemm_output_data = alloc->Alloc(SafeInt<size_t>(sizeof(int32_t)) * Y_offset);
|
||||
BufferUniquePtr gemm_output_buffer(gemm_output_data, BufferDeleter(alloc));
|
||||
auto* gemm_output = static_cast<int32_t*>(gemm_output_buffer.get());
|
||||
|
||||
const auto* Xdata = X->template Data<uint8_t>();
|
||||
const auto* Bdata = B != nullptr ? B->template Data<int32_t>() : nullptr;
|
||||
auto* Ydata = Y->template MutableData<uint8_t>();
|
||||
|
||||
auto* transpose_input = static_cast<uint8_t*>(alloc->Alloc(SafeInt<size_t>(sizeof(uint8_t)) * X_offset));
|
||||
BufferUniquePtr transpose_input_buffer(transpose_input, BufferDeleter(alloc));
|
||||
|
||||
auto* transpose_output = static_cast<uint8_t*>(alloc->Alloc(SafeInt<size_t>(sizeof(uint8_t)) * Y_offset));
|
||||
BufferUniquePtr transpose_output_buffer(transpose_output, BufferDeleter(alloc));
|
||||
|
||||
// Handle the case of a dynamic weight filter.
|
||||
BufferUniquePtr reordered_W_buffer;
|
||||
uint8_t* reordered_W = nullptr;
|
||||
bool use_reordered_W = true;
|
||||
#ifdef MLAS_SUPPORTS_PACKED_GEMM_U8X8
|
||||
if (packed_W_buffer_) {
|
||||
use_reordered_W = false;
|
||||
}
|
||||
#endif
|
||||
if (use_reordered_W) {
|
||||
if (reordered_W_buffer_) {
|
||||
reordered_W = static_cast<uint8_t*>(reordered_W_buffer_.get());
|
||||
} else {
|
||||
// Weight tensor was not constant or prepacking is disabled.
|
||||
reordered_W = static_cast<uint8_t*>(alloc->Alloc(SafeInt<size_t>(sizeof(uint8_t)) * W_shape.Size()));
|
||||
reordered_W_buffer = BufferUniquePtr(reordered_W, BufferDeleter(alloc));
|
||||
ReorderFilter(static_cast<const uint8_t*>(W->DataRaw()),
|
||||
reordered_W,
|
||||
static_cast<size_t>(M),
|
||||
static_cast<size_t>(group_input_channels),
|
||||
static_cast<size_t>(kernel_size));
|
||||
}
|
||||
}
|
||||
|
||||
// Pointwise convolutions can use the original input tensor in place,
|
||||
// otherwise a temporary buffer is required for the im2col transform.
|
||||
BufferUniquePtr col_buffer;
|
||||
if (kernel_size != 1 || !conv_attrs_.HasStridesOneAndNoPadding()) {
|
||||
auto* col_data = alloc->Alloc(SafeInt<size_t>(sizeof(uint8_t)) * col_buffer_size);
|
||||
col_buffer = BufferUniquePtr(col_data, BufferDeleter(alloc));
|
||||
}
|
||||
auto* col_buffer_data = static_cast<uint8_t*>(col_buffer.get());
|
||||
|
||||
// Replicate the logic from MlasGemmU8X8Schedule to control the number of
|
||||
// worker threads used for the convolution.
|
||||
constexpr int32_t maximum_thread_count = 16;
|
||||
constexpr double thread_complexity = static_cast<double>(64 * 1024);
|
||||
|
||||
const double complexity = static_cast<double>(output_image_size) *
|
||||
static_cast<double>(group_output_channels) *
|
||||
static_cast<double>(kernel_dim);
|
||||
|
||||
int32_t thread_count = maximum_thread_count;
|
||||
if (complexity < thread_complexity * maximum_thread_count) {
|
||||
thread_count = static_cast<int32_t>(complexity / thread_complexity) + 1;
|
||||
}
|
||||
if (thread_count > output_image_size) {
|
||||
// Ensure that every thread produces at least one output.
|
||||
thread_count = static_cast<int32_t>(output_image_size);
|
||||
}
|
||||
|
||||
concurrency::ThreadPool* thread_pool = context->GetOperatorThreadPool();
|
||||
thread_count = std::min(thread_count, concurrency::ThreadPool::DegreeOfParallelism(thread_pool));
|
||||
|
||||
for (int64_t image_id = 0; image_id < N; ++image_id) {
|
||||
for (int64_t group_id = 0; group_id < group_count; ++group_id) {
|
||||
// Transpose the input from channels first (NCHW) to channels last (NHWC).
|
||||
MlasTranspose(Xdata,
|
||||
transpose_input,
|
||||
static_cast<size_t>(group_input_channels),
|
||||
static_cast<size_t>(input_image_size));
|
||||
|
||||
auto conv_worker = [&](ptrdiff_t batch) {
|
||||
auto work = concurrency::ThreadPool::PartitionWork(batch, thread_count, static_cast<ptrdiff_t>(output_image_size));
|
||||
int64_t output_start = static_cast<int64_t>(work.start);
|
||||
int64_t output_count = static_cast<int64_t>(work.end - work.start);
|
||||
|
||||
// Prepare the im2col transformation or use the input buffer directly for
|
||||
// pointwise convolutions.
|
||||
uint8_t* worker_gemm_input;
|
||||
if (col_buffer_data != nullptr) {
|
||||
worker_gemm_input = col_buffer_data + output_start * kernel_dim;
|
||||
math::Im2col<uint8_t, StorageOrder::NHWC>()(
|
||||
transpose_input,
|
||||
group_input_channels,
|
||||
input_shape[0],
|
||||
input_shape[1],
|
||||
kernel_shape[0],
|
||||
kernel_shape[1],
|
||||
dilations[0],
|
||||
dilations[1],
|
||||
pads[0],
|
||||
pads[1],
|
||||
strides[0],
|
||||
strides[1],
|
||||
output_shape[1],
|
||||
output_start,
|
||||
output_count,
|
||||
worker_gemm_input,
|
||||
X_zero_point_value);
|
||||
} else {
|
||||
worker_gemm_input = transpose_input + output_start * kernel_dim;
|
||||
}
|
||||
|
||||
auto* worker_gemm_output = gemm_output + output_start * group_output_channels;
|
||||
auto* worker_transpose_output = transpose_output + output_start * group_output_channels;
|
||||
|
||||
#ifdef MLAS_SUPPORTS_PACKED_GEMM_U8X8
|
||||
if (packed_W_buffer_) {
|
||||
MlasGemm(static_cast<size_t>(output_count),
|
||||
static_cast<size_t>(group_output_channels),
|
||||
static_cast<size_t>(kernel_dim),
|
||||
worker_gemm_input,
|
||||
static_cast<size_t>(kernel_dim),
|
||||
X_zero_point_value,
|
||||
static_cast<const int8_t*>(packed_W_buffer_.get()) + group_id * packed_W_size_,
|
||||
0,
|
||||
true,
|
||||
worker_gemm_output,
|
||||
static_cast<size_t>(group_output_channels),
|
||||
nullptr);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
MlasGemm(static_cast<size_t>(output_count),
|
||||
static_cast<size_t>(group_output_channels),
|
||||
static_cast<size_t>(kernel_dim),
|
||||
worker_gemm_input,
|
||||
static_cast<size_t>(kernel_dim),
|
||||
X_zero_point_value,
|
||||
reordered_W + group_id * group_output_channels,
|
||||
static_cast<size_t>(M),
|
||||
0,
|
||||
true,
|
||||
worker_gemm_output,
|
||||
static_cast<size_t>(group_output_channels),
|
||||
nullptr);
|
||||
}
|
||||
|
||||
if (output_scales.size() == 1) {
|
||||
MlasRequantizeOutputColumn(worker_gemm_output,
|
||||
worker_transpose_output,
|
||||
Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr,
|
||||
static_cast<size_t>(output_count),
|
||||
static_cast<size_t>(group_output_channels),
|
||||
output_scales[0],
|
||||
Y_zero_point_value);
|
||||
} else {
|
||||
MlasRequantizeOutputColumn(worker_gemm_output,
|
||||
worker_transpose_output,
|
||||
Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr,
|
||||
static_cast<size_t>(output_count),
|
||||
static_cast<size_t>(group_output_channels),
|
||||
output_scales.data() + group_id * group_output_channels,
|
||||
Y_zero_point_value);
|
||||
}
|
||||
};
|
||||
|
||||
concurrency::ThreadPool::TrySimpleParallelFor(thread_pool, thread_count, conv_worker);
|
||||
|
||||
// Transpose the output from channels last (NHWC) to channels first (NCHW).
|
||||
MlasTranspose(transpose_output,
|
||||
Ydata,
|
||||
static_cast<size_t>(output_image_size),
|
||||
static_cast<size_t>(group_output_channels));
|
||||
|
||||
Xdata += X_offset;
|
||||
Ydata += Y_offset;
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -234,6 +234,24 @@ struct Im2col<T, StorageOrder::NHWC> {
|
|||
int64_t stride_w,
|
||||
T* data_col,
|
||||
T padding_value = 0);
|
||||
void operator()(
|
||||
const T* data_im,
|
||||
int64_t channels,
|
||||
int64_t input_h,
|
||||
int64_t input_w,
|
||||
int64_t kernel_h,
|
||||
int64_t kernel_w,
|
||||
int64_t dilation_h,
|
||||
int64_t dilation_w,
|
||||
int64_t pad_t,
|
||||
int64_t pad_l,
|
||||
int64_t stride_h,
|
||||
int64_t stride_w,
|
||||
int64_t output_w,
|
||||
int64_t output_start,
|
||||
int64_t output_count,
|
||||
T* data_col,
|
||||
T padding_value = 0);
|
||||
};
|
||||
|
||||
template <typename T, int order>
|
||||
|
|
|
|||
|
|
@ -297,7 +297,6 @@ DELEGATE_SIMPLE_UNARY_FUNCTION(float, Log, log)
|
|||
DELEGATE_SIMPLE_UNARY_FUNCTION(float, Sqr, square)
|
||||
#undef DELEGATE_SIMPLE_UNARY_FUNCTION
|
||||
|
||||
|
||||
#define EIGEN_SIMPLE_BINARY_FUNCTION(T, Funcname, expr) \
|
||||
template <> \
|
||||
void Funcname<T, CPUMathUtil>(int N, const T* a, const T* b, T* y, CPUMathUtil*) { \
|
||||
|
|
@ -412,12 +411,12 @@ void Im2col<T, StorageOrder::NCHW>::operator()(const T* data_im, int64_t channel
|
|||
template struct Im2col<float, StorageOrder::NCHW>;
|
||||
template struct Im2col<uint8_t, StorageOrder::NCHW>;
|
||||
|
||||
template <>
|
||||
void Im2col<float, StorageOrder::NHWC>::operator()(const float* data_im, int64_t channels, int64_t height,
|
||||
int64_t width, int64_t kernel_h, int64_t kernel_w,
|
||||
int64_t dilation_h, int64_t dilation_w, int64_t pad_t,
|
||||
int64_t pad_l, int64_t pad_b, int64_t pad_r, int64_t stride_h,
|
||||
int64_t stride_w, float* data_col, float padding_value) {
|
||||
template <typename T>
|
||||
void Im2col<T, StorageOrder::NHWC>::operator()(const T* data_im, int64_t channels, int64_t height,
|
||||
int64_t width, int64_t kernel_h, int64_t kernel_w,
|
||||
int64_t dilation_h, int64_t dilation_w, int64_t pad_t,
|
||||
int64_t pad_l, int64_t pad_b, int64_t pad_r, int64_t stride_h,
|
||||
int64_t stride_w, T* data_col, T padding_value) {
|
||||
const int64_t dkernel_h = dilation_h * (kernel_h - 1) + 1;
|
||||
const int64_t dkernel_w = dilation_w * (kernel_w - 1) + 1;
|
||||
|
||||
|
|
@ -430,13 +429,11 @@ void Im2col<float, StorageOrder::NHWC>::operator()(const float* data_im, int64_t
|
|||
for (int64_t w = 0; w < width_col; ++w) {
|
||||
for (int64_t ih = h_pad; ih < h_pad + dkernel_h; ih += dilation_h) {
|
||||
for (int64_t iw = w_pad; iw < w_pad + dkernel_w; iw += dilation_w) {
|
||||
if (ih >= 0 && ih < height && iw >= 0 && iw < width) {
|
||||
memcpy(data_col, data_im + (ih * width + iw) * channels,
|
||||
sizeof(float) * channels);
|
||||
if (is_a_ge_zero_and_a_lt_b(ih, height) && is_a_ge_zero_and_a_lt_b(iw, width)) {
|
||||
data_col = std::copy_n(data_im + (ih * width + iw) * channels, channels, data_col);
|
||||
} else {
|
||||
std::fill_n(data_col, channels, padding_value);
|
||||
data_col = std::fill_n(data_col, channels, padding_value);
|
||||
}
|
||||
data_col += channels;
|
||||
}
|
||||
}
|
||||
w_pad += stride_w;
|
||||
|
|
@ -445,6 +442,59 @@ void Im2col<float, StorageOrder::NHWC>::operator()(const float* data_im, int64_t
|
|||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void Im2col<T, StorageOrder::NHWC>::operator()(const T* data_im, int64_t channels, int64_t input_h,
|
||||
int64_t input_w, int64_t kernel_h, int64_t kernel_w,
|
||||
int64_t dilation_h, int64_t dilation_w, int64_t pad_t,
|
||||
int64_t pad_l, int64_t stride_h, int64_t stride_w,
|
||||
int64_t output_w, int64_t output_start, int64_t output_count,
|
||||
T* data_col, T padding_value) {
|
||||
for (int64_t m = output_start; m < output_start + output_count; m++) {
|
||||
int64_t mh = m / output_w;
|
||||
int64_t mw = m % output_w;
|
||||
|
||||
int64_t oh = mh * stride_h;
|
||||
int64_t ow = mw * stride_w;
|
||||
|
||||
for (int64_t kh = 0; kh < kernel_h; kh++) {
|
||||
int64_t ih = kh * dilation_h + oh - pad_t;
|
||||
|
||||
if (is_a_ge_zero_and_a_lt_b(ih, input_h)) {
|
||||
if (dilation_w == 1) {
|
||||
int64_t kw = kernel_w;
|
||||
int64_t iw = ow - pad_l;
|
||||
while (kw > 0) {
|
||||
if (is_a_ge_zero_and_a_lt_b(iw, input_w)) {
|
||||
// Increase the copy count size to reduce the number of copy calls.
|
||||
int64_t batch_w = std::min(kw, input_w - iw);
|
||||
data_col = std::copy_n(data_im + (ih * input_w + iw) * channels, batch_w * channels, data_col);
|
||||
iw += batch_w;
|
||||
kw -= batch_w;
|
||||
} else {
|
||||
data_col = std::fill_n(data_col, channels, padding_value);
|
||||
iw++;
|
||||
kw--;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (int64_t kw = 0; kw < kernel_w; kw++) {
|
||||
int64_t iw = kw * dilation_w + ow - pad_l;
|
||||
if (is_a_ge_zero_and_a_lt_b(iw, input_w)) {
|
||||
data_col = std::copy_n(data_im + (ih * input_w + iw) * channels, channels, data_col);
|
||||
} else {
|
||||
data_col = std::fill_n(data_col, channels, padding_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
data_col = std::fill_n(data_col, kernel_w * channels, padding_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template struct Im2col<uint8_t, StorageOrder::NHWC>;
|
||||
|
||||
template <>
|
||||
void Col2im<float, CPUMathUtil, StorageOrder::NCHW>(const float* data_col, int64_t channels, int64_t height,
|
||||
int64_t width, int64_t kernel_h, int64_t kernel_w,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
|
||||
#include "gtest/gtest.h"
|
||||
#include "test/providers/provider_test_utils.h"
|
||||
#include "core/mlas/inc/mlas.h"
|
||||
#include <random>
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace test {
|
||||
|
|
@ -258,6 +260,337 @@ TEST(QLinearConvTest, WithGroup_2D) {
|
|||
{kNGraphExecutionProvider});
|
||||
}
|
||||
|
||||
#if defined(MLAS_TARGET_AMD64_IX86)
|
||||
|
||||
template <typename T1, typename T2>
|
||||
class QLinearConvOpTester {
|
||||
private:
|
||||
template <typename T>
|
||||
struct QuantizedTensor {
|
||||
std::vector<T> data_;
|
||||
std::vector<int64_t> shape_;
|
||||
std::vector<float> scale_;
|
||||
T zero_point_{0};
|
||||
};
|
||||
|
||||
std::default_random_engine generator_{1234};
|
||||
QuantizedTensor<T1> X_;
|
||||
QuantizedTensor<T2> W_;
|
||||
std::vector<int32_t> B_;
|
||||
std::vector<int64_t> pads_;
|
||||
std::vector<int64_t> strides_;
|
||||
std::vector<int64_t> dilations_;
|
||||
int64_t groups_{0};
|
||||
float output_scale_{1.0f};
|
||||
T1 output_zero_point_{0};
|
||||
|
||||
static size_t ShapeSize(const std::vector<int64_t>& shape) {
|
||||
return static_cast<size_t>(std::accumulate(shape.cbegin(), shape.cend(), 1LL, std::multiplies<int64_t>()));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void GenerateRandom(QuantizedTensor<T>& tensor,
|
||||
const std::vector<int64_t>& shape,
|
||||
float scale,
|
||||
T zero_point,
|
||||
int32_t min_value,
|
||||
int32_t max_value) {
|
||||
std::uniform_int_distribution<int32_t> distribution(min_value, max_value);
|
||||
size_t shape_size = ShapeSize(shape);
|
||||
tensor.data_.resize(shape_size);
|
||||
for (size_t n = 0; n < shape_size; n++) {
|
||||
tensor.data_[n] = static_cast<T>(distribution(generator_));
|
||||
}
|
||||
tensor.shape_ = shape;
|
||||
tensor.scale_ = {scale};
|
||||
tensor.zero_point_ = {zero_point};
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct RequantizeValues {
|
||||
RequantizeValues(int32_t zero_point) {
|
||||
min_value_ = static_cast<float>(static_cast<int32_t>(std::numeric_limits<T>::min()) - zero_point);
|
||||
max_value_ = static_cast<float>(static_cast<int32_t>(std::numeric_limits<T>::max()) - zero_point);
|
||||
zero_point_ = static_cast<float>(zero_point);
|
||||
}
|
||||
float min_value_;
|
||||
float max_value_;
|
||||
float zero_point_;
|
||||
};
|
||||
|
||||
inline float RoundHalfToEven(float input) {
|
||||
if (!std::isfinite(input)) {
|
||||
return input;
|
||||
}
|
||||
// std::remainder returns x - n, where n is the integral value nearest to x. When |x - n| = 0.5, n is chosen to be even
|
||||
return input - std::remainderf(input, 1.f);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T RequantizeOutput(int32_t sum, float scale, RequantizeValues<T>& requantize_values) {
|
||||
float f = static_cast<float>(sum) * scale;
|
||||
f = std::min(f, requantize_values.max_value_);
|
||||
f = std::max(f, requantize_values.min_value_);
|
||||
return static_cast<T>(RoundHalfToEven(f) + requantize_values.zero_point_);
|
||||
}
|
||||
|
||||
void ComputeExpectedOutput(std::vector<T1>& Y_data, std::vector<int64_t>& Y_shape) {
|
||||
ORT_ENFORCE(W_.shape_.size() > 2);
|
||||
ORT_ENFORCE(X_.shape_.size() == W_.shape_.size());
|
||||
|
||||
const size_t kernel_rank = W_.shape_.size() - 2;
|
||||
|
||||
const int64_t batch_count = X_.shape_[0];
|
||||
const int64_t input_channels = X_.shape_[1];
|
||||
const int64_t output_channels = W_.shape_[0];
|
||||
const int64_t group_count = std::max<int64_t>(groups_, 1LL);
|
||||
const int64_t group_input_channels = W_.shape_[1];
|
||||
const int64_t group_output_channels = output_channels / group_count;
|
||||
|
||||
ORT_ENFORCE(input_channels == group_input_channels * group_count);
|
||||
ORT_ENFORCE(output_channels == group_output_channels * group_count);
|
||||
|
||||
const int64_t* input_shape = X_.shape_.data() + 2;
|
||||
const int64_t* kernel_shape = W_.shape_.data() + 2;
|
||||
|
||||
std::vector<int64_t> pads(pads_);
|
||||
if (pads.empty()) {
|
||||
pads.resize(kernel_rank * 2, 0);
|
||||
}
|
||||
std::vector<int64_t> dilations(dilations_);
|
||||
if (dilations.empty()) {
|
||||
dilations.resize(kernel_rank, 1);
|
||||
}
|
||||
std::vector<int64_t> strides(strides_);
|
||||
if (strides.empty()) {
|
||||
strides.resize(kernel_rank, 1);
|
||||
}
|
||||
|
||||
// Compute the expected shape of the output.
|
||||
Y_shape.reserve(kernel_rank + 2);
|
||||
Y_shape.push_back(batch_count);
|
||||
Y_shape.push_back(output_channels);
|
||||
for (size_t n = 0; n < kernel_rank; n++) {
|
||||
Y_shape.push_back(((input_shape[n] + pads[n] + pads[kernel_rank + n]) -
|
||||
(dilations[n] * (kernel_shape[n] - 1) + 1)) / strides[n] + 1);
|
||||
}
|
||||
const int64_t* output_shape = Y_shape.data() + 2;
|
||||
Y_data.resize(ShapeSize(Y_shape));
|
||||
|
||||
const int64_t input_h = input_shape[0];
|
||||
const int64_t input_w = input_shape[1];
|
||||
const int64_t input_image_size = input_h * input_w;
|
||||
const int64_t kernel_h = kernel_shape[0];
|
||||
const int64_t kernel_w = kernel_shape[1];
|
||||
const int64_t kernel_size = kernel_h * kernel_w;
|
||||
const int64_t output_h = output_shape[0];
|
||||
const int64_t output_w = output_shape[1];
|
||||
const int64_t pad_t = pads[0];
|
||||
const int64_t pad_l = pads[1];
|
||||
const int64_t dilation_h = dilations[0];
|
||||
const int64_t dilation_w = dilations[1];
|
||||
const int64_t stride_h = strides[0];
|
||||
const int64_t stride_w = strides[1];
|
||||
const int32_t X_zero_point = X_.zero_point_;
|
||||
|
||||
const T1* Xdata = X_.data_.data();
|
||||
T1* Ydata = Y_data.data();
|
||||
|
||||
RequantizeValues<T1> requantize_values(output_zero_point_);
|
||||
|
||||
for (int64_t batch = 0; batch < batch_count; batch++) {
|
||||
const T2* weight_group = W_.data_.data();
|
||||
for (int64_t group = 0; group < group_count; group++) {
|
||||
const T2* weight_row = weight_group;
|
||||
|
||||
for (int64_t oc = 0; oc < group_output_channels; oc++) {
|
||||
int64_t channel_index = group * group_output_channels + oc;
|
||||
int32_t bias = B_.empty() ? 0 : B_[channel_index];
|
||||
float weight_scale = W_.scale_[(W_.scale_.size() == 1) ? 0 : channel_index];
|
||||
float requantize_scale = (X_.scale_[0] * weight_scale) / output_scale_;
|
||||
|
||||
for (int64_t oh = 0; oh < output_h; oh++) {
|
||||
for (int64_t ow = 0; ow < output_w; ow++) {
|
||||
int32_t sum = bias;
|
||||
const T1* input_image = Xdata;
|
||||
const T2* weight_data = weight_row;
|
||||
for (int64_t ic = 0; ic < group_input_channels; ic++) {
|
||||
for (int64_t kh = 0; kh < kernel_h; kh++) {
|
||||
int64_t ih = kh * dilation_h + oh * stride_h - pad_t;
|
||||
for (int64_t kw = 0; kw < kernel_w; kw++) {
|
||||
int64_t iw = kw * dilation_w + ow * stride_w - pad_l;
|
||||
int32_t w_value = static_cast<int32_t>(*weight_data++);
|
||||
if (static_cast<uint64_t>(ih) < static_cast<uint64_t>(input_h) &&
|
||||
static_cast<uint64_t>(iw) < static_cast<uint64_t>(input_w)) {
|
||||
int32_t x_value = static_cast<int32_t>(input_image[ih * input_w + iw]) - X_zero_point;
|
||||
sum += x_value * w_value;
|
||||
}
|
||||
}
|
||||
}
|
||||
input_image += input_image_size;
|
||||
}
|
||||
*Ydata++ = RequantizeOutput<T1>(sum, requantize_scale, requantize_values);
|
||||
}
|
||||
}
|
||||
|
||||
weight_row += group_input_channels * kernel_size;
|
||||
}
|
||||
|
||||
Xdata += group_input_channels * input_image_size;
|
||||
weight_group += group_output_channels * group_input_channels * kernel_size;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Run(bool all_input_initializer_except_x) {
|
||||
OpTester test("QLinearConv", 10);
|
||||
|
||||
std::vector<T1> Y_data;
|
||||
std::vector<int64_t> Y_shape;
|
||||
ComputeExpectedOutput(Y_data, Y_shape);
|
||||
|
||||
test.AddInput<T1>("x", X_.shape_, X_.data_);
|
||||
test.AddInput<float>("x_scale", {}, X_.scale_, all_input_initializer_except_x);
|
||||
test.AddInput<T1>("x_zero_point", {}, {X_.zero_point_});
|
||||
|
||||
const std::vector<int64_t> W_scale_shape{static_cast<int64_t>(W_.scale_.size())};
|
||||
test.AddInput<T2>("w", W_.shape_, W_.data_, all_input_initializer_except_x);
|
||||
test.AddInput<float>("w_scale", W_scale_shape, W_.scale_, all_input_initializer_except_x);
|
||||
test.AddInput<T2>("w_zero_point", {}, {W_.zero_point_});
|
||||
|
||||
test.AddInput<float>("y_scale", {}, {output_scale_}, all_input_initializer_except_x);
|
||||
test.AddInput<T1>("y_zero_point", {}, {output_zero_point_});
|
||||
|
||||
if (!B_.empty()) {
|
||||
const std::vector<int64_t> B_shape{static_cast<int64_t>(B_.size())};
|
||||
test.AddInput<int32_t>("b", B_shape, B_);
|
||||
}
|
||||
|
||||
test.AddOutput<uint8_t>("y", Y_shape, Y_data);
|
||||
|
||||
if (!pads_.empty()) {
|
||||
test.AddAttribute("pads", pads_);
|
||||
}
|
||||
if (!strides_.empty()) {
|
||||
test.AddAttribute("strides", strides_);
|
||||
}
|
||||
if (!dilations_.empty()) {
|
||||
test.AddAttribute("dilations", dilations_);
|
||||
}
|
||||
if (groups_ > 0) {
|
||||
test.AddAttribute("group", groups_);
|
||||
}
|
||||
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "");
|
||||
}
|
||||
|
||||
public:
|
||||
QLinearConvOpTester() {
|
||||
}
|
||||
|
||||
void GenerateRandomInput(const std::vector<int64_t>& shape, float scale, T1 zero_point) {
|
||||
GenerateRandom(X_, shape, scale, zero_point, 0, 63);
|
||||
}
|
||||
|
||||
void GenerateRandomWeights(const std::vector<int64_t>& shape, float scale, T2 zero_point) {
|
||||
GenerateRandom(W_, shape, scale, zero_point, -63, 63);
|
||||
}
|
||||
|
||||
void SetWeightScales(const std::vector<float>& scales) {
|
||||
W_.scale_ = scales;
|
||||
}
|
||||
|
||||
void GenerateRandomBias() {
|
||||
ORT_ENFORCE(W_.shape_.size() >= 1);
|
||||
const size_t output_channels = static_cast<size_t>(W_.shape_[0]);
|
||||
B_.resize(output_channels);
|
||||
std::uniform_int_distribution<int32_t> distribution(-423, 423);
|
||||
for (size_t n = 0; n < output_channels; n++) {
|
||||
B_[n] = distribution(generator_);
|
||||
}
|
||||
}
|
||||
|
||||
void SetPads(const std::vector<int64_t>& pads) {
|
||||
pads_ = pads;
|
||||
}
|
||||
|
||||
void SetStrides(const std::vector<int64_t>& strides) {
|
||||
strides_ = strides;
|
||||
}
|
||||
|
||||
void SetDilations(const std::vector<int64_t>& dilations) {
|
||||
dilations_ = dilations;
|
||||
}
|
||||
|
||||
void SetGroups(int64_t groups) {
|
||||
groups_ = groups;
|
||||
}
|
||||
|
||||
void SetOutputScaleAndZeroPoint(float output_scale, T1 output_zero_point) {
|
||||
output_scale_ = output_scale;
|
||||
output_zero_point_ = output_zero_point;
|
||||
}
|
||||
|
||||
void Run() {
|
||||
for (bool all_input_initializer_except_x : std::initializer_list<bool>{false, true}) {
|
||||
Run(all_input_initializer_except_x);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
TEST(QLinearConvTest, Conv2D_U8S8) {
|
||||
QLinearConvOpTester<uint8_t, int8_t> test;
|
||||
test.GenerateRandomInput({3, 24, 15, 11}, .05f, 4);
|
||||
test.GenerateRandomWeights({32, 24, 3, 3}, .125f, 0);
|
||||
test.GenerateRandomBias();
|
||||
test.SetPads({1, 1, 1, 1});
|
||||
test.SetOutputScaleAndZeroPoint(.55f, 54);
|
||||
test.Run();
|
||||
}
|
||||
|
||||
TEST(QLinearConvTest, Conv2D_U8S8_Dilations) {
|
||||
QLinearConvOpTester<uint8_t, int8_t> test;
|
||||
test.GenerateRandomInput({1, 4, 19, 16}, .02f, 20);
|
||||
test.GenerateRandomWeights({6, 4, 3, 2}, .11f, 0);
|
||||
test.SetDilations({2, 2});
|
||||
test.SetOutputScaleAndZeroPoint(.24f, 15);
|
||||
test.Run();
|
||||
}
|
||||
|
||||
TEST(QLinearConvTest, Conv2D_U8S8_Strides) {
|
||||
QLinearConvOpTester<uint8_t, int8_t> test;
|
||||
test.GenerateRandomInput({1, 7, 18, 24}, .04f, 16);
|
||||
test.GenerateRandomWeights({5, 7, 2, 3}, .14f, 0);
|
||||
test.SetStrides({2, 2});
|
||||
test.SetOutputScaleAndZeroPoint(.31f, 30);
|
||||
test.Run();
|
||||
}
|
||||
|
||||
TEST(QLinearConvTest, Conv2D_U8S8_Groups) {
|
||||
QLinearConvOpTester<uint8_t, int8_t> test;
|
||||
test.GenerateRandomInput({1, 8, 13, 17}, .03f, 7);
|
||||
test.GenerateRandomWeights({12, 4, 3, 3}, .10f, 0);
|
||||
test.GenerateRandomBias();
|
||||
test.SetPads({1, 1, 1, 1});
|
||||
test.SetGroups(2);
|
||||
test.SetOutputScaleAndZeroPoint(.76f, 88);
|
||||
test.Run();
|
||||
}
|
||||
|
||||
TEST(QLinearConvTest, Conv2D_U8S8_Groups_PerChannel) {
|
||||
QLinearConvOpTester<uint8_t, int8_t> test;
|
||||
test.GenerateRandomInput({1, 8, 13, 17}, .03f, 7);
|
||||
test.GenerateRandomWeights({10, 4, 3, 3}, .10f, 0);
|
||||
test.SetWeightScales({.15f, .14f, .11f, .13f, .15f, .09f, .12f, .16f, .17f, .07f});
|
||||
test.GenerateRandomBias();
|
||||
test.SetPads({1, 1, 1, 1});
|
||||
test.SetGroups(2);
|
||||
test.SetOutputScaleAndZeroPoint(.76f, 88);
|
||||
test.Run();
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace
|
||||
} // namespace test
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
Loading…
Reference in a new issue