saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixed point based requantization on arm64 (#11540)

Browse source 
* fixed point based requantization on arm64

* reverse MlasConvSymDepthwiseKernel u8s8 and s8s8 order
This commit is contained in:
Yufeng Li 2022-06-02 12:34:17 -07:00 committed by GitHub
parent 453c57f92f
commit 1f2c92673b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
49 changed files with 3776 additions and 1102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
View file

@ -119,3 +119,25 @@ static const char* const kOrtSessionOptionsConfigDynamicBlockBase = "session.dyn
// "0": in some cases warnings will be logged but processing will continue. The default.
// May be useful to expose bugs in models.
static const char* const kOrtSessionOptionsConfigStrictShapeTypeInference = "session.strict_shape_type_inference";
// SessionOption 'fixed_point_requant_on_arm64' controls the requantization method on ARM devices.
// Requantization is computed with formula:
// v = round(clamp(S * (I - Z), min, max))
// where v is the target value with type TOutput, which is either int8_t or uint8_t
// I is the input value with type int32_t
// S is the scale with type float
// Z is the zero point with type same as TOutput.
// min is the minimum value of type TOutput.
// max is the maximum value of type TOutput.
// For considerations of power consumption and some ARM devices don't even have FPUs, it is import to to be able to run
// quantization with integer instructions only.FixedPoint Requantization is introduced to support this feature.
// Its general idea is to convert scale S to fixed point. Ruy and XNNPack's method are referred for the implementation.
// https://github.com/google/ruy/blob/a09683b8da7164b9c5704f88aef2dc65aa583e5d/ruy/apply_multiplier.cc#L48
// https://github.com/google/XNNPACK/blob/1e37b200d3f4ba19151eb30c1c329873d541326c/src/params-init.c#L211
// "0": disable. ORT uses float point based requantization on ARM devices.
// "1": enable. ORT uses fixed point based requantization on ARM devices.
// Its default value is "0"
// **NOTE** that fixed point requantization rounds half to up, whereas ONNX spec rounds half to even, so for identical
// model and input the inference results may not be exactly same with this option on and off. The impact should be
// small in practice (NNApi EP uses same rounding).
static const char* const kOrtSessionOptionsConfigFixedPointRequantOnARM64 = "session.fixed_point_requant_on_arm64";

11
onnxruntime/contrib_ops/cpu/quantization/quant_gemm.cc
View file

@ -104,7 +104,8 @@ class QGemm : protected GemmBase, public MatMulIntegerBase {
std::vector<float> output_scales = ComputeOutputScale(a_scale, b_scale, y_scale);
std::unique_ptr<MLAS_QGEMM_SCALE_BIAS_OUTPUT_PROCESSOR> scale_bias_proc_ptr;
std::unique_ptr<MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR> requant_proc_ptr;
SetPostProcessor(y_zp, N, output_scales, y, gemm_param, scale_bias_proc_ptr, requant_proc_ptr);
std::unique_ptr<MLAS_REQUANT_PARAM> requant_param;
SetPostProcessor(y_zp, N, output_scales, y, gemm_param, scale_bias_proc_ptr, requant_proc_ptr, requant_param);
MlasGemmBatch(gemm_shape, &gemm_param, 1, context->GetOperatorThreadPool());
return Status::OK();
@ -181,17 +182,17 @@ class QGemm : protected GemmBase, public MatMulIntegerBase {
Tensor* y,
MLAS_GEMM_QUANT_DATA_PARAMS& gemm_param,
std::unique_ptr<MLAS_QGEMM_SCALE_BIAS_OUTPUT_PROCESSOR>& scale_bias_proc_ptr,
std::unique_ptr<MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR>& requant_proc_ptr) {
std::unique_ptr<MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR>& requant_proc_ptr,
std::unique_ptr<MLAS_REQUANT_PARAM>& requant_param) {
if (nullptr != y_zp) {
bool is_y_signed = y->IsDataType<int8_t>();
int32_t y_zero_point = is_y_signed ? *y_zp->template Data<int8_t>() : *y_zp->template Data<uint8_t>();
requant_param = std::make_unique<MLAS_REQUANT_PARAM>(output_scales.data(), output_scales.size(), y_zero_point);
requant_proc_ptr = std::make_unique<MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR>(
y->MutableDataRaw(),
out_lda,
nullptr,
output_scales.data(),
output_scales.size() > 1,
y_zero_point,
requant_param.get(),
is_y_signed);
gemm_param.OutputProcessor = requant_proc_ptr.get();
} else {

131
onnxruntime/core/mlas/inc/mlas.h
View file

@ -535,7 +535,7 @@ private:
/**
* @brief Supply matrices shape and data type information to quantized gemm functions
*
** NOTE: AIsSigned == true is not supported on non-ARM devices for now.
** NOTE: AIsSigned == true is not supported on non-ARM devices for now.
** AIsSigned == true is supported on ARM devices when BIsSigned is also true.
*
*/
@ -623,10 +623,10 @@ struct MLAS_SYMM_QGEMM_DATA_PARAMS {
* @param [IN] Shape A single shape descriptor for all multiplicatons.
Currently A and B must be signed, and accumulation
mode not supported
* @param [IN] DataParams Array of data descriptors, one for each mutliplication
* @param [IN] DataParams Array of data descriptors, one for each multiplication
* B must be prepacked
* @param [IN] BatchN Number of multiplications
* @param [IN] ThreadPool
* @param [IN] ThreadPool
*/
void
MLASCALL
@ -683,8 +683,8 @@ MlasGemmPackB(
/**
* @brief For symmetric quantized GEMM, returns size of the
* packing buffer needed for right hand side
* @param N Number of columns
* packing buffer needed for right hand side
* @param N Number of columns
* @param K Number of rows
* @param AIsSigned Whether left hand size is signed int8_t
* @return size of the packing buffer,
@ -694,7 +694,7 @@ size_t
MLASCALL
MlasSymmQgemmPackBSize(
size_t N,
size_t K,
size_t K,
bool AIsSigned
);
@ -829,6 +829,95 @@ MlasConvSymFixupInputZeroPoint(
bool InputIsSigned
);
inline
uint32_t
BitsOfFp32(float FloatValue)
{
union {
uint32_t IntegerValue;
float FloatValue;
} u;
u.FloatValue = FloatValue;
return u.IntegerValue;
}
inline
void
MlasFloatToFixedPoint(
float Scale,
int32_t& Multiplier,
int32_t& PreShift,
int32_t& PostShift)
{
const uint32_t ScaleBits = BitsOfFp32(Scale);
// Multiplier is in [0x40000000, 0x7FFFFF80] range.
Multiplier = (int32_t)(((ScaleBits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000)) << 7);
// Shift is in [-8, 31] range.
const int32_t Shift = 127 + 31 - 32 - (ScaleBits >> 23);
// Split shift into PreShift + PostShift, PostShift in [1, 31] range.
PostShift = Shift > 1 ? Shift : 1;
PreShift = Shift - PostShift;
PreShift = -PreShift;
PostShift = -PostShift;
}
inline
void
MlasFloatToFixedPoint(
const float* Scale,
int32_t* Multiplier,
int32_t* PreShift,
int32_t* PostShift,
size_t N)
{
for(size_t i = 0; i < N; i++) {
MlasFloatToFixedPoint(Scale[i], Multiplier[i], PreShift[i], PostShift[i]);
}
}
enum MLAS_ROUND_KIND {
MlasRoundHalfEven,
MlasRoundHalfUp,
};
/**
* @brief Supply parameters to requantize a tensor. It supports 2 modes:
** RequantRoundKind == MlasRoundHalfEven: Scale is used.
** RequantRoundKind == MlasRoundHalfUp: Multiplier, PreShift and PostShift are used.
*
*/
struct MLAS_REQUANT_PARAM {
MLAS_REQUANT_PARAM()=default;
MLAS_REQUANT_PARAM(const float* Scale,
size_t Size,
int32_t ZeroPoint):
RequantRoundKind(MLAS_ROUND_KIND::MlasRoundHalfEven),
Scale(Scale), Size(Size), ZeroPoint(ZeroPoint){}
MLAS_REQUANT_PARAM(const int32_t* Multiplier,
const int32_t* PreShift,
const int32_t* PostShift,
size_t Size,
int32_t ZeroPoint):
RequantRoundKind(MLAS_ROUND_KIND::MlasRoundHalfUp),
Multiplier(Multiplier), PreShift(PreShift), PostShift(PostShift),
Size(Size), ZeroPoint(ZeroPoint){}
MLAS_ROUND_KIND RequantRoundKind;
const float* Scale;
const int32_t* Multiplier;
const int32_t* PreShift;
const int32_t* PostShift;
size_t Size;
int32_t ZeroPoint;
};
struct MLAS_CONV_SYM_PARAMS {
const void* InputDirect;
const void* const* InputIndirection;
@ -839,9 +928,7 @@ struct MLAS_CONV_SYM_PARAMS {
size_t OutputCount;
size_t KernelSize;
const int32_t* Bias;
const float* Scale;
bool PerChannelScale;
int32_t OutputZeroPoint;
const MLAS_REQUANT_PARAM* RequantParam;
bool InputIsSigned;
};
@ -1137,9 +1224,7 @@ MlasQuantizeLinear(
* @param OutputLeadingDimension Output matrix leading dimension
* @param Bias Optional bias vector, to be added
to the input before quantization
* @param Scale Quantization scale
* @param PerColumnScale true if scale is per-column
* @param ZeroPoint quantization zero point value
* @param MLAS_REQUANT_PARAM Requantization parameters
* @param StartM
* @param StartN
* @param CountM
@ -1155,9 +1240,7 @@ MlasRequantizeOutput(
OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
OutputType ZeroPoint,
const MLAS_REQUANT_PARAM* RequantParam,
size_t StartM,
size_t StartN,
size_t CountM,
@ -1171,16 +1254,12 @@ class MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR : public MLAS_QGEMM_OUTPUT_PROCESSOR
void* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
int32_t ZeroPoint,
const MLAS_REQUANT_PARAM* RequantParam,
bool OutputIsSigned)
: Output_(Output),
OutputLeadingDimension_(OutputLeadingDimension),
Bias_(Bias),
Scale_(Scale),
PerColumnScale_(PerColumnScale),
ZeroPoint_(ZeroPoint),
RequantParam_(RequantParam),
OutputIsSigned_(OutputIsSigned)
{
}
@ -1194,12 +1273,10 @@ class MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR : public MLAS_QGEMM_OUTPUT_PROCESSOR
{
if(OutputIsSigned_){
MlasRequantizeOutput(C, ldc, reinterpret_cast<int8_t*>(Output_), OutputLeadingDimension_,
Bias_, Scale_, PerColumnScale_, static_cast<int8_t>(ZeroPoint_),
StartM, StartN, CountM, CountN);
Bias_, RequantParam_, StartM, StartN, CountM, CountN);
} else {
MlasRequantizeOutput(C, ldc, reinterpret_cast<uint8_t*>(Output_), OutputLeadingDimension_,
Bias_, Scale_, PerColumnScale_, static_cast<uint8_t>(ZeroPoint_),
StartM, StartN, CountM, CountN);
Bias_, RequantParam_, StartM, StartN, CountM, CountN);
}
}
@ -1208,9 +1285,7 @@ class MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR : public MLAS_QGEMM_OUTPUT_PROCESSOR
void* Output_;
size_t OutputLeadingDimension_;
const int32_t* Bias_;
const float* Scale_;
bool PerColumnScale_;
int32_t ZeroPoint_;
const MLAS_REQUANT_PARAM* RequantParam_;
bool OutputIsSigned_;
};

142
onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelDot.S
View file

@ -19,20 +19,43 @@ Abstract:
#include "AssembleDotProduct.h"
.equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2
.equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4
//
// Stack frame layout for the symmetric convolution kernel.
// d8-d15, x19-x30 need to be preserved if used
//
.equ .LConvSymFrame_SavedRegisters, (6 * 8)
.equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters
.equ .LConvSymPostProcessParams_Bias, 0
.equ .LConvSymPostProcessParams_Scale, 8
.equ .LConvSymPostProcessParams_Min, 16
.equ .LConvSymPostProcessParams_Max, 20
.equ .LConvSymPostProcessParams_ZeroPoint, 24
.equ .LConvSymFrame_SavedRegisters_d8_d9, 0
.equ .LConvSymFrame_SavedRegisters_d10_d11, 16
.equ .LConvSymFrame_SavedRegisters_x19_x20, 32
.equ .LConvSymFrame_SavedRegisters_x21_x22, 48
.equ .LConvSymFrame_SavedRegisters, 64
.equ .LConvSymFrame_SavedRegisters_Neg, -64
.equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
.equ .LConvSymPostProcessParams_Bias, 0
.equ .LConvSymPostProcessParams_Scale, 8
.equ .LConvSymPostProcessParams_Min, 16
.equ .LConvSymPostProcessParams_Max, 20
.equ .LConvSymPostProcessParams_ZeroPoint, 24
.equ .LConvSymPostProcessParams_Multiplier, 32
.equ .LConvSymPostProcessParams_PreShift, 40
.equ .LConvSymPostProcessParams_PostShift, 48
.text
@ -78,13 +101,13 @@ Return Value:
--*/
FUNCTION_ENTRY MlasConvSymS8KernelDot
stp d8,d9,[sp,#-.LConvSymFrame_SavedRegisters]!
stp d8,d9,[sp,#.LConvSymFrame_SavedRegisters_Neg]!
ldr x8,[sp,#.LConvSymFrame_PostProcessParams]
str d10,[sp,#16]
cmp x7,2 // OutputCount < 2 ?
str d11,[sp,#24]
add x16,x2,x5 // x16 -> C1
str x19,[sp,#32]
stp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11]
stp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20]
stp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22]
lsl x3,x3,#3 // KernelSize * sizeof(int8_t*)
csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0
add x4,x4,3 // InputChannels align to 4
@ -97,6 +120,9 @@ Return Value:
add x5,x17,x5 // x5 -> C3
ldr x19,[x8,#.LConvSymPostProcessParams_Scale]
csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2
ldr x20,[x8,#.LConvSymPostProcessParams_PreShift]
ldr x21,[x8,#.LConvSymPostProcessParams_Multiplier]
ldr x22,[x8,#.LConvSymPostProcessParams_PostShift]
// TODO!! tiptoe around loading biases if we need to support
// output channels none divisible by 16
@ -347,12 +373,15 @@ InChLoopEpilogue:
b.hi KernelSizeLoop
Requantize:
tst w10,#.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale
FloatPointScale:
tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq BroadcastScaleValue
ldp q0,q1,[x19],32 // load scale vector
ldp q2,q3,[x19],32
b AccumulatorsToFloat
b ScaleWithFloatPoint
BroadcastScaleValue:
ld1r {v0.4s},[x19] // load scale Value
@ -360,7 +389,7 @@ BroadcastScaleValue:
mov v2.16b, v0.16b
mov v3.16b, v0.16b
AccumulatorsToFloat:
ScaleWithFloatPoint:
scvtf v16.4s,v16.4s // convert to float
scvtf v17.4s,v17.4s
scvtf v18.4s,v18.4s
@ -409,7 +438,87 @@ AccumulatorsToFloat:
fcvtns v29.4s,v29.4s
fcvtns v30.4s,v30.4s
fcvtns v31.4s,v31.4s
b Quantize
FixedPointScale:
tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq BroadcastFixedPointValue
ldp q0,q1,[x20],32 // load preshift vector
ldp q2,q3,[x20],32
ldp q4,q5,[x21],32 // load multiplier vector
ldp q6,q7,[x21],32
ldp q8,q9,[x22],32 // load postshift vector
ldp q10,q11,[x22],32
b ScaleWithFixedPoint
BroadcastFixedPointValue:
ld1r {v0.4s},[x20] // load preshift Value
mov v1.16b, v0.16b
mov v2.16b, v0.16b
mov v3.16b, v0.16b
ld1r {v4.4s},[x21] // load multiplier Value
mov v5.16b, v4.16b
mov v6.16b, v4.16b
mov v7.16b, v4.16b
ld1r {v8.4s},[x22] // load postshift Value
mov v9.16b, v8.16b
mov v10.16b, v8.16b
mov v11.16b, v8.16b
ScaleWithFixedPoint:
sqshl v16.4s, v16.4s, v0.4s // preshift
sqshl v17.4s, v17.4s, v0.4s
sqshl v18.4s, v18.4s, v0.4s
sqshl v19.4s, v19.4s, v0.4s
sqshl v20.4s, v20.4s, v1.4s
sqshl v21.4s, v21.4s, v1.4s
sqshl v22.4s, v22.4s, v1.4s
sqshl v23.4s, v23.4s, v1.4s
sqshl v24.4s, v24.4s, v2.4s
sqshl v25.4s, v25.4s, v2.4s
sqshl v26.4s, v26.4s, v2.4s
sqshl v27.4s, v27.4s, v2.4s
sqshl v28.4s, v28.4s, v3.4s
sqshl v29.4s, v29.4s, v3.4s
sqshl v30.4s, v30.4s, v3.4s
sqshl v31.4s, v31.4s, v3.4s
sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale
sqdmulh v17.4s, v17.4s, v4.4s
sqdmulh v18.4s, v18.4s, v4.4s
sqdmulh v19.4s, v19.4s, v4.4s
sqdmulh v20.4s, v20.4s, v5.4s
sqdmulh v21.4s, v21.4s, v5.4s
sqdmulh v22.4s, v22.4s, v5.4s
sqdmulh v23.4s, v23.4s, v5.4s
sqdmulh v24.4s, v24.4s, v6.4s
sqdmulh v25.4s, v25.4s, v6.4s
sqdmulh v26.4s, v26.4s, v6.4s
sqdmulh v27.4s, v27.4s, v6.4s
sqdmulh v28.4s, v28.4s, v7.4s
sqdmulh v29.4s, v29.4s, v7.4s
sqdmulh v30.4s, v30.4s, v7.4s
sqdmulh v31.4s, v31.4s, v7.4s
srshl v16.4s,v16.4s, v8.4s // post shift
srshl v17.4s,v17.4s, v8.4s
srshl v18.4s,v18.4s, v8.4s
srshl v19.4s,v19.4s, v8.4s
srshl v20.4s,v20.4s, v9.4s
srshl v21.4s,v21.4s, v9.4s
srshl v22.4s,v22.4s, v9.4s
srshl v23.4s,v23.4s, v9.4s
srshl v24.4s,v24.4s, v10.4s
srshl v25.4s,v25.4s, v10.4s
srshl v26.4s,v26.4s, v10.4s
srshl v27.4s,v27.4s, v10.4s
srshl v28.4s,v28.4s, v11.4s
srshl v29.4s,v29.4s, v11.4s
srshl v30.4s,v30.4s, v11.4s
srshl v31.4s,v31.4s, v11.4s
Quantize:
sqxtn v16.4h,v16.4s
sqxtn v17.4h,v17.4s
sqxtn v18.4h,v18.4s
@ -454,8 +563,9 @@ AccumulatorsToFloat:
b.hi OutputChannelLoop
ExitKernel:
ldr x19,[sp,#32]
ldp d10,d11,[sp,#16]
ldp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22]
ldp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20]
ldp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11]
ldp d8,d9,[sp],#.LConvSymFrame_SavedRegisters
ret

171
onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelDotLd64.S
View file

@ -18,22 +18,51 @@ Abstract:
#include "asmmacro.h"
#include "AssembleDotProduct.h"
.equ .LMLAS_CONV_SYM_FLAG_INPUT_DIRECT, 1
.equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2
.equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4
//
// Stack frame layout for the symmetric convolution kernel.
// d8-d15, x19-x30 need to be preserved if used
//
.equ .LConvSymFrame_SavedRegisters, (10 * 8)
.equ .LConvSymFrame_SavedRegisters_d8, 0
.equ .LConvSymFrame_SavedRegisters_d9, 8
.equ .LConvSymFrame_SavedRegisters_d10, 16
.equ .LConvSymFrame_SavedRegisters_d11, 24
.equ .LConvSymFrame_SavedRegisters_x19, 32
.equ .LConvSymFrame_SavedRegisters_x20, 40
.equ .LConvSymFrame_SavedRegisters_x21, 48
.equ .LConvSymFrame_SavedRegisters_x22, 56
.equ .LConvSymFrame_SavedRegisters_x23, 64
.equ .LConvSymFrame_SavedRegisters_x24, 72
.equ .LConvSymFrame_SavedRegisters_x25, 80
.equ .LConvSymFrame_SavedRegisters_x26, 88
.equ .LConvSymFrame_SavedRegisters, 96
.equ .LConvSymFrame_SavedRegisters_Neg, -96
.equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
.equ .LConvSymPostProcessParams_Bias, 0
.equ .LConvSymPostProcessParams_Scale, 8
.equ .LConvSymPostProcessParams_Min, 16
.equ .LConvSymPostProcessParams_Max, 20
.equ .LConvSymPostProcessParams_ZeroPoint, 24
.equ .LConvSymPostProcessParams_Multiplier,32
.equ .LConvSymPostProcessParams_PreShift, 40
.equ .LConvSymPostProcessParams_PostShift, 48
.text
@ -46,24 +75,17 @@ Routine Description:
Arguments:
Input (x0) - Points to the input buffer.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points
directly at the input tensor.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an
indirection buffer. Every pointer in the indirection buffer points at a
InputChannels length vector (either from the input tensor or a vector of
padding values). These are grouped in batches of length KernelSize.
These batches are then repeated OutputCount times.
Input (x0) - Points to the indirection buffer. Every pointer in the indirection
buffer points at a InputChannels length vector (either from the input tensor
or a vector of padding values). These are grouped in batches of length
KernelSize. These batches are then repeated OutputCount times.
Filter (x1) - Points to the filter buffer.
Output (x2) - Points the output buffer.
KernelSize (x3/x9) - Size of the kernel (most commonly. 3x3=9, 5x5=25).
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1.
Must be > 1
InputChannels (x4/x7) - Number of input channels.
@ -86,30 +108,35 @@ Return Value:
--*/
FUNCTION_ENTRY MlasConvSymS8KernelDotLd64
stp d8,d9,[sp,#-.LConvSymFrame_SavedRegisters]!
stp d8,d9,[sp,#.LConvSymFrame_SavedRegisters_Neg]!
ldr x8,[sp,#.LConvSymFrame_PostProcessParams]
str d10,[sp,#16]
str d10,[sp,#.LConvSymFrame_SavedRegisters_d10]
cmp x7,2 // OutputCount < 2 ?
str d11,[sp,#24]
str d11,[sp,#.LConvSymFrame_SavedRegisters_d11]
add x16,x2,x5 // x16 -> C1
str x19,[sp,#32]
str x19,[sp,#.LConvSymFrame_SavedRegisters_x19]
lsl x3,x3,#3 // KernelSize * sizeof(int8_t*)
str x20,[sp,#40]
str x20,[sp,#.LConvSymFrame_SavedRegisters_x20]
csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0
str x21,[sp,#48]
str x21,[sp,#.LConvSymFrame_SavedRegisters_x21]
add x4,x4,3 // InputChannels align to 4
str x22,[sp,#56]
str x22,[sp,#.LConvSymFrame_SavedRegisters_x22]
add x17,x16,x5 // x17 -> C2
str x23,[sp,#64]
str x23,[sp,#.LConvSymFrame_SavedRegisters_x23]
ldr x11,[x8,#.LConvSymPostProcessParams_Bias]
str x24,[sp,#.LConvSymFrame_SavedRegisters_x24]
csel x17,x16,x17,ls // if OutputCount <= 2 x17/C2 -> C1
str x25,[sp,#.LConvSymFrame_SavedRegisters_x25]
bic x4,x4,3
str x26,[sp,#.LConvSymFrame_SavedRegisters_x26]
cmp x7,4 // OutputCount < 4 ?
ldr w10,[sp,#.LConvSymFrame_KernelFlags]
add x5,x17,x5 // x5 -> C3
ldr x19,[x8,#.LConvSymPostProcessParams_Scale]
csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2
ldr x24,[x8,#.LConvSymPostProcessParams_PreShift]
ldr x25,[x8,#.LConvSymPostProcessParams_Multiplier]
ldr x26,[x8,#.LConvSymPostProcessParams_PostShift]
// TODO!! tiptoe around loading biases if we need to support
// output channels none divisible by 16
OutputChannelLoop:
@ -423,12 +450,15 @@ InChLoopEpilogue:
b.hi KernelSizeLoop
Requantize:
tst w10,#.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale
FloatPointScale:
tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq BroadcastScaleValue
ldp q0,q1,[x19],32 // load scale vector
ldp q2,q3,[x19],32
b AccumulatorsToFloat
b ScaleWithFloatPoint
BroadcastScaleValue:
ld1r {v0.4s},[x19] // load scale Value
@ -436,7 +466,7 @@ BroadcastScaleValue:
mov v2.16b, v0.16b
mov v3.16b, v0.16b
AccumulatorsToFloat:
ScaleWithFloatPoint:
scvtf v16.4s,v16.4s // convert to float
scvtf v17.4s,v17.4s
scvtf v18.4s,v18.4s
@ -485,7 +515,87 @@ AccumulatorsToFloat:
fcvtns v29.4s,v29.4s
fcvtns v30.4s,v30.4s
fcvtns v31.4s,v31.4s
b Quantize
FixedPointScale:
tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq BroadcastFixedPointValue
ldp q0,q1,[x24],32 // load preshift vector
ldp q2,q3,[x24],32
ldp q4,q5,[x25],32 // load multiplier vector
ldp q6,q7,[x25],32
ldp q8,q9,[x26],32 // load postshift vector
ldp q10,q11,[x26],32
b ScaleWithFixedPoint
BroadcastFixedPointValue:
ld1r {v0.4s},[x24] // load preshift Value
mov v1.16b, v0.16b
mov v2.16b, v0.16b
mov v3.16b, v0.16b
ld1r {v4.4s},[x25] // load multiplier Value
mov v5.16b, v4.16b
mov v6.16b, v4.16b
mov v7.16b, v4.16b
ld1r {v8.4s},[x26] // load postshift Value
mov v9.16b, v8.16b
mov v10.16b, v8.16b
mov v11.16b, v8.16b
ScaleWithFixedPoint:
sqshl v16.4s, v16.4s, v0.4s // preshift
sqshl v17.4s, v17.4s, v0.4s
sqshl v18.4s, v18.4s, v0.4s
sqshl v19.4s, v19.4s, v0.4s
sqshl v20.4s, v20.4s, v1.4s
sqshl v21.4s, v21.4s, v1.4s
sqshl v22.4s, v22.4s, v1.4s
sqshl v23.4s, v23.4s, v1.4s
sqshl v24.4s, v24.4s, v2.4s
sqshl v25.4s, v25.4s, v2.4s
sqshl v26.4s, v26.4s, v2.4s
sqshl v27.4s, v27.4s, v2.4s
sqshl v28.4s, v28.4s, v3.4s
sqshl v29.4s, v29.4s, v3.4s
sqshl v30.4s, v30.4s, v3.4s
sqshl v31.4s, v31.4s, v3.4s
sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale
sqdmulh v17.4s, v17.4s, v4.4s
sqdmulh v18.4s, v18.4s, v4.4s
sqdmulh v19.4s, v19.4s, v4.4s
sqdmulh v20.4s, v20.4s, v5.4s
sqdmulh v21.4s, v21.4s, v5.4s
sqdmulh v22.4s, v22.4s, v5.4s
sqdmulh v23.4s, v23.4s, v5.4s
sqdmulh v24.4s, v24.4s, v6.4s
sqdmulh v25.4s, v25.4s, v6.4s
sqdmulh v26.4s, v26.4s, v6.4s
sqdmulh v27.4s, v27.4s, v6.4s
sqdmulh v28.4s, v28.4s, v7.4s
sqdmulh v29.4s, v29.4s, v7.4s
sqdmulh v30.4s, v30.4s, v7.4s
sqdmulh v31.4s, v31.4s, v7.4s
srshl v16.4s,v16.4s, v8.4s // post shift
srshl v17.4s,v17.4s, v8.4s
srshl v18.4s,v18.4s, v8.4s
srshl v19.4s,v19.4s, v8.4s
srshl v20.4s,v20.4s, v9.4s
srshl v21.4s,v21.4s, v9.4s
srshl v22.4s,v22.4s, v9.4s
srshl v23.4s,v23.4s, v9.4s
srshl v24.4s,v24.4s, v10.4s
srshl v25.4s,v25.4s, v10.4s
srshl v26.4s,v26.4s, v10.4s
srshl v27.4s,v27.4s, v10.4s
srshl v28.4s,v28.4s, v11.4s
srshl v29.4s,v29.4s, v11.4s
srshl v30.4s,v30.4s, v11.4s
srshl v31.4s,v31.4s, v11.4s
Quantize:
sqxtn v16.4h,v16.4s
sqxtn v17.4h,v17.4s
sqxtn v18.4h,v18.4s
@ -530,10 +640,11 @@ AccumulatorsToFloat:
b.hi OutputChannelLoop
ExitKernel:
ldr x23,[sp,#64]
ldp x21,x22,[sp,#48]
ldp x19,x20,[sp,#32]
ldp d10,d11,[sp,#16]
ldp x25,x26,[sp,#.LConvSymFrame_SavedRegisters_x25]
ldp x23,x24,[sp,#.LConvSymFrame_SavedRegisters_x23]
ldp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21]
ldp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19]
ldp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10]
ldp d8,d9,[sp],#.LConvSymFrame_SavedRegisters
ret
@ -588,7 +699,7 @@ InChannels4:
ldr s1,[x13],4
ldr s2,[x14],4
ldr s3,[x15],4
ldr q5, [x1], 16
ldr q5,[x1],16
SdotByElement 16, 4, 0,0
SdotByElement 17, 4, 1,0
ldp q6, q7, [x1], 32

117
onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelNeon.S
View file

@ -18,21 +18,46 @@ Abstract:
#include "asmmacro.h"
.equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2
.equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4
//
// Stack frame layout for the symmetric convolution kernel.
// d8-d15, x19-x30 need to be preserved if used
//
.equ .LConvSymFrame_SavedNeonRegisters, (8 * 8)
.equ .LConvSymFrame_SavedRegisters, .LConvSymFrame_SavedNeonRegisters
.equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_SavedRegisters_d8_d9, 0
.equ .LConvSymFrame_SavedRegisters_d10_d11, 16
.equ .LConvSymFrame_SavedRegisters_d12_d13, 32
.equ .LConvSymFrame_SavedRegisters_d14_d15, 48
.equ .LConvSymFrame_SavedNeonRegisters, 64
.equ .LConvSymFrame_SavedRegisters_x19_x20, 64
.equ .LConvSymFrame_SavedRegisters_x21_x22, 80
.equ .LConvSymFrame_SavedRegisters, 96
.equ .LConvSymFrame_SavedRegisters_Neg, -96
.equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
.equ .LConvSymPostProcessParams_Bias, 0
.equ .LConvSymPostProcessParams_Scale, 8
.equ .LConvSymPostProcessParams_Min, 16
.equ .LConvSymPostProcessParams_Max, 20
.equ .LConvSymPostProcessParams_ZeroPoint, 24
.equ .LConvSymPostProcessParams_Multiplier,32
.equ .LConvSymPostProcessParams_PreShift, 40
.equ .LConvSymPostProcessParams_PostShift, 48
.text
@ -82,12 +107,15 @@ Return Value:
--*/
FUNCTION_ENTRY MlasConvSymS8KernelNeon
stp d8,d9,[sp,#-.LConvSymFrame_SavedRegisters]!
stp d8,d9,[sp,#.LConvSymFrame_SavedRegisters_Neg]!
ldr x8,[sp,#.LConvSymFrame_PostProcessParams]
ldrb w10,[sp,#.LConvSymFrame_KernelFlags]
stp d10,d11,[sp,#16]
stp d12,d13,[sp,#32]
stp d14,d15,[sp,#48]
stp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11]
stp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13]
stp d14,d15,[sp,#.LConvSymFrame_SavedRegisters_d14_d15]
stp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20]
stp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22]
mov x9,x3 // save kernel size
ldr x11,[x8,#.LConvSymPostProcessParams_Bias]
mov x16,x4 // save input channels
@ -96,6 +124,9 @@ Return Value:
add x5,x2,x5 // c1 = c0 + ldc
add x4,x4,7 // kc = (kc + 7) & ~7
csel x5,x2,x5,lo // if OutputCount < 2 c1 = c0
ldr x19,[x8,#.LConvSymPostProcessParams_PreShift]
ldr x20,[x8,#.LConvSymPostProcessParams_Multiplier]
ldr x21,[x8,#.LConvSymPostProcessParams_PostShift]
bic x4,x4,7
ldp s16,s18,[x11],8 // init accumulators with bias
ldp s20,s22,[x11],8
@ -296,17 +327,20 @@ Return Value:
b.hi .LConvSym.KernelSizeLoop
.LConvSym.Requantize:
ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint]
tst w10,#.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne .LConvSym.FixedPointScale
.LConvSym.FloatPointScale:
tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint]
beq .LConvSym.BroadcastScaleValue
ld1 {v4.4s,v5.4s},[x12] // load scale vector
b .LConvSym.AccumulatorsToFloat
b .LConvSym.ScaleWithFloatPoint
.LConvSym.BroadcastScaleValue:
ld1r {v4.4s},[x12] // load scale Value
mov v5.16b, v4.16b
.LConvSym.AccumulatorsToFloat:
.LConvSym.ScaleWithFloatPoint:
addp v16.4s,v16.4s,v18.4s
addp v20.4s,v20.4s,v22.4s
addp v24.4s,v24.4s,v26.4s
@ -329,12 +363,57 @@ Return Value:
fmul v3.4s,v3.4s,v5.4s
fcvtns v0.4s,v0.4s // convert to int
fcvtns v1.4s,v1.4s
dup v9.8h,w11
fcvtns v2.4s,v2.4s
fcvtns v3.4s,v3.4s
b .LConvSym.Quantize
.LConvSym.FixedPointScale:
tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint]
beq .LConvSym.BroadcastFixedPointValue
ld1 {v4.4s,v5.4s},[x19] // load preshift
ld1 {v6.4s,v7.4s},[x20] // load multiplier
ld1 {v8.4s,v9.4s},[x21] // load postshift
b .LConvSym.ScaleWithFixedPoint
.LConvSym.BroadcastFixedPointValue:
ld1r {v4.4s},[x19] // load preshift Value
mov v5.16b, v4.16b
ld1r {v6.4s},[x20] // load multiplier Value
mov v7.16b, v6.16b
ld1r {v8.4s},[x21] // load postshift Value
mov v9.16b, v8.16b
.LConvSym.ScaleWithFixedPoint:
addp v16.4s,v16.4s,v18.4s
addp v20.4s,v20.4s,v22.4s
addp v24.4s,v24.4s,v26.4s
addp v28.4s,v28.4s,v30.4s
addp v17.4s,v17.4s,v19.4s
addp v21.4s,v21.4s,v23.4s
addp v25.4s,v25.4s,v27.4s
addp v29.4s,v29.4s,v31.4s
addp v0.4s,v16.4s,v20.4s
addp v1.4s,v24.4s,v28.4s
addp v2.4s,v17.4s,v21.4s
addp v3.4s,v25.4s,v29.4s
sqshl v0.4s, v0.4s, v4.4s // preshift
sqshl v1.4s, v1.4s, v5.4s
sqshl v2.4s, v2.4s, v4.4s
sqshl v3.4s, v3.4s, v5.4s
sqdmulh v0.4s, v0.4s, v6.4s // multiplier
sqdmulh v1.4s, v1.4s, v7.4s
sqdmulh v2.4s, v2.4s, v6.4s
sqdmulh v3.4s, v3.4s, v7.4s
srshl v0.4s,v0.4s, v8.4s // postshift
srshl v1.4s,v1.4s, v9.4s
srshl v2.4s,v2.4s, v8.4s
srshl v3.4s,v3.4s, v9.4s
.LConvSym.Quantize:
dup v9.8h,w11
sqxtn v0.4h,v0.4s
sqxtn2 v0.8h,v1.4s
sqxtn v2.4h,v2.4s
sqxtn2 v0.8h,v1.4s
sqxtn2 v2.8h,v3.4s
subs x6, x6, 8
sqadd v0.8h,v0.8h,v9.8h
@ -343,14 +422,16 @@ Return Value:
sqxtn2 v0.16b,v2.8h
b.lo .LConvSym.PartialStore
st1 {v0.d}[1],[x5] // full 2x8 store to c
st1 {v0.d}[1],[x5] // full 2x8 store to c
st1 {v0.8b},[x2]
.LConvSym.ExitKernel:
ldp d14,d15,[sp,#48]
ldp d12,d13,[sp,#32]
ldp d10,d11,[sp,#16]
ldp d8,d9,[sp],#64
ldp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22]
ldp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20]
ldp d14,d15,[sp,#.LConvSymFrame_SavedRegisters_d14_d15]
ldp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13]
ldp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11]
ldp d8,d9,[sp],#.LConvSymFrame_SavedRegisters
ret
.LConvSym.8InputChannels:

169
onnxruntime/core/mlas/lib/aarch64/ConvSymU8KernelDot.S
View file

@ -18,22 +18,44 @@ Abstract:
#include "asmmacro.h"
#include "AssembleDotProduct.h"
.equ .LMLAS_CONV_SYM_FLAG_INPUT_DIRECT, 1
.equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2
.equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4
//
// Stack frame layout for the symmetric convolution kernel.
// d8-d15, x19-x30 need to be preserved if used
//
.equ .LConvSymFrame_SavedRegisters, (8 * 8)
.equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_SavedRegisters_d8_d9, 0
.equ .LConvSymFrame_SavedRegisters_d10_d11, 16
.equ .LConvSymFrame_SavedRegisters_d12_d13, 32
.equ .LConvSymFrame_SavedRegisters_x19_x20, 48
.equ .LConvSymFrame_SavedRegisters_x21_x22, 64
.equ .LConvSymFrame_SavedRegisters, 80
.equ .LConvSymFrame_SavedRegisters_Neg, -80
.equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
.equ .LConvSymPostProcessParams_Bias, 0
.equ .LConvSymPostProcessParams_Scale, 8
.equ .LConvSymPostProcessParams_Min, 16
.equ .LConvSymPostProcessParams_Max, 20
.equ .LConvSymPostProcessParams_ZeroPoint, 24
.equ .LConvSymPostProcessParams_Multiplier,32
.equ .LConvSymPostProcessParams_PreShift, 40
.equ .LConvSymPostProcessParams_PostShift, 48
.text
@ -46,15 +68,10 @@ Routine Description:
Arguments:
Input (x0) - Points to the input buffer.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points
directly at the input tensor.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an
indirection buffer. Every pointer in the indirection buffer points at a
InputChannels length vector (either from the input tensor or a vector of
padding values). These are grouped in batches of length KernelSize.
Input (x0) - Supplies the address of the indirect buffer. Every pointer in
the indirection buffer points at a InputChannels length vector (either
from the input tensor or a vector of padding values). These are grouped
in batches of length KernelSize.
These batches are then repeated OutputCount times.
Filter (x1) - Points to the filter buffer.
@ -63,8 +80,6 @@ Arguments:
KernelSize (x3/x9) - Size of the kernel (most commonly. 3x3=9, 5x5=25).
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1.
InputChannels (x4/x7) - Number of input channels.
OutputChannels (x5) - Number of output channels.
@ -86,18 +101,18 @@ Return Value:
--*/
FUNCTION_ENTRY MlasConvSymU8KernelDot
stp d8,d9,[sp,#-.LConvSymFrame_SavedRegisters]!
stp d8,d9,[sp,#.LConvSymFrame_SavedRegisters_Neg]!
ldr x8,[sp,#.LConvSymFrame_PostProcessParams]
ldr w10,[sp,#.LConvSymFrame_KernelFlags]
stp d10,d11,[sp,#16]
stp d12,d13,[sp,#32]
stp x19,x20,[sp,#48]
stp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11]
stp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13]
stp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20]
stp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22]
cmp x7,2 // OutputCount < 2 ?
add x16,x2,x5 // x16 -> C1
lsl x3,x3,#3 // KernelSize * sizeof(int8_t*)
csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0
mov x20,x4
add x4,x4,3 // InputChannels align to 4
add x17,x16,x5 // x17 -> C2
ldr x11,[x8,#.LConvSymPostProcessParams_Bias]
@ -108,6 +123,9 @@ Return Value:
ldr x19,[x8,#.LConvSymPostProcessParams_Scale]
csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2
movi v12.16b,128 // for top bit flipping
ldr x20,[x8,#.LConvSymPostProcessParams_PreShift]
ldr x21,[x8,#.LConvSymPostProcessParams_Multiplier]
ldr x22,[x8,#.LConvSymPostProcessParams_PostShift]
OutputChannelLoop:
ldp q16,q20,[x11],32 // Init accumulators with biases
@ -127,23 +145,6 @@ OutputChannelLoop:
mov x9,x3 // restore KernelSize * sizeof(int8_t*)
KernelSizeLoop:
tst w10,#.LMLAS_CONV_SYM_FLAG_INPUT_DIRECT
beq InputIndirection
InputDirect:
cmp x16,x2
mov x12,x0 // x12 -> A0
add x13,x0,x20 // x13 -> A1 = A0 + input channels
csel x13,x0,x13,eq
cmp x17,x16
add x14,x0,x20,lsl#1 // x14 -> A2
csel x14,x13,x14,eq
cmp x5,x17
add x15,x13,x20,lsl#1 // x15 -> A3
csel x15,x14,x15,eq
b FinishLoadAPtr
InputIndirection:
ldr x12,[x0] // x12 -> A0
cmp x16,x2
b.eq SkipLoadA1 // C1==C0 -> A0=A1=A2=A3
@ -391,12 +392,15 @@ InChLoopEpilogue:
b.hi KernelSizeLoop
Requantize:
tst w10,#.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale
FloatPointScale:
tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq BroadcastScaleValue
ldp q0,q1,[x19],32 // load scale vector
ldp q2,q3,[x19],32
b AccumulatorsToFloat
b ScaleWithFloatPoint
BroadcastScaleValue:
ld1r {v0.4s},[x19] // load scale Value
@ -404,7 +408,7 @@ BroadcastScaleValue:
mov v2.16b, v0.16b
mov v3.16b, v0.16b
AccumulatorsToFloat:
ScaleWithFloatPoint:
scvtf v16.4s,v16.4s // convert to float
scvtf v17.4s,v17.4s
scvtf v18.4s,v18.4s
@ -453,7 +457,87 @@ AccumulatorsToFloat:
fcvtns v29.4s,v29.4s
fcvtns v30.4s,v30.4s
fcvtns v31.4s,v31.4s
b Quantize
FixedPointScale:
tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq BroadcastFixedPointValue
ldp q0,q1,[x20],32 // load preshift vector
ldp q2,q3,[x20],32
ldp q4,q5,[x21],32 // load multiplier vector
ldp q6,q7,[x21],32
ldp q8,q9,[x22],32 // load postshift vector
ldp q10,q11,[x22],32
b ScaleWithFixedPoint
BroadcastFixedPointValue:
ld1r {v0.4s},[x20] // load preshift Value
mov v1.16b, v0.16b
mov v2.16b, v0.16b
mov v3.16b, v0.16b
ld1r {v4.4s},[x21] // load multiplier Value
mov v5.16b, v4.16b
mov v6.16b, v4.16b
mov v7.16b, v4.16b
ld1r {v8.4s},[x22] // load postshift Value
mov v9.16b, v8.16b
mov v10.16b, v8.16b
mov v11.16b, v8.16b
ScaleWithFixedPoint:
sqshl v16.4s, v16.4s, v0.4s // preshift
sqshl v17.4s, v17.4s, v0.4s
sqshl v18.4s, v18.4s, v0.4s
sqshl v19.4s, v19.4s, v0.4s
sqshl v20.4s, v20.4s, v1.4s
sqshl v21.4s, v21.4s, v1.4s
sqshl v22.4s, v22.4s, v1.4s
sqshl v23.4s, v23.4s, v1.4s
sqshl v24.4s, v24.4s, v2.4s
sqshl v25.4s, v25.4s, v2.4s
sqshl v26.4s, v26.4s, v2.4s
sqshl v27.4s, v27.4s, v2.4s
sqshl v28.4s, v28.4s, v3.4s
sqshl v29.4s, v29.4s, v3.4s
sqshl v30.4s, v30.4s, v3.4s
sqshl v31.4s, v31.4s, v3.4s
sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale
sqdmulh v17.4s, v17.4s, v4.4s
sqdmulh v18.4s, v18.4s, v4.4s
sqdmulh v19.4s, v19.4s, v4.4s
sqdmulh v20.4s, v20.4s, v5.4s
sqdmulh v21.4s, v21.4s, v5.4s
sqdmulh v22.4s, v22.4s, v5.4s
sqdmulh v23.4s, v23.4s, v5.4s
sqdmulh v24.4s, v24.4s, v6.4s
sqdmulh v25.4s, v25.4s, v6.4s
sqdmulh v26.4s, v26.4s, v6.4s
sqdmulh v27.4s, v27.4s, v6.4s
sqdmulh v28.4s, v28.4s, v7.4s
sqdmulh v29.4s, v29.4s, v7.4s
sqdmulh v30.4s, v30.4s, v7.4s
sqdmulh v31.4s, v31.4s, v7.4s
srshl v16.4s,v16.4s, v8.4s // post shift
srshl v17.4s,v17.4s, v8.4s
srshl v18.4s,v18.4s, v8.4s
srshl v19.4s,v19.4s, v8.4s
srshl v20.4s,v20.4s, v9.4s
srshl v21.4s,v21.4s, v9.4s
srshl v22.4s,v22.4s, v9.4s
srshl v23.4s,v23.4s, v9.4s
srshl v24.4s,v24.4s, v10.4s
srshl v25.4s,v25.4s, v10.4s
srshl v26.4s,v26.4s, v10.4s
srshl v27.4s,v27.4s, v10.4s
srshl v28.4s,v28.4s, v11.4s
srshl v29.4s,v29.4s, v11.4s
srshl v30.4s,v30.4s, v11.4s
srshl v31.4s,v31.4s, v11.4s
Quantize:
sqxtn v16.4h,v16.4s
sqxtn v17.4h,v17.4s
sqxtn v18.4h,v18.4s
@ -498,9 +582,10 @@ AccumulatorsToFloat:
b.hi OutputChannelLoop
ExitKernel:
ldp x19,x20,[sp,#48]
ldp d12,d13,[sp,#32]
ldp d10,d11,[sp,#16]
ldp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22]
ldp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20]
ldp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13]
ldp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11]
ldp d8,d9,[sp],#.LConvSymFrame_SavedRegisters
ret

156
onnxruntime/core/mlas/lib/aarch64/ConvSymU8KernelNeon.S
View file

@ -17,23 +17,45 @@ Abstract:
#include "asmmacro.h"
.equ .LMLAS_CONV_SYM_FLAG_INPUT_DIRECT, 1
.equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2
.equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4
//
// Stack frame layout for the symmetric convolution kernel.
// d8-d15, x19-x30 need to be preserved if used
//
.equ .LConvSymFrame_SavedNeonRegisters, (8 * 8)
.equ .LConvSymFrame_SavedRegisters, .LConvSymFrame_SavedNeonRegisters
.equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_SavedRegisters_d8_d9, 0
.equ .LConvSymFrame_SavedRegisters_d10_d11, 16
.equ .LConvSymFrame_SavedRegisters_d12_d13, 32
.equ .LConvSymFrame_SavedRegisters_d14_d15, 48
.equ .LConvSymFrame_SavedRegisters_x19_x20, 64
.equ .LConvSymFrame_SavedRegisters_x21_x22, 80
.equ .LConvSymFrame_SavedRegisters, 96
.equ .LConvSymFrame_SavedRegisters_Neg, -96
.equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters
.equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
.equ .LConvSymPostProcessParams_Bias, 0
.equ .LConvSymPostProcessParams_Scale, 8
.equ .LConvSymPostProcessParams_Min, 16
.equ .LConvSymPostProcessParams_Max, 20
.equ .LConvSymPostProcessParams_ZeroPoint, 24
.equ .LConvSymPostProcessParams_Multiplier,32
.equ .LConvSymPostProcessParams_PreShift, 40
.equ .LConvSymPostProcessParams_PostShift, 48
.text
@ -46,15 +68,10 @@ Routine Description:
Arguments:
Input (x0) - Supplies the address of the input buffer.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points
directly at the input tensor.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an
indirection buffer. Every pointer in the indirection buffer points at a
InputChannels length vector (either from the input tensor or a vector of
padding values). These are grouped in batches of length KernelSize.
Input (x0) - Supplies the address of the indirect buffer. Every pointer in
the indirection buffer points at a InputChannels length vector (either
from the input tensor or a vector of padding values). These are grouped
in batches of length KernelSize.
These batches are then repeated OutputCount times.
Filter (x1) - Supplies the address of the filter buffer.
@ -63,8 +80,6 @@ Arguments:
KernelSize (x3) - Supplies the size of the kernel.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1.
InputChannels (x4) - Supplies the number of input channels.
This implementation requires the count to be a multiple of 8.
@ -90,12 +105,14 @@ Return Value:
--*/
FUNCTION_ENTRY MlasConvSymU8KernelNeon
stp d8,d9,[sp,#-64]!
stp d8,d9,[sp,#.LConvSymFrame_SavedRegisters_Neg]!
ldr x8,[sp,#.LConvSymFrame_PostProcessParams]
ldrb w10,[sp,#.LConvSymFrame_KernelFlags]
stp d10,d11,[sp,#16]
stp d12,d13,[sp,#32]
stp d14,d15,[sp,#48]
stp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11]
stp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13]
stp d14,d15,[sp,#.LConvSymFrame_SavedRegisters_d14_d15]
stp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20]
stp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22]
mov x9,x3 // save kernel size
ldr x11,[x8,#.LConvSymPostProcessParams_Bias]
mov x16,x4 // save input channels
@ -104,6 +121,9 @@ Return Value:
add x5,x2,x5 // c1 = c0 + ldc
add x4,x4,7 // kc = (kc + 7) & ~7
csel x5,x2,x5,lo // if OutputCount < 2 c1 = c0
ldr x19,[x8,#.LConvSymPostProcessParams_PreShift]
ldr x20,[x8,#.LConvSymPostProcessParams_Multiplier]
ldr x21,[x8,#.LConvSymPostProcessParams_PostShift]
bic x4,x4,7
ldp s16,s18,[x11],8 // init accumulators with bias
ldp s20,s22,[x11],8
@ -137,30 +157,18 @@ Return Value:
//
.LConvSym.KernelSizeLoop:
# Load next 2 A pointers
tst w10,#.LMLAS_CONV_SYM_FLAG_INPUT_DIRECT
ldr d4,[x1]
ldr d5,[x1,8]
beq .LConvSym.InputIndirection
.LConvSym.InputDirect:
mov x13,x0 // x13 -> A0
add x15,x0,x16 // x15 -> A1 = A0 + input channels
b .LConvSym.BlockLoopPrologue
.LConvSym.InputIndirection:
cmp x7,2 // test if OutputCount < 2
ldr x13,[x0] // x13 -> A0
blo .LConvSym.SkipLoadA1
bhs .LConvSym.LoadA1
ldr x15,[x0],#8 // x15 -> A0
b .LConvSym.BlockLoopPrologue
.LConvSym.LoadA1:
ldr x15,[x0,x3,lsl#3] // x15 -> A1
.LConvSym.SkipLoadA1:
.LConvSym.BlockLoopPrologue:
cmp x7,2 // test if OutputCount < 2
add x0,x0,8 // indirect A advance to next pointer, prepare for kernel size loop
csel x15,x13,x15,lo // if OutputCount < 2 x15 -> A0
.LConvSym.BlockLoopPrologue:
ldr d4,[x1]
subs x14,x4,16 // input channel - 16
ldr d5,[x1,8]
movi v12.8b,128
blo .LConvSym.8InputChannels // less than 16 deep, no unroll
@ -325,17 +333,20 @@ Return Value:
b.hi .LConvSym.KernelSizeLoop
.LConvSym.Requantize:
ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint]
tst w10,#.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne .LConvSym.FixedPointScale
.LConvSym.FloatPointScale:
tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint]
beq .LConvSym.BroadcastScaleValue
ld1 {v4.4s,v5.4s},[x12] // load scale vector
b .LConvSym.AccumulatorsToFloat
b .LConvSym.ScaleWithFloatPoint
.LConvSym.BroadcastScaleValue:
ld1r {v4.4s},[x12] // load scale Value
mov v5.16b, v4.16b
.LConvSym.AccumulatorsToFloat:
.LConvSym.ScaleWithFloatPoint:
addp v16.4s,v16.4s,v18.4s
addp v20.4s,v20.4s,v22.4s
addp v24.4s,v24.4s,v26.4s
@ -358,12 +369,57 @@ Return Value:
fmul v3.4s,v3.4s,v5.4s
fcvtns v0.4s,v0.4s // convert to int
fcvtns v1.4s,v1.4s
dup v9.8h,w11
fcvtns v2.4s,v2.4s
fcvtns v3.4s,v3.4s
b .LConvSym.Quantize
.LConvSym.FixedPointScale:
tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint]
beq .LConvSym.BroadcastFixedPointValue
ld1 {v4.4s,v5.4s},[x19] // load preshift
ld1 {v6.4s,v7.4s},[x20] // load multiplier
ld1 {v8.4s,v9.4s},[x21] // load postshift
b .LConvSym.ScaleWithFixedPoint
.LConvSym.BroadcastFixedPointValue:
ld1r {v4.4s},[x19] // load preshift Value
mov v5.16b, v4.16b
ld1r {v6.4s},[x20] // load multiplier Value
mov v7.16b, v6.16b
ld1r {v8.4s},[x21] // load postshift Value
mov v9.16b, v8.16b
.LConvSym.ScaleWithFixedPoint:
addp v16.4s,v16.4s,v18.4s
addp v20.4s,v20.4s,v22.4s
addp v24.4s,v24.4s,v26.4s
addp v28.4s,v28.4s,v30.4s
addp v17.4s,v17.4s,v19.4s
addp v21.4s,v21.4s,v23.4s
addp v25.4s,v25.4s,v27.4s
addp v29.4s,v29.4s,v31.4s
addp v0.4s,v16.4s,v20.4s
addp v1.4s,v24.4s,v28.4s
addp v2.4s,v17.4s,v21.4s
addp v3.4s,v25.4s,v29.4s
sqshl v0.4s, v0.4s, v4.4s // preshift
sqshl v1.4s, v1.4s, v5.4s
sqshl v2.4s, v2.4s, v4.4s
sqshl v3.4s, v3.4s, v5.4s
sqdmulh v0.4s, v0.4s, v6.4s // multiplier
sqdmulh v1.4s, v1.4s, v7.4s
sqdmulh v2.4s, v2.4s, v6.4s
sqdmulh v3.4s, v3.4s, v7.4s
srshl v0.4s,v0.4s, v8.4s // postshift
srshl v1.4s,v1.4s, v9.4s
srshl v2.4s,v2.4s, v8.4s
srshl v3.4s,v3.4s, v9.4s
.LConvSym.Quantize:
dup v9.8h,w11
sqxtn v0.4h,v0.4s
sqxtn2 v0.8h,v1.4s
sqxtn v2.4h,v2.4s
sqxtn2 v0.8h,v1.4s
sqxtn2 v2.8h,v3.4s
subs x6, x6, 8
sqadd v0.8h,v0.8h,v9.8h
@ -372,14 +428,16 @@ Return Value:
sqxtun2 v0.16b,v2.8h
b.lo .LConvSym.PartialStore
st1 {v0.d}[1],[x5] // full 2x8 store to c
st1 {v0.d}[1],[x5] // full 2x8 store to c
st1 {v0.8b},[x2]
.LConvSym.ExitKernel:
ldp d14,d15,[sp,#48]
ldp d12,d13,[sp,#32]
ldp d10,d11,[sp,#16]
ldp d8,d9,[sp],#64
ldp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22]
ldp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20]
ldp d14,d15,[sp,#.LConvSymFrame_SavedRegisters_d14_d15]
ldp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13]
ldp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11]
ldp d8,d9,[sp],#.LConvSymFrame_SavedRegisters
ret
.LConvSym.8InputChannels:

269
onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvKernelSize9Neon.S
View file

@ -18,28 +18,47 @@ Abstract:
#include "asmmacro.h"
.equ .LConvSymDepthwisePostProcessParams_Bias, 0
.equ .LConvSymDepthwisePostProcessParams_Scale, 8
.equ .LConvSymDepthwisePostProcessParams_ZeroPoint, 24
.equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, 1
.equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4
//
// Stack frame layout for the depthwise conv kernel. d8-d15, x19-x30 need save
//
.equ .LMlasConvSymDepthwiseKernelSize9_backup_x19_x20, 0
.equ .LMlasConvSymDepthwiseKernelSize9_backup_x21_x22, 16
.equ .LMlasConvSymDepthwiseKernelSize9_backup_x23_x24, 32
.equ .LMlasConvSymDepthwiseKernelSize9_backup_x25_x26, 48
.equ .LMlasConvSymDepthwiseKernelSize9_backup_x27_x28, 64
.equ .LMlasConvSymDepthwiseKernelSize9_backup_d8_d9, 80
.equ .LMlasConvSymDepthwiseKernelSize9_backup_d10_d11, 96
.equ .LMlasConvSymDepthwiseKernelSize9_backup_d12_d13, 112
.equ .LMlasConvSymDepthwiseKernelSize9_backup_d14_d15, 128
.equ .LMlasConvSymDepthwiseKernelSize9_SavedRegisters, 144
.equ .LMlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg, -144
.equ .LMlasConvSymDepthwiseKernelSize9_x19_x20, 0
.equ .LMlasConvSymDepthwiseKernelSize9_x21_x22, 16
.equ .LMlasConvSymDepthwiseKernelSize9_x23_x24, 32
.equ .LMlasConvSymDepthwiseKernelSize9_x25_x26, 48
.equ .LMlasConvSymDepthwiseKernelSize9_x27_x28, 64
.equ .LMlasConvSymDepthwiseKernelSize9_d8_d9, 80
.equ .LMlasConvSymDepthwiseKernelSize9_d10_d11, 96
.equ .LMlasConvSymDepthwiseKernelSize9_d12_d13, 112
.equ .LMlasConvSymDepthwiseKernelSize9_d14_d15, 128
.equ .LMlasConvSymDepthwiseKernelSize9_x0_x4, 144
.equ .LMlasConvSymDepthwiseKernelSize9_multiplier_postshift, 160
.equ .LMlasConvSymDepthwiseKernelSize9_SavedRegisters, 176
.equ .LMlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg, -176
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
.equ .LConvSymPostProcessParams_Bias, 0
.equ .LConvSymPostProcessParams_Scale, 8
.equ .LConvSymPostProcessParams_Min, 16
.equ .LConvSymPostProcessParams_Max, 20
.equ .LConvSymPostProcessParams_ZeroPoint, 24
.equ .LConvSymPostProcessParams_Multiplier, 32
.equ .LConvSymPostProcessParams_PreShift, 40
.equ .LConvSymPostProcessParams_PostShift, 48
.text
@ -75,26 +94,32 @@ Return Value:
FUNCTION_ENTRY MlasConvSymDepthwiseKernelSize9Arm64U8S8
stp x19, x20, [sp, #.LMlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg]!
stp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x21_x22]
stp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x23_x24]
stp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x25_x26]
stp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x27_x28]
stp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d8_d9]
stp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d10_d11]
stp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d12_d13]
stp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d14_d15]
stp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_x21_x22]
stp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_x23_x24]
stp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_x25_x26]
stp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_x27_x28]
stp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_d8_d9]
stp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_d10_d11]
stp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_d12_d13]
stp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_d14_d15]
ldr x9, [x5, #.LConvSymDepthwisePostProcessParams_Bias]
ldr x8, [x5, #.LConvSymDepthwisePostProcessParams_Scale]
add x5, x5, #.LConvSymDepthwisePostProcessParams_ZeroPoint
tst x6, #.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
ldr x11, [x5, #.LConvSymPostProcessParams_Multiplier]
ldr x12, [x5, #.LConvSymPostProcessParams_PostShift]
ldr x10, [x5, #.LConvSymPostProcessParams_PreShift]
ldr x9, [x5, #.LConvSymPostProcessParams_Bias]
ldr x8, [x5, #.LConvSymPostProcessParams_Scale]
add x5, x5, #.LConvSymPostProcessParams_ZeroPoint
stp x11, x12,[sp, #.LMlasConvSymDepthwiseKernelSize9_multiplier_postshift]
csel x8, x8, x10, eq
ins v12.d[0], x1 // Filter
ins v13.d[0], x9 // Bias
ins v13.d[1], x8 // Scale
ins v13.d[1], x8 // Scale/PreShift
ld1r {v0.8h}, [x5] // zero point
movi v5.16b, #0x80 // flip 0x80
tbnz x6, #.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LMlasConvSymDepthwiseKernelSize9_SkipPerTensorScaleInit
ld1r {v1.4s}, [x8] // load scale value
ld1r {v1.4s}, [x8] // load scale/PreShift value
mov v2.16b, v1.16b
mov v3.16b, v1.16b
mov v4.16b, v1.16b
@ -113,7 +138,7 @@ Return Value:
ldur x28, [x0, #-8]
cbz x4, .LMlasConvSymDepthwiseKernelSize9_Dup_Inputs
ldp x10, x11, [x0], #72 // input ptrs for Output0
ldp x10, x11, [x0], #72 // input ptrs for Output1
ldp x12, x13, [x0, #-56]
sub x4, x4, #1
ldp x14, x15, [x0, #-40]
@ -123,7 +148,7 @@ Return Value:
.LMlasConvSymDepthwiseKernelSize9_Dup_Inputs:
mov x9, x3 // Output1 <-- Output0
mov x10, x20
mov x10, x20 // Input1 <-- Input0
mov x11, x21
mov x12, x22
mov x13, x23
@ -135,10 +160,12 @@ Return Value:
.LMlasConvSymDepthwiseKernelSize9_Loaded_Input:
stp x0, x4, [sp, #.LMlasConvSymDepthwiseKernelSize9_x0_x4]
eor x8, x8, x8 // Processed channels
umov x1, v12.D[0] // filter
umov x5, v13.D[0] // bias
umov x7, v13.D[1] // scale
umov x7, v13.D[1] // scale/PreShift
ldp x0, x4, [sp, #.LMlasConvSymDepthwiseKernelSize9_multiplier_postshift]
cmp x8, x2 // Save one register by not using count down to zero here
bhs .LMlasConvSymDepthwiseKernelSize9_Finish_Channels16_Loop
@ -232,7 +259,7 @@ Return Value:
ldr q23, [x17, x8] // out1 vi7
tbz x6, #.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LDonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9
ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales 0-15 for outs
ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales/preshift 0-15 for outs
.LDonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9:
eor v20.16b, v20.16b, v5.16b
@ -288,6 +315,9 @@ Return Value:
saddw v8.4s, v8.4s, v27.4h
saddw2 v9.4s, v9.4s, v27.8h
tst x6, #.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne .LFixedPointScale_U8S8
.LFloatPointScale_U8S8:
scvtf v16.4s, v16.4s // Requantize
scvtf v17.4s, v17.4s
scvtf v18.4s, v18.4s
@ -314,7 +344,55 @@ Return Value:
fcvtns v7.4s, v7.4s
fcvtns v8.4s, v8.4s
fcvtns v9.4s, v9.4s
b .LQuantize_U8S8
.LFixedPointScale_U8S8:
tbz x6,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LBroadcastFixedPointValue_U8S8
ldp q20,q21,[x0],32 // load multiplier vector
ldp q22,q23,[x0],32
ldp q24,q25,[x4],32 // load postshift vector
ldp q26,q27,[x4],32
b .LScaleWithFixedPoint_U8S8
.LBroadcastFixedPointValue_U8S8:
ld1r {v20.4s},[x0] // load multiplier Value
mov v21.16b, v20.16b
mov v22.16b, v20.16b
mov v23.16b, v20.16b
ld1r {v24.4s},[x4] // load postshift Value
mov v25.16b, v24.16b
mov v26.16b, v24.16b
mov v27.16b, v24.16b
.LScaleWithFixedPoint_U8S8:
sqshl v6.4s, v6.4s, v1.4s // preshift
sqshl v7.4s, v7.4s, v2.4s
sqshl v8.4s, v8.4s, v3.4s
sqshl v9.4s, v9.4s, v4.4s
sqshl v16.4s, v16.4s, v1.4s
sqshl v17.4s, v17.4s, v2.4s
sqshl v18.4s, v18.4s, v3.4s
sqshl v19.4s, v19.4s, v4.4s
sqdmulh v6.4s, v6.4s, v20.4s // multiply by multiplier
sqdmulh v7.4s, v7.4s, v21.4s
sqdmulh v8.4s, v8.4s, v22.4s
sqdmulh v9.4s, v9.4s, v23.4s
sqdmulh v16.4s, v16.4s, v20.4s
sqdmulh v17.4s, v17.4s, v21.4s
sqdmulh v18.4s, v18.4s, v22.4s
sqdmulh v19.4s, v19.4s, v23.4s
srshl v6.4s,v6.4s, v24.4s // post shift
srshl v7.4s,v7.4s, v25.4s
srshl v8.4s,v8.4s, v26.4s
srshl v9.4s,v9.4s, v27.4s
srshl v16.4s,v16.4s, v24.4s
srshl v17.4s,v17.4s, v25.4s
srshl v18.4s,v18.4s, v26.4s
srshl v19.4s,v19.4s, v27.4s
.LQuantize_U8S8:
sqxtn v16.4h, v16.4s // +zp, narrow and combine
sqxtn v18.4h, v18.4s
sqxtn v6.4h, v6.4s
@ -341,19 +419,20 @@ Return Value:
blo .LMlasConvSymDepthwiseKernelSize9_Channels16_Loop
.LMlasConvSymDepthwiseKernelSize9_Finish_Channels16_Loop:
ldp x0, x4, [sp, #.LMlasConvSymDepthwiseKernelSize9_x0_x4]
add x3, x3, x2, LSL #1
add x9, x9, x2, LSL #1
cbnz x4, .LMlasConvSymDepthwiseKernelSize9_OutputLoop
.LMlasConvSymDepthwiseKernelSize9_Exit:
ldp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d14_d15]
ldp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d12_d13]
ldp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d10_d11]
ldp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d8_d9]
ldp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x27_x28]
ldp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x25_x26]
ldp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x23_x24]
ldp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x21_x22]
ldp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_d14_d15]
ldp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_d12_d13]
ldp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_d10_d11]
ldp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_d8_d9]
ldp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_x27_x28]
ldp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_x25_x26]
ldp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_x23_x24]
ldp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_x21_x22]
ldp x19, x20, [sp], #.LMlasConvSymDepthwiseKernelSize9_SavedRegisters
ret
@ -390,25 +469,32 @@ Return Value:
FUNCTION_ENTRY MlasConvSymDepthwiseKernelSize9Arm64S8S8
stp x19, x20, [sp, #.LMlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg]!
stp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x21_x22]
stp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x23_x24]
stp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x25_x26]
stp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x27_x28]
stp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d8_d9]
stp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d10_d11]
stp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d12_d13]
stp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d14_d15]
stp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_x21_x22]
stp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_x23_x24]
stp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_x25_x26]
stp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_x27_x28]
stp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_d8_d9]
stp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_d10_d11]
stp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_d12_d13]
stp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_d14_d15]
ldr x9, [x5, #.LConvSymDepthwisePostProcessParams_Bias]
ldr x8, [x5, #.LConvSymDepthwisePostProcessParams_Scale]
add x5, x5, #.LConvSymDepthwisePostProcessParams_ZeroPoint
tst x6, #.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
ldr x9, [x5, #.LConvSymPostProcessParams_Bias]
ldr x8, [x5, #.LConvSymPostProcessParams_Scale]
ldr x10, [x5, #.LConvSymPostProcessParams_PreShift]
ldr x11, [x5, #.LConvSymPostProcessParams_Multiplier]
ldr x12, [x5, #.LConvSymPostProcessParams_PostShift]
add x5, x5, #.LConvSymPostProcessParams_ZeroPoint
csel x8, x8, x10, eq
ins v12.d[0], x1 // Filter
ins v13.d[0], x9 // Bias
ins v13.d[1], x8 // Scale
ins v13.d[1], x8 // Scale/PreShift
ins v5.d[0], x11 // Multiplier
ins v5.d[1], x12 // PostShift
ld1r {v0.8h}, [x5] // zero point
tbnz x6, #.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LMlasConvSymDepthwiseKernelSize9S8S8_SkipPerTensorScaleInit
ld1r {v1.4s}, [x8] // load scale value
ld1r {v1.4s}, [x8] // load scale/PreShift value
mov v2.16b, v1.16b
mov v3.16b, v1.16b
mov v4.16b, v1.16b
@ -449,10 +535,13 @@ Return Value:
.LMlasConvSymDepthwiseKernelSize9S8S8_Loaded_Input:
stp x0, x4, [sp, #.LMlasConvSymDepthwiseKernelSize9_x0_x4]
eor x8, x8, x8 // Processed channels
umov x1, v12.D[0] // filter
umov x5, v13.D[0] // bias
umov x7, v13.D[1] // scale
umov x7, v13.D[1] // scale/PreShift
umov x0, v5.D[0]
umov x4, v5.D[1]
cmp x8, x2 // Save one register by not using count down to zero here
bhs .LMlasConvSymDepthwiseKernelSize9S8S8_Finish_Channels16_Loop
@ -534,7 +623,7 @@ Return Value:
ldr q23, [x17, x8] // out1 vi7
tbz x6, #.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LDonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8
ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales 0-15 for outs
ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales/preshift 0-15 for outs
.LDonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8:
ldr q10, [x1] // vk8
@ -584,6 +673,9 @@ Return Value:
saddw v8.4s, v8.4s, v27.4h
saddw2 v9.4s, v9.4s, v27.8h
tst x6, #.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale
FloatPointScale:
scvtf v16.4s, v16.4s // Requantize
scvtf v17.4s, v17.4s
scvtf v18.4s, v18.4s
@ -610,7 +702,55 @@ Return Value:
fcvtns v7.4s, v7.4s
fcvtns v8.4s, v8.4s
fcvtns v9.4s, v9.4s
b Quantize
FixedPointScale:
tbz x6,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LBroadcastFixedPointValue
ldp q20,q21,[x0],32 // load multiplier vector
ldp q22,q23,[x0],32
ldp q24,q25,[x4],32 // load postshift vector
ldp q26,q27,[x4],32
b ScaleWithFixedPoint
.LBroadcastFixedPointValue:
ld1r {v20.4s},[x0] // load multiplier Value
mov v21.16b, v20.16b
mov v22.16b, v20.16b
mov v23.16b, v20.16b
ld1r {v24.4s},[x4] // load postshift Value
mov v25.16b, v24.16b
mov v26.16b, v24.16b
mov v27.16b, v24.16b
ScaleWithFixedPoint:
sqshl v6.4s, v6.4s, v1.4s // preshift
sqshl v7.4s, v7.4s, v2.4s
sqshl v8.4s, v8.4s, v3.4s
sqshl v9.4s, v9.4s, v4.4s
sqshl v16.4s, v16.4s, v1.4s
sqshl v17.4s, v17.4s, v2.4s
sqshl v18.4s, v18.4s, v3.4s
sqshl v19.4s, v19.4s, v4.4s
sqdmulh v6.4s, v6.4s, v20.4s // multiply by multiplier
sqdmulh v7.4s, v7.4s, v21.4s
sqdmulh v8.4s, v8.4s, v22.4s
sqdmulh v9.4s, v9.4s, v23.4s
sqdmulh v16.4s, v16.4s, v20.4s
sqdmulh v17.4s, v17.4s, v21.4s
sqdmulh v18.4s, v18.4s, v22.4s
sqdmulh v19.4s, v19.4s, v23.4s
srshl v6.4s,v6.4s, v24.4s // post shift
srshl v7.4s,v7.4s, v25.4s
srshl v8.4s,v8.4s, v26.4s
srshl v9.4s,v9.4s, v27.4s
srshl v16.4s,v16.4s, v24.4s
srshl v17.4s,v17.4s, v25.4s
srshl v18.4s,v18.4s, v26.4s
srshl v19.4s,v19.4s, v27.4s
Quantize:
sqxtn v16.4h, v16.4s // +zp, narrow and combine
sqxtn v18.4h, v18.4s
sqxtn v6.4h, v6.4s
@ -637,19 +777,20 @@ Return Value:
blo .LMlasConvSymDepthwiseKernelSize9S8S8_Channels16_Loop
.LMlasConvSymDepthwiseKernelSize9S8S8_Finish_Channels16_Loop:
ldp x0, x4, [sp, #.LMlasConvSymDepthwiseKernelSize9_x0_x4]
add x3, x3, x2, LSL #1
add x9, x9, x2, LSL #1
cbnz x4, .LMlasConvSymDepthwiseKernelSize9S8S8_OutputLoop
.LMlasConvSymDepthwiseKernelSize9S8S8_Exit:
ldp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d14_d15]
ldp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d12_d13]
ldp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d10_d11]
ldp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d8_d9]
ldp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x27_x28]
ldp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x25_x26]
ldp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x23_x24]
ldp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x21_x22]
ldp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_d14_d15]
ldp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_d12_d13]
ldp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_d10_d11]
ldp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_d8_d9]
ldp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_x27_x28]
ldp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_x25_x26]
ldp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_x23_x24]
ldp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_x21_x22]
ldp x19, x20, [sp], #.LMlasConvSymDepthwiseKernelSize9_SavedRegisters
ret

299
onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvSymS8KernelNeon.S
View file

@ -17,23 +17,44 @@ Abstract:
#include "asmmacro.h"
.equ MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2
.equ MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4
//
// Stack frame layout for the depthwise conv kernel.
// d8-d15, x19-x30 need to be preserved if used
//
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d8_d9, 0
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11, 16
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13, 32
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15, 48
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters, 64
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters_Neg, -64
.equ .LConvSymDepthwiseKernelFrame_PostProcessParams, 0 + .LConvSymDepthwiseKernelFrame_SavedRegisters
.equ .LConvSymDepthwiseKernelFrame_KernelFlags, 8 + .LConvSymDepthwiseKernelFrame_SavedRegisters
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters, (4 * 8)
.equ .LConvSymDepthwiseKernelFrame_PostProcessParams, 0 + .LConvSymDepthwiseKernelFrame_SavedRegisters
.equ .LConvSymDepthwiseKernelFrame_KernelFlags, 8 + .LConvSymDepthwiseKernelFrame_SavedRegisters
.equ .LConvSymDepthwisePostProcessParams_Bias, 0
.equ .LConvSymDepthwisePostProcessParams_Scale, 8
.equ .LConvSymDepthwisePostProcessParams_Min, 16
.equ .LConvSymDepthwisePostProcessParams_Max, 20
.equ .LConvSymDepthwisePostProcessParams_ZeroPoint, 24
.equ MLAS_CONV_SYM_FLAG_INPUT_DIRECT, 1
.equ MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
.equ .LConvSymPostProcessParams_Bias, 0
.equ .LConvSymPostProcessParams_Scale, 8
.equ .LConvSymPostProcessParams_Min, 16
.equ .LConvSymPostProcessParams_Max, 20
.equ .LConvSymPostProcessParams_ZeroPoint, 24
.equ .LConvSymPostProcessParams_Multiplier, 32
.equ .LConvSymPostProcessParams_PreShift, 40
.equ .LConvSymPostProcessParams_PostShift, 48
.text
@ -47,28 +68,28 @@ Routine Description:
Arguments:
Input (x0) - Supplies the address of the indirection buffer.
Filter (x1) - Supplies the address of the filter buffer.
Output (x2) - Supplies the address of the output buffer.
KernelSize (x3) - Supplies the size of the kernel.
Channels (x4) - Supplies the number of input and output channels.
ChannelOffset (x5) - Supplies the byte offset from the indirection buffer base
address for this iteration.
ChannelCount (x6) - Supplies the number of channels this iteration produces.
This implementation requires the count to be 16 or 8
OutputCount (x7)- Supplies the number of output elements this iteration produces.
This implementation requires the count to be in the range 1 to 2.
This implementation requires the count to be in the range 1 to 4.
PostProcessParams - Supplies the address of the post process parameter block.
KernelFlags - Supplies additional flags controlling the operation.
Return Value:
@ -79,14 +100,16 @@ Return Value:
FUNCTION_ENTRY MlasConvSymDepthwiseS8KernelNeon
stp d12,d13,[sp,#-.LConvSymDepthwiseKernelFrame_SavedRegisters]!
stp d8,d9,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_Neg]!
ldr x8,[sp,#.LConvSymDepthwiseKernelFrame_PostProcessParams]
stp d14,d15,[sp,#16]
stp d10,d11,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11]
stp d12,d13,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13]
stp d14,d15,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15]
cmp x7,2
add x9,x0,x3,lsl#3 // x9 -> &A1
add x14,x0,x3,lsl#4 // x14 -> &A2
add x15,x9,x3,lsl#4 // x15 -> &A3
ldr x16,[x8,#.LConvSymDepthwisePostProcessParams_Bias]
ldr x16,[x8,#.LConvSymPostProcessParams_Bias]
csel x9,x0,x9,lo // x9 -> &A0 if OutputCount < 2
csel x14,x0,x14,ls // x14 -> &A0 if OutputCount <= 2
ldr x11,[x9],#8 // x11 -> A1 iter 0
@ -161,7 +184,7 @@ Return Value:
smlal v12.8h,v1.8b,v2.8b
ldr x11,[x9],#8 // x11 -> A1 iter 2
smlal2 v13.8h,v1.16b,v2.16b
b.eq .LEpilogueC16P3 // 3 pixel remains
b.eq .LEpilogueC16P3 // 3 pixel remains
ldr q2,[x12,x5] // A2 iter 1
smlal v14.8h,v1.8b,v3.8b
ldr x12,[x0],#8 // x12 -> A0 iter 3
@ -228,22 +251,15 @@ Return Value:
saddw v30.4s,v30.4s,v15.4h
saddw2 v31.4s,v31.4s,v15.8h
ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags]
ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
smull2 v13.8h,v0.16b,v4.16b
ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
smull2 v15.8h,v0.16b,v6.16b
smlal v12.8h,v1.8b,v2.8b
smlal2 v13.8h,v1.16b,v2.16b
smlal v14.8h,v1.8b,v3.8b
smlal2 v15.8h,v1.16b,v3.16b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq .LSkipScaleVecLoad2
ldp q4,q5,[x12],#32 // load scale vector if per channel
ldp q6,q3,[x12]
.LSkipScaleVecLoad2:
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v18.4s,v18.4s,v13.4h
@ -252,7 +268,7 @@ Return Value:
saddw2 v21.4s,v21.4s,v14.8h
saddw v22.4s,v22.4s,v15.4h
saddw2 v23.4s,v23.4s,v15.8h
b .LDequantization
b .LRequantization
.LProcC16P1:
//
@ -327,10 +343,8 @@ Return Value:
// Loop epilogue (process last single pixel) mixed with loading of dequantization params
//
ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags]
ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
smull2 v13.8h,v0.16b,v4.16b
ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
smull2 v15.8h,v0.16b,v6.16b
saddw v24.4s,v24.4s,v12.4h
@ -345,12 +359,7 @@ Return Value:
smull2 v13.8h,v0.16b,v2.16b
smull v14.8h,v0.8b,v3.8b
smull2 v15.8h,v0.16b,v3.16b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq .LSkipScaleVecLoad
ldp q4,q5,[x12],#32 // load scale vector if per channel
ldp q6,q3,[x12]
.LSkipScaleVecLoad:
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v18.4s,v18.4s,v13.4h
@ -360,7 +369,24 @@ Return Value:
saddw v22.4s,v22.4s,v15.4h
saddw2 v23.4s,v23.4s,v15.8h
.LDequantization:
.LRequantization:
b.ne .LFixedPointScale
.LFloatingPointScale:
ldr x12,[x8,#.LConvSymPostProcessParams_Scale]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq .LBroadcastScaleValue
ldp q4,q5,[x12],#32 // load scale vector if per channel
ldp q6,q3,[x12]
b .LScaleWithFloatPoint
.LBroadcastScaleValue:
ld1r {v4.4s},[x12] // load scale val
mov v5.16b,v4.16b // broadcast scale val if not per channel
mov v6.16b,v4.16b
mov v3.16b,v4.16b
.LScaleWithFloatPoint:
scvtf v24.4s,v24.4s // convert to float
scvtf v25.4s,v25.4s
scvtf v26.4s,v26.4s
@ -377,11 +403,6 @@ Return Value:
scvtf v21.4s,v21.4s
scvtf v22.4s,v22.4s
scvtf v23.4s,v23.4s
b.ne .LSkipScaleBroadcast
mov v5.16b,v4.16b // broadcast scale val if not per channel
mov v6.16b,v4.16b
mov v3.16b,v4.16b
.LSkipScaleBroadcast:
fmul v24.4s,v24.4s,v4.4s // multiply by scale
fmul v25.4s,v25.4s,v5.4s
fmul v26.4s,v26.4s,v6.4s
@ -414,6 +435,90 @@ Return Value:
fcvtns v21.4s,v21.4s
fcvtns v22.4s,v22.4s
fcvtns v23.4s,v23.4s
b .LQuantize
.LFixedPointScale:
ldr x10,[x8,#.LConvSymPostProcessParams_PreShift]
ldr x11,[x8,#.LConvSymPostProcessParams_Multiplier]
ldr x12,[x8,#.LConvSymPostProcessParams_PostShift]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq .LBroadcastFixedPointValue
ldp q0,q1,[x10],32 // load preshift vector
ldp q2,q3,[x10],32
ldp q4,q5,[x11],32 // load multiplier vector
ldp q6,q7,[x11],32
ldp q8,q9,[x12],32 // load postshift vector
ldp q10,q11,[x12],32
b .LScaleWithFixedPoint
.LBroadcastFixedPointValue:
ld1r {v0.4s},[x10] // load preshift Value
mov v1.16b, v0.16b
mov v2.16b, v0.16b
mov v3.16b, v0.16b
ld1r {v4.4s},[x11] // load multiplier Value
mov v5.16b, v4.16b
mov v6.16b, v4.16b
mov v7.16b, v4.16b
ld1r {v8.4s},[x12] // load postshift Value
mov v9.16b, v8.16b
mov v10.16b, v8.16b
mov v11.16b, v8.16b
.LScaleWithFixedPoint:
sqshl v24.4s, v24.4s, v0.4s // preshift
sqshl v25.4s, v25.4s, v1.4s
sqshl v26.4s, v26.4s, v2.4s
sqshl v27.4s, v27.4s, v3.4s
sqshl v28.4s, v28.4s, v0.4s
sqshl v29.4s, v29.4s, v1.4s
sqshl v30.4s, v30.4s, v2.4s
sqshl v31.4s, v31.4s, v3.4s
sqshl v16.4s, v16.4s, v0.4s
sqshl v17.4s, v17.4s, v1.4s
sqshl v18.4s, v18.4s, v2.4s
sqshl v19.4s, v19.4s, v3.4s
sqshl v20.4s, v20.4s, v0.4s
sqshl v21.4s, v21.4s, v1.4s
sqshl v22.4s, v22.4s, v2.4s
sqshl v23.4s, v23.4s, v3.4s
sqdmulh v24.4s, v24.4s, v4.4s // multiplier
sqdmulh v25.4s, v25.4s, v5.4s
sqdmulh v26.4s, v26.4s, v6.4s
sqdmulh v27.4s, v27.4s, v7.4s
sqdmulh v28.4s, v28.4s, v4.4s
sqdmulh v29.4s, v29.4s, v5.4s
sqdmulh v30.4s, v30.4s, v6.4s
sqdmulh v31.4s, v31.4s, v7.4s
sqdmulh v16.4s, v16.4s, v4.4s
sqdmulh v17.4s, v17.4s, v5.4s
sqdmulh v18.4s, v18.4s, v6.4s
sqdmulh v19.4s, v19.4s, v7.4s
sqdmulh v20.4s, v20.4s, v4.4s
sqdmulh v21.4s, v21.4s, v5.4s
sqdmulh v22.4s, v22.4s, v6.4s
sqdmulh v23.4s, v23.4s, v7.4s
srshl v24.4s, v24.4s, v8.4s // post shift
srshl v25.4s, v25.4s, v9.4s
srshl v26.4s, v26.4s, v10.4s
srshl v27.4s, v27.4s, v11.4s
srshl v28.4s, v28.4s, v8.4s
srshl v29.4s, v29.4s, v9.4s
srshl v30.4s, v30.4s, v10.4s
srshl v31.4s, v31.4s, v11.4s
srshl v16.4s, v16.4s, v8.4s
srshl v17.4s, v17.4s, v9.4s
srshl v18.4s, v18.4s, v10.4s
srshl v19.4s, v19.4s, v11.4s
srshl v20.4s, v20.4s, v8.4s
srshl v21.4s, v21.4s, v9.4s
srshl v22.4s, v22.4s, v10.4s
srshl v23.4s, v23.4s, v11.4s
.LQuantize:
sqxtn v24.4h,v24.4s // shorten to int16
sqxtn v26.4h,v26.4s
sqxtn2 v24.8h,v25.4s
@ -458,8 +563,10 @@ Return Value:
str q20,[x2]
.LExitKernel:
ldp d14,d15,[sp,#16]
ldp d12,d13,[sp],#.LConvSymDepthwiseKernelFrame_SavedRegisters
ldp d14,d15,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15]
ldp d12,d13,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13]
ldp d10,d11,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11]
ldp d8,d9,[sp], #.LConvSymDepthwiseKernelFrame_SavedRegisters
ret
.LProcess8Channels:
@ -501,7 +608,7 @@ Return Value:
ldr x11,[x9],#8 // x11 -> A1 iter 2
smlal v14.8h,v1.8b,v3.8b
ldr d2,[x12,x5] // A2 iter 1
b.eq .LEpilogueC8P3 // 3 pixel remains
b.eq .LEpilogueC8P3 // 3 pixel remains
ldr d3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
ldr x12,[x0],#8 // x12 -> A0 iter 3
@ -546,21 +653,15 @@ Return Value:
saddw2 v29.4s,v29.4s,v14.8h
ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags]
smull v12.8h,v0.8b,v4.8b
ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale]
smull v14.8h,v0.8b,v6.8b
ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint]
smlal v12.8h,v1.8b,v2.8b
smlal v14.8h,v1.8b,v3.8b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq .LSkipScaleVecLoad2C8
ldp q4,q5,[x12],#32 // load scale vector if per channel
.LSkipScaleVecLoad2C8:
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v20.4s,v20.4s,v14.4h
saddw2 v21.4s,v21.4s,v14.8h
b .LDequantC8
b .LRequantizationC8
.LProcC8P1:
//
@ -613,9 +714,7 @@ Return Value:
// Loop epilogue (process last single pixel) mixed with loading of dequantization params
//
ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags]
ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
@ -623,17 +722,27 @@ Return Value:
saddw2 v29.4s,v29.4s,v14.8h
smull v12.8h,v0.8b,v2.8b
smull v14.8h,v0.8b,v3.8b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq .LSkipScaleVecLoadC8
ldp q4,q5,[x12] // load scale vector if per channel
.LSkipScaleVecLoadC8:
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v20.4s,v20.4s,v14.4h
saddw2 v21.4s,v21.4s,v14.8h
.LDequantC8:
.LRequantizationC8:
b.ne .LFixedPointScaleC8
.LFloatingPointScaleC8:
ldr x12,[x8,#.LConvSymPostProcessParams_Scale]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq .LBroadcastScaleValueC8
ldp q4,q5,[x12] // load scale vector if per channel
b .LScaleWithFloatPointC8
.LBroadcastScaleValueC8:
ld1r {v4.4s},[x12] // load scale val
mov v5.16b,v4.16b // broadcast scale val if not per channel
.LScaleWithFloatPointC8:
scvtf v24.4s,v24.4s // convert to float
scvtf v25.4s,v25.4s
scvtf v28.4s,v28.4s
@ -642,9 +751,6 @@ Return Value:
scvtf v17.4s,v17.4s
scvtf v20.4s,v20.4s
scvtf v21.4s,v21.4s
b.ne .LSkipScaleBroadcastC8
mov v5.16b,v4.16b // broadcast scale val if not per channel
.LSkipScaleBroadcastC8:
fmul v24.4s,v24.4s,v4.4s // multiply by scale
fmul v25.4s,v25.4s,v5.4s
fmul v28.4s,v28.4s,v4.4s
@ -661,6 +767,57 @@ Return Value:
fcvtns v17.4s,v17.4s
fcvtns v20.4s,v20.4s
fcvtns v21.4s,v21.4s
b .LQuantizeC8
.LFixedPointScaleC8:
ldr x10,[x8,#.LConvSymPostProcessParams_PreShift]
ldr x11,[x8,#.LConvSymPostProcessParams_Multiplier]
ldr x12,[x8,#.LConvSymPostProcessParams_PostShift]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq .LBroadcastFixedPointValueC8
ldp q0,q1,[x10],32 // load preshift vector
ldp q4,q5,[x11],32 // load multiplier vector
ldp q8,q9,[x12],32 // load postshift vector
b .LScaleWithFixedPointC8
.LBroadcastFixedPointValueC8:
ld1r {v0.4s},[x10] // load preshift Value
mov v1.16b, v0.16b
ld1r {v4.4s},[x11] // load multiplier Value
mov v5.16b, v4.16b
ld1r {v8.4s},[x12] // load postshift Value
mov v9.16b, v8.16b
.LScaleWithFixedPointC8:
sqshl v24.4s, v24.4s, v0.4s // preshift
sqshl v25.4s, v25.4s, v1.4s
sqshl v28.4s, v28.4s, v0.4s
sqshl v29.4s, v29.4s, v1.4s
sqshl v16.4s, v16.4s, v0.4s
sqshl v17.4s, v17.4s, v1.4s
sqshl v20.4s, v20.4s, v0.4s
sqshl v21.4s, v21.4s, v1.4s
sqdmulh v24.4s, v24.4s, v4.4s // multiplier
sqdmulh v25.4s, v25.4s, v5.4s
sqdmulh v28.4s, v28.4s, v4.4s
sqdmulh v29.4s, v29.4s, v5.4s
sqdmulh v16.4s, v16.4s, v4.4s
sqdmulh v17.4s, v17.4s, v5.4s
sqdmulh v20.4s, v20.4s, v4.4s
sqdmulh v21.4s, v21.4s, v5.4s
srshl v24.4s, v24.4s, v8.4s // post shift
srshl v25.4s, v25.4s, v9.4s
srshl v28.4s, v28.4s, v8.4s
srshl v29.4s, v29.4s, v9.4s
srshl v16.4s, v16.4s, v8.4s
srshl v17.4s, v17.4s, v9.4s
srshl v20.4s, v20.4s, v8.4s
srshl v21.4s, v21.4s, v9.4s
.LQuantizeC8:
dup v0.8h,w15
sqxtn v24.4h,v24.4s // shorten to int16
sqxtn2 v24.8h,v25.4s

424
onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvSymU8KernelNeon.S
View file

@ -17,24 +17,43 @@ Abstract:
#include "asmmacro.h"
.equ MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2
.equ MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4
//
// Stack frame layout for the depthwise conv kernel.
// d8-d15, x19-x30 need to be preserved if used
//
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d8_d9, 0
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11, 16
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13, 32
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15, 48
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters, 64
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters_Neg, -64
.equ .LConvSymDepthwiseKernelFrame_PostProcessParams, 0 + .LConvSymDepthwiseKernelFrame_SavedRegisters
.equ .LConvSymDepthwiseKernelFrame_KernelFlags, 8 + .LConvSymDepthwiseKernelFrame_SavedRegisters
.equ .LConvSymDepthwiseKernelFrame_SavedNeonRegisters, (8 * 8)
.equ .LConvSymDepthwiseKernelFrame_SavedRegisters, .LConvSymDepthwiseKernelFrame_SavedNeonRegisters
.equ .LConvSymDepthwiseKernelFrame_PostProcessParams, 0 + .LConvSymDepthwiseKernelFrame_SavedRegisters
.equ .LConvSymDepthwiseKernelFrame_KernelFlags, 8 + .LConvSymDepthwiseKernelFrame_SavedRegisters
.equ .LConvSymDepthwisePostProcessParams_Bias, 0
.equ .LConvSymDepthwisePostProcessParams_Scale, 8
.equ .LConvSymDepthwisePostProcessParams_Min, 16
.equ .LConvSymDepthwisePostProcessParams_Max, 20
.equ .LConvSymDepthwisePostProcessParams_ZeroPoint, 24
.equ MLAS_CONV_SYM_FLAG_INPUT_DIRECT, 1
.equ MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
.equ .LConvSymPostProcessParams_Bias, 0
.equ .LConvSymPostProcessParams_Scale, 8
.equ .LConvSymPostProcessParams_Min, 16
.equ .LConvSymPostProcessParams_Max, 20
.equ .LConvSymPostProcessParams_ZeroPoint, 24
.equ .LConvSymPostProcessParams_Multiplier, 32
.equ .LConvSymPostProcessParams_PreShift, 40
.equ .LConvSymPostProcessParams_PostShift, 48
.text
@ -48,28 +67,28 @@ Routine Description:
Arguments:
Input (x0) - Supplies the address of the indirection buffer.
Filter (x1) - Supplies the address of the filter buffer.
Output (x2) - Supplies the address of the output buffer.
KernelSize (x3) - Supplies the size of the kernel.
Channels (x4) - Supplies the number of input and output channels.
ChannelOffset (x5) - Supplies the byte offset from the indirection buffer base
address for this iteration.
ChannelCount (x6) - Supplies the number of channels this iteration produces.
This implementation requires the count to be 16 or 8
OutputCount (x7)- Supplies the number of output elements this iteration produces.
This implementation requires the count to be in the range 1 to 2.
PostProcessParams - Supplies the address of the post process parameter block.
KernelFlags - Supplies additional flags controlling the operation.
Return Value:
@ -79,15 +98,14 @@ Return Value:
--*/
FUNCTION_ENTRY MlasConvSymDepthwiseU8KernelNeon
stp d8,d9,[sp,#-64]!
stp d8,d9,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_Neg]!
ldr x8,[sp,#.LConvSymDepthwiseKernelFrame_PostProcessParams]
mov w10,#0x80808080
stp d10,d11,[sp,#16]
stp d12,d13,[sp,#32]
stp d14,d15,[sp,#48]
stp d10,d11,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11]
stp d12,d13,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13]
stp d14,d15,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15]
dup v8.4s,w10 // bit flip vector
ldr x16,[x8,#.LConvSymDepthwisePostProcessParams_Bias]
ldr x16,[x8,#.LConvSymPostProcessParams_Bias]
cmp x7,2
add x9,x0,x3,lsl#3 // x9 -> &A1
add x14,x0,x3,lsl#4 // x14 -> &A2
@ -111,11 +129,11 @@ Return Value:
// x0 -> x10 -> v4
// -> x12 -> v2 // unroll
// x9 -> x11 -> v6
// -> x13 -> v10 // unroll
// -> x13 -> v3 // unroll
// x14 -> x10 -> v4
// -> x12 -> v2 // unroll
// x15 -> x11 -> v6
// -> x13 -> v10 // unroll
// -> x13 -> v3 // unroll
//
.LProcess16Channels:
@ -144,7 +162,7 @@ Return Value:
mov v18.16b,v26.16b
ldr x12,[x14],#8 // x12 -> A2 iter 1
mov v19.16b,v27.16b
ldr q10,[x13,x5] // A1 iter 1
ldr q3,[x13,x5] // A1 iter 1
mov v22.16b,v26.16b
ldr x13,[x15],#8 // x13 -> A3 iter 1
mov v23.16b,v27.16b
@ -169,13 +187,13 @@ Return Value:
smlal v12.8h,v1.8b,v2.8b
ldr x11,[x9],#8 // x11 -> A1 iter 2
smlal2 v13.8h,v1.16b,v2.16b
b.eq .LEpilogueC16P3 // 3 pixel remains
eor v10.16b,v10.16b,v8.16b
b.eq .LEpilogueC16P3 // 3 pixel remains
eor v3.16b,v3.16b,v8.16b
ldr q2,[x12,x5] // A2 iter 1
smlal v14.8h,v1.8b,v10.8b
smlal v14.8h,v1.8b,v3.8b
ldr x12,[x0],#8 // x12 -> A0 iter 3
smlal2 v15.8h,v1.16b,v10.16b
ldr q10,[x13,x5] // A3 iter 1
smlal2 v15.8h,v1.16b,v3.16b
ldr q3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
ldr x13,[x9],#8 // x13 -> A1 iter 3
@ -200,12 +218,12 @@ Return Value:
smlal v12.8h,v1.8b,v2.8b
ldr x11,[x15],#8 // x11 -> A3 iter 2
smlal2 v13.8h,v1.16b,v2.16b
eor v10.16b,v10.16b,v8.16b
eor v3.16b,v3.16b,v8.16b
ldr q2,[x12,x5] // A0 iter 3
smlal v14.8h,v1.8b,v10.8b
smlal v14.8h,v1.8b,v3.8b
ldr x12,[x14],#8 // x12 -> A2 iter 3
smlal2 v15.8h,v1.16b,v10.16b
ldr q10,[x13,x5] // A1 iter 3
smlal2 v15.8h,v1.16b,v3.16b
ldr q3,[x13,x5] // A1 iter 3
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
ld1 {v1.16b},[x1],x4 // filter iter 3
@ -229,11 +247,11 @@ Return Value:
eor v2.16b,v2.16b,v8.16b
smlal v12.8h,v1.8b,v2.8b
smlal2 v13.8h,v1.16b,v2.16b
eor v10.16b,v10.16b,v8.16b
eor v3.16b,v3.16b,v8.16b
ldr q2,[x12,x5] // A2 iter 1
smlal v14.8h,v1.8b,v10.8b
smlal2 v15.8h,v1.16b,v10.16b
ldr q10,[x13,x5] // A3 iter 1
smlal v14.8h,v1.8b,v3.8b
smlal2 v15.8h,v1.16b,v3.16b
ldr q3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
saddw v26.4s,v26.4s,v13.4h
@ -244,25 +262,18 @@ Return Value:
saddw2 v31.4s,v31.4s,v15.8h
ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags]
eor v4.16b,v4.16b,v8.16b
ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
smull2 v13.8h,v0.16b,v4.16b
eor v6.16b,v6.16b,v8.16b
ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
smull2 v15.8h,v0.16b,v6.16b
eor v2.16b,v2.16b,v8.16b
smlal v12.8h,v1.8b,v2.8b
smlal2 v13.8h,v1.16b,v2.16b
eor v10.16b,v10.16b,v8.16b
smlal v14.8h,v1.8b,v10.8b
smlal2 v15.8h,v1.16b,v10.16b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq .LSkipScaleVecLoad2
ldp q4,q11,[x12],#32 // load scale vector if per channel
ldp q6,q9,[x12]
.LSkipScaleVecLoad2:
eor v3.16b,v3.16b,v8.16b
smlal v14.8h,v1.8b,v3.8b
smlal2 v15.8h,v1.16b,v3.16b
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v18.4s,v18.4s,v13.4h
@ -271,7 +282,7 @@ Return Value:
saddw2 v21.4s,v21.4s,v14.8h
saddw v22.4s,v22.4s,v15.4h
saddw2 v23.4s,v23.4s,v15.8h
b .LDequantization
b .LRequantization
.LProcC16P1:
//
@ -295,7 +306,7 @@ Return Value:
subs x3,x3,2 // decrement input blocks remaining
mov v18.16b,v26.16b
mov v19.16b,v27.16b
ldr q10,[x13,x5]
ldr q3,[x13,x5]
mov v22.16b,v26.16b
mov v23.16b,v27.16b
b .LEpilogueC16P1
@ -305,12 +316,12 @@ Return Value:
// Loop epilogue (process last 2 pixels) mixed
// with loading of dequantization params
//
eor v10.16b,v10.16b,v8.16b
eor v3.16b,v3.16b,v8.16b
ldr q2,[x12,x5] // A2 iter 1
smlal v14.8h,v1.8b,v10.8b
smlal v14.8h,v1.8b,v3.8b
ldr x12,[x14],#8 // x12 -> A2 iter 2
smlal2 v15.8h,v1.16b,v10.16b
ldr q10,[x13,x5] // A3 iter 1
smlal2 v15.8h,v1.16b,v3.16b
ldr q3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
ldr x13,[x15],#8 // x13 -> A3 iter 2
@ -332,11 +343,11 @@ Return Value:
eor v2.16b,v2.16b,v8.16b
smlal v12.8h,v1.8b,v2.8b
smlal2 v13.8h,v1.16b,v2.16b
eor v10.16b,v10.16b,v8.16b
eor v3.16b,v3.16b,v8.16b
ldr q2,[x12,x5] // A2 iter 2
smlal v14.8h,v1.8b,v10.8b
smlal2 v15.8h,v1.16b,v10.16b
ldr q10,[x13,x5] // A3 iter 2
smlal v14.8h,v1.8b,v3.8b
smlal2 v15.8h,v1.16b,v3.16b
ldr q3,[x13,x5] // A3 iter 2
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v18.4s,v18.4s,v13.4h
@ -352,11 +363,9 @@ Return Value:
//
ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags]
eor v4.16b,v4.16b,v8.16b
ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
smull2 v13.8h,v0.16b,v4.16b
eor v6.16b,v6.16b,v8.16b
ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
smull2 v15.8h,v0.16b,v6.16b
saddw v24.4s,v24.4s,v12.4h
@ -370,15 +379,10 @@ Return Value:
eor v2.16b,v2.16b,v8.16b
smull v12.8h,v0.8b,v2.8b
smull2 v13.8h,v0.16b,v2.16b
eor v10.16b,v10.16b,v8.16b
smull v14.8h,v0.8b,v10.8b
smull2 v15.8h,v0.16b,v10.16b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq .LSkipScaleVecLoad
ldp q4,q11,[x12],#32 // load scale vector if per channel
ldp q6,q9,[x12]
.LSkipScaleVecLoad:
eor v3.16b,v3.16b,v8.16b
smull v14.8h,v0.8b,v3.8b
smull2 v15.8h,v0.16b,v3.16b
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v18.4s,v18.4s,v13.4h
@ -388,7 +392,24 @@ Return Value:
saddw v22.4s,v22.4s,v15.4h
saddw2 v23.4s,v23.4s,v15.8h
.LDequantization:
.LRequantization:
b.ne .LFixedPointScale
.LFloatingPointScale:
ldr x12,[x8,#.LConvSymPostProcessParams_Scale]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq .LBroadcastScaleValue
ldp q4,q5,[x12],#32 // load scale vector if per channel
ldp q6,q3,[x12]
b .LScaleWithFloatPoint
.LBroadcastScaleValue:
ld1r {v4.4s},[x12] // load scale val
mov v5.16b,v4.16b // broadcast scale val if not per channel
mov v6.16b,v4.16b
mov v3.16b,v4.16b
.LScaleWithFloatPoint:
scvtf v24.4s,v24.4s // convert to float
scvtf v25.4s,v25.4s
scvtf v26.4s,v26.4s
@ -405,27 +426,22 @@ Return Value:
scvtf v21.4s,v21.4s
scvtf v22.4s,v22.4s
scvtf v23.4s,v23.4s
b.ne .LSkipScaleBroadcast
mov v11.16b,v4.16b // broadcast scale val if not per channel
mov v6.16b,v4.16b
mov v9.16b,v4.16b
.LSkipScaleBroadcast:
fmul v24.4s,v24.4s,v4.4s // multiply by scale
fmul v25.4s,v25.4s,v11.4s
fmul v25.4s,v25.4s,v5.4s
fmul v26.4s,v26.4s,v6.4s
fmul v27.4s,v27.4s,v9.4s
fmul v27.4s,v27.4s,v3.4s
fmul v28.4s,v28.4s,v4.4s
fmul v29.4s,v29.4s,v11.4s
fmul v29.4s,v29.4s,v5.4s
fmul v30.4s,v30.4s,v6.4s
fmul v31.4s,v31.4s,v9.4s
fmul v31.4s,v31.4s,v3.4s
fmul v16.4s,v16.4s,v4.4s
fmul v17.4s,v17.4s,v11.4s
fmul v17.4s,v17.4s,v5.4s
fmul v18.4s,v18.4s,v6.4s
fmul v19.4s,v19.4s,v9.4s
fmul v19.4s,v19.4s,v3.4s
fmul v20.4s,v20.4s,v4.4s
fmul v21.4s,v21.4s,v11.4s
fmul v21.4s,v21.4s,v5.4s
fmul v22.4s,v22.4s,v6.4s
fmul v23.4s,v23.4s,v9.4s
fmul v23.4s,v23.4s,v3.4s
fcvtns v24.4s,v24.4s // convert to int
fcvtns v25.4s,v25.4s
fcvtns v26.4s,v26.4s
@ -442,6 +458,90 @@ Return Value:
fcvtns v21.4s,v21.4s
fcvtns v22.4s,v22.4s
fcvtns v23.4s,v23.4s
b .LQuantize
.LFixedPointScale:
ldr x10,[x8,#.LConvSymPostProcessParams_PreShift]
ldr x11,[x8,#.LConvSymPostProcessParams_Multiplier]
ldr x12,[x8,#.LConvSymPostProcessParams_PostShift]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq .LBroadcastFixedPointValue
ldp q0,q1,[x10],32 // load preshift vector
ldp q2,q3,[x10],32
ldp q4,q5,[x11],32 // load multiplier vector
ldp q6,q7,[x11],32
ldp q8,q9,[x12],32 // load postshift vector
ldp q10,q11,[x12],32
b .LScaleWithFixedPoint
.LBroadcastFixedPointValue:
ld1r {v0.4s},[x10] // load preshift Value
mov v1.16b, v0.16b
mov v2.16b, v0.16b
mov v3.16b, v0.16b
ld1r {v4.4s},[x11] // load multiplier Value
mov v5.16b, v4.16b
mov v6.16b, v4.16b
mov v7.16b, v4.16b
ld1r {v8.4s},[x12] // load postshift Value
mov v9.16b, v8.16b
mov v10.16b, v8.16b
mov v11.16b, v8.16b
.LScaleWithFixedPoint:
sqshl v24.4s, v24.4s, v0.4s // preshift
sqshl v25.4s, v25.4s, v1.4s
sqshl v26.4s, v26.4s, v2.4s
sqshl v27.4s, v27.4s, v3.4s
sqshl v28.4s, v28.4s, v0.4s
sqshl v29.4s, v29.4s, v1.4s
sqshl v30.4s, v30.4s, v2.4s
sqshl v31.4s, v31.4s, v3.4s
sqshl v16.4s, v16.4s, v0.4s
sqshl v17.4s, v17.4s, v1.4s
sqshl v18.4s, v18.4s, v2.4s
sqshl v19.4s, v19.4s, v3.4s
sqshl v20.4s, v20.4s, v0.4s
sqshl v21.4s, v21.4s, v1.4s
sqshl v22.4s, v22.4s, v2.4s
sqshl v23.4s, v23.4s, v3.4s
sqdmulh v24.4s, v24.4s, v4.4s // multiplier
sqdmulh v25.4s, v25.4s, v5.4s
sqdmulh v26.4s, v26.4s, v6.4s
sqdmulh v27.4s, v27.4s, v7.4s
sqdmulh v28.4s, v28.4s, v4.4s
sqdmulh v29.4s, v29.4s, v5.4s
sqdmulh v30.4s, v30.4s, v6.4s
sqdmulh v31.4s, v31.4s, v7.4s
sqdmulh v16.4s, v16.4s, v4.4s
sqdmulh v17.4s, v17.4s, v5.4s
sqdmulh v18.4s, v18.4s, v6.4s
sqdmulh v19.4s, v19.4s, v7.4s
sqdmulh v20.4s, v20.4s, v4.4s
sqdmulh v21.4s, v21.4s, v5.4s
sqdmulh v22.4s, v22.4s, v6.4s
sqdmulh v23.4s, v23.4s, v7.4s
srshl v24.4s, v24.4s, v8.4s // post shift
srshl v25.4s, v25.4s, v9.4s
srshl v26.4s, v26.4s, v10.4s
srshl v27.4s, v27.4s, v11.4s
srshl v28.4s, v28.4s, v8.4s
srshl v29.4s, v29.4s, v9.4s
srshl v30.4s, v30.4s, v10.4s
srshl v31.4s, v31.4s, v11.4s
srshl v16.4s, v16.4s, v8.4s
srshl v17.4s, v17.4s, v9.4s
srshl v18.4s, v18.4s, v10.4s
srshl v19.4s, v19.4s, v11.4s
srshl v20.4s, v20.4s, v8.4s
srshl v21.4s, v21.4s, v9.4s
srshl v22.4s, v22.4s, v10.4s
srshl v23.4s, v23.4s, v11.4s
.LQuantize:
sqxtn v24.4h,v24.4s // shorten to int16
sqxtn v26.4h,v26.4s
sqxtn2 v24.8h,v25.4s
@ -486,10 +586,10 @@ Return Value:
str q20,[x2]
.LExitKernel:
ldp d14,d15,[sp,#48]
ldp d12,d13,[sp,#32]
ldp d10,d11,[sp,#16]
ldp d8,d9,[sp],#64
ldp d14,d15,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15]
ldp d12,d13,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13]
ldp d10,d11,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11]
ldp d8,d9,[sp], #.LConvSymDepthwiseKernelFrame_SavedRegisters
ret
.LProcess8Channels:
@ -511,7 +611,7 @@ Return Value:
mov v17.16b,v25.16b
ldr x12,[x14],#8 // x12 -> A2 iter 1
subs x3,x3,2 // decrement input blocks remaining
ldr d10,[x13,x5] // A1 iter 1
ldr d3,[x13,x5] // A1 iter 1
mov v20.16b,v24.16b
ldr x13,[x15],#8 // x13 -> A3 iter 1
mov v21.16b,v25.16b
@ -528,15 +628,15 @@ Return Value:
b.eq .LEpilogueC8P2
ldr x10,[x0],#8 // x10 -> A0 iter 2
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
ldr d6,[x11,x5] // A3 iter 0
cmp x3,1
smlal v12.8h,v1.8b,v2.8b
ldr x11,[x9],#8 // x11 -> A1 iter 2
smlal v14.8h,v1.8b,v10.8b
smlal v14.8h,v1.8b,v3.8b
ldr d2,[x12,x5] // A2 iter 1
b.eq .LEpilogueC8P3 // 3 pixel remains
ldr d10,[x13,x5] // A3 iter 1
b.eq .LEpilogueC8P3 // 3 pixel remains
ldr d3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
ldr x12,[x0],#8 // x12 -> A0 iter 3
saddw2 v25.4s,v25.4s,v12.8h
@ -552,15 +652,15 @@ Return Value:
ldr x10,[x14],#8 // x10 -> A2 iter 2
ldr d6,[x11,x5] // A1 iter 2
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
ld1 {v0.8b},[x1],x4 // filter iter 2
smlal v12.8h,v1.8b,v2.8b
ldr x11,[x15],#8 // x11 -> A3 iter 2
ldr d2,[x12,x5] // A0 iter 3
smlal v14.8h,v1.8b,v10.8b
smlal v14.8h,v1.8b,v3.8b
ldr x12,[x14],#8 // x12 -> A2 iter 3
saddw v16.4s,v16.4s,v12.4h
ldr d10,[x13,x5] // A1 iter 3
ldr d3,[x13,x5] // A1 iter 3
saddw2 v17.4s,v17.4s,v12.8h
ld1 {v1.8b},[x1],x4 // filter iter 3
saddw v20.4s,v20.4s,v14.4h
@ -575,11 +675,11 @@ Return Value:
//
ldr d6,[x11,x5] // A3 iter 0
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
smlal v12.8h,v1.8b,v2.8b
ldr d2,[x12,x5] // A2 iter 1
smlal v14.8h,v1.8b,v10.8b
ldr d10,[x13,x5] // A3 iter 1
smlal v14.8h,v1.8b,v3.8b
ldr d3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
saddw v28.4s,v28.4s,v14.4h
@ -588,23 +688,17 @@ Return Value:
eor v4.8b,v4.8b,v8.8b
eor v6.8b,v6.8b,v8.8b
smull v12.8h,v0.8b,v4.8b
ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale]
smull v14.8h,v0.8b,v6.8b
ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint]
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
smlal v12.8h,v1.8b,v2.8b
smlal v14.8h,v1.8b,v10.8b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq .LSkipScaleVecLoad2C8
ldp q4,q11,[x12],#32 // load scale vector if per channel
.LSkipScaleVecLoad2C8:
smlal v14.8h,v1.8b,v3.8b
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v20.4s,v20.4s,v14.4h
saddw2 v21.4s,v21.4s,v14.8h
b .LDequantC8
b .LRequantizationC8
.LProcC8P1:
//
@ -623,7 +717,7 @@ Return Value:
mov v20.16b,v24.16b
ldr d2,[x12,x5]
subs x3,x3,2 // decrement input blocks remaining
ldr d10,[x13,x5]
ldr d3,[x13,x5]
mov v21.16b,v25.16b
b .LEpilogueC8P1
@ -632,7 +726,7 @@ Return Value:
// Loop epilogue (process 2 of last 3 pixels)
//
ldr x12,[x14],#8 // x12 -> A2 iter 2
ldr d10,[x13,x5] // A3 iter 1
ldr d3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
ldr x13,[x15],#8 // x13 -> A3 iter 2
@ -645,12 +739,12 @@ Return Value:
smull v14.8h,v0.8b,v6.8b
ld1 {v0.8b},[x1] // filter iter 2
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
ldr d6,[x11,x5] // A1 iter 2
smlal v12.8h,v1.8b,v2.8b
ldr d2,[x12,x5] // A2 iter 2
smlal v14.8h,v1.8b,v10.8b
ldr d10,[x13,x5] // A3 iter 2
smlal v14.8h,v1.8b,v3.8b
ldr d3,[x13,x5] // A3 iter 2
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v20.4s,v20.4s,v14.4h
@ -663,29 +757,37 @@ Return Value:
ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags]
eor v4.8b,v4.8b,v8.8b
eor v6.8b,v6.8b,v8.8b
ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
saddw v28.4s,v28.4s,v14.4h
saddw2 v29.4s,v29.4s,v14.8h
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
smull v12.8h,v0.8b,v2.8b
smull v14.8h,v0.8b,v10.8b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq .LSkipScaleVecLoadC8
ldp q4,q11,[x12] // load scale vector if per channel
.LSkipScaleVecLoadC8:
smull v14.8h,v0.8b,v3.8b
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v20.4s,v20.4s,v14.4h
saddw2 v21.4s,v21.4s,v14.8h
.LDequantC8:
.LRequantizationC8:
b.ne .LFixedPointScaleC8
.LFloatingPointScaleC8:
ldr x12,[x8,#.LConvSymPostProcessParams_Scale]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq .LBroadcastScaleValueC8
ldp q4,q5,[x12] // load scale vector if per channel
b .LScaleWithFloatPointC8
.LBroadcastScaleValueC8:
ld1r {v4.4s},[x12] // load scale val
mov v5.16b,v4.16b // broadcast scale val if not per channel
.LScaleWithFloatPointC8:
scvtf v24.4s,v24.4s // convert to float
scvtf v25.4s,v25.4s
scvtf v28.4s,v28.4s
@ -694,17 +796,14 @@ Return Value:
scvtf v17.4s,v17.4s
scvtf v20.4s,v20.4s
scvtf v21.4s,v21.4s
b.ne .LSkipScaleBroadcastC8
mov v11.16b,v4.16b // broadcast scale val if not per channel
.LSkipScaleBroadcastC8:
fmul v24.4s,v24.4s,v4.4s // multiply by scale
fmul v25.4s,v25.4s,v11.4s
fmul v25.4s,v25.4s,v5.4s
fmul v28.4s,v28.4s,v4.4s
fmul v29.4s,v29.4s,v11.4s
fmul v29.4s,v29.4s,v5.4s
fmul v16.4s,v16.4s,v4.4s
fmul v17.4s,v17.4s,v11.4s
fmul v17.4s,v17.4s,v5.4s
fmul v20.4s,v20.4s,v4.4s
fmul v21.4s,v21.4s,v11.4s
fmul v21.4s,v21.4s,v5.4s
fcvtns v24.4s,v24.4s // convert to int
fcvtns v25.4s,v25.4s
fcvtns v28.4s,v28.4s
@ -713,6 +812,57 @@ Return Value:
fcvtns v17.4s,v17.4s
fcvtns v20.4s,v20.4s
fcvtns v21.4s,v21.4s
b .LQuantizeC8
.LFixedPointScaleC8:
ldr x10,[x8,#.LConvSymPostProcessParams_PreShift]
ldr x11,[x8,#.LConvSymPostProcessParams_Multiplier]
ldr x12,[x8,#.LConvSymPostProcessParams_PostShift]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint]
beq .LBroadcastFixedPointValueC8
ldp q0,q1,[x10],32 // load preshift vector
ldp q4,q5,[x11],32 // load multiplier vector
ldp q8,q9,[x12],32 // load postshift vector
b .LScaleWithFixedPointC8
.LBroadcastFixedPointValueC8:
ld1r {v0.4s},[x10] // load preshift Value
mov v1.16b, v0.16b
ld1r {v4.4s},[x11] // load multiplier Value
mov v5.16b, v4.16b
ld1r {v8.4s},[x12] // load postshift Value
mov v9.16b, v8.16b
.LScaleWithFixedPointC8:
sqshl v24.4s, v24.4s, v0.4s // preshift
sqshl v25.4s, v25.4s, v1.4s
sqshl v28.4s, v28.4s, v0.4s
sqshl v29.4s, v29.4s, v1.4s
sqshl v16.4s, v16.4s, v0.4s
sqshl v17.4s, v17.4s, v1.4s
sqshl v20.4s, v20.4s, v0.4s
sqshl v21.4s, v21.4s, v1.4s
sqdmulh v24.4s, v24.4s, v4.4s // multiplier
sqdmulh v25.4s, v25.4s, v5.4s
sqdmulh v28.4s, v28.4s, v4.4s
sqdmulh v29.4s, v29.4s, v5.4s
sqdmulh v16.4s, v16.4s, v4.4s
sqdmulh v17.4s, v17.4s, v5.4s
sqdmulh v20.4s, v20.4s, v4.4s
sqdmulh v21.4s, v21.4s, v5.4s
srshl v24.4s, v24.4s, v8.4s // post shift
srshl v25.4s, v25.4s, v9.4s
srshl v28.4s, v28.4s, v8.4s
srshl v29.4s, v29.4s, v9.4s
srshl v16.4s, v16.4s, v8.4s
srshl v17.4s, v17.4s, v9.4s
srshl v20.4s, v20.4s, v8.4s
srshl v21.4s, v21.4s, v9.4s
.LQuantizeC8:
dup v0.8h,w15
sqxtn v24.4h,v24.4s // shorten to int16
sqxtn2 v24.8h,v25.4s

142
onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelDot.asm
View file

@ -19,20 +19,43 @@ Abstract:
#include "AssembleDotProduct.h"
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2
#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4
//
// Stack frame layout for the symmetric convolution kernel.
// d8-d15, x19-x30 need to be preserved if used
//
#define ConvSymFrame_SavedRegisters (6 * 8)
#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
#define ConvSymFrame_SavedRegisters_d8_d9 0
#define ConvSymFrame_SavedRegisters_d10_d11 16
#define ConvSymFrame_SavedRegisters_x19_x20 32
#define ConvSymFrame_SavedRegisters_x21_x22 48
#define ConvSymFrame_SavedRegisters 64
#define ConvSymFrame_SavedRegisters_Neg -64
#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
#define ConvSymPostProcessParams_Multiplier 32
#define ConvSymPostProcessParams_PreShift 40
#define ConvSymPostProcessParams_PostShift 48
TEXTAREA
@ -78,13 +101,13 @@ Return Value:
--*/
NESTED_ENTRY MlasConvSymS8KernelDot
PROLOG_SAVE_REG_PAIR d8,d9,#-ConvSymFrame_SavedRegisters!
PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters_Neg!
PROLOG_NOP ldr x8,[sp,#ConvSymFrame_PostProcessParams]
PROLOG_SAVE_REG d10,#16
PROLOG_NOP cmp x7,2 // OutputCount < 2 ?
PROLOG_SAVE_REG d11,#24
PROLOG_NOP add x16,x2,x5 // x16 -> C1
PROLOG_SAVE_REG x19,#32
PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11
PROLOG_SAVE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20
PROLOG_SAVE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22
lsl x3,x3,#3 // KernelSize * sizeof(int8_t*)
csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0
add x4,x4,3 // InputChannels align to 4
@ -97,6 +120,9 @@ Return Value:
add x5,x17,x5 // x5 -> C3
ldr x19,[x8,#ConvSymPostProcessParams_Scale]
csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2
ldr x20,[x8,#ConvSymPostProcessParams_PreShift]
ldr x21,[x8,#ConvSymPostProcessParams_Multiplier]
ldr x22,[x8,#ConvSymPostProcessParams_PostShift]
// TODO!! tiptoe around loading biases if we need to support
// output channels none divisible by 16
@ -347,12 +373,15 @@ InChLoopEpilogue
b.hi KernelSizeLoop
Requantize
tst w10,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale
FloatPointScale
tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastScaleValue
ldp q0,q1,[x19],32 // load scale vector
ldp q2,q3,[x19],32
b AccumulatorsToFloat
b ScaleWithFloatPoint
BroadcastScaleValue
ld1r {v0.4s},[x19] // load scale Value
@ -360,7 +389,7 @@ BroadcastScaleValue
mov v2.16b, v0.16b
mov v3.16b, v0.16b
AccumulatorsToFloat
ScaleWithFloatPoint
scvtf v16.4s,v16.4s // convert to float
scvtf v17.4s,v17.4s
scvtf v18.4s,v18.4s
@ -409,7 +438,87 @@ AccumulatorsToFloat
fcvtns v29.4s,v29.4s
fcvtns v30.4s,v30.4s
fcvtns v31.4s,v31.4s
b Quantize
FixedPointScale
tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastFixedPointValue
ldp q0,q1,[x20],32 // load preshift vector
ldp q2,q3,[x20],32
ldp q4,q5,[x21],32 // load multiplier vector
ldp q6,q7,[x21],32
ldp q8,q9,[x22],32 // load postshift vector
ldp q10,q11,[x22],32
b ScaleWithFixedPoint
BroadcastFixedPointValue
ld1r {v0.4s},[x20] // load preshift Value
mov v1.16b, v0.16b
mov v2.16b, v0.16b
mov v3.16b, v0.16b
ld1r {v4.4s},[x21] // load multiplier Value
mov v5.16b, v4.16b
mov v6.16b, v4.16b
mov v7.16b, v4.16b
ld1r {v8.4s},[x22] // load postshift Value
mov v9.16b, v8.16b
mov v10.16b, v8.16b
mov v11.16b, v8.16b
ScaleWithFixedPoint
sqshl v16.4s, v16.4s, v0.4s // preshift
sqshl v17.4s, v17.4s, v0.4s
sqshl v18.4s, v18.4s, v0.4s
sqshl v19.4s, v19.4s, v0.4s
sqshl v20.4s, v20.4s, v1.4s
sqshl v21.4s, v21.4s, v1.4s
sqshl v22.4s, v22.4s, v1.4s
sqshl v23.4s, v23.4s, v1.4s
sqshl v24.4s, v24.4s, v2.4s
sqshl v25.4s, v25.4s, v2.4s
sqshl v26.4s, v26.4s, v2.4s
sqshl v27.4s, v27.4s, v2.4s
sqshl v28.4s, v28.4s, v3.4s
sqshl v29.4s, v29.4s, v3.4s
sqshl v30.4s, v30.4s, v3.4s
sqshl v31.4s, v31.4s, v3.4s
sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale
sqdmulh v17.4s, v17.4s, v4.4s
sqdmulh v18.4s, v18.4s, v4.4s
sqdmulh v19.4s, v19.4s, v4.4s
sqdmulh v20.4s, v20.4s, v5.4s
sqdmulh v21.4s, v21.4s, v5.4s
sqdmulh v22.4s, v22.4s, v5.4s
sqdmulh v23.4s, v23.4s, v5.4s
sqdmulh v24.4s, v24.4s, v6.4s
sqdmulh v25.4s, v25.4s, v6.4s
sqdmulh v26.4s, v26.4s, v6.4s
sqdmulh v27.4s, v27.4s, v6.4s
sqdmulh v28.4s, v28.4s, v7.4s
sqdmulh v29.4s, v29.4s, v7.4s
sqdmulh v30.4s, v30.4s, v7.4s
sqdmulh v31.4s, v31.4s, v7.4s
srshl v16.4s,v16.4s, v8.4s // post shift
srshl v17.4s,v17.4s, v8.4s
srshl v18.4s,v18.4s, v8.4s
srshl v19.4s,v19.4s, v8.4s
srshl v20.4s,v20.4s, v9.4s
srshl v21.4s,v21.4s, v9.4s
srshl v22.4s,v22.4s, v9.4s
srshl v23.4s,v23.4s, v9.4s
srshl v24.4s,v24.4s, v10.4s
srshl v25.4s,v25.4s, v10.4s
srshl v26.4s,v26.4s, v10.4s
srshl v27.4s,v27.4s, v10.4s
srshl v28.4s,v28.4s, v11.4s
srshl v29.4s,v29.4s, v11.4s
srshl v30.4s,v30.4s, v11.4s
srshl v31.4s,v31.4s, v11.4s
Quantize
sqxtn v16.4h,v16.4s
sqxtn v17.4h,v17.4s
sqxtn v18.4h,v18.4s
@ -454,8 +563,9 @@ AccumulatorsToFloat
b.hi OutputChannelLoop
ExitKernel
EPILOG_RESTORE_REG x19,#32
EPILOG_RESTORE_REG_PAIR d10,d11,#16
EPILOG_RESTORE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22
EPILOG_RESTORE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20
EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11
EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters!
EPILOG_RETURN

177
onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelDotLd64.asm
View file

@ -19,20 +19,50 @@ Abstract:
#include "AssembleDotProduct.h"
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2
#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4
//
// Stack frame layout for the symmetric convolution kernel.
// d8-d15, x19-x30 need to be preserved if used
//
#define ConvSymFrame_SavedRegisters (10 * 8)
#define ConvSymFrame_PostProcessParams (0 + ConvSymFrame_SavedRegisters)
#define ConvSymFrame_KernelFlags (8 + ConvSymFrame_SavedRegisters)
#define ConvSymFrame_SavedRegisters_d8 0
#define ConvSymFrame_SavedRegisters_d9 8
#define ConvSymFrame_SavedRegisters_d10 16
#define ConvSymFrame_SavedRegisters_d11 24
#define ConvSymFrame_SavedRegisters_x19 32
#define ConvSymFrame_SavedRegisters_x20 40
#define ConvSymFrame_SavedRegisters_x21 48
#define ConvSymFrame_SavedRegisters_x22 56
#define ConvSymFrame_SavedRegisters_x23 64
#define ConvSymFrame_SavedRegisters_x24 72
#define ConvSymFrame_SavedRegisters_x25 80
#define ConvSymFrame_SavedRegisters_x26 88
#define ConvSymFrame_SavedRegisters 96
#define ConvSymFrame_SavedRegisters_Neg -96
#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
#define ConvSymPostProcessParams_Multiplier 32
#define ConvSymPostProcessParams_PreShift 40
#define ConvSymPostProcessParams_PostShift 48
TEXTAREA
@ -45,13 +75,7 @@ Routine Description:
Arguments:
Input (x0) - Points to the input buffer.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points
directly at the input tensor.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an
indirection buffer. Every pointer in the indirection buffer points at a
Input (x0) - Points to the input indirection buffer. Every pointer in the indirection buffer points at a
InputChannels length vector (either from the input tensor or a vector of
padding values). These are grouped in batches of length KernelSize.
These batches are then repeated OutputCount times.
@ -61,8 +85,7 @@ Arguments:
Output (x2) - Points the output buffer.
KernelSize (x3/x9) - Size of the kernel (most commonly. 3x3=9, 5x5=25).
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1.
Must be > 1
InputChannels (x4/x7) - Number of input channels.
@ -85,29 +108,35 @@ Return Value:
--*/
NESTED_ENTRY MlasConvSymS8KernelDotLd64
PROLOG_SAVE_REG_PAIR d8,d9,#-ConvSymFrame_SavedRegisters!
PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters_Neg!
PROLOG_NOP ldr x8,[sp,#ConvSymFrame_PostProcessParams]
PROLOG_SAVE_REG d10,#16
PROLOG_SAVE_REG d10,#ConvSymFrame_SavedRegisters_d10
PROLOG_NOP cmp x7,2 // OutputCount < 2 ?
PROLOG_SAVE_REG d11,#24
PROLOG_SAVE_REG d11,#ConvSymFrame_SavedRegisters_d11
PROLOG_NOP add x16,x2,x5 // x16 -> C1
PROLOG_SAVE_REG x19,#32
PROLOG_SAVE_REG x19,#ConvSymFrame_SavedRegisters_x19
PROLOG_NOP lsl x3,x3,#3 // KernelSize * sizeof(int8_t*)
PROLOG_SAVE_REG x20,#40
PROLOG_SAVE_REG x20,#ConvSymFrame_SavedRegisters_x20
PROLOG_NOP csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0
PROLOG_SAVE_REG x21,#48
PROLOG_SAVE_REG x21,#ConvSymFrame_SavedRegisters_x21
PROLOG_NOP add x4,x4,3 // InputChannels align to 4
PROLOG_SAVE_REG x22,#56
PROLOG_SAVE_REG x22,#ConvSymFrame_SavedRegisters_x22
PROLOG_NOP add x17,x16,x5 // x17 -> C2
PROLOG_SAVE_REG x23,#64
PROLOG_SAVE_REG x23,#ConvSymFrame_SavedRegisters_x23
ldr x11,[x8,#ConvSymPostProcessParams_Bias]
PROLOG_SAVE_REG x24,#ConvSymFrame_SavedRegisters_x24
csel x17,x16,x17,ls // if OutputCount <= 2 x17/C2 -> C1
PROLOG_SAVE_REG x25,#ConvSymFrame_SavedRegisters_x25
bic x4,x4,3
PROLOG_SAVE_REG x26,#ConvSymFrame_SavedRegisters_x26
cmp x7,4 // OutputCount < 4 ?
ldr w10,[sp,#ConvSymFrame_KernelFlags]
add x5,x17,x5 // x5 -> C3
ldr x19,[x8,#ConvSymPostProcessParams_Scale]
csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2
ldr x24,[x8,#ConvSymPostProcessParams_PreShift]
ldr x25,[x8,#ConvSymPostProcessParams_Multiplier]
ldr x26,[x8,#ConvSymPostProcessParams_PostShift]
// TODO!! tiptoe around loading biases if we need to support
// output channels none divisible by 16
@ -422,12 +451,15 @@ InChLoopEpilogue
b.hi KernelSizeLoop
Requantize
tst w10,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale
FloatPointScale
tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastScaleValue
ldp q0,q1,[x19],32 // load scale vector
ldp q2,q3,[x19],32
b AccumulatorsToFloat
b ScaleWithFloatPoint
BroadcastScaleValue
ld1r {v0.4s},[x19] // load scale Value
@ -435,7 +467,7 @@ BroadcastScaleValue
mov v2.16b, v0.16b
mov v3.16b, v0.16b
AccumulatorsToFloat
ScaleWithFloatPoint
scvtf v16.4s,v16.4s // convert to float
scvtf v17.4s,v17.4s
scvtf v18.4s,v18.4s
@ -484,7 +516,87 @@ AccumulatorsToFloat
fcvtns v29.4s,v29.4s
fcvtns v30.4s,v30.4s
fcvtns v31.4s,v31.4s
b Quantize
FixedPointScale
tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastFixedPointValue
ldp q0,q1,[x24],32 // load preshift vector
ldp q2,q3,[x24],32
ldp q4,q5,[x25],32 // load multiplier vector
ldp q6,q7,[x25],32
ldp q8,q9,[x26],32 // load postshift vector
ldp q10,q11,[x26],32
b ScaleWithFixedPoint
BroadcastFixedPointValue
ld1r {v0.4s},[x24] // load preshift Value
mov v1.16b, v0.16b
mov v2.16b, v0.16b
mov v3.16b, v0.16b
ld1r {v4.4s},[x25] // load multiplier Value
mov v5.16b, v4.16b
mov v6.16b, v4.16b
mov v7.16b, v4.16b
ld1r {v8.4s},[x26] // load postshift Value
mov v9.16b, v8.16b
mov v10.16b, v8.16b
mov v11.16b, v8.16b
ScaleWithFixedPoint
sqshl v16.4s, v16.4s, v0.4s // preshift
sqshl v17.4s, v17.4s, v0.4s
sqshl v18.4s, v18.4s, v0.4s
sqshl v19.4s, v19.4s, v0.4s
sqshl v20.4s, v20.4s, v1.4s
sqshl v21.4s, v21.4s, v1.4s
sqshl v22.4s, v22.4s, v1.4s
sqshl v23.4s, v23.4s, v1.4s
sqshl v24.4s, v24.4s, v2.4s
sqshl v25.4s, v25.4s, v2.4s
sqshl v26.4s, v26.4s, v2.4s
sqshl v27.4s, v27.4s, v2.4s
sqshl v28.4s, v28.4s, v3.4s
sqshl v29.4s, v29.4s, v3.4s
sqshl v30.4s, v30.4s, v3.4s
sqshl v31.4s, v31.4s, v3.4s
sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale
sqdmulh v17.4s, v17.4s, v4.4s
sqdmulh v18.4s, v18.4s, v4.4s
sqdmulh v19.4s, v19.4s, v4.4s
sqdmulh v20.4s, v20.4s, v5.4s
sqdmulh v21.4s, v21.4s, v5.4s
sqdmulh v22.4s, v22.4s, v5.4s
sqdmulh v23.4s, v23.4s, v5.4s
sqdmulh v24.4s, v24.4s, v6.4s
sqdmulh v25.4s, v25.4s, v6.4s
sqdmulh v26.4s, v26.4s, v6.4s
sqdmulh v27.4s, v27.4s, v6.4s
sqdmulh v28.4s, v28.4s, v7.4s
sqdmulh v29.4s, v29.4s, v7.4s
sqdmulh v30.4s, v30.4s, v7.4s
sqdmulh v31.4s, v31.4s, v7.4s
srshl v16.4s,v16.4s, v8.4s // post shift
srshl v17.4s,v17.4s, v8.4s
srshl v18.4s,v18.4s, v8.4s
srshl v19.4s,v19.4s, v8.4s
srshl v20.4s,v20.4s, v9.4s
srshl v21.4s,v21.4s, v9.4s
srshl v22.4s,v22.4s, v9.4s
srshl v23.4s,v23.4s, v9.4s
srshl v24.4s,v24.4s, v10.4s
srshl v25.4s,v25.4s, v10.4s
srshl v26.4s,v26.4s, v10.4s
srshl v27.4s,v27.4s, v10.4s
srshl v28.4s,v28.4s, v11.4s
srshl v29.4s,v29.4s, v11.4s
srshl v30.4s,v30.4s, v11.4s
srshl v31.4s,v31.4s, v11.4s
Quantize
sqxtn v16.4h,v16.4s
sqxtn v17.4h,v17.4s
sqxtn v18.4h,v18.4s
@ -529,10 +641,11 @@ AccumulatorsToFloat
b.hi OutputChannelLoop
ExitKernel
EPILOG_RESTORE_REG x23,#64
EPILOG_RESTORE_REG_PAIR x21,x22,#48
EPILOG_RESTORE_REG_PAIR x19,x20,#32
EPILOG_RESTORE_REG_PAIR d10,d11,#16
EPILOG_RESTORE_REG_PAIR x25,x26,#ConvSymFrame_SavedRegisters_x25
EPILOG_RESTORE_REG_PAIR x23,x24,#ConvSymFrame_SavedRegisters_x23
EPILOG_RESTORE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21
EPILOG_RESTORE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19
EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10
EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters!
EPILOG_RETURN
@ -587,7 +700,7 @@ InChannels4
ldr s1,[x13],4
ldr s2,[x14],4
ldr s3,[x15],4
ldr q5, [x1], 16
ldr q5, [x1],16
SdotByElement 16, 4, 0,0
SdotByElement 17, 4, 1,0
ldp q6, q7, [x1], 32

124
onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelNeon.asm
View file

@ -18,21 +18,45 @@ Abstract:
#include "kxarm64.h"
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2
#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4
//
// Stack frame layout for the symmetric convolution kernel.
// d8-d15, x19-x30 need to be preserved if used
//
#define ConvSymFrame_SavedNeonRegisters (8 * 8)
#define ConvSymFrame_SavedRegisters ConvSymFrame_SavedNeonRegisters
#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
#define ConvSymFrame_SavedRegisters_d8_d9 0
#define ConvSymFrame_SavedRegisters_d10_d11 16
#define ConvSymFrame_SavedRegisters_d12_d13 32
#define ConvSymFrame_SavedRegisters_d14_d15 48
#define ConvSymFrame_SavedRegisters_x19_x20 64
#define ConvSymFrame_SavedRegisters_x21_x22 80
#define ConvSymFrame_SavedRegisters 96
#define ConvSymFrame_SavedRegisters_Neg -96
#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
#define ConvSymPostProcessParams_Multiplier 32
#define ConvSymPostProcessParams_PreShift 40
#define ConvSymPostProcessParams_PostShift 48
TEXTAREA
@ -82,12 +106,14 @@ Return Value:
--*/
NESTED_ENTRY MlasConvSymS8KernelNeon
PROLOG_SAVE_REG_PAIR d8,d9,#-ConvSymFrame_SavedRegisters!
PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters_Neg!
PROLOG_NOP ldr x8,[sp,#ConvSymFrame_PostProcessParams]
PROLOG_NOP ldrb w10,[sp,#ConvSymFrame_KernelFlags]
PROLOG_SAVE_REG_PAIR d10,d11,#16
PROLOG_SAVE_REG_PAIR d12,d13,#32
PROLOG_SAVE_REG_PAIR d14,d15,#48
PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11
PROLOG_SAVE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13
PROLOG_SAVE_REG_PAIR d14,d15,#ConvSymFrame_SavedRegisters_d14_d15
PROLOG_SAVE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20
PROLOG_SAVE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22
mov x9,x3 // save kernel size
ldr x11,[x8,#ConvSymPostProcessParams_Bias]
mov x16,x4 // save input channels
@ -96,6 +122,9 @@ Return Value:
add x5,x2,x5 // c1 = c0 + ldc
add x4,x4,7 // kc = (kc + 7) & ~7
csel x5,x2,x5,lo // if OutputCount < 2 c1 = c0
ldr x19,[x8,#ConvSymPostProcessParams_PreShift]
ldr x20,[x8,#ConvSymPostProcessParams_Multiplier]
ldr x21,[x8,#ConvSymPostProcessParams_PostShift]
bic x4,x4,7
ldp s16,s18,[x11],8 // init accumulators with bias
ldp s20,s22,[x11],8
@ -296,17 +325,19 @@ BlockLoopEpilogue // remaining 16 input channels
b.hi KernelSizeLoop
Requantize
ldr w11,[x8,#ConvSymPostProcessParams_ZeroPoint]
tst w10,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale
FloatPointScale
tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
beq BroadcastScaleValue
ld1 {v4.4s,v5.4s},[x12] // load scale vector
b AccumulatorsToFloat
b ScaleWithFloatPoint
BroadcastScaleValue
ld1r {v4.4s},[x12] // load scale Value
mov v5.16b, v4.16b
AccumulatorsToFloat
ScaleWithFloatPoint
addp v16.4s,v16.4s,v18.4s
addp v20.4s,v20.4s,v22.4s
addp v24.4s,v24.4s,v26.4s
@ -329,25 +360,72 @@ AccumulatorsToFloat
fmul v3.4s,v3.4s,v5.4s
fcvtns v0.4s,v0.4s // convert to int
fcvtns v1.4s,v1.4s
dup v9.8h,w11
fcvtns v2.4s,v2.4s
fcvtns v3.4s,v3.4s
b Quantize
FixedPointScale
tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w11, [x8, #ConvSymPostProcessParams_ZeroPoint]
beq BroadcastFixedPointValue
ld1 {v4.4s,v5.4s},[x19] // load preshift
ld1 {v6.4s,v7.4s},[x20] // load multiplier
ld1 {v8.4s,v9.4s},[x21] // load postshift
b ScaleWithFixedPoint
BroadcastFixedPointValue
ld1r {v4.4s},[x19] // load preshift Value
mov v5.16b, v4.16b
ld1r {v6.4s},[x20] // load multiplier Value
mov v7.16b, v6.16b
ld1r {v8.4s},[x21] // load postshift Value
mov v9.16b, v8.16b
ScaleWithFixedPoint
addp v16.4s,v16.4s,v18.4s
addp v20.4s,v20.4s,v22.4s
addp v24.4s,v24.4s,v26.4s
addp v28.4s,v28.4s,v30.4s
addp v17.4s,v17.4s,v19.4s
addp v21.4s,v21.4s,v23.4s
addp v25.4s,v25.4s,v27.4s
addp v29.4s,v29.4s,v31.4s
addp v0.4s,v16.4s,v20.4s
addp v1.4s,v24.4s,v28.4s
addp v2.4s,v17.4s,v21.4s
addp v3.4s,v25.4s,v29.4s
sqshl v0.4s, v0.4s, v4.4s // preshift
sqshl v1.4s, v1.4s, v5.4s
sqshl v2.4s, v2.4s, v4.4s
sqshl v3.4s, v3.4s, v5.4s
sqdmulh v0.4s, v0.4s, v6.4s // multiplier
sqdmulh v1.4s, v1.4s, v7.4s
sqdmulh v2.4s, v2.4s, v6.4s
sqdmulh v3.4s, v3.4s, v7.4s
srshl v0.4s,v0.4s, v8.4s // postshift
srshl v1.4s,v1.4s, v9.4s
srshl v2.4s,v2.4s, v8.4s
srshl v3.4s,v3.4s, v9.4s
Quantize
dup v9.8h,w11
sqxtn v0.4h,v0.4s
sqxtn2 v0.8h,v1.4s
sqxtn v2.4h,v2.4s
sqxtn2 v0.8h,v1.4s
sqxtn2 v2.8h,v3.4s
sqadd v0.8h,v0.8h,v9.8h
sqadd v2.8h,v2.8h,v9.8h
sqxtn v0.8b,v0.8h // shorten to int8
sqxtn2 v0.16b,v2.8h
st1 {v0.d}[1],[x5] // full 2x8 store to c
st1 {v0.d}[1],[x5] // full 2x8 store to c
st1 {v0.8b},[x2]
ExitKernel
EPILOG_RESTORE_REG_PAIR d14,d15,#48
EPILOG_RESTORE_REG_PAIR d12,d13,#32
EPILOG_RESTORE_REG_PAIR d10,d11,#16
EPILOG_RESTORE_REG_PAIR d8,d9,#64!
EPILOG_RESTORE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22
EPILOG_RESTORE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20
EPILOG_RESTORE_REG_PAIR d14,d15,#ConvSymFrame_SavedRegisters_d14_d15
EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13
EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11
EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters!
EPILOG_RETURN
InputChannel8

182
onnxruntime/core/mlas/lib/arm64/ConvSymU8KernelDot.asm
View file

@ -17,23 +17,44 @@ Abstract:
#include "kxarm64.h"
#define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 1
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2
#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4
//
// Stack frame layout for the symmetric convolution kernel.
// d8-d15, x19-x30 need to be preserved if used
//
#define ConvSymFrame_SavedNeonRegisters (8 * 8)
#define ConvSymFrame_SavedRegisters ConvSymFrame_SavedNeonRegisters
#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_SavedRegisters_d8_d9 0
#define ConvSymFrame_SavedRegisters_d10_d11 16
#define ConvSymFrame_SavedRegisters_d12_d13 32
#define ConvSymFrame_SavedRegisters_x19_x20 48
#define ConvSymFrame_SavedRegisters_x21_x22 64
#define ConvSymFrame_SavedRegisters 80
#define ConvSymFrame_SavedRegisters_Neg -80
#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
#define ConvSymPostProcessParams_Multiplier 32
#define ConvSymPostProcessParams_PreShift 40
#define ConvSymPostProcessParams_PostShift 48
TEXTAREA
@ -46,15 +67,10 @@ Routine Description:
Arguments:
Input (x0) - Points to the input buffer.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points
directly at the input tensor.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an
indirection buffer. Every pointer in the indirection buffer points at a
InputChannels length vector (either from the input tensor or a vector of
padding values). These are grouped in batches of length KernelSize.
Input (x0) - Supplies the address of the indirect buffer. Every pointer in
the indirection buffer points at a InputChannels length vector (either
from the input tensor or a vector of padding values). These are grouped
in batches of length KernelSize.
These batches are then repeated OutputCount times.
Filter (x1) - Points to the filter buffer.
@ -63,8 +79,6 @@ Arguments:
KernelSize (x3/x9) - Size of the kernel (most commonly. 3x3=9, 5x5=25).
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1.
InputChannels (x4/x7) - Number of input channels.
OutputChannels (x5) - Number of output channels.
@ -86,19 +100,19 @@ Return Value:
--*/
NESTED_ENTRY MlasConvSymU8KernelDot
PROLOG_SAVE_REG_PAIR d8,d9,#-64!
PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters_Neg!
PROLOG_NOP ldr x8,[sp,#ConvSymFrame_PostProcessParams]
PROLOG_NOP ldr w10,[sp,#ConvSymFrame_KernelFlags]
PROLOG_SAVE_REG_PAIR d10,d11,#16
PROLOG_SAVE_REG_PAIR d12,d13,#32
PROLOG_SAVE_REG_PAIR x19,x20,#48
PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11
PROLOG_SAVE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13
PROLOG_SAVE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20
PROLOG_SAVE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22
// compute C pointers: x2, x16, x17, x5
cmp x7,2 // OutputCount < 2 ?
add x16,x2,x5 // x16 -> C1
lsl x3,x3,#3 // KernelSize * sizeof(int8_t*)
csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0
mov x20,x4
add x4,x4,3 // InputChannels align to 4
add x17,x16,x5 // x17 -> C2
ldr x11,[x8,#ConvSymPostProcessParams_Bias]
@ -109,6 +123,9 @@ Return Value:
ldr x19,[x8,#ConvSymPostProcessParams_Scale]
csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2
movi v12.16b,128 // for top bit flipping
ldr x20,[x8,#ConvSymPostProcessParams_PreShift]
ldr x21,[x8,#ConvSymPostProcessParams_Multiplier]
ldr x22,[x8,#ConvSymPostProcessParams_PostShift]
OutputChannelLoop
ldp q16,q20,[x11],32 // Init accumulators with biases
@ -128,23 +145,6 @@ OutputChannelLoop
mov x9,x3 // restore KernelSize * sizeof(int8_t*)
KernelSizeLoop
tst w10,#MLAS_CONV_SYM_FLAG_INPUT_DIRECT
beq InputIndirection
InputDirect
cmp x16,x2
mov x12,x0 // x12 -> A0
add x13,x0,x20 // x13 -> A1 = A0 + input channels
csel x13,x0,x13,eq
cmp x17,x16
add x14,x0,x20,lsl#1 // x14 -> A2
csel x14,x13,x14,eq
cmp x5,x17
add x15,x13,x20,lsl#1 // x15 -> A3
csel x15,x14,x15,eq
b FinishLoadAPtr
InputIndirection
ldr x12,[x0] // x12 -> A0
cmp x16,x2
b.eq SkipLoadA1 // C1==C0 -> A0=A1=A2=A3
@ -392,12 +392,15 @@ InChLoopEpilogue
b.hi KernelSizeLoop
Requantize
tst w10,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale
FloatPointScale
tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastScaleValue
ldp q0,q1,[x19],32 // load scale vector
ldp q2,q3,[x19],32
b AccumulatorsToFloat
b ScaleWithFloatPoint
BroadcastScaleValue
ld1r {v0.4s},[x19] // load scale Value
@ -405,7 +408,7 @@ BroadcastScaleValue
mov v2.16b, v0.16b
mov v3.16b, v0.16b
AccumulatorsToFloat
ScaleWithFloatPoint
scvtf v16.4s,v16.4s // convert to float
scvtf v17.4s,v17.4s
scvtf v18.4s,v18.4s
@ -454,7 +457,87 @@ AccumulatorsToFloat
fcvtns v29.4s,v29.4s
fcvtns v30.4s,v30.4s
fcvtns v31.4s,v31.4s
b Quantize
FixedPointScale
tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastFixedPointValue
ldp q0,q1,[x20],32 // load preshift vector
ldp q2,q3,[x20],32
ldp q4,q5,[x21],32 // load multiplier vector
ldp q6,q7,[x21],32
ldp q8,q9,[x22],32 // load postshift vector
ldp q10,q11,[x22],32
b ScaleWithFixedPoint
BroadcastFixedPointValue
ld1r {v0.4s},[x20] // load preshift Value
mov v1.16b, v0.16b
mov v2.16b, v0.16b
mov v3.16b, v0.16b
ld1r {v4.4s},[x21] // load multiplier Value
mov v5.16b, v4.16b
mov v6.16b, v4.16b
mov v7.16b, v4.16b
ld1r {v8.4s},[x22] // load postshift Value
mov v9.16b, v8.16b
mov v10.16b, v8.16b
mov v11.16b, v8.16b
ScaleWithFixedPoint
sqshl v16.4s, v16.4s, v0.4s // preshift
sqshl v17.4s, v17.4s, v0.4s
sqshl v18.4s, v18.4s, v0.4s
sqshl v19.4s, v19.4s, v0.4s
sqshl v20.4s, v20.4s, v1.4s
sqshl v21.4s, v21.4s, v1.4s
sqshl v22.4s, v22.4s, v1.4s
sqshl v23.4s, v23.4s, v1.4s
sqshl v24.4s, v24.4s, v2.4s
sqshl v25.4s, v25.4s, v2.4s
sqshl v26.4s, v26.4s, v2.4s
sqshl v27.4s, v27.4s, v2.4s
sqshl v28.4s, v28.4s, v3.4s
sqshl v29.4s, v29.4s, v3.4s
sqshl v30.4s, v30.4s, v3.4s
sqshl v31.4s, v31.4s, v3.4s
sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale
sqdmulh v17.4s, v17.4s, v4.4s
sqdmulh v18.4s, v18.4s, v4.4s
sqdmulh v19.4s, v19.4s, v4.4s
sqdmulh v20.4s, v20.4s, v5.4s
sqdmulh v21.4s, v21.4s, v5.4s
sqdmulh v22.4s, v22.4s, v5.4s
sqdmulh v23.4s, v23.4s, v5.4s
sqdmulh v24.4s, v24.4s, v6.4s
sqdmulh v25.4s, v25.4s, v6.4s
sqdmulh v26.4s, v26.4s, v6.4s
sqdmulh v27.4s, v27.4s, v6.4s
sqdmulh v28.4s, v28.4s, v7.4s
sqdmulh v29.4s, v29.4s, v7.4s
sqdmulh v30.4s, v30.4s, v7.4s
sqdmulh v31.4s, v31.4s, v7.4s
srshl v16.4s,v16.4s, v8.4s // post shift
srshl v17.4s,v17.4s, v8.4s
srshl v18.4s,v18.4s, v8.4s
srshl v19.4s,v19.4s, v8.4s
srshl v20.4s,v20.4s, v9.4s
srshl v21.4s,v21.4s, v9.4s
srshl v22.4s,v22.4s, v9.4s
srshl v23.4s,v23.4s, v9.4s
srshl v24.4s,v24.4s, v10.4s
srshl v25.4s,v25.4s, v10.4s
srshl v26.4s,v26.4s, v10.4s
srshl v27.4s,v27.4s, v10.4s
srshl v28.4s,v28.4s, v11.4s
srshl v29.4s,v29.4s, v11.4s
srshl v30.4s,v30.4s, v11.4s
srshl v31.4s,v31.4s, v11.4s
Quantize
sqxtn v16.4h,v16.4s
sqxtn v17.4h,v17.4s
sqxtn v18.4h,v18.4s
@ -499,10 +582,11 @@ AccumulatorsToFloat
b.hi OutputChannelLoop
ExitKernel
EPILOG_RESTORE_REG_PAIR x19,x20,#48
EPILOG_RESTORE_REG_PAIR d12,d13,#32
EPILOG_RESTORE_REG_PAIR d10,d11,#16
EPILOG_RESTORE_REG_PAIR d8,d9,#64!
EPILOG_RESTORE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22
EPILOG_RESTORE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20
EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13
EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11
EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters!
EPILOG_RETURN
InChannels8

167
onnxruntime/core/mlas/lib/arm64/ConvSymU8KernelNeon.asm
View file

@ -17,23 +17,45 @@ Abstract:
#include "kxarm64.h"
#define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 1
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2
#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4
//
// Stack frame layout for the symmetric convolution kernel.
// d8-d15, x19-x30 need to be preserved if used
//
#define ConvSymFrame_SavedNeonRegisters (8 * 8)
#define ConvSymFrame_SavedRegisters ConvSymFrame_SavedNeonRegisters
#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_SavedRegisters_d8_d9 0
#define ConvSymFrame_SavedRegisters_d10_d11 16
#define ConvSymFrame_SavedRegisters_d12_d13 32
#define ConvSymFrame_SavedRegisters_d14_d15 48
#define ConvSymFrame_SavedRegisters_x19_x20 64
#define ConvSymFrame_SavedRegisters_x21_x22 80
#define ConvSymFrame_SavedRegisters 96
#define ConvSymFrame_SavedRegisters_Neg -96
#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters
#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
#define ConvSymPostProcessParams_Multiplier 32
#define ConvSymPostProcessParams_PreShift 40
#define ConvSymPostProcessParams_PostShift 48
TEXTAREA
@ -46,16 +68,10 @@ Routine Description:
Arguments:
Input (x0) - Supplies the address of the input buffer.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points
directly at the input tensor.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an
indirection buffer. Every pointer in the indirection buffer points at a
InputChannels length vector (either from the input tensor or a vector of
padding values). These are grouped in batches of length KernelSize.
These batches are then repeated OutputCount times.
Input (x0) - Supplies the address of the indirect buffer. Every pointer in
the indirection buffer points at a InputChannels length vector (either
from the input tensor or a vector of padding values). These are grouped
in batches of length KernelSize.
Filter (x1) - Supplies the address of the filter buffer.
@ -63,8 +79,6 @@ Arguments:
KernelSize (x3) - Supplies the size of the kernel.
If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1.
InputChannels (x4) - Supplies the number of input channels.
This implementation requires the count to be a multiple of 8.
@ -90,12 +104,14 @@ Return Value:
--*/
NESTED_ENTRY MlasConvSymU8KernelNeon
PROLOG_SAVE_REG_PAIR d8,d9,#-64!
PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters_Neg!
PROLOG_NOP ldr x8,[sp,#ConvSymFrame_PostProcessParams]
PROLOG_NOP ldrb w10,[sp,#ConvSymFrame_KernelFlags]
PROLOG_SAVE_REG_PAIR d10,d11,#16
PROLOG_SAVE_REG_PAIR d12,d13,#32
PROLOG_SAVE_REG_PAIR d14,d15,#48
PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11
PROLOG_SAVE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13
PROLOG_SAVE_REG_PAIR d14,d15,#ConvSymFrame_SavedRegisters_d14_d15
PROLOG_SAVE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20
PROLOG_SAVE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22
mov x9,x3 // save kernel size
ldr x11,[x8,#ConvSymPostProcessParams_Bias]
mov x16,x4 // save input channels
@ -104,6 +120,9 @@ Return Value:
add x5,x2,x5 // c1 = c0 + ldc
add x4,x4,7 // kc = (kc + 7) & ~7
csel x5,x2,x5,lo // if OutputCount < 2 c1 = c0
ldr x19,[x8,#ConvSymPostProcessParams_PreShift]
ldr x20,[x8,#ConvSymPostProcessParams_Multiplier]
ldr x21,[x8,#ConvSymPostProcessParams_PostShift]
bic x4,x4,7
ldp s16,s18,[x11],8 // init accumulators with bias
ldp s20,s22,[x11],8
@ -137,30 +156,18 @@ Return Value:
//
KernelSizeLoop
// Load next 2 A pointers
tst w10,#MLAS_CONV_SYM_FLAG_INPUT_DIRECT
ldr d4,[x1]
ldr d5,[x1,8]
beq InputIndirection
InputDirect
mov x13,x0 // x13 -> A0
add x15,x0,x16 // x15 -> A1 = A0 + input channels
b BlockLoopPrologue
InputIndirection
cmp x7,2 // test if OutputCount < 2
ldr x13,[x0] // x13 -> A0
blo SkipLoadA1
bhs LoadA1
ldr x15,[x0],#8 // x15 -> A0
b BlockLoopPrologue
LoadA1
ldr x15,[x0,x3,lsl#3] // x15 -> A1
SkipLoadA1
BlockLoopPrologue
cmp x7,2 // test if OutputCount < 2
add x0,x0,8 // indirect A advance to next pointer, prepare for kernel size loop
csel x15,x13,x15,lo // if OutputCount < 2 x15 -> A0
BlockLoopPrologue
ldr d4,[x1]
subs x14,x4,16 // input channel - 16
ldr d5,[x1,8]
movi v12.8b,128
blo InputChannel8 // less than 16 deep, no unroll
@ -325,17 +332,20 @@ BlockLoopEpilogue // remaining 16 input channels
b.hi KernelSizeLoop
Requantize
ldr w11,[x8,#ConvSymPostProcessParams_ZeroPoint]
tst w10,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale
FloatPointScale
tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w11,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastScaleValue
ld1 {v4.4s,v5.4s},[x12] // load scale vector
b AccumulatorsToFloat
b ScaleWithFloatPoint
BroadcastScaleValue
ld1r {v4.4s},[x12] // load scale Value
mov v5.16b, v4.16b
AccumulatorsToFloat
ScaleWithFloatPoint
addp v16.4s,v16.4s,v18.4s
addp v20.4s,v20.4s,v22.4s
addp v24.4s,v24.4s,v26.4s
@ -358,25 +368,72 @@ AccumulatorsToFloat
fmul v3.4s,v3.4s,v5.4s
fcvtns v0.4s,v0.4s // convert to int
fcvtns v1.4s,v1.4s
dup v9.8h,w11
fcvtns v2.4s,v2.4s
fcvtns v3.4s,v3.4s
b Quantize
FixedPointScale
tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w11, [x8, #ConvSymPostProcessParams_ZeroPoint]
beq BroadcastFixedPointValue
ld1 {v4.4s,v5.4s},[x19] // load preshift
ld1 {v6.4s,v7.4s},[x20] // load multiplier
ld1 {v8.4s,v9.4s},[x21] // load postshift
b ScaleWithFixedPoint
BroadcastFixedPointValue
ld1r {v4.4s},[x19] // load preshift Value
mov v5.16b, v4.16b
ld1r {v6.4s},[x20] // load multiplier Value
mov v7.16b, v6.16b
ld1r {v8.4s},[x21] // load postshift Value
mov v9.16b, v8.16b
ScaleWithFixedPoint
addp v16.4s,v16.4s,v18.4s
addp v20.4s,v20.4s,v22.4s
addp v24.4s,v24.4s,v26.4s
addp v28.4s,v28.4s,v30.4s
addp v17.4s,v17.4s,v19.4s
addp v21.4s,v21.4s,v23.4s
addp v25.4s,v25.4s,v27.4s
addp v29.4s,v29.4s,v31.4s
addp v0.4s,v16.4s,v20.4s
addp v1.4s,v24.4s,v28.4s
addp v2.4s,v17.4s,v21.4s
addp v3.4s,v25.4s,v29.4s
sqshl v0.4s, v0.4s, v4.4s // preshift
sqshl v1.4s, v1.4s, v5.4s
sqshl v2.4s, v2.4s, v4.4s
sqshl v3.4s, v3.4s, v5.4s
sqdmulh v0.4s, v0.4s, v6.4s // multiplier
sqdmulh v1.4s, v1.4s, v7.4s
sqdmulh v2.4s, v2.4s, v6.4s
sqdmulh v3.4s, v3.4s, v7.4s
srshl v0.4s,v0.4s, v8.4s // postshift
srshl v1.4s,v1.4s, v9.4s
srshl v2.4s,v2.4s, v8.4s
srshl v3.4s,v3.4s, v9.4s
Quantize
dup v9.8h,w11
sqxtn v0.4h,v0.4s
sqxtn2 v0.8h,v1.4s
sqxtn v2.4h,v2.4s
sqxtn2 v0.8h,v1.4s
sqxtn2 v2.8h,v3.4s
sqadd v0.8h,v0.8h,v9.8h
sqadd v2.8h,v2.8h,v9.8h
sqxtun v0.8b,v0.8h // shorten to int8
sqxtun2 v0.16b,v2.8h
st1 {v0.d}[1],[x5] // full 2x8 store to c
st1 {v0.d}[1],[x5] // full 2x8 store to c
st1 {v0.8b},[x2]
ExitKernel
EPILOG_RESTORE_REG_PAIR d14,d15,#48
EPILOG_RESTORE_REG_PAIR d12,d13,#32
EPILOG_RESTORE_REG_PAIR d10,d11,#16
EPILOG_RESTORE_REG_PAIR d8,d9,#64!
EPILOG_RESTORE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22
EPILOG_RESTORE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20
EPILOG_RESTORE_REG_PAIR d14,d15,#ConvSymFrame_SavedRegisters_d14_d15
EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13
EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11
EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters!
EPILOG_RETURN
InputChannel8

261
onnxruntime/core/mlas/lib/arm64/DepthwiseQConvKernelSize9Neon.asm
View file

@ -18,29 +18,48 @@ Abstract:
#include "kxarm64.h"
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX 1
#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4
//
// Stack frame layout for the depthwise conv kernel.
// d8-d15, x19-x30 need to be preserved if used
//
#define MlasConvSymDepthwiseKernelSize9_x19_x20 0
#define MlasConvSymDepthwiseKernelSize9_x21_x22 16
#define MlasConvSymDepthwiseKernelSize9_x23_x24 32
#define MlasConvSymDepthwiseKernelSize9_x25_x26 48
#define MlasConvSymDepthwiseKernelSize9_x27_x28 64
#define MlasConvSymDepthwiseKernelSize9_d8_d9 80
#define MlasConvSymDepthwiseKernelSize9_d10_d11 96
#define MlasConvSymDepthwiseKernelSize9_d12_d13 112
#define MlasConvSymDepthwiseKernelSize9_d14_d15 128
#define MlasConvSymDepthwiseKernelSize9_x0_x4 144
#define MlasConvSymDepthwiseKernelSize9_multiplier_postshift 160
#define MlasConvSymDepthwiseKernelSize9_SavedRegisters 176
#define MlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg -176
#define ConvSymDepthwisePostProcessParams_Bias 0
#define ConvSymDepthwisePostProcessParams_Scale 8
#define ConvSymDepthwisePostProcessParams_ZeroPoint 24
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX 1
#define MlasConvSymDepthwiseKernelSize9_backup_x19_x20 0
#define MlasConvSymDepthwiseKernelSize9_backup_x21_x22 16
#define MlasConvSymDepthwiseKernelSize9_backup_x23_x24 32
#define MlasConvSymDepthwiseKernelSize9_backup_x25_x26 48
#define MlasConvSymDepthwiseKernelSize9_backup_x27_x28 64
#define MlasConvSymDepthwiseKernelSize9_backup_d8_d9 80
#define MlasConvSymDepthwiseKernelSize9_backup_d10_d11 96
#define MlasConvSymDepthwiseKernelSize9_backup_d12_d13 112
#define MlasConvSymDepthwiseKernelSize9_backup_d14_d15 128
#define MlasConvSymDepthwiseKernelSize9_SavedRegisters 144
#define MlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg -144
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
#define ConvSymDepthwisePostProcessParams_Bias 0
#define ConvSymDepthwisePostProcessParams_Scale 8
#define ConvSymDepthwisePostProcessParams_Min 16
#define ConvSymDepthwisePostProcessParams_Max 20
#define ConvSymDepthwisePostProcessParams_ZeroPoint 24
#define ConvSymPostProcessParams_Multiplier 32
#define ConvSymPostProcessParams_PreShift 40
#define ConvSymPostProcessParams_PostShift 48
TEXTAREA
@ -76,26 +95,32 @@ Return Value:
LEAF_ENTRY MlasConvSymDepthwiseKernelSize9Arm64U8S8
PROLOG_SAVE_REG_PAIR x19, x20, #MlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg !
PROLOG_SAVE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_backup_x21_x22
PROLOG_SAVE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_backup_x23_x24
PROLOG_SAVE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_backup_x25_x26
PROLOG_SAVE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_backup_x27_x28
PROLOG_SAVE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_backup_d8_d9
PROLOG_SAVE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_backup_d10_d11
PROLOG_SAVE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_backup_d12_d13
PROLOG_SAVE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_backup_d14_d15
PROLOG_SAVE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_x21_x22
PROLOG_SAVE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_x23_x24
PROLOG_SAVE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_x25_x26
PROLOG_SAVE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_x27_x28
PROLOG_SAVE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_d8_d9
PROLOG_SAVE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_d10_d11
PROLOG_SAVE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_d12_d13
PROLOG_SAVE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_d14_d15
tst x6, #MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
ldr x11, [x5, #ConvSymPostProcessParams_Multiplier]
ldr x12, [x5, #ConvSymPostProcessParams_PostShift]
ldr x10, [x5, #ConvSymPostProcessParams_PreShift]
ldr x9, [x5, #ConvSymDepthwisePostProcessParams_Bias]
ldr x8, [x5, #ConvSymDepthwisePostProcessParams_Scale]
add x5, x5, #ConvSymDepthwisePostProcessParams_ZeroPoint
stp x11, x12,[sp, #MlasConvSymDepthwiseKernelSize9_multiplier_postshift]
csel x8, x8, x10, eq
ins v12.d[0], x1 // Filter
ins v13.d[0], x9 // Bias
ins v13.d[1], x8 // Scale
ins v13.d[1], x8 // Scale/PreShift
ld1r {v0.8h}, [x5] // v0.8h <--- vector for output zero point
movi v5.16b, #0x80
tbnz x6, #MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, MlasConvSymDepthwiseKernelSize9_SkipPerTensorScaleInit
ld1r {v1.4s}, [x8] // load and dup scale value
ld1r {v1.4s}, [x8] // load and dup scale/PreShift value
mov v2.16b, v1.16b
mov v3.16b, v1.16b
mov v4.16b, v1.16b
@ -114,7 +139,7 @@ MlasConvSymDepthwiseKernelSize9_OutputLoop
ldur x28, [x0, #-8]
cbz x4, MlasConvSymDepthwiseKernelSize9_Dup_Inputs
ldp x10, x11, [x0], #72 // input ptrs for Output0
ldp x10, x11, [x0], #72 // input ptrs for Output1
ldp x12, x13, [x0, #-56]
sub x4, x4, #1
ldp x14, x15, [x0, #-40]
@ -124,7 +149,7 @@ MlasConvSymDepthwiseKernelSize9_OutputLoop
MlasConvSymDepthwiseKernelSize9_Dup_Inputs
mov x9, x3 // Output1 <-- Output0
mov x10, x20
mov x10, x20 // Input1 <-- Input0
mov x11, x21
mov x12, x22
mov x13, x23
@ -136,10 +161,12 @@ MlasConvSymDepthwiseKernelSize9_Dup_Inputs
MlasConvSymDepthwiseKernelSize9_Loaded_Input
stp x0, x4, [sp, #MlasConvSymDepthwiseKernelSize9_x0_x4]
eor x8, x8, x8 // Processed channels
umov x1, v12.d[0] // filter
umov x5, v13.d[0] // bias
umov x7, v13.d[1] // scale
umov x7, v13.d[1] // scale/PreShift
ldp x0, x4, [sp, #MlasConvSymDepthwiseKernelSize9_multiplier_postshift]
cmp x8, x2 // Save one register by not using count down to zero here
bhs MlasConvSymDepthwiseKernelSize9_Finish_Channels16_Loop
@ -232,7 +259,7 @@ MlasConvSymDepthwiseKernelSize9_Channels16_Loop
ldr q22, [x27, x8] // out0 vi7
ldr q23, [x17, x8] // out1 vi7
tbz x6, #MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9
ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales 0-15 for outs
ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales/preshift 0-15 for outs
DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9
eor v20.16b, v20.16b, v5.16b
@ -289,6 +316,9 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9
saddw v8.4s, v8.4s, v27.4h
saddw2 v9.4s, v9.4s, v27.8h
tst x6, #MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale_U8S8
FloatPointScale_U8S8
scvtf v16.4s, v16.4s // Requantize
scvtf v17.4s, v17.4s
scvtf v18.4s, v18.4s
@ -315,7 +345,55 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9
fcvtns v7.4s, v7.4s
fcvtns v8.4s, v8.4s
fcvtns v9.4s, v9.4s
b Quantize_U8S8
FixedPointScale_U8S8
tbz x6,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, BroadcastFixedPointValue_U8S8
ldp q20,q21,[x0],32 // load multiplier vector
ldp q22,q23,[x0],32
ldp q24,q25,[x4],32 // load postshift vector
ldp q26,q27,[x4],32
b ScaleWithFixedPoint_U8S8
BroadcastFixedPointValue_U8S8
ld1r {v20.4s},[x0] // load multiplier Value
mov v21.16b, v20.16b
mov v22.16b, v20.16b
mov v23.16b, v20.16b
ld1r {v24.4s},[x4] // load postshift Value
mov v25.16b, v24.16b
mov v26.16b, v24.16b
mov v27.16b, v24.16b
ScaleWithFixedPoint_U8S8
sqshl v6.4s, v6.4s, v1.4s // preshift
sqshl v7.4s, v7.4s, v2.4s
sqshl v8.4s, v8.4s, v3.4s
sqshl v9.4s, v9.4s, v4.4s
sqshl v16.4s, v16.4s, v1.4s
sqshl v17.4s, v17.4s, v2.4s
sqshl v18.4s, v18.4s, v3.4s
sqshl v19.4s, v19.4s, v4.4s
sqdmulh v6.4s, v6.4s, v20.4s // multiply by multiplier
sqdmulh v7.4s, v7.4s, v21.4s
sqdmulh v8.4s, v8.4s, v22.4s
sqdmulh v9.4s, v9.4s, v23.4s
sqdmulh v16.4s, v16.4s, v20.4s
sqdmulh v17.4s, v17.4s, v21.4s
sqdmulh v18.4s, v18.4s, v22.4s
sqdmulh v19.4s, v19.4s, v23.4s
srshl v6.4s,v6.4s, v24.4s // post shift
srshl v7.4s,v7.4s, v25.4s
srshl v8.4s,v8.4s, v26.4s
srshl v9.4s,v9.4s, v27.4s
srshl v16.4s,v16.4s, v24.4s
srshl v17.4s,v17.4s, v25.4s
srshl v18.4s,v18.4s, v26.4s
srshl v19.4s,v19.4s, v27.4s
Quantize_U8S8
sqxtn v16.4h, v16.4s // +zp, narrow and combine
sqxtn v18.4h, v18.4s
sqxtn v6.4h, v6.4s
@ -342,19 +420,20 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9
blo MlasConvSymDepthwiseKernelSize9_Channels16_Loop
MlasConvSymDepthwiseKernelSize9_Finish_Channels16_Loop
ldp x0, x4, [sp, #MlasConvSymDepthwiseKernelSize9_x0_x4]
add x3, x3, x2, LSL #1
add x9, x9, x2, LSL #1
cbnz x4, MlasConvSymDepthwiseKernelSize9_OutputLoop
MlasConvSymDepthwiseKernelSize9_Exit
EPILOG_RESTORE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_backup_d14_d15
EPILOG_RESTORE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_backup_d12_d13
EPILOG_RESTORE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_backup_d10_d11
EPILOG_RESTORE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_backup_d8_d9
EPILOG_RESTORE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_backup_x27_x28
EPILOG_RESTORE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_backup_x25_x26
EPILOG_RESTORE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_backup_x23_x24
EPILOG_RESTORE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_backup_x21_x22
EPILOG_RESTORE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_d14_d15
EPILOG_RESTORE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_d12_d13
EPILOG_RESTORE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_d10_d11
EPILOG_RESTORE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_d8_d9
EPILOG_RESTORE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_x27_x28
EPILOG_RESTORE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_x25_x26
EPILOG_RESTORE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_x23_x24
EPILOG_RESTORE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_x21_x22
EPILOG_RESTORE_REG_PAIR x19, x20, #MlasConvSymDepthwiseKernelSize9_SavedRegisters !
EPILOG_RETURN
@ -394,25 +473,32 @@ Return Value:
LEAF_ENTRY MlasConvSymDepthwiseKernelSize9Arm64S8S8
PROLOG_SAVE_REG_PAIR x19, x20, #MlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg !
PROLOG_SAVE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_backup_x21_x22
PROLOG_SAVE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_backup_x23_x24
PROLOG_SAVE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_backup_x25_x26
PROLOG_SAVE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_backup_x27_x28
PROLOG_SAVE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_backup_d8_d9
PROLOG_SAVE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_backup_d10_d11
PROLOG_SAVE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_backup_d12_d13
PROLOG_SAVE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_backup_d14_d15
PROLOG_SAVE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_x21_x22
PROLOG_SAVE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_x23_x24
PROLOG_SAVE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_x25_x26
PROLOG_SAVE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_x27_x28
PROLOG_SAVE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_d8_d9
PROLOG_SAVE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_d10_d11
PROLOG_SAVE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_d12_d13
PROLOG_SAVE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_d14_d15
tst x6, #MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
ldr x9, [x5, #ConvSymDepthwisePostProcessParams_Bias]
ldr x8, [x5, #ConvSymDepthwisePostProcessParams_Scale]
ldr x10, [x5, #ConvSymPostProcessParams_PreShift]
ldr x11, [x5, #ConvSymPostProcessParams_Multiplier]
ldr x12, [x5, #ConvSymPostProcessParams_PostShift]
add x5, x5, #ConvSymDepthwisePostProcessParams_ZeroPoint
csel x8, x8, x10, eq
ins v12.d[0], x1 // Filter
ins v13.d[0], x9 // Bias
ins v13.d[1], x8 // Scale
ins v13.d[1], x8 // Scale/PreShift
ins v5.d[0], x11 // Multiplier
ins v5.d[1], x12 // PostShift
ld1r {v0.8h}, [x5] // v0.8h <--- vector for output zero point
tbnz x6, #MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, MlasConvSymDepthwiseKernelSize9S8S8_SkipPerTensorScaleInit
ld1r {v1.4s}, [x8] // load and dup scale value
ld1r {v1.4s}, [x8] // load and dup scale/PreShift value
mov v2.16b, v1.16b
mov v3.16b, v1.16b
mov v4.16b, v1.16b
@ -453,10 +539,13 @@ MlasConvSymDepthwiseKernelSize9S8S8_Dup_Inputs
MlasConvSymDepthwiseKernelSize9S8S8_Loaded_Input
stp x0, x4, [sp, #MlasConvSymDepthwiseKernelSize9_x0_x4]
eor x8, x8, x8 // Processed channels
umov x1, v12.d[0] // filter
umov x5, v13.d[0] // bias
umov x7, v13.d[1] // scale
umov x7, v13.d[1] // scale/PreShift
umov x0, v5.D[0]
umov x4, v5.D[1]
cmp x8, x2 // Save one register by not using count down to zero here
bhs MlasConvSymDepthwiseKernelSize9S8S8_Finish_Channels16_Loop
@ -537,7 +626,7 @@ MlasConvSymDepthwiseKernelSize9S8S8_Channels16_Loop
ldr q22, [x27, x8] // out0 vi7
ldr q23, [x17, x8] // out1 vi7
tbz x6, #MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8
ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales 0-15 for outs
ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales/preshift 0-15 for outs
DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8
ldr q10, [x1] // vk8
@ -588,6 +677,9 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8
saddw v8.4s, v8.4s, v27.4h
saddw2 v9.4s, v9.4s, v27.8h
tst x6, #MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
bne FixedPointScale
FloatPointScale
scvtf v16.4s, v16.4s // Requantize
scvtf v17.4s, v17.4s
scvtf v18.4s, v18.4s
@ -614,7 +706,55 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8
fcvtns v7.4s, v7.4s
fcvtns v8.4s, v8.4s
fcvtns v9.4s, v9.4s
b Quantize
FixedPointScale
tbz x6,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, BroadcastFixedPointValue
ldp q20,q21,[x0],32 // load multiplier vector
ldp q22,q23,[x0],32
ldp q24,q25,[x4],32 // load postshift vector
ldp q26,q27,[x4],32
b ScaleWithFixedPoint
BroadcastFixedPointValue
ld1r {v20.4s},[x0] // load multiplier Value
mov v21.16b, v20.16b
mov v22.16b, v20.16b
mov v23.16b, v20.16b
ld1r {v24.4s},[x4] // load postshift Value
mov v25.16b, v24.16b
mov v26.16b, v24.16b
mov v27.16b, v24.16b
ScaleWithFixedPoint
sqshl v6.4s, v6.4s, v1.4s // preshift
sqshl v7.4s, v7.4s, v2.4s
sqshl v8.4s, v8.4s, v3.4s
sqshl v9.4s, v9.4s, v4.4s
sqshl v16.4s, v16.4s, v1.4s
sqshl v17.4s, v17.4s, v2.4s
sqshl v18.4s, v18.4s, v3.4s
sqshl v19.4s, v19.4s, v4.4s
sqdmulh v6.4s, v6.4s, v20.4s // multiply by multiplier
sqdmulh v7.4s, v7.4s, v21.4s
sqdmulh v8.4s, v8.4s, v22.4s
sqdmulh v9.4s, v9.4s, v23.4s
sqdmulh v16.4s, v16.4s, v20.4s
sqdmulh v17.4s, v17.4s, v21.4s
sqdmulh v18.4s, v18.4s, v22.4s
sqdmulh v19.4s, v19.4s, v23.4s
srshl v6.4s,v6.4s, v24.4s // post shift
srshl v7.4s,v7.4s, v25.4s
srshl v8.4s,v8.4s, v26.4s
srshl v9.4s,v9.4s, v27.4s
srshl v16.4s,v16.4s, v24.4s
srshl v17.4s,v17.4s, v25.4s
srshl v18.4s,v18.4s, v26.4s
srshl v19.4s,v19.4s, v27.4s
Quantize
sqxtn v16.4h, v16.4s // +zp, narrow and combine
sqxtn v18.4h, v18.4s
sqxtn v6.4h, v6.4s
@ -641,19 +781,20 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8
blo MlasConvSymDepthwiseKernelSize9S8S8_Channels16_Loop
MlasConvSymDepthwiseKernelSize9S8S8_Finish_Channels16_Loop
ldp x0, x4, [sp, #MlasConvSymDepthwiseKernelSize9_x0_x4]
add x3, x3, x2, LSL #1
add x9, x9, x2, LSL #1
cbnz x4, MlasConvSymDepthwiseKernelSize9S8S8_OutputLoop
MlasConvSymDepthwiseKernelSize9S8S8_Exit
EPILOG_RESTORE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_backup_d14_d15
EPILOG_RESTORE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_backup_d12_d13
EPILOG_RESTORE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_backup_d10_d11
EPILOG_RESTORE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_backup_d8_d9
EPILOG_RESTORE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_backup_x27_x28
EPILOG_RESTORE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_backup_x25_x26
EPILOG_RESTORE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_backup_x23_x24
EPILOG_RESTORE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_backup_x21_x22
EPILOG_RESTORE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_d14_d15
EPILOG_RESTORE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_d12_d13
EPILOG_RESTORE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_d10_d11
EPILOG_RESTORE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_d8_d9
EPILOG_RESTORE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_x27_x28
EPILOG_RESTORE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_x25_x26
EPILOG_RESTORE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_x23_x24
EPILOG_RESTORE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_x21_x22
EPILOG_RESTORE_REG_PAIR x19, x20, #MlasConvSymDepthwiseKernelSize9_SavedRegisters !
EPILOG_RETURN

294
onnxruntime/core/mlas/lib/arm64/DepthwiseQConvSymS8KernelNeon.asm
View file

@ -17,23 +17,45 @@ Abstract:
#include "kxarm64.h"
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2
#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4
//
// Stack frame layout for the depthwise conv kernel.
// d8-d15, x19-x30 need to be preserved if used
//
#define ConvSymDepthwiseKernelFrame_SavedRegisters (4 * 8)
#define ConvSymDepthwiseKernelFrame_PostProcessParams 0 + ConvSymDepthwiseKernelFrame_SavedRegisters
#define ConvSymDepthwiseKernelFrame_KernelFlags 8 + ConvSymDepthwiseKernelFrame_SavedRegisters
#define ConvSymDepthwiseKernelFrame_SavedRegisters_d8_d9 0
#define ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11 16
#define ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13 32
#define ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15 48
#define ConvSymDepthwiseKernelFrame_SavedRegisters 64
#define ConvSymDepthwiseKernelFrame_SavedRegisters_Neg -64
#define ConvSymDepthwiseKernelFrame_PostProcessParams 0 + ConvSymDepthwiseKernelFrame_SavedRegisters
#define ConvSymDepthwiseKernelFrame_KernelFlags 8 + ConvSymDepthwiseKernelFrame_SavedRegisters
#define ConvSymDepthwisePostProcessParams_Bias 0
#define ConvSymDepthwisePostProcessParams_Scale 8
#define ConvSymDepthwisePostProcessParams_Min 16
#define ConvSymDepthwisePostProcessParams_Max 20
#define ConvSymDepthwisePostProcessParams_ZeroPoint 24
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
#define ConvSymDepthwisePostProcessParams_Bias 0
#define ConvSymDepthwisePostProcessParams_Scale 8
#define ConvSymDepthwisePostProcessParams_Min 16
#define ConvSymDepthwisePostProcessParams_Max 20
#define ConvSymDepthwisePostProcessParams_ZeroPoint 24
#define ConvSymDepthwisePostProcessParams_Multiplier 32
#define ConvSymDepthwisePostProcessParams_PreShift 40
#define ConvSymDepthwisePostProcessParams_PostShift 48
#define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 1
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2
TEXTAREA
@ -47,28 +69,28 @@ Routine Description:
Arguments:
Input (x0) - Supplies the address of the indirection buffer.
Filter (x1) - Supplies the address of the filter buffer.
Output (x2) - Supplies the address of the output buffer.
KernelSize (x3) - Supplies the size of the kernel.
Channels (x4) - Supplies the number of input and output channels.
ChannelOffset (x5) - Supplies the byte offset from the indirection buffer base
address for this iteration.
ChannelCount (x6) - Supplies the number of channels this iteration produces.
This implementation requires the count to be 16 or 8
OutputCount (x7)- Supplies the number of output elements this iteration produces.
This implementation requires the count to be in the range 1 to 2.
This implementation requires the count to be in the range 1 to 4.
PostProcessParams - Supplies the address of the post process parameter block.
KernelFlags - Supplies additional flags controlling the operation.
Return Value:
@ -79,9 +101,11 @@ Return Value:
NESTED_ENTRY MlasConvSymDepthwiseS8KernelNeon
PROLOG_SAVE_REG_PAIR d12,d13,#-ConvSymDepthwiseKernelFrame_SavedRegisters!
PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymDepthwiseKernelFrame_SavedRegisters_Neg!
PROLOG_NOP ldr x8,[sp,#ConvSymDepthwiseKernelFrame_PostProcessParams]
PROLOG_SAVE_REG_PAIR d14,d15,#16
PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11
PROLOG_SAVE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13
PROLOG_SAVE_REG_PAIR d14,d15,#ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15
cmp x7,2
add x9,x0,x3,lsl#3 // x9 -> &A1
add x14,x0,x3,lsl#4 // x14 -> &A2
@ -161,7 +185,7 @@ BlockLoopC16
smlal v12.8h,v1.8b,v2.8b
ldr x11,[x9],#8 // x11 -> A1 iter 2
smlal2 v13.8h,v1.16b,v2.16b
b.eq EpilogueC16P3 // 3 pixel remains
b.eq EpilogueC16P3 // 3 pixel remains
ldr q2,[x12,x5] // A2 iter 1
smlal v14.8h,v1.8b,v3.8b
ldr x12,[x0],#8 // x12 -> A0 iter 3
@ -228,22 +252,15 @@ EpilogueC16P2
saddw v30.4s,v30.4s,v15.4h
saddw2 v31.4s,v31.4s,v15.8h
ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
smull2 v13.8h,v0.16b,v4.16b
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
smull2 v15.8h,v0.16b,v6.16b
smlal v12.8h,v1.8b,v2.8b
smlal2 v13.8h,v1.16b,v2.16b
smlal v14.8h,v1.8b,v3.8b
smlal2 v15.8h,v1.16b,v3.16b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq SkipScaleVecLoad2
ldp q4,q5,[x12],#32 // load scale vector if per channel
ldp q6,q3,[x12]
SkipScaleVecLoad2
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v18.4s,v18.4s,v13.4h
@ -252,7 +269,7 @@ SkipScaleVecLoad2
saddw2 v21.4s,v21.4s,v14.8h
saddw v22.4s,v22.4s,v15.4h
saddw2 v23.4s,v23.4s,v15.8h
b Dequantization
b Requantization
ProcC16P1
//
@ -327,10 +344,8 @@ EpilogueC16P1
// Loop epilogue (process last single pixel) mixed with loading of dequantization params
//
ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
smull2 v13.8h,v0.16b,v4.16b
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
smull2 v15.8h,v0.16b,v6.16b
saddw v24.4s,v24.4s,v12.4h
@ -345,12 +360,7 @@ EpilogueC16P1
smull2 v13.8h,v0.16b,v2.16b
smull v14.8h,v0.8b,v3.8b
smull2 v15.8h,v0.16b,v3.16b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq SkipScaleVecLoad
ldp q4,q5,[x12],#32 // load scale vector if per channel
ldp q6,q3,[x12]
SkipScaleVecLoad
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v18.4s,v18.4s,v13.4h
@ -360,7 +370,24 @@ SkipScaleVecLoad
saddw v22.4s,v22.4s,v15.4h
saddw2 v23.4s,v23.4s,v15.8h
Dequantization
Requantization
b.ne FixedPointScale
FloatingPointScale
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
beq BroadcastScaleValue
ldp q4,q5,[x12],#32 // load scale vector if per channel
ldp q6,q3,[x12]
b ScaleWithFloatPoint
BroadcastScaleValue
ld1r {v4.4s},[x12] // load scale val
mov v5.16b,v4.16b // broadcast scale val if not per channel
mov v6.16b,v4.16b
mov v3.16b,v4.16b
ScaleWithFloatPoint
scvtf v24.4s,v24.4s // convert to float
scvtf v25.4s,v25.4s
scvtf v26.4s,v26.4s
@ -377,11 +404,6 @@ Dequantization
scvtf v21.4s,v21.4s
scvtf v22.4s,v22.4s
scvtf v23.4s,v23.4s
b.ne SkipScaleBroadcast
mov v5.16b,v4.16b // broadcast scale val if not per channel
mov v6.16b,v4.16b
mov v3.16b,v4.16b
SkipScaleBroadcast
fmul v24.4s,v24.4s,v4.4s // multiply by scale
fmul v25.4s,v25.4s,v5.4s
fmul v26.4s,v26.4s,v6.4s
@ -414,6 +436,90 @@ SkipScaleBroadcast
fcvtns v21.4s,v21.4s
fcvtns v22.4s,v22.4s
fcvtns v23.4s,v23.4s
b Quantize
FixedPointScale
ldr x10,[x8,#ConvSymDepthwisePostProcessParams_PreShift]
ldr x11,[x8,#ConvSymDepthwisePostProcessParams_Multiplier]
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_PostShift]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
beq BroadcastFixedPointValue
ldp q0,q1,[x10],32 // load preshift vector
ldp q2,q3,[x10],32
ldp q4,q5,[x11],32 // load multiplier vector
ldp q6,q7,[x11],32
ldp q8,q9,[x12],32 // load postshift vector
ldp q10,q11,[x12],32
b ScaleWithFixedPoint
BroadcastFixedPointValue
ld1r {v0.4s},[x10] // load preshift Value
mov v1.16b, v0.16b
mov v2.16b, v0.16b
mov v3.16b, v0.16b
ld1r {v4.4s},[x11] // load multiplier Value
mov v5.16b, v4.16b
mov v6.16b, v4.16b
mov v7.16b, v4.16b
ld1r {v8.4s},[x12] // load postshift Value
mov v9.16b, v8.16b
mov v10.16b, v8.16b
mov v11.16b, v8.16b
ScaleWithFixedPoint
sqshl v24.4s, v24.4s, v0.4s // preshift
sqshl v25.4s, v25.4s, v1.4s
sqshl v26.4s, v26.4s, v2.4s
sqshl v27.4s, v27.4s, v3.4s
sqshl v28.4s, v28.4s, v0.4s
sqshl v29.4s, v29.4s, v1.4s
sqshl v30.4s, v30.4s, v2.4s
sqshl v31.4s, v31.4s, v3.4s
sqshl v16.4s, v16.4s, v0.4s
sqshl v17.4s, v17.4s, v1.4s
sqshl v18.4s, v18.4s, v2.4s
sqshl v19.4s, v19.4s, v3.4s
sqshl v20.4s, v20.4s, v0.4s
sqshl v21.4s, v21.4s, v1.4s
sqshl v22.4s, v22.4s, v2.4s
sqshl v23.4s, v23.4s, v3.4s
sqdmulh v24.4s, v24.4s, v4.4s // multiplier
sqdmulh v25.4s, v25.4s, v5.4s
sqdmulh v26.4s, v26.4s, v6.4s
sqdmulh v27.4s, v27.4s, v7.4s
sqdmulh v28.4s, v28.4s, v4.4s
sqdmulh v29.4s, v29.4s, v5.4s
sqdmulh v30.4s, v30.4s, v6.4s
sqdmulh v31.4s, v31.4s, v7.4s
sqdmulh v16.4s, v16.4s, v4.4s
sqdmulh v17.4s, v17.4s, v5.4s
sqdmulh v18.4s, v18.4s, v6.4s
sqdmulh v19.4s, v19.4s, v7.4s
sqdmulh v20.4s, v20.4s, v4.4s
sqdmulh v21.4s, v21.4s, v5.4s
sqdmulh v22.4s, v22.4s, v6.4s
sqdmulh v23.4s, v23.4s, v7.4s
srshl v24.4s, v24.4s, v8.4s // post shift
srshl v25.4s, v25.4s, v9.4s
srshl v26.4s, v26.4s, v10.4s
srshl v27.4s, v27.4s, v11.4s
srshl v28.4s, v28.4s, v8.4s
srshl v29.4s, v29.4s, v9.4s
srshl v30.4s, v30.4s, v10.4s
srshl v31.4s, v31.4s, v11.4s
srshl v16.4s, v16.4s, v8.4s
srshl v17.4s, v17.4s, v9.4s
srshl v18.4s, v18.4s, v10.4s
srshl v19.4s, v19.4s, v11.4s
srshl v20.4s, v20.4s, v8.4s
srshl v21.4s, v21.4s, v9.4s
srshl v22.4s, v22.4s, v10.4s
srshl v23.4s, v23.4s, v11.4s
Quantize
sqxtn v24.4h,v24.4s // shorten to int16
sqxtn v26.4h,v26.4s
sqxtn2 v24.8h,v25.4s
@ -458,8 +564,10 @@ SkipScaleBroadcast
str q20,[x2]
ExitKernel
EPILOG_RESTORE_REG_PAIR d14,d15,#16
EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters!
EPILOG_RESTORE_REG_PAIR d14,d15,#ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15
EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13
EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11
EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymDepthwiseKernelFrame_SavedRegisters!
EPILOG_RETURN
Process8Channels
@ -501,7 +609,7 @@ BlockLoopC8
ldr x11,[x9],#8 // x11 -> A1 iter 2
smlal v14.8h,v1.8b,v3.8b
ldr d2,[x12,x5] // A2 iter 1
b.eq EpilogueC8P3 // 3 pixel remains
b.eq EpilogueC8P3 // 3 pixel remains
ldr d3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
ldr x12,[x0],#8 // x12 -> A0 iter 3
@ -546,21 +654,15 @@ EpilogueC8P2
saddw2 v29.4s,v29.4s,v14.8h
ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
smull v12.8h,v0.8b,v4.8b
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
smull v14.8h,v0.8b,v6.8b
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
smlal v12.8h,v1.8b,v2.8b
smlal v14.8h,v1.8b,v3.8b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq SkipScaleVecLoad2C8
ldp q4,q5,[x12],#32 // load scale vector if per channel
SkipScaleVecLoad2C8
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v20.4s,v20.4s,v14.4h
saddw2 v21.4s,v21.4s,v14.8h
b DequantC8
b RequantizationC8
ProcC8P1
//
@ -613,9 +715,7 @@ EpilogueC8P1
// Loop epilogue (process last single pixel) mixed with loading of dequantization params
//
ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
@ -623,17 +723,27 @@ EpilogueC8P1
saddw2 v29.4s,v29.4s,v14.8h
smull v12.8h,v0.8b,v2.8b
smull v14.8h,v0.8b,v3.8b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq SkipScaleVecLoadC8
ldp q4,q5,[x12] // load scale vector if per channel
SkipScaleVecLoadC8
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v20.4s,v20.4s,v14.4h
saddw2 v21.4s,v21.4s,v14.8h
DequantC8
RequantizationC8
b.ne FixedPointScaleC8
FloatingPointScaleC8:
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
beq BroadcastScaleValueC8
ldp q4,q5,[x12] // load scale vector if per channel
b ScaleWithFloatPointC8
BroadcastScaleValueC8
ld1r {v4.4s},[x12] // load scale val
mov v5.16b,v4.16b // broadcast scale val if not per channel
ScaleWithFloatPointC8
scvtf v24.4s,v24.4s // convert to float
scvtf v25.4s,v25.4s
scvtf v28.4s,v28.4s
@ -642,9 +752,6 @@ DequantC8
scvtf v17.4s,v17.4s
scvtf v20.4s,v20.4s
scvtf v21.4s,v21.4s
b.ne SkipScaleBroadcastC8
mov v5.16b,v4.16b // broadcast scale val if not per channel
SkipScaleBroadcastC8
fmul v24.4s,v24.4s,v4.4s // multiply by scale
fmul v25.4s,v25.4s,v5.4s
fmul v28.4s,v28.4s,v4.4s
@ -661,6 +768,57 @@ SkipScaleBroadcastC8
fcvtns v17.4s,v17.4s
fcvtns v20.4s,v20.4s
fcvtns v21.4s,v21.4s
b QuantizeC8
FixedPointScaleC8
ldr x10,[x8,#ConvSymDepthwisePostProcessParams_PreShift]
ldr x11,[x8,#ConvSymDepthwisePostProcessParams_Multiplier]
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_PostShift]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
beq BroadcastFixedPointValueC8
ldp q0,q1,[x10],32 // load preshift vector
ldp q4,q5,[x11],32 // load multiplier vector
ldp q8,q9,[x12],32 // load postshift vector
b ScaleWithFixedPointC8
BroadcastFixedPointValueC8
ld1r {v0.4s},[x10] // load preshift Value
mov v1.16b, v0.16b
ld1r {v4.4s},[x11] // load multiplier Value
mov v5.16b, v4.16b
ld1r {v8.4s},[x12] // load postshift Value
mov v9.16b, v8.16b
ScaleWithFixedPointC8
sqshl v24.4s, v24.4s, v0.4s // preshift
sqshl v25.4s, v25.4s, v1.4s
sqshl v28.4s, v28.4s, v0.4s
sqshl v29.4s, v29.4s, v1.4s
sqshl v16.4s, v16.4s, v0.4s
sqshl v17.4s, v17.4s, v1.4s
sqshl v20.4s, v20.4s, v0.4s
sqshl v21.4s, v21.4s, v1.4s
sqdmulh v24.4s, v24.4s, v4.4s // multiplier
sqdmulh v25.4s, v25.4s, v5.4s
sqdmulh v28.4s, v28.4s, v4.4s
sqdmulh v29.4s, v29.4s, v5.4s
sqdmulh v16.4s, v16.4s, v4.4s
sqdmulh v17.4s, v17.4s, v5.4s
sqdmulh v20.4s, v20.4s, v4.4s
sqdmulh v21.4s, v21.4s, v5.4s
srshl v24.4s, v24.4s, v8.4s // post shift
srshl v25.4s, v25.4s, v9.4s
srshl v28.4s, v28.4s, v8.4s
srshl v29.4s, v29.4s, v9.4s
srshl v16.4s, v16.4s, v8.4s
srshl v17.4s, v17.4s, v9.4s
srshl v20.4s, v20.4s, v8.4s
srshl v21.4s, v21.4s, v9.4s
QuantizeC8
dup v0.8h,w15
sqxtn v24.4h,v24.4s // shorten to int16
sqxtn2 v24.8h,v25.4s

427
onnxruntime/core/mlas/lib/arm64/DepthwiseQConvSymU8KernelNeon.asm
View file

@ -17,24 +17,45 @@ Abstract:
#include "kxarm64.h"
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2
#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4
//
// Stack frame layout for the depthwise conv kernel.
// d8-d15, x19-x30 need to be preserved if used
//
#define ConvSymDepthwiseKernelFrame_SavedNeonRegisters (8 * 8)
#define ConvSymDepthwiseKernelFrame_SavedRegisters ConvSymDepthwiseKernelFrame_SavedNeonRegisters
#define ConvSymDepthwiseKernelFrame_PostProcessParams 0 + ConvSymDepthwiseKernelFrame_SavedRegisters
#define ConvSymDepthwiseKernelFrame_KernelFlags 8 + ConvSymDepthwiseKernelFrame_SavedRegisters
#define ConvSymDepthwiseKernelFrame_SavedRegisters_d8_d9 0
#define ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11 16
#define ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13 32
#define ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15 48
#define ConvSymDepthwiseKernelFrame_SavedRegisters 64
#define ConvSymDepthwiseKernelFrame_SavedRegisters_Neg -64
#define ConvSymDepthwiseKernelFrame_PostProcessParams 0 + ConvSymDepthwiseKernelFrame_SavedRegisters
#define ConvSymDepthwiseKernelFrame_KernelFlags 8 + ConvSymDepthwiseKernelFrame_SavedRegisters
#define ConvSymDepthwisePostProcessParams_Bias 0
#define ConvSymDepthwisePostProcessParams_Scale 8
#define ConvSymDepthwisePostProcessParams_Min 16
#define ConvSymDepthwisePostProcessParams_Max 20
#define ConvSymDepthwisePostProcessParams_ZeroPoint 24
/*
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
*/
#define ConvSymPostProcessParams_Bias 0
#define ConvSymPostProcessParams_Scale 8
#define ConvSymPostProcessParams_Min 16
#define ConvSymPostProcessParams_Max 20
#define ConvSymPostProcessParams_ZeroPoint 24
#define ConvSymPostProcessParams_Multiplier 32
#define ConvSymPostProcessParams_PreShift 40
#define ConvSymPostProcessParams_PostShift 48
#define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 1
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2
TEXTAREA
@ -48,28 +69,28 @@ Routine Description:
Arguments:
Input (x0) - Supplies the address of the indirection buffer.
Filter (x1) - Supplies the address of the filter buffer.
Output (x2) - Supplies the address of the output buffer.
KernelSize (x3) - Supplies the size of the kernel.
Channels (x4) - Supplies the number of input and output channels.
ChannelOffset (x5) - Supplies the byte offset from the indirection buffer base
address for this iteration.
ChannelCount (x6) - Supplies the number of channels this iteration produces.
This implementation requires the count to be 16 or 8
OutputCount (x7)- Supplies the number of output elements this iteration produces.
This implementation requires the count to be in the range 1 to 2.
PostProcessParams - Supplies the address of the post process parameter block.
KernelFlags - Supplies additional flags controlling the operation.
Return Value:
@ -80,14 +101,14 @@ Return Value:
NESTED_ENTRY MlasConvSymDepthwiseU8KernelNeon
PROLOG_SAVE_REG_PAIR d8,d9,#-64!
PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymDepthwiseKernelFrame_SavedRegisters_Neg!
PROLOG_NOP ldr x8,[sp,#ConvSymDepthwiseKernelFrame_PostProcessParams]
PROLOG_NOP mov w10,#0x80808080
PROLOG_SAVE_REG_PAIR d10,d11,#16
PROLOG_SAVE_REG_PAIR d12,d13,#32
PROLOG_SAVE_REG_PAIR d14,d15,#48
PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11
PROLOG_SAVE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13
PROLOG_SAVE_REG_PAIR d14,d15,#ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15
dup v8.4s,w10 // bit flip vector
ldr x16,[x8,#ConvSymDepthwisePostProcessParams_Bias]
ldr x16,[x8,#ConvSymPostProcessParams_Bias]
cmp x7,2
add x9,x0,x3,lsl#3 // x9 -> &A1
add x14,x0,x3,lsl#4 // x14 -> &A2
@ -111,11 +132,11 @@ Return Value:
// x0 -> x10 -> v4
// -> x12 -> v2 // unroll
// x9 -> x11 -> v6
// -> x13 -> v10 // unroll
// -> x13 -> v3 // unroll
// x14 -> x10 -> v4
// -> x12 -> v2 // unroll
// x15 -> x11 -> v6
// -> x13 -> v10 // unroll
// -> x13 -> v3 // unroll
//
Process16Channels
@ -144,7 +165,7 @@ Process16Channels
mov v18.16b,v26.16b
ldr x12,[x14],#8 // x12 -> A2 iter 1
mov v19.16b,v27.16b
ldr q10,[x13,x5] // A1 iter 1
ldr q3,[x13,x5] // A1 iter 1
mov v22.16b,v26.16b
ldr x13,[x15],#8 // x13 -> A3 iter 1
mov v23.16b,v27.16b
@ -169,13 +190,13 @@ BlockLoopC16
smlal v12.8h,v1.8b,v2.8b
ldr x11,[x9],#8 // x11 -> A1 iter 2
smlal2 v13.8h,v1.16b,v2.16b
b.eq EpilogueC16P3 // 3 pixel remains
eor v10.16b,v10.16b,v8.16b
b.eq EpilogueC16P3 // 3 pixel remains
eor v3.16b,v3.16b,v8.16b
ldr q2,[x12,x5] // A2 iter 1
smlal v14.8h,v1.8b,v10.8b
smlal v14.8h,v1.8b,v3.8b
ldr x12,[x0],#8 // x12 -> A0 iter 3
smlal2 v15.8h,v1.16b,v10.16b
ldr q10,[x13,x5] // A3 iter 1
smlal2 v15.8h,v1.16b,v3.16b
ldr q3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
ldr x13,[x9],#8 // x13 -> A1 iter 3
@ -200,12 +221,12 @@ BlockLoopC16
smlal v12.8h,v1.8b,v2.8b
ldr x11,[x15],#8 // x11 -> A3 iter 2
smlal2 v13.8h,v1.16b,v2.16b
eor v10.16b,v10.16b,v8.16b
eor v3.16b,v3.16b,v8.16b
ldr q2,[x12,x5] // A0 iter 3
smlal v14.8h,v1.8b,v10.8b
smlal v14.8h,v1.8b,v3.8b
ldr x12,[x14],#8 // x12 -> A2 iter 3
smlal2 v15.8h,v1.16b,v10.16b
ldr q10,[x13,x5] // A1 iter 3
smlal2 v15.8h,v1.16b,v3.16b
ldr q3,[x13,x5] // A1 iter 3
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
ld1 {v1.16b},[x1],x4 // filter iter 3
@ -229,11 +250,11 @@ EpilogueC16P2
eor v2.16b,v2.16b,v8.16b
smlal v12.8h,v1.8b,v2.8b
smlal2 v13.8h,v1.16b,v2.16b
eor v10.16b,v10.16b,v8.16b
eor v3.16b,v3.16b,v8.16b
ldr q2,[x12,x5] // A2 iter 1
smlal v14.8h,v1.8b,v10.8b
smlal2 v15.8h,v1.16b,v10.16b
ldr q10,[x13,x5] // A3 iter 1
smlal v14.8h,v1.8b,v3.8b
smlal2 v15.8h,v1.16b,v3.16b
ldr q3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
saddw v26.4s,v26.4s,v13.4h
@ -244,25 +265,20 @@ EpilogueC16P2
saddw2 v31.4s,v31.4s,v15.8h
ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
eor v4.16b,v4.16b,v8.16b
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
ldr x12,[x8,#ConvSymPostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
smull2 v13.8h,v0.16b,v4.16b
eor v6.16b,v6.16b,v8.16b
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
smull2 v15.8h,v0.16b,v6.16b
eor v2.16b,v2.16b,v8.16b
smlal v12.8h,v1.8b,v2.8b
smlal2 v13.8h,v1.16b,v2.16b
eor v10.16b,v10.16b,v8.16b
smlal v14.8h,v1.8b,v10.8b
smlal2 v15.8h,v1.16b,v10.16b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq SkipScaleVecLoad2
ldp q4,q11,[x12],#32 // load scale vector if per channel
ldp q6,q9,[x12]
SkipScaleVecLoad2
eor v3.16b,v3.16b,v8.16b
smlal v14.8h,v1.8b,v3.8b
smlal2 v15.8h,v1.16b,v3.16b
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v18.4s,v18.4s,v13.4h
@ -271,7 +287,7 @@ SkipScaleVecLoad2
saddw2 v21.4s,v21.4s,v14.8h
saddw v22.4s,v22.4s,v15.4h
saddw2 v23.4s,v23.4s,v15.8h
b Dequantization
b Requantization
ProcC16P1
//
@ -295,7 +311,7 @@ ProcC16P1
subs x3,x3,2 // decrement input blocks remaining
mov v18.16b,v26.16b
mov v19.16b,v27.16b
ldr q10,[x13,x5]
ldr q3,[x13,x5]
mov v22.16b,v26.16b
mov v23.16b,v27.16b
b EpilogueC16P1
@ -305,12 +321,12 @@ EpilogueC16P3
// Loop epilogue (process last 2 pixels) mixed
// with loading of dequantization params
//
eor v10.16b,v10.16b,v8.16b
eor v3.16b,v3.16b,v8.16b
ldr q2,[x12,x5] // A2 iter 1
smlal v14.8h,v1.8b,v10.8b
smlal v14.8h,v1.8b,v3.8b
ldr x12,[x14],#8 // x12 -> A2 iter 2
smlal2 v15.8h,v1.16b,v10.16b
ldr q10,[x13,x5] // A3 iter 1
smlal2 v15.8h,v1.16b,v3.16b
ldr q3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
ldr x13,[x15],#8 // x13 -> A3 iter 2
@ -332,11 +348,11 @@ EpilogueC16P3
eor v2.16b,v2.16b,v8.16b
smlal v12.8h,v1.8b,v2.8b
smlal2 v13.8h,v1.16b,v2.16b
eor v10.16b,v10.16b,v8.16b
eor v3.16b,v3.16b,v8.16b
ldr q2,[x12,x5] // A2 iter 2
smlal v14.8h,v1.8b,v10.8b
smlal2 v15.8h,v1.16b,v10.16b
ldr q10,[x13,x5] // A3 iter 2
smlal v14.8h,v1.8b,v3.8b
smlal2 v15.8h,v1.16b,v3.16b
ldr q3,[x13,x5] // A3 iter 2
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v18.4s,v18.4s,v13.4h
@ -352,11 +368,9 @@ EpilogueC16P1
//
ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
eor v4.16b,v4.16b,v8.16b
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
smull2 v13.8h,v0.16b,v4.16b
eor v6.16b,v6.16b,v8.16b
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
smull2 v15.8h,v0.16b,v6.16b
saddw v24.4s,v24.4s,v12.4h
@ -370,15 +384,10 @@ EpilogueC16P1
eor v2.16b,v2.16b,v8.16b
smull v12.8h,v0.8b,v2.8b
smull2 v13.8h,v0.16b,v2.16b
eor v10.16b,v10.16b,v8.16b
smull v14.8h,v0.8b,v10.8b
smull2 v15.8h,v0.16b,v10.16b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq SkipScaleVecLoad
ldp q4,q11,[x12],#32 // load scale vector if per channel
ldp q6,q9,[x12]
SkipScaleVecLoad
eor v3.16b,v3.16b,v8.16b
smull v14.8h,v0.8b,v3.8b
smull2 v15.8h,v0.16b,v3.16b
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v18.4s,v18.4s,v13.4h
@ -388,7 +397,24 @@ SkipScaleVecLoad
saddw v22.4s,v22.4s,v15.4h
saddw2 v23.4s,v23.4s,v15.8h
Dequantization
Requantization
b.ne FixedPointScale
FloatingPointScale
ldr x12,[x8,#ConvSymPostProcessParams_Scale]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastScaleValue
ldp q4,q5,[x12],#32 // load scale vector if per channel
ldp q6,q3,[x12]
b ScaleWithFloatPoint
BroadcastScaleValue
ld1r {v4.4s},[x12] // load scale val
mov v5.16b,v4.16b // broadcast scale val if not per channel
mov v6.16b,v4.16b
mov v3.16b,v4.16b
ScaleWithFloatPoint
scvtf v24.4s,v24.4s // convert to float
scvtf v25.4s,v25.4s
scvtf v26.4s,v26.4s
@ -405,27 +431,22 @@ Dequantization
scvtf v21.4s,v21.4s
scvtf v22.4s,v22.4s
scvtf v23.4s,v23.4s
b.ne SkipScaleBroadcast
mov v11.16b,v4.16b // broadcast scale val if not per channel
mov v6.16b,v4.16b
mov v9.16b,v4.16b
SkipScaleBroadcast
fmul v24.4s,v24.4s,v4.4s // multiply by scale
fmul v25.4s,v25.4s,v11.4s
fmul v25.4s,v25.4s,v5.4s
fmul v26.4s,v26.4s,v6.4s
fmul v27.4s,v27.4s,v9.4s
fmul v27.4s,v27.4s,v3.4s
fmul v28.4s,v28.4s,v4.4s
fmul v29.4s,v29.4s,v11.4s
fmul v29.4s,v29.4s,v5.4s
fmul v30.4s,v30.4s,v6.4s
fmul v31.4s,v31.4s,v9.4s
fmul v31.4s,v31.4s,v3.4s
fmul v16.4s,v16.4s,v4.4s
fmul v17.4s,v17.4s,v11.4s
fmul v17.4s,v17.4s,v5.4s
fmul v18.4s,v18.4s,v6.4s
fmul v19.4s,v19.4s,v9.4s
fmul v19.4s,v19.4s,v3.4s
fmul v20.4s,v20.4s,v4.4s
fmul v21.4s,v21.4s,v11.4s
fmul v21.4s,v21.4s,v5.4s
fmul v22.4s,v22.4s,v6.4s
fmul v23.4s,v23.4s,v9.4s
fmul v23.4s,v23.4s,v3.4s
fcvtns v24.4s,v24.4s // convert to int
fcvtns v25.4s,v25.4s
fcvtns v26.4s,v26.4s
@ -442,6 +463,90 @@ SkipScaleBroadcast
fcvtns v21.4s,v21.4s
fcvtns v22.4s,v22.4s
fcvtns v23.4s,v23.4s
b Quantize
FixedPointScale
ldr x10,[x8,#ConvSymPostProcessParams_PreShift]
ldr x11,[x8,#ConvSymPostProcessParams_Multiplier]
ldr x12,[x8,#ConvSymPostProcessParams_PostShift]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastFixedPointValue
ldp q0,q1,[x10],32 // load preshift vector
ldp q2,q3,[x10],32
ldp q4,q5,[x11],32 // load multiplier vector
ldp q6,q7,[x11],32
ldp q8,q9,[x12],32 // load postshift vector
ldp q10,q11,[x12],32
b ScaleWithFixedPoint
BroadcastFixedPointValue
ld1r {v0.4s},[x10] // load preshift Value
mov v1.16b, v0.16b
mov v2.16b, v0.16b
mov v3.16b, v0.16b
ld1r {v4.4s},[x11] // load multiplier Value
mov v5.16b, v4.16b
mov v6.16b, v4.16b
mov v7.16b, v4.16b
ld1r {v8.4s},[x12] // load postshift Value
mov v9.16b, v8.16b
mov v10.16b, v8.16b
mov v11.16b, v8.16b
ScaleWithFixedPoint
sqshl v24.4s, v24.4s, v0.4s // preshift
sqshl v25.4s, v25.4s, v1.4s
sqshl v26.4s, v26.4s, v2.4s
sqshl v27.4s, v27.4s, v3.4s
sqshl v28.4s, v28.4s, v0.4s
sqshl v29.4s, v29.4s, v1.4s
sqshl v30.4s, v30.4s, v2.4s
sqshl v31.4s, v31.4s, v3.4s
sqshl v16.4s, v16.4s, v0.4s
sqshl v17.4s, v17.4s, v1.4s
sqshl v18.4s, v18.4s, v2.4s
sqshl v19.4s, v19.4s, v3.4s
sqshl v20.4s, v20.4s, v0.4s
sqshl v21.4s, v21.4s, v1.4s
sqshl v22.4s, v22.4s, v2.4s
sqshl v23.4s, v23.4s, v3.4s
sqdmulh v24.4s, v24.4s, v4.4s // multiplier
sqdmulh v25.4s, v25.4s, v5.4s
sqdmulh v26.4s, v26.4s, v6.4s
sqdmulh v27.4s, v27.4s, v7.4s
sqdmulh v28.4s, v28.4s, v4.4s
sqdmulh v29.4s, v29.4s, v5.4s
sqdmulh v30.4s, v30.4s, v6.4s
sqdmulh v31.4s, v31.4s, v7.4s
sqdmulh v16.4s, v16.4s, v4.4s
sqdmulh v17.4s, v17.4s, v5.4s
sqdmulh v18.4s, v18.4s, v6.4s
sqdmulh v19.4s, v19.4s, v7.4s
sqdmulh v20.4s, v20.4s, v4.4s
sqdmulh v21.4s, v21.4s, v5.4s
sqdmulh v22.4s, v22.4s, v6.4s
sqdmulh v23.4s, v23.4s, v7.4s
srshl v24.4s, v24.4s, v8.4s // post shift
srshl v25.4s, v25.4s, v9.4s
srshl v26.4s, v26.4s, v10.4s
srshl v27.4s, v27.4s, v11.4s
srshl v28.4s, v28.4s, v8.4s
srshl v29.4s, v29.4s, v9.4s
srshl v30.4s, v30.4s, v10.4s
srshl v31.4s, v31.4s, v11.4s
srshl v16.4s, v16.4s, v8.4s
srshl v17.4s, v17.4s, v9.4s
srshl v18.4s, v18.4s, v10.4s
srshl v19.4s, v19.4s, v11.4s
srshl v20.4s, v20.4s, v8.4s
srshl v21.4s, v21.4s, v9.4s
srshl v22.4s, v22.4s, v10.4s
srshl v23.4s, v23.4s, v11.4s
Quantize
sqxtn v24.4h,v24.4s // shorten to int16
sqxtn v26.4h,v26.4s
sqxtn2 v24.8h,v25.4s
@ -486,10 +591,10 @@ SkipScaleBroadcast
str q20,[x2]
ExitKernel
EPILOG_RESTORE_REG_PAIR d14,d15,#48
EPILOG_RESTORE_REG_PAIR d12,d13,#32
EPILOG_RESTORE_REG_PAIR d10,d11,#16
EPILOG_RESTORE_REG_PAIR d8,d9,#64!
EPILOG_RESTORE_REG_PAIR d14,d15,#ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15
EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13
EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11
EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymDepthwiseKernelFrame_SavedRegisters!
EPILOG_RETURN
Process8Channels
@ -511,7 +616,7 @@ Process8Channels
mov v17.16b,v25.16b
ldr x12,[x14],#8 // x12 -> A2 iter 1
subs x3,x3,2 // decrement input blocks remaining
ldr d10,[x13,x5] // A1 iter 1
ldr d3,[x13,x5] // A1 iter 1
mov v20.16b,v24.16b
ldr x13,[x15],#8 // x13 -> A3 iter 1
mov v21.16b,v25.16b
@ -528,15 +633,15 @@ BlockLoopC8
b.eq EpilogueC8P2
ldr x10,[x0],#8 // x10 -> A0 iter 2
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
ldr d6,[x11,x5] // A3 iter 0
cmp x3,1
smlal v12.8h,v1.8b,v2.8b
ldr x11,[x9],#8 // x11 -> A1 iter 2
smlal v14.8h,v1.8b,v10.8b
smlal v14.8h,v1.8b,v3.8b
ldr d2,[x12,x5] // A2 iter 1
b.eq EpilogueC8P3 // 3 pixel remains
ldr d10,[x13,x5] // A3 iter 1
b.eq EpilogueC8P3 // 3 pixel remains
ldr d3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
ldr x12,[x0],#8 // x12 -> A0 iter 3
saddw2 v25.4s,v25.4s,v12.8h
@ -552,15 +657,15 @@ BlockLoopC8
ldr x10,[x14],#8 // x10 -> A2 iter 2
ldr d6,[x11,x5] // A1 iter 2
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
ld1 {v0.8b},[x1],x4 // filter iter 2
smlal v12.8h,v1.8b,v2.8b
ldr x11,[x15],#8 // x11 -> A3 iter 2
ldr d2,[x12,x5] // A0 iter 3
smlal v14.8h,v1.8b,v10.8b
smlal v14.8h,v1.8b,v3.8b
ldr x12,[x14],#8 // x12 -> A2 iter 3
saddw v16.4s,v16.4s,v12.4h
ldr d10,[x13,x5] // A1 iter 3
ldr d3,[x13,x5] // A1 iter 3
saddw2 v17.4s,v17.4s,v12.8h
ld1 {v1.8b},[x1],x4 // filter iter 3
saddw v20.4s,v20.4s,v14.4h
@ -575,11 +680,11 @@ EpilogueC8P2
//
ldr d6,[x11,x5] // A3 iter 0
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
smlal v12.8h,v1.8b,v2.8b
ldr d2,[x12,x5] // A2 iter 1
smlal v14.8h,v1.8b,v10.8b
ldr d10,[x13,x5] // A3 iter 1
smlal v14.8h,v1.8b,v3.8b
ldr d3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
saddw v28.4s,v28.4s,v14.4h
@ -588,23 +693,19 @@ EpilogueC8P2
eor v4.8b,v4.8b,v8.8b
eor v6.8b,v6.8b,v8.8b
smull v12.8h,v0.8b,v4.8b
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
ldr x12,[x8,#ConvSymPostProcessParams_Scale]
smull v14.8h,v0.8b,v6.8b
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint]
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
smlal v12.8h,v1.8b,v2.8b
smlal v14.8h,v1.8b,v10.8b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq SkipScaleVecLoad2C8
ldp q4,q11,[x12],#32 // load scale vector if per channel
SkipScaleVecLoad2C8
smlal v14.8h,v1.8b,v3.8b
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v20.4s,v20.4s,v14.4h
saddw2 v21.4s,v21.4s,v14.8h
b DequantC8
b RequantizationC8
ProcC8P1
//
@ -623,7 +724,7 @@ ProcC8P1
mov v20.16b,v24.16b
ldr d2,[x12,x5]
subs x3,x3,2 // decrement input blocks remaining
ldr d10,[x13,x5]
ldr d3,[x13,x5]
mov v21.16b,v25.16b
b EpilogueC8P1
@ -632,7 +733,7 @@ EpilogueC8P3
// Loop epilogue (process 2 of last 3 pixels)
//
ldr x12,[x14],#8 // x12 -> A2 iter 2
ldr d10,[x13,x5] // A3 iter 1
ldr d3,[x13,x5] // A3 iter 1
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
ldr x13,[x15],#8 // x13 -> A3 iter 2
@ -645,12 +746,12 @@ EpilogueC8P3
smull v14.8h,v0.8b,v6.8b
ld1 {v0.8b},[x1] // filter iter 2
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
ldr d6,[x11,x5] // A1 iter 2
smlal v12.8h,v1.8b,v2.8b
ldr d2,[x12,x5] // A2 iter 2
smlal v14.8h,v1.8b,v10.8b
ldr d10,[x13,x5] // A3 iter 2
smlal v14.8h,v1.8b,v3.8b
ldr d3,[x13,x5] // A3 iter 2
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v20.4s,v20.4s,v14.4h
@ -663,29 +764,39 @@ EpilogueC8P1
ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
eor v4.8b,v4.8b,v8.8b
eor v6.8b,v6.8b,v8.8b
ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
ldr x12,[x8,#ConvSymPostProcessParams_Scale]
smull v12.8h,v0.8b,v4.8b
ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint]
smull v14.8h,v0.8b,v6.8b
saddw v24.4s,v24.4s,v12.4h
saddw2 v25.4s,v25.4s,v12.8h
saddw v28.4s,v28.4s,v14.4h
saddw2 v29.4s,v29.4s,v14.8h
eor v2.8b,v2.8b,v8.8b
eor v10.8b,v10.8b,v8.8b
eor v3.8b,v3.8b,v8.8b
smull v12.8h,v0.8b,v2.8b
smull v14.8h,v0.8b,v10.8b
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ld1r {v4.4s},[x12] // load scale val
b.eq SkipScaleVecLoadC8
ldp q4,q11,[x12] // load scale vector if per channel
SkipScaleVecLoadC8
smull v14.8h,v0.8b,v3.8b
tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE
saddw v16.4s,v16.4s,v12.4h
saddw2 v17.4s,v17.4s,v12.8h
saddw v20.4s,v20.4s,v14.4h
saddw2 v21.4s,v21.4s,v14.8h
DequantC8
RequantizationC8
b.ne FixedPointScaleC8
FloatingPointScaleC8
ldr x12,[x8,#ConvSymPostProcessParams_Scale]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastScaleValueC8
ldp q4,q5,[x12] // load scale vector if per channel
b ScaleWithFloatPointC8
BroadcastScaleValueC8
ld1r {v4.4s},[x12] // load scale val
mov v5.16b,v4.16b // broadcast scale val if not per channel
ScaleWithFloatPointC8
scvtf v24.4s,v24.4s // convert to float
scvtf v25.4s,v25.4s
scvtf v28.4s,v28.4s
@ -694,17 +805,14 @@ DequantC8
scvtf v17.4s,v17.4s
scvtf v20.4s,v20.4s
scvtf v21.4s,v21.4s
b.ne SkipScaleBroadcastC8
mov v11.16b,v4.16b // broadcast scale val if not per channel
SkipScaleBroadcastC8
fmul v24.4s,v24.4s,v4.4s // multiply by scale
fmul v25.4s,v25.4s,v11.4s
fmul v25.4s,v25.4s,v5.4s
fmul v28.4s,v28.4s,v4.4s
fmul v29.4s,v29.4s,v11.4s
fmul v29.4s,v29.4s,v5.4s
fmul v16.4s,v16.4s,v4.4s
fmul v17.4s,v17.4s,v11.4s
fmul v17.4s,v17.4s,v5.4s
fmul v20.4s,v20.4s,v4.4s
fmul v21.4s,v21.4s,v11.4s
fmul v21.4s,v21.4s,v5.4s
fcvtns v24.4s,v24.4s // convert to int
fcvtns v25.4s,v25.4s
fcvtns v28.4s,v28.4s
@ -713,6 +821,57 @@ SkipScaleBroadcastC8
fcvtns v17.4s,v17.4s
fcvtns v20.4s,v20.4s
fcvtns v21.4s,v21.4s
b QuantizeC8
FixedPointScaleC8
ldr x10,[x8,#ConvSymPostProcessParams_PreShift]
ldr x11,[x8,#ConvSymPostProcessParams_Multiplier]
ldr x12,[x8,#ConvSymPostProcessParams_PostShift]
tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint]
beq BroadcastFixedPointValueC8
ldp q0,q1,[x10],32 // load preshift vector
ldp q4,q5,[x11],32 // load multiplier vector
ldp q8,q9,[x12],32 // load postshift vector
b ScaleWithFixedPointC8
BroadcastFixedPointValueC8
ld1r {v0.4s},[x10] // load preshift Value
mov v1.16b, v0.16b
ld1r {v4.4s},[x11] // load multiplier Value
mov v5.16b, v4.16b
ld1r {v8.4s},[x12] // load postshift Value
mov v9.16b, v8.16b
ScaleWithFixedPointC8
sqshl v24.4s, v24.4s, v0.4s // preshift
sqshl v25.4s, v25.4s, v1.4s
sqshl v28.4s, v28.4s, v0.4s
sqshl v29.4s, v29.4s, v1.4s
sqshl v16.4s, v16.4s, v0.4s
sqshl v17.4s, v17.4s, v1.4s
sqshl v20.4s, v20.4s, v0.4s
sqshl v21.4s, v21.4s, v1.4s
sqdmulh v24.4s, v24.4s, v4.4s // multiplier
sqdmulh v25.4s, v25.4s, v5.4s
sqdmulh v28.4s, v28.4s, v4.4s
sqdmulh v29.4s, v29.4s, v5.4s
sqdmulh v16.4s, v16.4s, v4.4s
sqdmulh v17.4s, v17.4s, v5.4s
sqdmulh v20.4s, v20.4s, v4.4s
sqdmulh v21.4s, v21.4s, v5.4s
srshl v24.4s, v24.4s, v8.4s // post shift
srshl v25.4s, v25.4s, v9.4s
srshl v28.4s, v28.4s, v8.4s
srshl v29.4s, v29.4s, v9.4s
srshl v16.4s, v16.4s, v8.4s
srshl v17.4s, v17.4s, v9.4s
srshl v20.4s, v20.4s, v8.4s
srshl v21.4s, v21.4s, v9.4s
QuantizeC8
dup v0.8h,w15
sqxtn v24.4h,v24.4s // shorten to int16
sqxtn2 v24.8h,v25.4s

55
onnxruntime/core/mlas/lib/convsym.cpp
View file

@ -478,7 +478,7 @@ MlasConvSym(
{
const MLAS_CONV_SYM_DISPATCH* ConvSymDispatch = GetConvSymDispatch(Params.InputIsSigned);
// Pick the suitable kernel for current core. Currently we only have specialized core for
// Pick the suitable kernel for current core. Currently we only have specialized core for
// s8s8 under ARM64
#if defined(MLAS_TARGET_ARM64)
const auto Kernel =
@ -490,8 +490,16 @@ MlasConvSym(
#endif
int32_t KernelFlags = 0;
const MLAS_REQUANT_PARAM* RequantParam = Params.RequantParam;
bool PerChannelScale = RequantParam->Size > 1;
int32_t OutputZeroPoint = RequantParam->ZeroPoint;
MLAS_ROUND_KIND RequantRoundKind = RequantParam->RequantRoundKind;
if (Params.PerChannelScale) {
if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfUp) {
KernelFlags |= MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE;
}
if (PerChannelScale) {
KernelFlags |= MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE;
}
@ -500,8 +508,7 @@ MlasConvSym(
}
MLAS_CONV_SYM_POST_PROCESS_PARAMS PostProcessParams = {};
MlasConvSymSetOutputZeroPoint(PostProcessParams, Params.OutputZeroPoint, Params.InputIsSigned);
MlasConvSymSetOutputZeroPoint(PostProcessParams, OutputZeroPoint, Params.InputIsSigned);
const size_t KernelChannelCount = (ConvSymDispatch->KernelChannelCount == 0)
? std::numeric_limits<size_t>::max()
@ -523,7 +530,14 @@ MlasConvSym(
void* conv_out = static_cast<int8_t*>(Params.Output) + (oc_outside * OutputChannels) + co;
PostProcessParams.Bias = Params.Bias + co;
PostProcessParams.Scale = Params.Scale + (Params.PerChannelScale ? co : 0);
if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfEven) {
PostProcessParams.Scale = RequantParam->Scale + (PerChannelScale ? co : 0);
} else {
PostProcessParams.Multiplier = RequantParam->Multiplier + (PerChannelScale ? co : 0);
PostProcessParams.PreShift = RequantParam->PreShift + (PerChannelScale ? co : 0);
PostProcessParams.PostShift = RequantParam->PostShift + (PerChannelScale ? co : 0);
}
for (size_t oc = 0; oc < oc_outside_block_size;) {
@ -567,23 +581,38 @@ MlasConvSymDepthwise(
unsigned KernelFlags = 0;
if (Params.PerChannelScale) {
const MLAS_REQUANT_PARAM* RequantParam = Params.RequantParam;
bool PerChannelScale = RequantParam->Size > 1;
int32_t OutputZeroPoint = RequantParam->ZeroPoint;
MLAS_ROUND_KIND RequantRoundKind = RequantParam->RequantRoundKind;
if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfUp) {
KernelFlags |= MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE;
}
if (PerChannelScale) {
KernelFlags |= MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE;
}
MLAS_CONV_SYM_POST_PROCESS_PARAMS PostProcessParams = {};
MlasConvSymSetOutputZeroPoint(PostProcessParams, Params.OutputZeroPoint, Params.InputIsSigned);
MlasConvSymSetOutputZeroPoint(PostProcessParams, OutputZeroPoint, Params.InputIsSigned);
if ((Params.OutputChannels & 15) == 0) {
PostProcessParams.Bias = Params.Bias;
PostProcessParams.Scale = Params.Scale;
if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfEven) {
PostProcessParams.Scale = RequantParam->Scale;
} else {
PostProcessParams.Multiplier = RequantParam->Multiplier;
PostProcessParams.PreShift = RequantParam->PreShift;
PostProcessParams.PostShift = RequantParam->PostShift;
}
if (ConvSymDispatch->Depthwise3x3Proc && Params.KernelSize == 9) {
ConvSymDispatch->Depthwise3x3Proc(Params.InputIndirection, (int8_t const*)Params.Filter,
Params.OutputChannels, Params.Output,
Params.OutputCount, &PostProcessParams, KernelFlags);
return;
}
if (ConvSymDispatch->Depthwise5x5Proc && Params.KernelSize == 25) {
ConvSymDispatch->Depthwise5x5Proc(Params.InputIndirection, (int8_t const*)Params.Filter,
Params.OutputChannels, Params.Output,
@ -610,7 +639,13 @@ MlasConvSymDepthwise(
const size_t ChannelCount = std::min(OutputChannels - ChannelOffset, KernelChannelCount);
PostProcessParams.Bias = Params.Bias + ChannelOffset;
PostProcessParams.Scale = Params.Scale + (Params.PerChannelScale ? ChannelOffset : 0);
if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfEven) {
PostProcessParams.Scale = RequantParam->Scale + (PerChannelScale ? ChannelOffset : 0);
} else {
PostProcessParams.Multiplier = RequantParam->Multiplier + (PerChannelScale ? ChannelOffset : 0);
PostProcessParams.PreShift = RequantParam->PreShift + (PerChannelScale ? ChannelOffset : 0);
PostProcessParams.PostShift = RequantParam->PostShift + (PerChannelScale ? ChannelOffset : 0);
}
ConvSymDispatch->DepthwiseKernel(
InputIndirection,

11
onnxruntime/core/mlas/lib/mlasi.h
View file

@ -804,17 +804,22 @@ MlasConvDepthwiseKernelAvx2(
#define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 0x00000001
#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 0x00000002
#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 0x00000004 // Only available on ARM64
//
// Define the post-processing parameters for conv sym: bias and re-quant params
//
struct MLAS_CONV_SYM_POST_PROCESS_PARAMS {
const int32_t* Bias;
const float* Scale;
const int32_t* Bias{nullptr};
const float* Scale{nullptr};
float MinimumValue;
float MaximumValue;
int32_t OutputZeroPoint;
int32_t Padding;
const int32_t* Multiplier{nullptr};
const int32_t* PreShift{nullptr};
const int32_t* PostShift{nullptr};
};
//
@ -988,7 +993,7 @@ MlasPartitionWork(
#if defined(_MSC_VER) && !defined(__clang__)
#pragma warning(push)
// VC++ suggests we can attempt to make 'MlasBitsOfFp32' constexpr, but it is not valid.
#pragma warning(disable:26497)
#pragma warning(disable:26497)
#endif
MLAS_FORCEINLINE

267
onnxruntime/core/mlas/lib/qdwconv_kernelsize.cpp
View file

@ -43,13 +43,12 @@ Abstract:
#include "mlasi.h"
extern "C" {
#if defined(MLAS_TARGET_ARM64)
template<bool IsFixedPoint>
void
MLASCALL
MlasConvSymDepthwiseKernelSize25ArmU8S8(
MlasConvSymDepthwiseKernelSize25ArmU8S8Impl(
void const* const* InputIndirection,
int8_t const* Filter,
size_t Channels,
@ -64,10 +63,44 @@ MlasConvSymDepthwiseKernelSize25ArmU8S8(
const uint8x16_t vu128 = vdupq_n_u8(128);
const int16x8_t voutput_zero_point = vld1q_dup_s16((int16_t const*)&PostProcessParams->OutputZeroPoint);
float32x4_t vscale_0123, vscale_4567, vscale_89AB, vscale_CDEF;
int32x4_t vpreshift_0123, vpreshift_4567, vpreshift_89AB, vpreshift_CDEF;
int32x4_t vmultiplier_0123, vmultiplier_4567, vmultiplier_89AB, vmultiplier_CDEF;
int32x4_t vpostshift_0123, vpostshift_4567, vpostshift_89AB, vpostshift_CDEF;
const bool is_per_channel = ((KernelFlags & MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE) != 0);
// Init them anyway due to some compiler will generate uninitialized warnings.
vscale_0123 = vscale_4567 = vscale_89AB = vscale_CDEF = vld1q_dup_f32(PostProcessParams->Scale);
if constexpr (IsFixedPoint) {
vpreshift_0123 = vpreshift_4567 = vpreshift_89AB = vpreshift_CDEF = vld1q_dup_s32(PostProcessParams->PreShift);
vmultiplier_0123 = vmultiplier_4567 = vmultiplier_89AB = vmultiplier_CDEF = vld1q_dup_s32(PostProcessParams->Multiplier);
vpostshift_0123 = vpostshift_4567 = vpostshift_89AB = vpostshift_CDEF = vld1q_dup_s32(PostProcessParams->PostShift);
MLAS_UNREFERENCED_PARAMETER(vscale_0123);
MLAS_UNREFERENCED_PARAMETER(vscale_4567);
MLAS_UNREFERENCED_PARAMETER(vscale_89AB);
MLAS_UNREFERENCED_PARAMETER(vscale_CDEF);
} else {
vscale_0123 = vscale_4567 = vscale_89AB = vscale_CDEF = vld1q_dup_f32(PostProcessParams->Scale);
MLAS_UNREFERENCED_PARAMETER(vpreshift_0123);
MLAS_UNREFERENCED_PARAMETER(vpreshift_4567);
MLAS_UNREFERENCED_PARAMETER(vpreshift_89AB);
MLAS_UNREFERENCED_PARAMETER(vpreshift_CDEF);
MLAS_UNREFERENCED_PARAMETER(vmultiplier_0123);
MLAS_UNREFERENCED_PARAMETER(vmultiplier_4567);
MLAS_UNREFERENCED_PARAMETER(vmultiplier_89AB);
MLAS_UNREFERENCED_PARAMETER(vmultiplier_CDEF);
MLAS_UNREFERENCED_PARAMETER(vpostshift_0123);
MLAS_UNREFERENCED_PARAMETER(vpostshift_4567);
MLAS_UNREFERENCED_PARAMETER(vpostshift_89AB);
MLAS_UNREFERENCED_PARAMETER(vpostshift_CDEF);
}
while (OutputCount-- > 0) {
const uint8_t* i00 = IndirectBuf[0];
const uint8_t* i01 = IndirectBuf[1];
const uint8_t* i02 = IndirectBuf[2];
@ -99,7 +132,11 @@ MlasConvSymDepthwiseKernelSize25ArmU8S8(
IndirectBuf += 25;
int32_t const* bias = PostProcessParams->Bias;
float const* scale = PostProcessParams->Scale;
int32_t const* preshift = PostProcessParams->PreShift;
int32_t const* multiplier = PostProcessParams->Multiplier;
int32_t const* postshift = PostProcessParams->PostShift;
for (size_t c = 0; c < Channels; c += 16) {
int8_t const* w = Filter + c;
int32x4_t vacc_0123 = vld1q_s32(bias); bias += 4;
int32x4_t vacc_4567 = vld1q_s32(bias); bias += 4;
@ -309,18 +346,50 @@ MlasConvSymDepthwiseKernelSize25ArmU8S8(
vacc_89AB = vaddw_s16(vacc_89AB, vget_low_s16(vprod_89ABCDEF));
vacc_CDEF = vaddw_s16(vacc_CDEF, vget_high_s16(vprod_89ABCDEF));
if (is_per_channel) {
vscale_0123 = vld1q_f32(scale); scale += 4;
vscale_4567 = vld1q_f32(scale); scale += 4;
vscale_89AB = vld1q_f32(scale); scale += 4;
vscale_CDEF = vld1q_f32(scale); scale += 4;
}
if constexpr (IsFixedPoint) {
// requantize
vacc_0123 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_0123), vscale_0123));
vacc_4567 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_4567), vscale_4567));
vacc_89AB = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_89AB), vscale_89AB));
vacc_CDEF = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_CDEF), vscale_CDEF));
if (is_per_channel) {
vpreshift_0123 = vld1q_s32(preshift); preshift += 4;
vpreshift_4567 = vld1q_s32(preshift); preshift += 4;
vpreshift_89AB = vld1q_s32(preshift); preshift += 4;
vpreshift_CDEF = vld1q_s32(preshift); preshift += 4;
vmultiplier_0123 = vld1q_s32(multiplier); multiplier += 4;
vmultiplier_4567 = vld1q_s32(multiplier); multiplier += 4;
vmultiplier_89AB = vld1q_s32(multiplier); multiplier += 4;
vmultiplier_CDEF = vld1q_s32(multiplier); multiplier += 4;
vpostshift_0123 = vld1q_s32(postshift); postshift += 4;
vpostshift_4567 = vld1q_s32(postshift); postshift += 4;
vpostshift_89AB = vld1q_s32(postshift); postshift += 4;
vpostshift_CDEF = vld1q_s32(postshift); postshift += 4;
}
vacc_0123 = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_0123, vpreshift_0123), vmultiplier_0123), vpostshift_0123);
vacc_4567 = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_4567, vpreshift_4567), vmultiplier_4567), vpostshift_4567);
vacc_89AB = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_89AB, vpreshift_89AB), vmultiplier_89AB), vpostshift_89AB);
vacc_CDEF = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_CDEF, vpreshift_CDEF), vmultiplier_CDEF), vpostshift_CDEF);
} else {
if (is_per_channel) {
vscale_0123 = vld1q_f32(scale); scale += 4;
vscale_4567 = vld1q_f32(scale); scale += 4;
vscale_89AB = vld1q_f32(scale); scale += 4;
vscale_CDEF = vld1q_f32(scale); scale += 4;
}
// requantize
vacc_0123 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_0123), vscale_0123));
vacc_4567 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_4567), vscale_4567));
vacc_89AB = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_89AB), vscale_89AB));
vacc_CDEF = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_CDEF), vscale_CDEF));
}
const int16x8_t vacc_01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc_0123), vacc_4567), voutput_zero_point);
const int16x8_t vacc_89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc_89AB), vacc_CDEF), voutput_zero_point);
@ -332,9 +401,11 @@ MlasConvSymDepthwiseKernelSize25ArmU8S8(
}
}
template<bool IsFixedPoint>
void
MLASCALL
MlasConvSymDepthwiseKernelSize25ArmS8S8(
MlasConvSymDepthwiseKernelSize25ArmS8S8Impl(
void const* const* InputIndirection,
int8_t const* Filter,
size_t Channels,
@ -349,10 +420,43 @@ MlasConvSymDepthwiseKernelSize25ArmS8S8(
const int16x8_t voutput_zero_point =
vld1q_dup_s16((int16_t const*)&PostProcessParams->OutputZeroPoint);
float32x4_t vscale_0123, vscale_4567, vscale_89AB, vscale_CDEF;
int32x4_t vpreshift_0123, vpreshift_4567, vpreshift_89AB, vpreshift_CDEF;
int32x4_t vmultiplier_0123, vmultiplier_4567, vmultiplier_89AB, vmultiplier_CDEF;
int32x4_t vpostshift_0123, vpostshift_4567, vpostshift_89AB, vpostshift_CDEF;
const bool is_per_channel = ((KernelFlags & MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE) != 0);
// Init them anyway due to some compiler will generate uninitialized warnings.
vscale_0123 = vscale_4567 = vscale_89AB = vscale_CDEF = vld1q_dup_f32(PostProcessParams->Scale);
if constexpr (IsFixedPoint) {
vpreshift_0123 = vpreshift_4567 = vpreshift_89AB = vpreshift_CDEF = vld1q_dup_s32(PostProcessParams->PreShift);
vmultiplier_0123 = vmultiplier_4567 = vmultiplier_89AB = vmultiplier_CDEF = vld1q_dup_s32(PostProcessParams->Multiplier);
vpostshift_0123 = vpostshift_4567 = vpostshift_89AB = vpostshift_CDEF = vld1q_dup_s32(PostProcessParams->PostShift);
MLAS_UNREFERENCED_PARAMETER(vscale_0123);
MLAS_UNREFERENCED_PARAMETER(vscale_4567);
MLAS_UNREFERENCED_PARAMETER(vscale_89AB);
MLAS_UNREFERENCED_PARAMETER(vscale_CDEF);
} else {
vscale_0123 = vscale_4567 = vscale_89AB = vscale_CDEF = vld1q_dup_f32(PostProcessParams->Scale);
MLAS_UNREFERENCED_PARAMETER(vpreshift_0123);
MLAS_UNREFERENCED_PARAMETER(vpreshift_4567);
MLAS_UNREFERENCED_PARAMETER(vpreshift_89AB);
MLAS_UNREFERENCED_PARAMETER(vpreshift_CDEF);
MLAS_UNREFERENCED_PARAMETER(vmultiplier_0123);
MLAS_UNREFERENCED_PARAMETER(vmultiplier_4567);
MLAS_UNREFERENCED_PARAMETER(vmultiplier_89AB);
MLAS_UNREFERENCED_PARAMETER(vmultiplier_CDEF);
MLAS_UNREFERENCED_PARAMETER(vpostshift_0123);
MLAS_UNREFERENCED_PARAMETER(vpostshift_4567);
MLAS_UNREFERENCED_PARAMETER(vpostshift_89AB);
MLAS_UNREFERENCED_PARAMETER(vpostshift_CDEF);
}
while (OutputCount-- > 0) {
const int8_t* i00 = IndirectBuf[0];
const int8_t* i01 = IndirectBuf[1];
const int8_t* i02 = IndirectBuf[2];
@ -383,8 +487,12 @@ MlasConvSymDepthwiseKernelSize25ArmS8S8(
IndirectBuf += 25;
int32_t const* bias = PostProcessParams->Bias;
float const* scale = PostProcessParams->Scale;
const float* scale = PostProcessParams->Scale;
const int32_t* preshift = PostProcessParams->PreShift;
const int32_t* multiplier = PostProcessParams->Multiplier;
const int32_t* postshift = PostProcessParams->PostShift;
for (size_t c = 0; c < Channels; c += 16) {
int8_t const* w = Filter + c;
int32x4_t vacc_0123 = vld1q_s32(bias); bias += 4;
int32x4_t vacc_4567 = vld1q_s32(bias); bias += 4;
@ -594,18 +702,50 @@ MlasConvSymDepthwiseKernelSize25ArmS8S8(
vacc_89AB = vaddw_s16(vacc_89AB, vget_low_s16(vprod_89ABCDEF));
vacc_CDEF = vaddw_s16(vacc_CDEF, vget_high_s16(vprod_89ABCDEF));
if (is_per_channel) {
vscale_0123 = vld1q_f32(scale); scale += 4;
vscale_4567 = vld1q_f32(scale); scale += 4;
vscale_89AB = vld1q_f32(scale); scale += 4;
vscale_CDEF = vld1q_f32(scale); scale += 4;
}
if constexpr (IsFixedPoint) {
// requantize
vacc_0123 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_0123), vscale_0123));
vacc_4567 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_4567), vscale_4567));
vacc_89AB = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_89AB), vscale_89AB));
vacc_CDEF = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_CDEF), vscale_CDEF));
if (is_per_channel) {
vpreshift_0123 = vld1q_s32(preshift); preshift += 4;
vpreshift_4567 = vld1q_s32(preshift); preshift += 4;
vpreshift_89AB = vld1q_s32(preshift); preshift += 4;
vpreshift_CDEF = vld1q_s32(preshift); preshift += 4;
vmultiplier_0123 = vld1q_s32(multiplier); multiplier += 4;
vmultiplier_4567 = vld1q_s32(multiplier); multiplier += 4;
vmultiplier_89AB = vld1q_s32(multiplier); multiplier += 4;
vmultiplier_CDEF = vld1q_s32(multiplier); multiplier += 4;
vpostshift_0123 = vld1q_s32(postshift); postshift += 4;
vpostshift_4567 = vld1q_s32(postshift); postshift += 4;
vpostshift_89AB = vld1q_s32(postshift); postshift += 4;
vpostshift_CDEF = vld1q_s32(postshift); postshift += 4;
}
vacc_0123 = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_0123, vpreshift_0123), vmultiplier_0123), vpostshift_0123);
vacc_4567 = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_4567, vpreshift_4567), vmultiplier_4567), vpostshift_4567);
vacc_89AB = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_89AB, vpreshift_89AB), vmultiplier_89AB), vpostshift_89AB);
vacc_CDEF = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_CDEF, vpreshift_CDEF), vmultiplier_CDEF), vpostshift_CDEF);
} else {
if (is_per_channel) {
vscale_0123 = vld1q_f32(scale); scale += 4;
vscale_4567 = vld1q_f32(scale); scale += 4;
vscale_89AB = vld1q_f32(scale); scale += 4;
vscale_CDEF = vld1q_f32(scale); scale += 4;
}
// requantize
vacc_0123 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_0123), vscale_0123));
vacc_4567 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_4567), vscale_4567));
vacc_89AB = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_89AB), vscale_89AB));
vacc_CDEF = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_CDEF), vscale_CDEF));
}
const int16x8_t vacc_01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc_0123), vacc_4567), voutput_zero_point);
const int16x8_t vacc_89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc_89AB), vacc_CDEF), voutput_zero_point);
@ -617,5 +757,72 @@ MlasConvSymDepthwiseKernelSize25ArmS8S8(
}
}
extern "C" {
void
MLASCALL
MlasConvSymDepthwiseKernelSize25ArmU8S8(
void const* const* InputIndirection,
int8_t const* Filter,
size_t Channels,
void* Output,
size_t OutputCount,
MLAS_CONV_SYM_POST_PROCESS_PARAMS const* PostProcessParams,
unsigned KernelFlags
)
{
if((KernelFlags & MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE) != 0) {
MlasConvSymDepthwiseKernelSize25ArmU8S8Impl<true>(
InputIndirection,
Filter,
Channels,
Output,
OutputCount,
PostProcessParams,
KernelFlags);
} else {
MlasConvSymDepthwiseKernelSize25ArmU8S8Impl<false>(
InputIndirection,
Filter,
Channels,
Output,
OutputCount,
PostProcessParams,
KernelFlags);
}
}
void
MLASCALL
MlasConvSymDepthwiseKernelSize25ArmS8S8(
void const* const* InputIndirection,
int8_t const* Filter,
size_t Channels,
void* Output,
size_t OutputCount,
MLAS_CONV_SYM_POST_PROCESS_PARAMS const* PostProcessParams,
unsigned KernelFlags
) {
if((KernelFlags & MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE) != 0) {
MlasConvSymDepthwiseKernelSize25ArmS8S8Impl<true>(
InputIndirection,
Filter,
Channels,
Output,
OutputCount,
PostProcessParams,
KernelFlags);
} else {
MlasConvSymDepthwiseKernelSize25ArmS8S8Impl<false>(
InputIndirection,
Filter,
Channels,
Output,
OutputCount,
PostProcessParams,
KernelFlags);
}
}
}
#endif
}

18
onnxruntime/core/mlas/lib/qlgavgpool.cpp
View file

@ -153,8 +153,8 @@ MlasQLinearGlobalAveragePoolNchw(
*sum_buffer++ = vget_lane_s32(vpadd_s32(vacc, vacc), 0);
}
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false,
static_cast<T8Bits>(ZeroPointOutput), 0, 0, 1, Channels);
MLAS_REQUANT_PARAM RequantParam(&scale, 1, ZeroPointOutput);
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &RequantParam, 0, 0, 1, Channels);
}
template <typename T8Bits>
@ -321,8 +321,9 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
vst1q_s32(acc + 4, vacc_hi);
}
}
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &Scale, false,
Output_zero_point, 0, 0, 1, Channels);
MLAS_REQUANT_PARAM RequantParam(&Scale, 1, Output_zero_point);
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &RequantParam, 0, 0, 1, Channels);
}
#elif defined(MLAS_SSE2_INTRINSICS)
@ -427,8 +428,8 @@ MlasQLinearGlobalAveragePoolNchw(
*sum_buffer++ = _mm_cvtsi128_si32(vsums);
}
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false,
static_cast<T8Bits>(ZeroPointOutput), 0, 0, 1, Channels);
MLAS_REQUANT_PARAM RequantParam(&scale, 1, ZeroPointOutput);
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &RequantParam, 0, 0, 1, Channels);
}
template <typename T8Bits>
@ -697,8 +698,9 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
_mm_storeu_si128(((__m128i*)acc) + 1, vacc_hi);
}
}
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &Scale, false,
Output_zero_point, 0, 0, 1, Channels);
MLAS_REQUANT_PARAM RequantParam(&Scale, 1, Output_zero_point);
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &RequantParam, 0, 0, 1, Channels);
}
#else

357
onnxruntime/core/mlas/lib/quantize.cpp
View file

@ -396,15 +396,27 @@ MlasRequantizeOutput(
OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
OutputType ZeroPoint,
const MLAS_REQUANT_PARAM* RequantParam,
size_t StartM,
size_t StartN,
size_t CountM,
size_t CountN
)
{
MLAS_ROUND_KIND RequantRoundKind = RequantParam->RequantRoundKind;
if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfUp) {
#ifdef MLAS_NO_EXCEPTION
abort();
#else
throw std::invalid_argument("Requantization rounding to nearest and tie to up is only supported on ARM64.");
#endif
}
const float* Scale = RequantParam->Scale;
bool PerColumnScale = RequantParam->Size > 1;
OutputType ZeroPoint = static_cast<OutputType>(RequantParam->ZeroPoint);
const __m128 PerMatrixScaleVector = PerColumnScale ? _mm_setzero_ps() : _mm_load1_ps(Scale);
const __m128 MinimumValueVector = _mm_set1_ps(float(std::numeric_limits<OutputType>::lowest() - ZeroPoint));
const __m128 MaximumValueVector = _mm_set1_ps(float(std::numeric_limits<OutputType>::max() - ZeroPoint));
@ -631,21 +643,301 @@ MlasRequantizeOutput(
template<typename OutputType>
void
MLASCALL
MlasRequantizeOutput(
MlasRequantizeOutputRoundNearestUp(
const int32_t* Input,
size_t InputLeadingDimension,
OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
OutputType ZeroPoint,
const MLAS_REQUANT_PARAM* RequantParam,
size_t StartM,
size_t StartN,
size_t CountM,
size_t CountN
)
{
bool PerColumnScale = RequantParam->Size > 1;
const int32_t* Multiplier = RequantParam->Multiplier;
const int32_t* PreShift = RequantParam->PreShift;
const int32_t* PostShift = RequantParam->PostShift;
OutputType ZeroPoint = static_cast<OutputType>(RequantParam->ZeroPoint);
const int32x4_t PerTensorMultiplierVector = PerColumnScale ? vdupq_n_s32(0) : vld1q_dup_s32(Multiplier);
const int32x4_t PerTensorPreShiftVector = PerColumnScale ? vdupq_n_s32(0) : vld1q_dup_s32(PreShift);
const int32x4_t PerTensorPostShiftVector = PerColumnScale ? vdupq_n_s32(0) : vld1q_dup_s32(PostShift);
const int16x8_t ZeroPointVector = vdupq_n_s16(ZeroPoint);
if (nullptr != Bias) {
Bias += StartN;
}
if (PerColumnScale) {
Multiplier += StartN;
PreShift += StartN;
PostShift += StartN;
}
Input += StartM * InputLeadingDimension + StartN;
Output += StartM * OutputLeadingDimension + StartN;
//
// Step through each row of the output matrix.
//
while (CountM-- > 0) {
const int32_t* bias = Bias;
const int32_t* multiplier = PerColumnScale ? Multiplier : nullptr;
const int32_t* preShift = PerColumnScale ? PreShift : nullptr;
const int32_t* postShift = PerColumnScale ? PostShift : nullptr;
size_t n = CountN;
auto* RowInput = Input;
auto* RowOutput = Output;
//
// Process 16 columns of the matrices at a time.
//
while (n >= 16) {
//
// Load the input data and optionally add the per-column bias.
//
int32x4x4_t IntegerVector;
IntegerVector.val[0] = vld1q_s32(&RowInput[0]);
IntegerVector.val[1] = vld1q_s32(&RowInput[4]);
IntegerVector.val[2] = vld1q_s32(&RowInput[8]);
IntegerVector.val[3] = vld1q_s32(&RowInput[12]);
RowInput += 16;
if (bias != nullptr) {
IntegerVector.val[0] = vaddq_s32(IntegerVector.val[0], vld1q_s32(&bias[0]));
IntegerVector.val[1] = vaddq_s32(IntegerVector.val[1], vld1q_s32(&bias[4]));
IntegerVector.val[2] = vaddq_s32(IntegerVector.val[2], vld1q_s32(&bias[8]));
IntegerVector.val[3] = vaddq_s32(IntegerVector.val[3], vld1q_s32(&bias[12]));
bias += 16;
}
if (PerColumnScale) {
int32x4x4_t PerColumnPreShiftVector;
PerColumnPreShiftVector.val[0] = vld1q_s32(&preShift[0]);
PerColumnPreShiftVector.val[1] = vld1q_s32(&preShift[4]);
PerColumnPreShiftVector.val[2] = vld1q_s32(&preShift[8]);
PerColumnPreShiftVector.val[3] = vld1q_s32(&preShift[12]);
preShift += 16;
IntegerVector.val[0] = vqshlq_s32(IntegerVector.val[0], PerColumnPreShiftVector.val[0]);
IntegerVector.val[1] = vqshlq_s32(IntegerVector.val[1], PerColumnPreShiftVector.val[1]);
IntegerVector.val[2] = vqshlq_s32(IntegerVector.val[2], PerColumnPreShiftVector.val[2]);
IntegerVector.val[3] = vqshlq_s32(IntegerVector.val[3], PerColumnPreShiftVector.val[3]);
int32x4x4_t PerColumnMultiplierVector;
PerColumnMultiplierVector.val[0] = vld1q_s32(&multiplier[0]);
PerColumnMultiplierVector.val[1] = vld1q_s32(&multiplier[4]);
PerColumnMultiplierVector.val[2] = vld1q_s32(&multiplier[8]);
PerColumnMultiplierVector.val[3] = vld1q_s32(&multiplier[12]);
multiplier += 16;
IntegerVector.val[0] = vqdmulhq_s32(IntegerVector.val[0], PerColumnMultiplierVector.val[0]);
IntegerVector.val[1] = vqdmulhq_s32(IntegerVector.val[1], PerColumnMultiplierVector.val[1]);
IntegerVector.val[2] = vqdmulhq_s32(IntegerVector.val[2], PerColumnMultiplierVector.val[2]);
IntegerVector.val[3] = vqdmulhq_s32(IntegerVector.val[3], PerColumnMultiplierVector.val[3]);
int32x4x4_t PerColumnPostShiftVector;
PerColumnPostShiftVector.val[0] = vld1q_s32(&postShift[0]);
PerColumnPostShiftVector.val[1] = vld1q_s32(&postShift[4]);
PerColumnPostShiftVector.val[2] = vld1q_s32(&postShift[8]);
PerColumnPostShiftVector.val[3] = vld1q_s32(&postShift[12]);
postShift += 16;
IntegerVector.val[0] = vrshlq_s32(IntegerVector.val[0], PerColumnPostShiftVector.val[0]);
IntegerVector.val[1] = vrshlq_s32(IntegerVector.val[1], PerColumnPostShiftVector.val[1]);
IntegerVector.val[2] = vrshlq_s32(IntegerVector.val[2], PerColumnPostShiftVector.val[2]);
IntegerVector.val[3] = vrshlq_s32(IntegerVector.val[3], PerColumnPostShiftVector.val[3]);
} else {
IntegerVector.val[0] = vqshlq_s32(IntegerVector.val[0], PerTensorPreShiftVector);
IntegerVector.val[1] = vqshlq_s32(IntegerVector.val[1], PerTensorPreShiftVector);
IntegerVector.val[2] = vqshlq_s32(IntegerVector.val[2], PerTensorPreShiftVector);
IntegerVector.val[3] = vqshlq_s32(IntegerVector.val[3], PerTensorPreShiftVector);
IntegerVector.val[0] = vqdmulhq_s32(IntegerVector.val[0], PerTensorMultiplierVector);
IntegerVector.val[1] = vqdmulhq_s32(IntegerVector.val[1], PerTensorMultiplierVector);
IntegerVector.val[2] = vqdmulhq_s32(IntegerVector.val[2], PerTensorMultiplierVector);
IntegerVector.val[3] = vqdmulhq_s32(IntegerVector.val[3], PerTensorMultiplierVector);
IntegerVector.val[0] = vrshlq_s32(IntegerVector.val[0], PerTensorPostShiftVector);
IntegerVector.val[1] = vrshlq_s32(IntegerVector.val[1], PerTensorPostShiftVector);
IntegerVector.val[2] = vrshlq_s32(IntegerVector.val[2], PerTensorPostShiftVector);
IntegerVector.val[3] = vrshlq_s32(IntegerVector.val[3], PerTensorPostShiftVector);
}
//
// Pack the integers with saturation to 16-bit values and shift by
// the zero point, then pack the integers again to bytes.
//
int16x8x2_t WordVector;
WordVector.val[0] = vqmovn_high_s32(vqmovn_s32(IntegerVector.val[0]), IntegerVector.val[1]);
WordVector.val[1] = vqmovn_high_s32(vqmovn_s32(IntegerVector.val[2]), IntegerVector.val[3]);
WordVector.val[0] = vqaddq_s16(WordVector.val[0], ZeroPointVector);
WordVector.val[1] = vqaddq_s16(WordVector.val[1], ZeroPointVector);
if (std::is_signed<OutputType>::value) {
vst1q_s8(reinterpret_cast<int8_t*>(RowOutput),
vqmovn_high_s16(vqmovn_s16(WordVector.val[0]), WordVector.val[1]));
} else {
vst1q_u8(reinterpret_cast<uint8_t*>(RowOutput),
vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1]));
}
RowOutput += 16;
n -= 16;
}
//
// Process the remaining columns of the matrices.
//
while (n > 0) {
//
// Load the input data and optionally add the per-column bias.
//
int32x4_t IntegerVector;
if (n >= 4) {
IntegerVector = vld1q_s32(&RowInput[0]);
RowInput += 4;
if (bias != nullptr) {
IntegerVector = vaddq_s32(IntegerVector, vld1q_s32(&bias[0]));
bias += 4;
}
} else {
IntegerVector = vld1q_dup_s32(RowInput);
RowInput += 1;
if (bias != nullptr) {
IntegerVector = vaddq_s32(IntegerVector, vld1q_dup_s32(bias));
bias += 1;
}
}
//
// Convert to integer values to float and apply the per-tensor or
// per-column scaling.
//
int32x4_t MultiplierVector;
int32x4_t PreShiftVector;
int32x4_t PostShiftVector;
if (PerColumnScale) {
if (n >= 4) {
MultiplierVector = vld1q_s32(multiplier);
PreShiftVector = vld1q_s32(preShift);
PostShiftVector = vld1q_s32(postShift);
multiplier += 4;
preShift += 4;
postShift += 4;
} else {
MultiplierVector = vld1q_dup_s32(multiplier);
PreShiftVector = vld1q_dup_s32(preShift);
PostShiftVector = vld1q_dup_s32(postShift);
multiplier += 1;
preShift += 1;
postShift += 1;
}
IntegerVector = vqshlq_s32(IntegerVector, PreShiftVector);
IntegerVector = vqdmulhq_s32(IntegerVector, MultiplierVector);
IntegerVector = vrshlq_s32(IntegerVector, PostShiftVector);
} else {
IntegerVector = vqshlq_s32(IntegerVector, PerTensorPreShiftVector);
IntegerVector = vqdmulhq_s32(IntegerVector, PerTensorMultiplierVector);
IntegerVector = vrshlq_s32(IntegerVector, PerTensorPostShiftVector);
}
//
// Pack the integers with saturation to 16-bit values and shift by
// the zero point, then pack the integers again to unsigned bytes.
//
int16x8_t WordVector = vcombine_s16(vqmovn_s32(IntegerVector), vdup_n_s16(0));
WordVector = vqaddq_s16(WordVector, ZeroPointVector);
uint8x16_t ByteVector;
if (std::is_signed<OutputType>::value) {
ByteVector = vcombine_u8(vreinterpret_u8_s8(vqmovn_s16(WordVector)), vdup_n_u8(0));
} else {
ByteVector = vcombine_u8(vqmovun_s16(WordVector), vdup_n_u8(0));
}
if (n >= 4) {
vst1q_lane_u32(reinterpret_cast<uint32_t*>(RowOutput),
vreinterpretq_u32_u8(ByteVector), 0);
RowOutput += 4;
n -= 4;
} else {
vst1q_lane_u8(reinterpret_cast<uint8_t*>(RowOutput), ByteVector, 0);
RowOutput += 1;
n -= 1;
}
}
// Next Row
Input += InputLeadingDimension;
Output += OutputLeadingDimension;
}
}
template<typename OutputType>
void
MLASCALL
MlasRequantizeOutputRoundNearestEven(
const int32_t* Input,
size_t InputLeadingDimension,
OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const MLAS_REQUANT_PARAM* RequantParam,
size_t StartM,
size_t StartN,
size_t CountM,
size_t CountN
)
{
const float* Scale = RequantParam->Scale;
bool PerColumnScale = RequantParam->Size > 1;
OutputType ZeroPoint = static_cast<OutputType>(RequantParam->ZeroPoint);
const float32x4_t PerMatrixScaleVector = PerColumnScale ? vdupq_n_f32(0) : vld1q_dup_f32(Scale);
const int16x8_t ZeroPointVector = vdupq_n_s16(ZeroPoint);
@ -871,6 +1163,32 @@ MlasRequantizeOutput(
}
}
template<typename OutputType>
void
MLASCALL
MlasRequantizeOutput(
const int32_t* Input,
size_t InputLeadingDimension,
OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const MLAS_REQUANT_PARAM* RequantParam,
size_t StartM,
size_t StartN,
size_t CountM,
size_t CountN
)
{
if(RequantParam->RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfEven)
{
return MlasRequantizeOutputRoundNearestEven(Input, InputLeadingDimension, Output, OutputLeadingDimension, Bias,
RequantParam, StartM, StartN, CountM, CountN);
} else {
return MlasRequantizeOutputRoundNearestUp(Input, InputLeadingDimension, Output, OutputLeadingDimension, Bias,
RequantParam, StartM, StartN, CountM, CountN);
}
}
#else
template <typename OutputType>
@ -882,15 +1200,26 @@ MlasRequantizeOutput(
OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
OutputType ZeroPoint,
const MLAS_REQUANT_PARAM* RequantParam,
size_t StartM,
size_t StartN,
size_t CountM,
size_t CountN
)
{
MLAS_ROUND_KIND RequantRoundKind = RequantParam->RequantRoundKind;
if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfUp) {
#ifdef MLAS_NO_EXCEPTION
abort();
#else
throw std::invalid_argument("Requantization rounding to nearest and tie to up is only supported on ARM64.");
#endif
}
const float* Scale = RequantParam->Scale;
bool PerColumnScale = RequantParam->Size > 1;
OutputType ZeroPoint = static_cast<OutputType>(RequantParam->ZeroPoint);
const float PerMatrixScaleValue = PerColumnScale ? 0.0f : *Scale;
const float MinimumValue = float(std::numeric_limits<OutputType>::lowest() - ZeroPoint);
const float MaximumValue = float(std::numeric_limits<OutputType>::max() - ZeroPoint);
@ -966,9 +1295,7 @@ MlasRequantizeOutput<int8_t>(
int8_t* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
int8_t ZeroPoint,
const MLAS_REQUANT_PARAM* RequantParam,
size_t StartM,
size_t StartN,
size_t CountM,
@ -984,9 +1311,7 @@ MlasRequantizeOutput<uint8_t>(
uint8_t* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
uint8_t ZeroPoint,
const MLAS_REQUANT_PARAM* RequantParam,
size_t StartM,
size_t StartN,
size_t CountM,

17
onnxruntime/core/providers/cpu/cpu_execution_provider.h
View file

@ -12,9 +12,11 @@ namespace onnxruntime {
// Information needed to construct CPU execution providers.
struct CPUExecutionProviderInfo {
bool create_arena{true};
bool use_fixed_point_requant_on_arm64{false};
explicit CPUExecutionProviderInfo(bool use_arena)
: create_arena(use_arena) {}
explicit CPUExecutionProviderInfo(bool use_arena, bool use_fixed_point_requant_on_arm64)
: create_arena(use_arena),
use_fixed_point_requant_on_arm64(use_fixed_point_requant_on_arm64) {}
CPUExecutionProviderInfo() = default;
};
@ -41,12 +43,23 @@ class CPUExecutionProvider : public IExecutionProvider {
0, create_arena};
InsertAllocator(CreateAllocator(device_info));
#if defined(__aarch64__) || defined(_M_ARM64)
use_fixed_point_requant_on_arm64_ = info.use_fixed_point_requant_on_arm64;
#else
use_fixed_point_requant_on_arm64_ = false;
#endif
}
std::shared_ptr<KernelRegistry> GetKernelRegistry() const override;
std::unique_ptr<IDataTransfer> GetDataTransfer() const override;
bool UseFixedPointRequantOnARM64() const {
return use_fixed_point_requant_on_arm64_;
}
private:
bool use_fixed_point_requant_on_arm64_;
std::vector<FuseRuleFn> fuse_rules_;
};

17
onnxruntime/core/providers/cpu/cpu_provider_factory.cc
View file

@ -8,33 +8,36 @@
#include "core/providers/cpu/cpu_execution_provider.h"
#include "core/session/abi_session_options_impl.h"
#include "core/session/onnxruntime_session_options_config_keys.h"
#include "core/session/ort_apis.h"
namespace onnxruntime {
struct CpuProviderFactory : IExecutionProviderFactory {
CpuProviderFactory(bool create_arena) : create_arena_(create_arena) {}
CpuProviderFactory(const CPUExecutionProviderInfo& info) : info(info) {}
~CpuProviderFactory() override = default;
std::unique_ptr<IExecutionProvider> CreateProvider() override;
private:
bool create_arena_;
CPUExecutionProviderInfo info;
};
std::unique_ptr<IExecutionProvider> CpuProviderFactory::CreateProvider() {
CPUExecutionProviderInfo info;
info.create_arena = create_arena_;
return std::make_unique<CPUExecutionProvider>(info);
}
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CPU(int use_arena) {
return std::make_shared<onnxruntime::CpuProviderFactory>(use_arena != 0);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CPU(const CPUExecutionProviderInfo& info) {
return std::make_shared<onnxruntime::CpuProviderFactory>(info);
}
} // namespace onnxruntime
ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CPU, _In_ OrtSessionOptions* options, int use_arena) {
options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CPU(use_arena));
const bool use_fixed_point_requant_on_arm64 =
options->value.config_options.GetConfigOrDefault(kOrtSessionOptionsConfigFixedPointRequantOnARM64, "0") == "1";
options->provider_factories.push_back(
onnxruntime::CreateExecutionProviderFactory_CPU(
onnxruntime::CPUExecutionProviderInfo(use_arena, use_fixed_point_requant_on_arm64)));
return nullptr;
}
#if defined(_MSC_VER) && !defined(__clang__)

3
onnxruntime/core/providers/cpu/cpu_provider_factory_creator.h
View file

@ -5,10 +5,11 @@
#include <memory>
#include "core/providers/cpu/cpu_execution_provider.h"
#include "core/providers/providers.h"
namespace onnxruntime {
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CPU(int use_arena);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CPU(const CPUExecutionProviderInfo& info);
} // namespace onnxruntime

36
onnxruntime/core/providers/cpu/quantization/qlinearconv.cc
View file

@ -3,6 +3,7 @@
#include "core/framework/op_kernel.h"
#include "core/providers/cpu/nn/conv_attributes.h"
#include "core/providers/cpu/cpu_execution_provider.h"
#include "core/common/cpuid_info.h"
#include "core/common/safeint.h"
#include "core/providers/common.h"
@ -10,6 +11,7 @@
#include "core/util/math_cpuonly.h"
#include "core/util/qmath.h"
#include "core/mlas/inc/mlas.h"
#include "core/quantization/quantization.h"
#if defined(_M_ARM64) || defined(__aarch64__)
@ -36,6 +38,8 @@ class QLinearConv : public OpKernel {
public:
explicit QLinearConv(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info) {
channels_last_ = (info.GetAttrOrDefault<int64_t>("channels_last", static_cast<int64_t>(0)) != 0);
const CPUExecutionProvider* ep = static_cast<const CPUExecutionProvider*>(info.GetExecutionProvider());
use_fixed_point_requant_ = ep->UseFixedPointRequantOnARM64();
}
Status Compute(OpKernelContext* context) const override;
@ -283,6 +287,7 @@ class QLinearConv : public OpKernel {
bool is_symmetric_gemm_{false};
bool channels_last_{false};
std::vector<int32_t> column_sums_;
bool use_fixed_point_requant_{false};
};
// uint8_t kernel supports weight being either uint8_t or int8_t
@ -511,6 +516,26 @@ Status QLinearConv<ActType>::Compute(OpKernelContext* context) const {
uint8_t W_zero_point_value;
ComputeOffset(context, M, X_zero_point_value, Y_zero_point_value, W_zero_point_value);
std::vector<float> output_scales = ComputeOutputScale(context, M);
MLAS_REQUANT_PARAM RequantParam(output_scales.data(), output_scales.size(), Y_zero_point_value);
#if defined(_M_ARM64) || defined(__aarch64__)
std::vector<int32_t> pre_shifts;
std::vector<int32_t> multipliers;
std::vector<int32_t> post_shifts;
if (use_fixed_point_requant_) {
pre_shifts.resize(output_scales.size());
multipliers.resize(output_scales.size());
post_shifts.resize(output_scales.size());
MlasFloatToFixedPoint(output_scales.data(),
multipliers.data(),
pre_shifts.data(),
post_shifts.data(),
output_scales.size());
RequantParam.RequantRoundKind = MLAS_ROUND_KIND::MlasRoundHalfUp;
RequantParam.PreShift = pre_shifts.data();
RequantParam.Multiplier = multipliers.data();
RequantParam.PostShift = post_shifts.data();
}
#endif
const Tensor* B = context->Input<Tensor>(InputTensors::IN_BIAS);
@ -656,7 +681,7 @@ Status QLinearConv<ActType>::Compute(OpKernelContext* context) const {
concurrency::ThreadPool* thread_pool = context->GetOperatorThreadPool();
#if defined(_M_ARM64) || defined(__aarch64__)
int32_t task_count = (output_image_size + (GEMM_KERNEL_STRIDE_M - 1)) / GEMM_KERNEL_STRIDE_M;
int32_t task_count = static_cast<int32_t>((output_image_size + (GEMM_KERNEL_STRIDE_M - 1)) / GEMM_KERNEL_STRIDE_M);
#else
int32_t task_count = ComputeTaskCount(output_image_size, group_output_channels, kernel_dim);
task_count = std::min(task_count, concurrency::ThreadPool::DegreeOfParallelism(thread_pool));
@ -735,6 +760,7 @@ Status QLinearConv<ActType>::Compute(OpKernelContext* context) const {
} else {
conv_params.InputDirect = input_data + output_start * C;
}
conv_params.Filter = packed_W_buffer_.get();
conv_params.Output = worker_output;
conv_params.InputChannels = static_cast<size_t>(C);
@ -742,9 +768,7 @@ Status QLinearConv<ActType>::Compute(OpKernelContext* context) const {
conv_params.OutputCount = static_cast<size_t>(output_count);
conv_params.KernelSize = static_cast<size_t>(kernel_size);
conv_params.Bias = column_sums_.data();
conv_params.Scale = output_scales.data();
conv_params.PerChannelScale = output_scales.size() > 1;
conv_params.OutputZeroPoint = Y_zero_point_value;
conv_params.RequantParam = &RequantParam;
conv_params.InputIsSigned = std::is_signed<ActType>::value;
if (is_depthwise_conv) {
@ -871,9 +895,7 @@ Status QLinearConv<ActType>::Compute(OpKernelContext* context) const {
worker_output,
static_cast<size_t>(M),
Bdata,
output_scales.data(),
output_scales.size() > 1,
Y_zero_point_value,
&RequantParam,
0,
0,
static_cast<size_t>(output_count),

37
onnxruntime/core/providers/cpu/quantization/quantize_linear_matmul.cc
View file

@ -95,6 +95,22 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const {
output_scales[i] = (a_scale_data * b_scale_data[i] / y_scale_data);
}
#if defined(_M_ARM64) || defined(__aarch64__)
std::vector<int32_t> pre_shifts;
std::vector<int32_t> multipliers;
std::vector<int32_t> post_shifts;
if (use_fixed_point_requant_) {
pre_shifts.resize(output_scales.size());
multipliers.resize(output_scales.size());
post_shifts.resize(output_scales.size());
MlasFloatToFixedPoint(output_scales.data(),
multipliers.data(),
pre_shifts.data(),
post_shifts.data(),
output_scales.size());
}
#endif
const size_t num_gemms = helper.OutputOffsets().size();
MLAS_GEMM_QUANT_SHAPE_PARAMS gemm_shape;
gemm_shape.M = static_cast<size_t>(helper.M());
@ -111,6 +127,7 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const {
auto* gemm_output = static_cast<int32_t*>(gemm_output_buffer.get());
std::vector<MLAS_GEMM_QUANT_DATA_PARAMS> gemm_params(num_gemms);
std::vector<MLAS_REQUANT_PARAM> requant_params(num_gemms);
std::vector<MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR> requant_procs;
requant_procs.reserve(num_gemms);
@ -133,12 +150,26 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const {
gemm_params[i].PerColumnZeroPoints = !IsScalarOr1ElementVector(b_offset);
requant_params[i].Size = output_scales.size();
requant_params[i].ZeroPoint = output_offset;
#if defined(_M_ARM64) || defined(__aarch64__)
if (use_fixed_point_requant_) {
requant_params[i].RequantRoundKind = MLAS_ROUND_KIND::MlasRoundHalfUp;
requant_params[i].PreShift = pre_shifts.data() + helper.RightScaleOffsets()[i];
requant_params[i].Multiplier = multipliers.data() + helper.RightScaleOffsets()[i];
requant_params[i].PostShift = post_shifts.data() + helper.RightScaleOffsets()[i];
} else {
requant_params[i].RequantRoundKind = MLAS_ROUND_KIND::MlasRoundHalfEven;
requant_params[i].Scale = output_scales.data() + helper.RightScaleOffsets()[i];
}
#else
requant_params[i].RequantRoundKind = MLAS_ROUND_KIND::MlasRoundHalfEven;
requant_params[i].Scale = output_scales.data() + helper.RightScaleOffsets()[i];
#endif
requant_procs.emplace_back(static_cast<uint8_t*>(y->MutableDataRaw()) + helper.OutputOffsets()[i],
static_cast<size_t>(helper.N()),
nullptr,
output_scales.data() + helper.RightScaleOffsets()[i],
output_scales.size() > 1,
output_offset,
&requant_params[i],
is_output_signed);
gemm_params[i].OutputProcessor = &(requant_procs[i]);
}

8
onnxruntime/core/providers/cpu/quantization/quantize_linear_matmul.h
View file

@ -3,13 +3,17 @@
#include "matmul_integer_base.h"
#include "core/providers/cpu/cpu_execution_provider.h"
namespace onnxruntime {
// Allow subclassing for test only
class QLinearMatMul : public MatMulIntegerBase {
public:
QLinearMatMul(const OpKernelInfo& info) : MatMulIntegerBase(info) {}
QLinearMatMul(const OpKernelInfo& info) : MatMulIntegerBase(info) {
const CPUExecutionProvider* ep = static_cast<const CPUExecutionProvider*>(info.GetExecutionProvider());
use_fixed_point_requant_ = ep->UseFixedPointRequantOnARM64();
}
Status Compute(OpKernelContext* context) const override;
@ -36,6 +40,8 @@ class QLinearMatMul : public MatMulIntegerBase {
int GetBIdx() const override {
return IN_B;
}
bool use_fixed_point_requant_{false};
};
} // namespace onnxruntime

4
onnxruntime/core/session/inference_session.cc
View file

@ -1272,7 +1272,9 @@ common::Status InferenceSession::Initialize() {
// RegisterExecutionProvider locks the session_mutex_ so we can't be holding it when we call that
if (!have_cpu_ep) {
LOGS(*session_logger_, INFO) << "Adding default CPU execution provider.";
CPUExecutionProviderInfo epi{session_options_.enable_cpu_mem_arena};
const bool use_fixed_point_requant_on_arm64 =
session_options_.config_options.GetConfigOrDefault(kOrtSessionOptionsConfigFixedPointRequantOnARM64, "0") == "1";
CPUExecutionProviderInfo epi(session_options_.enable_cpu_mem_arena, use_fixed_point_requant_on_arm64);
auto p_cpu_exec_provider = std::make_unique<CPUExecutionProvider>(epi);
ORT_RETURN_IF_ERROR_SESSIONID_(RegisterExecutionProvider(std::move(p_cpu_exec_provider)));
}

2
onnxruntime/python/onnxruntime_pybind_schema.cc
View file

@ -22,7 +22,7 @@ void addGlobalSchemaFunctions(pybind11::module& m) {
std::vector<onnxruntime::KernelDef> result;
std::vector<std::shared_ptr<onnxruntime::IExecutionProviderFactory>> factories = {
onnxruntime::CreateExecutionProviderFactory_CPU(0),
onnxruntime::CreateExecutionProviderFactory_CPU(onnxruntime::CPUExecutionProviderInfo(false, false)),
#ifdef USE_CUDA
[]() {
OrtCUDAProviderOptions provider_options{};

6
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -365,8 +365,10 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
const std::string& type,
const ProviderOptionsMap& provider_options_map) {
if (type == kCpuExecutionProvider) {
const bool use_fixed_point_requant_on_arm64 =
session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsConfigFixedPointRequantOnARM64, "0") == "1";
return onnxruntime::CreateExecutionProviderFactory_CPU(
session_options.enable_cpu_mem_arena)
onnxruntime::CPUExecutionProviderInfo(session_options.enable_cpu_mem_arena, use_fixed_point_requant_on_arm64))
->CreateProvider();
} else if (type == kTensorrtExecutionProvider) {
#ifdef USE_TENSORRT
@ -1236,7 +1238,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc")
const OrtValue* ml_value = ml_value_pyobject.attr(PYTHON_ORTVALUE_NATIVE_OBJECT_ATTR).cast<OrtValue*>();
ORT_THROW_IF_ERROR(options->AddInitializer(name, ml_value));
})
.def("add_external_initializers", [](PySessionOptions* options, py::list& names,
.def("add_external_initializers", [](PySessionOptions* options, py::list& names,
const py::list& ort_values) -> void {
#if !defined(ORT_MINIMAL_BUILD) && !defined(DISABLE_EXTERNAL_INITIALIZERS)
const auto init_num = ort_values.size();

2
onnxruntime/test/contrib_ops/fastgelu_op_test.cc
View file

@ -254,7 +254,7 @@ TEST(FastGeluTest, FastGeluWithBias_BFloat16) {
execution_providers.push_back(DefaultCudaExecutionProvider());
#elif USE_ROCM
execution_providers.push_back(DefaultRocmExecutionProvider());
#endif
#endif
tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers);
}
#endif

33
onnxruntime/test/eager/ort_invoker_test.cc
View file

@ -16,13 +16,14 @@ namespace onnxruntime {
namespace test {
TEST(InvokerTest, Basic) {
std::unique_ptr<IExecutionProvider> cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false));
std::unique_ptr<IExecutionProvider> cpu_execution_provider =
std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false, false));
const std::string logger_id{"InvokerTest"};
auto logging_manager = std::make_unique<logging::LoggingManager>(
std::unique_ptr<logging::ISink>{new logging::CLogSink{}},
logging::Severity::kVERBOSE, false,
logging::LoggingManager::InstanceType::Default,
&logger_id);
&logger_id);
std::unique_ptr<Environment> env;
ASSERT_STATUS_OK(Environment::Create(std::move(logging_manager), env));
IOnnxRuntimeOpSchemaRegistryList tmp_op_registry = {};
@ -49,13 +50,13 @@ TEST(InvokerTest, Basic) {
}
TEST(InvokerTest, Inplace) {
std::unique_ptr<IExecutionProvider> cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false));
std::unique_ptr<IExecutionProvider> cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false, false));
const std::string logger_id{"InvokerTest"};
auto logging_manager = std::make_unique<logging::LoggingManager>(
std::unique_ptr<logging::ISink>{new logging::CLogSink{}},
logging::Severity::kVERBOSE, false,
logging::LoggingManager::InstanceType::Default,
&logger_id);
&logger_id);
std::unique_ptr<Environment> env;
ASSERT_STATUS_OK(Environment::Create(std::move(logging_manager), env));
IOnnxRuntimeOpSchemaRegistryList tmp_op_registry = {};
@ -93,26 +94,22 @@ Status CreateTestKernel(FuncManager&, const OpKernelInfo& info, std::unique_ptr<
}
TEST(InvokerTest, CustomOp) {
std::unique_ptr<IExecutionProvider> cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false));
std::unique_ptr<IExecutionProvider> cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false, false));
// register "Test" kernel to "FakeDomain"
auto kernel_registry = cpu_execution_provider->GetKernelRegistry();
auto kernel_def = KernelDefBuilder()
.MayInplace(0, 0)
.TypeConstraint("T", DataTypeImpl::GetTensorType<float>())
.SetName("Test")
.SetDomain("FakeDomain")
.SinceVersion(1)
.Provider(kCpuExecutionProvider)
.Build();
.MayInplace(0, 0)
.TypeConstraint("T", DataTypeImpl::GetTensorType<float>())
.SetName("Test")
.SetDomain("FakeDomain")
.SinceVersion(1)
.Provider(kCpuExecutionProvider)
.Build();
ASSERT_STATUS_OK(kernel_registry->Register(KernelCreateInfo(std::move(kernel_def), CreateTestKernel)));
// create custom op schema for "Test" op
std::shared_ptr<onnxruntime::OnnxRuntimeOpSchemaRegistry> schema_registry = std::make_shared<OnnxRuntimeOpSchemaRegistry>();
std::vector<ONNX_NAMESPACE::OpSchema> schema = {
ONNX_NAMESPACE::OpSchema().SetName("Test")
.Input(0, "X", "A N-D input tensor that is to be processed.", "T")
.Output(0, "Y", "desc", "T")
.TypeConstraint("T", ONNX_NAMESPACE::OpSchema::all_tensor_types(), "Constrain input and output types to any tensor type.")
.SetDomain("FakeDomain")};
ONNX_NAMESPACE::OpSchema().SetName("Test").Input(0, "X", "A N-D input tensor that is to be processed.", "T").Output(0, "Y", "desc", "T").TypeConstraint("T", ONNX_NAMESPACE::OpSchema::all_tensor_types(), "Constrain input and output types to any tensor type.").SetDomain("FakeDomain")};
ASSERT_STATUS_OK(schema_registry->RegisterOpSet(schema, "FakeDomain", 0, 1));
std::list<std::shared_ptr<IOnnxRuntimeOpSchemaCollection>> regs = {schema_registry};
@ -121,7 +118,7 @@ TEST(InvokerTest, CustomOp) {
std::unique_ptr<logging::ISink>{new logging::CLogSink{}},
logging::Severity::kVERBOSE, false,
logging::LoggingManager::InstanceType::Default,
&logger_id);
&logger_id);
std::unique_ptr<Environment> env;
ASSERT_STATUS_OK(Environment::Create(std::move(logging_manager), env));
ORTInvoker kernel_invoker(std::move(cpu_execution_provider), env->GetLoggingManager()->DefaultLogger(), regs);

19
onnxruntime/test/framework/parallel_executor_test.cc
View file

@ -93,8 +93,10 @@ TEST(ParallelExecutor, TestStatusPropagation) {
tester.AddInput<int64_t>("action", {1}, {/*success*/ 0});
tester.AddOutput<int64_t>("action_out", {1}, {0});
// TensorRT doesn't handle a custom op. Possibly it should, but that would be a separate PR
tester.Run(OpTester::ExpectResult::kExpectSuccess, {}, {kTensorrtExecutionProvider}, nullptr, nullptr,
ExecutionMode::ORT_PARALLEL);
SessionOptions so;
tester.SetUpDefaultSessionOptions(so);
so.execution_mode = ExecutionMode::ORT_PARALLEL;
tester.Run(so, OpTester::ExpectResult::kExpectSuccess, {}, {kTensorrtExecutionProvider});
}
{ // test failure
@ -103,8 +105,11 @@ TEST(ParallelExecutor, TestStatusPropagation) {
tester.AddInput<int64_t>("action", {1}, {/*failure*/ 1});
tester.AddOutput<int64_t>("action_out", {1}, {0});
tester.Run(OpTester::ExpectResult::kExpectFailure, "Action was 1", {kTensorrtExecutionProvider}, nullptr, nullptr,
ExecutionMode::ORT_PARALLEL);
SessionOptions so;
tester.SetUpDefaultSessionOptions(so);
so.execution_mode = ExecutionMode::ORT_PARALLEL;
tester.Run(so, OpTester::ExpectResult::kExpectFailure, "Action was 1", {kTensorrtExecutionProvider});
}
{ // test exception
@ -113,7 +118,11 @@ TEST(ParallelExecutor, TestStatusPropagation) {
tester.AddInput<int64_t>("action", {1}, {/*exception*/ 2});
tester.AddOutput<int64_t>("action_out", {1}, {0});
tester.Run(OpTester::ExpectResult::kExpectFailure, "Throwing as action was 2", {kTensorrtExecutionProvider}, nullptr, nullptr, ExecutionMode::ORT_PARALLEL);
SessionOptions so;
tester.SetUpDefaultSessionOptions(so);
so.execution_mode = ExecutionMode::ORT_PARALLEL;
tester.Run(so, OpTester::ExpectResult::kExpectFailure, "Throwing as action was 2", {kTensorrtExecutionProvider});
}
}

12
onnxruntime/test/framework/session_state_test.cc
View file

@ -55,7 +55,7 @@ TEST_P(SessionStateAddGetKernelTest, AddGetKernelTest) {
auto& graph = model.MainGraph();
ExecutionProviders execution_providers;
auto tmp_cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false));
auto tmp_cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false, false));
auto* cpu_execution_provider = tmp_cpu_execution_provider.get();
ASSERT_STATUS_OK(execution_providers.Add(kCpuExecutionProvider, std::move(tmp_cpu_execution_provider)));
@ -128,7 +128,7 @@ TEST_P(SessionStateTestP, TestInitializerProcessing) {
InitializedTensorSet initializers = graph.GetAllInitializedTensors();
ExecutionProviders execution_providers;
CPUExecutionProviderInfo epi{false};
CPUExecutionProviderInfo epi{false, false};
status =
execution_providers.Add(onnxruntime::kCpuExecutionProvider, std::make_unique<CPUExecutionProvider>(epi));
ASSERT_TRUE(status.IsOK()) << status;
@ -193,7 +193,7 @@ TEST(SessionStateTest, TestInitializerMemoryAllocatedUsingNonArenaMemory) {
Graph& graph = model->MainGraph();
ExecutionProviders execution_providers;
CPUExecutionProviderInfo epi{true}; // use an arena-based allocator for this EP
CPUExecutionProviderInfo epi{true, false}; // use an arena-based allocator for this EP
status = execution_providers.Add(onnxruntime::kCpuExecutionProvider, std::make_unique<CPUExecutionProvider>(epi));
ASSERT_TRUE(status.IsOK()) << status;
@ -243,7 +243,7 @@ TEST(SessionStateTest, TestInitializerMemoryAllocatedUsingNonArenaMemory) {
Graph& graph = model->MainGraph();
ExecutionProviders execution_providers;
CPUExecutionProviderInfo epi{true}; // use an arena-based allocator for this EP
CPUExecutionProviderInfo epi{true, false}; // use an arena-based allocator for this EP
status = execution_providers.Add(onnxruntime::kCpuExecutionProvider, std::make_unique<CPUExecutionProvider>(epi));
ASSERT_TRUE(status.IsOK()) << status;
@ -480,7 +480,7 @@ TEST_P(SessionStatePrepackingTest, PrePackingTest) {
.Output(0, "output_0", "docstr for output_0.", "tensor(float)");
ExecutionProviders execution_providers;
auto cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false));
auto cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false, false));
ASSERT_STATUS_OK(execution_providers.Add(kCpuExecutionProvider, std::move(cpu_execution_provider)));
DataTransferManager dtm;
@ -541,7 +541,7 @@ TEST(SessionStateTest, SharedInitalizersWithPrePackingTest) {
.Output(0, "output_0", "docstr for output_0.", "tensor(float)");
ExecutionProviders execution_providers;
auto cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false));
auto cpu_execution_provider = std::make_unique<CPUExecutionProvider>(CPUExecutionProviderInfo(false, false));
ASSERT_STATUS_OK(execution_providers.Add(kCpuExecutionProvider, std::move(cpu_execution_provider)));
DataTransferManager dtm;

6
onnxruntime/test/perftest/command_args_parser.cc
View file

@ -56,6 +56,7 @@ namespace perftest {
"\t-q: [CUDA only] use separate stream for copy. \n"
"\t-z: Set denormal as zero. When turning on this option reduces latency dramatically, a model may have denormals.\n"
"\t-i: Specify EP specific runtime options as key value pairs. Different runtime options available are: \n"
"\t-R: Use fixed point based requantization on ARM64."
"\t [OpenVINO only] [device_type]: Overrides the accelerator hardware type and precision with these values at runtime.\n"
"\t [OpenVINO only] [device_id]: Selects a particular hardware device for inference.\n"
"\t [OpenVINO only] [enable_vpu_fast_compile]: Optionally enabled to speeds up the model's compilation on VPU device targets.\n"
@ -114,7 +115,7 @@ static bool ParseDimensionOverride(std::basic_string<ORTCHAR_T>& dim_identifier,
/*static*/ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]) {
int ch;
while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:y:c:d:o:u:i:f:F:AMPIvhsqz"))) != -1) {
while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:y:c:d:o:u:i:f:F:AMPIvhsqzR"))) != -1) {
switch (ch) {
case 'f': {
std::basic_string<ORTCHAR_T> dim_name;
@ -272,6 +273,9 @@ static bool ParseDimensionOverride(std::basic_string<ORTCHAR_T>& dim_identifier,
case 'i':
test_config.run_config.ep_runtime_config_string = optarg;
break;
case 'R':
test_config.run_config.use_fixed_point_requant = true;
break;
case '?':
case 'h':
default:

4
onnxruntime/test/perftest/ort_test_session.cc
View file

@ -460,6 +460,10 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
}
}
if (performance_test_config.run_config.use_fixed_point_requant) {
session_options.AddConfigEntry(kOrtSessionOptionsConfigFixedPointRequantOnARM64, "1");
}
session_ = Ort::Session(env, performance_test_config.model_info.model_file_path.c_str(), session_options);
size_t output_count = session_.GetOutputCount();

1
onnxruntime/test/perftest/test_configuration.h
View file

@ -54,6 +54,7 @@ struct RunConfig {
int cudnn_conv_algo{0};
bool do_cuda_copy_in_separate_stream{false};
bool set_denormal_as_zero{false};
bool use_fixed_point_requant{false};
std::basic_string<ORTCHAR_T> ep_runtime_config_string;
std::map<std::basic_string<ORTCHAR_T>, int64_t> free_dim_name_overrides;
std::map<std::basic_string<ORTCHAR_T>, int64_t> free_dim_denotation_overrides;

5
onnxruntime/test/providers/cpu/controlflow/loop_test.cc
View file

@ -685,8 +685,9 @@ TEST(Loop, SubgraphTypeOverride) {
Graph::ResolveOptions options;
options.override_types = true;
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kTensorrtExecutionProvider}, &session_run_options, nullptr,
ExecutionMode::ORT_SEQUENTIAL, options);
{kTensorrtExecutionProvider},
&session_run_options, nullptr,
options);
}
// Regression test that a subgraph input overrides an outer scope value of the same name.

82
onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc
View file

@ -6,12 +6,68 @@
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
#include "core/session/onnxruntime_session_options_config_keys.h"
namespace onnxruntime {
namespace test {
class OpTesterRequant : public OpTester {
public:
explicit OpTesterRequant(
const char* op,
int opset_version = 7,
const char* domain = onnxruntime::kOnnxDomain,
bool verify_output = true) : OpTester(op, opset_version, domain, verify_output) {}
void Run(ExpectResult expect_result = ExpectResult::kExpectSuccess, const std::string& expected_failure_string = "",
const std::unordered_set<std::string>& excluded_provider_types = {},
const RunOptions* run_options = nullptr,
std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers = nullptr,
const Graph::ResolveOptions& resolve_options = {}) {
SessionOptions session_options;
SetUpDefaultSessionOptions(session_options);
for (auto use_fixed_point : {"0", "1"}) {
ASSERT_STATUS_OK(session_options.config_options.AddConfigEntry(
kOrtSessionOptionsConfigFixedPointRequantOnARM64,
use_fixed_point));
OpTester::Run(session_options,
expect_result,
expected_failure_string,
excluded_provider_types,
run_options,
execution_providers,
resolve_options);
}
}
void Run(SessionOptions session_options,
ExpectResult expect_result = ExpectResult::kExpectSuccess,
const std::string& expected_failure_string = "",
const std::unordered_set<std::string>& excluded_provider_types = {},
const RunOptions* run_options = nullptr,
std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers = nullptr,
const Graph::ResolveOptions& resolve_options = {},
/*out*/ size_t* number_of_pre_packed_weights_counter = nullptr,
/*out*/ size_t* number_of_shared_pre_packed_weights_counter = nullptr) {
for (auto use_fixed_point : {"0", "1"}) {
ASSERT_STATUS_OK(session_options.config_options.AddConfigEntry(
kOrtSessionOptionsConfigFixedPointRequantOnARM64,
use_fixed_point));
OpTester::Run(session_options,
expect_result,
expected_failure_string,
excluded_provider_types,
run_options,
execution_providers,
resolve_options,
number_of_pre_packed_weights_counter,
number_of_shared_pre_packed_weights_counter);
}
}
};
TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_U8U8) {
OpTester test("QLinearMatMul", 10);
OpTesterRequant test("QLinearMatMul", 10);
test.AddInput<uint8_t>("T1", {2, 2, 4},
{208, 236, 0, 238,
3, 214, 255, 29,
@ -49,7 +105,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_U8U8) {
}
TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_U8S8) {
OpTester test("QLinearMatMul", 10);
OpTesterRequant test("QLinearMatMul", 10);
test.AddInput<uint8_t>("T1", {2, 2, 4},
{208, 126, 0, 238,
3, 214, 255, 29,
@ -87,7 +143,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_U8S8) {
}
TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_S8S8) {
OpTester test("QLinearMatMul", 10);
OpTesterRequant test("QLinearMatMul", 10);
test.AddInput<int8_t>("T1", {2, 2, 4},
{80, -2, -128, 110,
-125, 86, 127, -99,
@ -126,7 +182,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_S8S8) {
TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_U8U8) {
auto run_test = [](bool only_t1_not_initializer) {
OpTester test("QLinearMatMul", 10);
OpTesterRequant test("QLinearMatMul", 10);
test.AddInput<uint8_t>("T1", {2, 4},
{208, 236, 0, 238,
3, 214, 255, 29});
@ -161,7 +217,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_U8U8) {
TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_U8S8) {
auto run_test = [](bool only_t1_not_initializer) {
OpTester test("QLinearMatMul", 10);
OpTesterRequant test("QLinearMatMul", 10);
test.AddInput<uint8_t>("T1", {2, 4},
{208, 126, 0, 238,
3, 214, 255, 29});
@ -196,7 +252,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_U8S8) {
TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_S8S8) {
auto run_test = [](bool only_t1_not_initializer) {
OpTester test("QLinearMatMul", 10);
OpTesterRequant test("QLinearMatMul", 10);
test.AddInput<int8_t>("T1", {2, 4},
{80, -2, -128, 110,
-125, 86, 127, -99});
@ -231,7 +287,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_S8S8) {
static void QLinearMatMul2DTest(bool only_t1_not_initializer) {
// Test non-empty inputs
OpTester test_non_empty("QLinearMatMul", 10);
OpTesterRequant test_non_empty("QLinearMatMul", 10);
test_non_empty.AddInput<uint8_t>("T1", {2, 4}, {208, 236, 0, 238, 3, 214, 255, 29});
test_non_empty.AddInput<float>("a_scale", {1}, {0.0066f}, only_t1_not_initializer);
test_non_empty.AddInput<uint8_t>("a_zero_point", {1}, {113}, only_t1_not_initializer);
@ -244,7 +300,7 @@ static void QLinearMatMul2DTest(bool only_t1_not_initializer) {
test_non_empty.Run();
// Test with an empty input
OpTester test_empty("QLinearMatMul", 10);
OpTesterRequant test_empty("QLinearMatMul", 10);
test_empty.AddInput<uint8_t>("T1", {0, 4}, {});
test_empty.AddInput<float>("a_scale", {1}, {0.0066f}, only_t1_not_initializer);
test_empty.AddInput<uint8_t>("a_zero_point", {1}, {113}, only_t1_not_initializer);
@ -269,7 +325,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMulAllInputExceptT1AreInitializers) {
}
TEST(QuantizeLinearMatmulOpTest, PerColumn_2D) {
OpTester test("QLinearMatMul", 10);
OpTesterRequant test("QLinearMatMul", 10);
test.AddInput<uint8_t>("a",
{2, 4},
{125, 135, 133, 122,
@ -299,7 +355,7 @@ TEST(QuantizeLinearMatmulOpTest, PerColumn_2D) {
}
TEST(QuantizeLinearMatmulOpTest, PerColumn_2D_S8S8) {
OpTester test("QLinearMatMul", 10);
OpTesterRequant test("QLinearMatMul", 10);
test.AddInput<int8_t>("a",
{2, 4},
{-3, 7, 5, -6,
@ -329,7 +385,7 @@ TEST(QuantizeLinearMatmulOpTest, PerColumn_2D_S8S8) {
}
TEST(QuantizeLinearMatmulOpTest, PerColumn_ND) {
OpTester test("QLinearMatMul", 10);
OpTesterRequant test("QLinearMatMul", 10);
test.AddInput<uint8_t>("a",
{2, 2, 4},
{125, 135, 133, 122,
@ -372,7 +428,7 @@ TEST(QuantizeLinearMatmulOpTest, PerColumn_ND) {
}
TEST(QuantizeLinearMatmulOpTest, PerColumn_ND_S8S8) {
OpTester test("QLinearMatMul", 10);
OpTesterRequant test("QLinearMatMul", 10);
test.AddInput<int8_t>("a",
{2, 2, 4},
{-3, 7, 5, -6,
@ -491,7 +547,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMulPrePack) {
auto kernel_def = PrePackTestOp::KernelDef();
ASSERT_TRUE((status = registry->RegisterCustomKernel(kernel_def, kernel_create_fn)).IsOK()) << status;
OpTester test_non_empty(PrePackTestOp::OpName, 10, PrePackTestOp::OpDomain);
OpTesterRequant test_non_empty(PrePackTestOp::OpName, 10, PrePackTestOp::OpDomain);
test_non_empty.AddCustomOpRegistry(registry);
test_non_empty.AddInput<uint8_t>("T1", {2, 4}, {208, 236, 0, 238, 3, 214, 255, 29});

97
onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc
View file

@ -9,6 +9,7 @@
#include "default_providers.h"
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
#include "core/session/onnxruntime_session_options_config_keys.h"
namespace onnxruntime {
namespace test {
@ -318,21 +319,49 @@ class QLinearConvOpTester {
float zero_point_;
};
inline float RoundHalfToEven(float input) {
if (!std::isfinite(input)) {
return input;
}
// std::remainder returns x - n, where n is the integral value nearest to x. When |x - n| = 0.5, n is chosen to be even
return input - std::remainderf(input, 1.f);
}
template <typename T, MLAS_ROUND_KIND RoundKind>
struct RequantizeOutput {};
template <typename T>
T RequantizeOutput(int32_t sum, float scale, RequantizeValues<T>& requantize_values) {
float f = static_cast<float>(sum) * scale;
f = std::min(f, requantize_values.max_value_);
f = std::max(f, requantize_values.min_value_);
return static_cast<T>(RoundHalfToEven(f) + requantize_values.zero_point_);
}
struct RequantizeOutput<T, MLAS_ROUND_KIND::MlasRoundHalfEven> {
T operator()(int32_t sum,
float scale,
RequantizeValues<T>& requantize_values) {
float f = static_cast<float>(sum) * scale;
f = std::min(f, requantize_values.max_value_);
f = std::max(f, requantize_values.min_value_);
if (std::isfinite(f)) {
f = f - std::remainderf(f, 1.f); // std::remainder returns x - n, where n is the integral value nearest to x.
// When |x - n| = 0.5, n is chosen to be even
}
return static_cast<T>(f + requantize_values.zero_point_);
}
};
// Compute RoundUp with double float precision to avoid corner case like:
// sum: 1140, scale: 0.00394736836.
// with single float precision, it is 4.50000000 and rounds to 5.
// with double float precision, it is 4.4999999303999996 and rounds to 4.
// Fixed point computation in arm64 is equivalent to double float precision,
// so use double float precision to get same result as fixed point.
template <typename T>
struct RequantizeOutput<T, MLAS_ROUND_KIND::MlasRoundHalfUp> {
T operator()(int32_t sum,
float scale,
RequantizeValues<T>& requantize_values) {
double d = static_cast<double>(sum) * static_cast<double>(scale);
d = std::min(d, static_cast<double>(requantize_values.max_value_));
d = std::max(d, static_cast<double>(requantize_values.min_value_));
if (std::isfinite(d)) {
d = std::round(d);
}
return static_cast<T>(d + requantize_values.zero_point_);
}
};
static bool NextPosition(int64_t N, const int64_t* shape, int64_t* dims) {
// Loop over spatial axes in reverse order to choose an index, like counting.
@ -351,7 +380,9 @@ class QLinearConvOpTester {
return incremented;
}
void ComputeExpectedOutput(std::vector<ActType>& Y_data, std::vector<int64_t>& Y_shape) {
void ComputeExpectedOutput(std::vector<ActType>& Y_data,
std::vector<int64_t>& Y_shape,
bool requant_with_fixed_point_on_arm64) {
ORT_ENFORCE(W_.shape_.size() > 2);
ORT_ENFORCE(X_.shape_.size() == W_.shape_.size());
@ -444,7 +475,17 @@ class QLinearConvOpTester {
input_image += input_image_size;
}
*Ydata++ = RequantizeOutput<ActType>(sum, requantize_scale, requantize_values);
if (requant_with_fixed_point_on_arm64) {
*Ydata++ = RequantizeOutput<ActType, MLAS_ROUND_KIND::MlasRoundHalfUp>()(
sum,
requantize_scale,
requantize_values);
} else {
*Ydata++ = RequantizeOutput<ActType, MLAS_ROUND_KIND::MlasRoundHalfEven>()(
sum,
requantize_scale,
requantize_values);
}
} while (NextPosition(kernel_rank, output_shape, d_output.data()));
@ -457,12 +498,12 @@ class QLinearConvOpTester {
}
}
void Run(bool all_input_initializer_except_x) {
void Run(bool all_input_initializer_except_x, bool requant_with_fixed_point_on_arm64) {
OpTester test("QLinearConv", 10);
std::vector<ActType> Y_data;
std::vector<int64_t> Y_shape;
ComputeExpectedOutput(Y_data, Y_shape);
ComputeExpectedOutput(Y_data, Y_shape, requant_with_fixed_point_on_arm64);
test.AddInput<ActType>("x", X_.shape_, X_.data_);
test.AddInput<float>("x_scale", {}, X_.scale_, all_input_initializer_except_x);
@ -512,7 +553,11 @@ class QLinearConvOpTester {
test.AddAttribute("group", groups_);
}
test.Run(OpTester::ExpectResult::kExpectSuccess, "");
SessionOptions so;
ASSERT_STATUS_OK(so.config_options.AddConfigEntry(
kOrtSessionOptionsConfigFixedPointRequantOnARM64,
requant_with_fixed_point_on_arm64 ? "1" : "0"));
test.Run(so, OpTester::ExpectResult::kExpectSuccess, "");
}
public:
@ -569,11 +614,16 @@ class QLinearConvOpTester {
}
void Run() {
for (bool all_input_initializer_except_x : std::initializer_list<bool>{false, true}) {
Run(all_input_initializer_except_x);
}
#if defined(_M_ARM64) || defined(__aarch64__)
for (bool requant_with_fixed_point_on_arm64 : std::initializer_list<bool>{false, true})
#else
bool requant_with_fixed_point_on_arm64 = false;
#endif
for (bool all_input_initializer_except_x : std::initializer_list<bool>{false, true}) {
Run(all_input_initializer_except_x, requant_with_fixed_point_on_arm64);
}
}
};
}; // namespace
TEST(QLinearConvTest, Conv1D_U8S8) {
QLinearConvOpTester<uint8_t, int8_t> test;
@ -637,7 +687,7 @@ TEST(QLinearConvTest, Conv2D_U8S8_Sym_M32_C32_Bias_Pads) {
TEST(QLinearConvTest, Conv2D_U8S8_Sym_M8_C8) {
// Targeting code processing 8 channels, with odd number
// of output pixels
// of output pixels
QLinearConvOpTester<uint8_t, int8_t> test;
test.GenerateRandomInput({1, 8, 3, 5}, .85f, 4);
test.GenerateRandomWeights({8, 8, 3, 3}, .125f, 0);
@ -1039,7 +1089,6 @@ TEST(QLinearConvTest, Conv2D_S8S8_Sym_M64_C64) {
test.Run();
}
TEST(QLinearConvTest, Conv2D_S8S8_Sym_M16_C4_Bias) {
QLinearConvOpTester<int8_t, int8_t> test;
test.GenerateRandomInput({1, 4, 3, 3}, .05f, 4);

62
onnxruntime/test/providers/provider_test_utils.cc
View file

@ -5,6 +5,7 @@
#include "core/common/logging/logging.h"
#include "core/common/logging/sinks/clog_sink.h"
#include "core/common/type_utils.h"
#include "core/framework/tensorprotoutils.h"
#include "core/framework/data_types_internal.h"
#include "core/session/inference_session.h"
@ -57,7 +58,7 @@ void sort_expected_and_actual_buffers(std::vector<T>& expected,
// The default implementation compares for equality, specialized versions for
// other types are below
template <typename T>
template <typename T, typename Enabled = void>
struct TensorCheck {
void operator()(const Tensor& expected_tensor, const Tensor& output_tensor,
const std::string& provider_type, const CheckParams& params) const {
@ -87,8 +88,8 @@ struct TensorCheck {
}
};
template <>
struct TensorCheck<uint8_t> {
template <typename T>
struct TensorCheck<T, typename std::enable_if<utils::IsByteType<T>::value>::type> {
void operator()(const Tensor& expected_tensor,
const Tensor& output_tensor,
const std::string& provider_type, const CheckParams& params) const {
@ -96,8 +97,8 @@ struct TensorCheck<uint8_t> {
const bool has_rel_err = params.relative_error_.has_value();
Tensor expected_sorted, output_sorted;
const uint8_t* expected;
const uint8_t* output;
const T* expected;
const T* output;
const auto size = output_tensor.Shape().Size();
if (params.sort_output_) {
// if order can be jumbled in the output of an operator, sort both the
@ -106,15 +107,15 @@ struct TensorCheck<uint8_t> {
// requirement for the few ops that do require this
// support without investing in a more sophisticated infrastructure for the
// same
sort_expected_and_actual_buffers<uint8_t>(expected_tensor, expected_sorted, output_tensor, output_sorted);
expected = expected_sorted.Data<uint8_t>();
output = output_sorted.Data<uint8_t>();
sort_expected_and_actual_buffers<T>(expected_tensor, expected_sorted, output_tensor, output_sorted);
expected = expected_sorted.Data<T>();
output = output_sorted.Data<T>();
} else {
expected = expected_tensor.template Data<uint8_t>();
output = output_tensor.template Data<uint8_t>();
expected = expected_tensor.template Data<T>();
output = output_tensor.template Data<T>();
}
// For uint8_t results, we only allow NNAPI EP to have an error tolerance, see below for the reason
// For int8_t/uint8_t results, we only allow NNAPI EP to have an error tolerance, see below for the reason
// For any other EPs, we still expect an exact match for the results
if (provider_type == kNnapiExecutionProvider && (has_abs_err || has_rel_err)) {
double threshold = has_abs_err
@ -897,15 +898,9 @@ void OpTester::Run(
const std::unordered_set<std::string>& excluded_provider_types,
const RunOptions* run_options,
std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers,
ExecutionMode execution_mode,
const Graph::ResolveOptions& options) {
SessionOptions so;
so.use_per_session_threads = false;
so.session_logid = op_;
so.session_log_verbosity_level = 1;
so.execution_mode = execution_mode;
so.use_deterministic_compute = use_determinism_;
so.graph_optimization_level = TransformerLevel::Default; // 'Default' == off
SetUpDefaultSessionOptions(so);
Run(so, expect_result, expected_failure_string, excluded_provider_types,
run_options, execution_providers, options);
}
@ -1088,30 +1083,35 @@ void OpTester::Run(
ASSERT_PROVIDER_STATUS_OK(session_object.RegisterCustomRegistry(custom_session_registry));
std::unique_ptr<IExecutionProvider> execution_provider;
if (provider_type == onnxruntime::kCpuExecutionProvider)
execution_provider = DefaultCpuExecutionProvider();
else if (provider_type == onnxruntime::kCudaExecutionProvider)
if (provider_type == onnxruntime::kCpuExecutionProvider) {
const bool use_fixed_point_requant_on_arm64 =
so.config_options.GetConfigOrDefault(kOrtSessionOptionsConfigFixedPointRequantOnARM64, "0") == "1";
execution_provider = DefaultCpuExecutionProvider(
CPUExecutionProviderInfo(so.enable_cpu_mem_arena,
use_fixed_point_requant_on_arm64));
} else if (provider_type == onnxruntime::kCudaExecutionProvider) {
execution_provider = DefaultCudaExecutionProvider();
else if (provider_type == onnxruntime::kDnnlExecutionProvider)
} else if (provider_type == onnxruntime::kDnnlExecutionProvider) {
execution_provider = DefaultDnnlExecutionProvider();
else if (provider_type == onnxruntime::kOpenVINOExecutionProvider)
} else if (provider_type == onnxruntime::kOpenVINOExecutionProvider) {
execution_provider = DefaultOpenVINOExecutionProvider();
else if (provider_type == onnxruntime::kNupharExecutionProvider)
} else if (provider_type == onnxruntime::kNupharExecutionProvider) {
execution_provider = DefaultNupharExecutionProvider();
else if (provider_type == onnxruntime::kTensorrtExecutionProvider)
} else if (provider_type == onnxruntime::kTensorrtExecutionProvider) {
execution_provider = DefaultTensorrtExecutionProvider();
else if (provider_type == onnxruntime::kNnapiExecutionProvider)
} else if (provider_type == onnxruntime::kNnapiExecutionProvider) {
execution_provider = DefaultNnapiExecutionProvider();
else if (provider_type == onnxruntime::kRknpuExecutionProvider)
} else if (provider_type == onnxruntime::kRknpuExecutionProvider) {
execution_provider = DefaultRknpuExecutionProvider();
else if (provider_type == onnxruntime::kAclExecutionProvider)
} else if (provider_type == onnxruntime::kAclExecutionProvider) {
execution_provider = DefaultAclExecutionProvider();
else if (provider_type == onnxruntime::kArmNNExecutionProvider)
} else if (provider_type == onnxruntime::kArmNNExecutionProvider) {
execution_provider = DefaultArmNNExecutionProvider();
else if (provider_type == onnxruntime::kRocmExecutionProvider)
} else if (provider_type == onnxruntime::kRocmExecutionProvider) {
execution_provider = DefaultRocmExecutionProvider();
else if (provider_type == onnxruntime::kCoreMLExecutionProvider)
} else if (provider_type == onnxruntime::kCoreMLExecutionProvider) {
execution_provider = DefaultCoreMLExecutionProvider();
}
// skip if execution provider is disabled
if (execution_provider == nullptr)
continue;

10
onnxruntime/test/providers/provider_test_utils.h
View file

@ -731,7 +731,6 @@ class OpTester {
const std::unordered_set<std::string>& excluded_provider_types = {},
const RunOptions* run_options = nullptr,
std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers = nullptr,
ExecutionMode execution_mode = ExecutionMode::ORT_SEQUENTIAL,
const Graph::ResolveOptions& resolve_options = {});
void Run(SessionOptions session_options,
@ -803,6 +802,15 @@ class OpTester {
return prepacked_weights_container_.GetNumberOfElements();
}
void SetUpDefaultSessionOptions(SessionOptions& so) {
so.use_per_session_threads = false;
so.session_logid = op_;
so.session_log_verbosity_level = 1;
so.execution_mode = ExecutionMode::ORT_SEQUENTIAL;
so.use_deterministic_compute = use_determinism_;
so.graph_optimization_level = TransformerLevel::Default; // 'Default' == off
}
bool test_allow_released_onnx_opset_only_ = true;
protected:

5
onnxruntime/test/util/default_providers.cc
View file

@ -5,6 +5,7 @@
#include "default_providers.h"
#include "providers.h"
#include "core/providers/cpu/cpu_provider_factory_creator.h"
#include "core/providers/cpu/cpu_execution_provider.h"
#ifdef USE_COREML
#include "core/providers/coreml/coreml_provider_factory.h"
#endif
@ -13,8 +14,8 @@
namespace onnxruntime {
namespace test {
std::unique_ptr<IExecutionProvider> DefaultCpuExecutionProvider(bool enable_arena) {
return CreateExecutionProviderFactory_CPU(enable_arena)->CreateProvider();
std::unique_ptr<IExecutionProvider> DefaultCpuExecutionProvider(const CPUExecutionProviderInfo& info ) {
return CreateExecutionProviderFactory_CPU(info)->CreateProvider();
}
std::unique_ptr<IExecutionProvider> DefaultTensorrtExecutionProvider() {

10
onnxruntime/test/util/include/default_providers.h
View file

@ -2,8 +2,9 @@
// Licensed under the MIT License.
#pragma once
#include "core/common/optional.h"
#include "core/providers/providers.h"
#include "core/framework/execution_provider.h"
#include "core/providers/cpu/cpu_execution_provider.h"
#include "core/providers/providers.h"
namespace onnxruntime {
@ -17,7 +18,7 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_MIGrap
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nnapi(
uint32_t flags, const optional<std::string>& partitioning_stop_ops_list);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nuphar(bool, const char*);
//std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tvm(const char*);
// std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tvm(const char*);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_OpenVINO(
const char* device_type, bool enable_vpu_fast_compile, const char* device_id, size_t num_of_threads, bool use_compiled_network, const char* blob_dump_path);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_OpenVINO(const OrtOpenVINOProviderOptions* params);
@ -32,11 +33,12 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Intern
namespace test {
// unique_ptr providers with default values for session registration
std::unique_ptr<IExecutionProvider> DefaultCpuExecutionProvider(bool enable_arena = true);
std::unique_ptr<IExecutionProvider> DefaultCpuExecutionProvider(const CPUExecutionProviderInfo& info =
CPUExecutionProviderInfo(true, false));
std::unique_ptr<IExecutionProvider> DefaultCudaExecutionProvider();
std::unique_ptr<IExecutionProvider> DefaultDnnlExecutionProvider(bool enable_arena = true);
std::unique_ptr<IExecutionProvider> DefaultNupharExecutionProvider(bool allow_unaligned_buffers = true);
//std::unique_ptr<IExecutionProvider> DefaultTvmExecutionProvider();
// std::unique_ptr<IExecutionProvider> DefaultTvmExecutionProvider();
std::unique_ptr<IExecutionProvider> DefaultTensorrtExecutionProvider();
std::unique_ptr<IExecutionProvider> TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptions* params);
std::unique_ptr<IExecutionProvider> TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptionsV2* params);

3
orttraining/orttraining/test/gradient/allreduce_op_test.cc
View file

@ -50,7 +50,6 @@ TEST(AllreduceTest, CPUAdasumAllreduceTestReduceTwoTensors) {
allreduce_test.Run(OpTester::ExpectResult::kExpectSuccess /*expect_result*/, "" /*expected_failure_string*/,
{} /*excluded_provider_types*/, nullptr /*run_options*/, &providers /*execution_providers*/,
ExecutionMode::ORT_SEQUENTIAL /*execution_mode*/,
{} /*resolve_options*/);
}
@ -93,7 +92,6 @@ TEST(AllreduceTest, CPUAdasumAllreduceTestReduceTwoTensorsFP16) {
allreduce_test.Run(OpTester::ExpectResult::kExpectSuccess /*expect_result*/, "" /*expected_failure_string*/,
{} /*excluded_provider_types*/, nullptr /*run_options*/, &providers /*execution_providers*/,
ExecutionMode::ORT_SEQUENTIAL /*execution_mode*/,
{} /*resolve_options*/);
}
@ -119,7 +117,6 @@ TEST(AllreduceTest, CPUAdasumAllreduceTestFailTensorCountMismatch) {
allreduce_test.Run(OpTester::ExpectResult::kExpectFailure /*expect_result*/, "" /*expected_failure_string*/,
{} /*excluded_provider_types*/, nullptr /*run_options*/, &providers /*execution_providers*/,
ExecutionMode::ORT_SEQUENTIAL /*execution_mode*/,
{} /*resolve_options*/);
}

4
orttraining/orttraining/test/training_ops/cpu/nn/broadcast_grad_args_test.cc
View file

@ -40,9 +40,9 @@ void RunBroadcastGradientArgsTest(const char* op,
std::vector<std::unique_ptr<IExecutionProvider>> execution_providers;
execution_providers.push_back(DefaultCpuExecutionProvider());
if (fail)
t.Run(OpTester::ExpectResult::kExpectFailure, "", {}, nullptr, &execution_providers, ExecutionMode::ORT_SEQUENTIAL);
t.Run(OpTester::ExpectResult::kExpectFailure, "", {}, nullptr, &execution_providers);
else
t.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers, ExecutionMode::ORT_SEQUENTIAL);
t.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers);
}
} // namespace