From 1f2c92673b2edf75460478d406f84aead7a140f3 Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Thu, 2 Jun 2022 12:34:17 -0700 Subject: [PATCH] fixed point based requantization on arm64 (#11540) * fixed point based requantization on arm64 * reverse MlasConvSymDepthwiseKernel u8s8 and s8s8 order --- .../onnxruntime_session_options_config_keys.h | 22 + .../cpu/quantization/quant_gemm.cc | 11 +- onnxruntime/core/mlas/inc/mlas.h | 131 ++++-- .../mlas/lib/aarch64/ConvSymS8KernelDot.S | 142 +++++- .../mlas/lib/aarch64/ConvSymS8KernelDotLd64.S | 171 +++++-- .../mlas/lib/aarch64/ConvSymS8KernelNeon.S | 117 ++++- .../mlas/lib/aarch64/ConvSymU8KernelDot.S | 169 +++++-- .../mlas/lib/aarch64/ConvSymU8KernelNeon.S | 156 +++++-- .../aarch64/DepthwiseQConvKernelSize9Neon.S | 269 ++++++++--- .../aarch64/DepthwiseQConvSymS8KernelNeon.S | 299 +++++++++--- .../aarch64/DepthwiseQConvSymU8KernelNeon.S | 424 +++++++++++------ .../mlas/lib/arm64/ConvSymS8KernelDot.asm | 142 +++++- .../mlas/lib/arm64/ConvSymS8KernelDotLd64.asm | 177 ++++++-- .../mlas/lib/arm64/ConvSymS8KernelNeon.asm | 124 ++++- .../mlas/lib/arm64/ConvSymU8KernelDot.asm | 182 ++++++-- .../mlas/lib/arm64/ConvSymU8KernelNeon.asm | 167 ++++--- .../arm64/DepthwiseQConvKernelSize9Neon.asm | 261 ++++++++--- .../arm64/DepthwiseQConvSymS8KernelNeon.asm | 294 +++++++++--- .../arm64/DepthwiseQConvSymU8KernelNeon.asm | 427 ++++++++++++------ onnxruntime/core/mlas/lib/convsym.cpp | 55 ++- onnxruntime/core/mlas/lib/mlasi.h | 11 +- .../core/mlas/lib/qdwconv_kernelsize.cpp | 267 +++++++++-- onnxruntime/core/mlas/lib/qlgavgpool.cpp | 18 +- onnxruntime/core/mlas/lib/quantize.cpp | 357 ++++++++++++++- .../providers/cpu/cpu_execution_provider.h | 17 +- .../providers/cpu/cpu_provider_factory.cc | 17 +- .../cpu/cpu_provider_factory_creator.h | 3 +- .../providers/cpu/quantization/qlinearconv.cc | 36 +- .../quantization/quantize_linear_matmul.cc | 37 +- .../cpu/quantization/quantize_linear_matmul.h | 8 +- onnxruntime/core/session/inference_session.cc | 4 +- .../python/onnxruntime_pybind_schema.cc | 2 +- .../python/onnxruntime_pybind_state.cc | 6 +- .../test/contrib_ops/fastgelu_op_test.cc | 2 +- onnxruntime/test/eager/ort_invoker_test.cc | 33 +- .../test/framework/parallel_executor_test.cc | 19 +- .../test/framework/session_state_test.cc | 12 +- .../test/perftest/command_args_parser.cc | 6 +- onnxruntime/test/perftest/ort_test_session.cc | 4 + .../test/perftest/test_configuration.h | 1 + .../providers/cpu/controlflow/loop_test.cc | 5 +- .../cpu/math/quantize_linear_matmul_test.cc | 82 +++- .../providers/cpu/nn/qlinearconv_op_test.cc | 97 +++- .../test/providers/provider_test_utils.cc | 62 +-- .../test/providers/provider_test_utils.h | 10 +- onnxruntime/test/util/default_providers.cc | 5 +- .../test/util/include/default_providers.h | 10 +- .../test/gradient/allreduce_op_test.cc | 3 - .../cpu/nn/broadcast_grad_args_test.cc | 4 +- 49 files changed, 3776 insertions(+), 1102 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h index f1a7009a89..63ef08a5bb 100644 --- a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h +++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h @@ -119,3 +119,25 @@ static const char* const kOrtSessionOptionsConfigDynamicBlockBase = "session.dyn // "0": in some cases warnings will be logged but processing will continue. The default. // May be useful to expose bugs in models. static const char* const kOrtSessionOptionsConfigStrictShapeTypeInference = "session.strict_shape_type_inference"; + +// SessionOption 'fixed_point_requant_on_arm64' controls the requantization method on ARM devices. +// Requantization is computed with formula: +// v = round(clamp(S * (I - Z), min, max)) +// where v is the target value with type TOutput, which is either int8_t or uint8_t +// I is the input value with type int32_t +// S is the scale with type float +// Z is the zero point with type same as TOutput. +// min is the minimum value of type TOutput. +// max is the maximum value of type TOutput. +// For considerations of power consumption and some ARM devices don't even have FPUs, it is import to to be able to run +// quantization with integer instructions only.FixedPoint Requantization is introduced to support this feature. +// Its general idea is to convert scale S to fixed point. Ruy and XNNPack's method are referred for the implementation. +// https://github.com/google/ruy/blob/a09683b8da7164b9c5704f88aef2dc65aa583e5d/ruy/apply_multiplier.cc#L48 +// https://github.com/google/XNNPACK/blob/1e37b200d3f4ba19151eb30c1c329873d541326c/src/params-init.c#L211 +// "0": disable. ORT uses float point based requantization on ARM devices. +// "1": enable. ORT uses fixed point based requantization on ARM devices. +// Its default value is "0" +// **NOTE** that fixed point requantization rounds half to up, whereas ONNX spec rounds half to even, so for identical +// model and input the inference results may not be exactly same with this option on and off. The impact should be +// small in practice (NNApi EP uses same rounding). +static const char* const kOrtSessionOptionsConfigFixedPointRequantOnARM64 = "session.fixed_point_requant_on_arm64"; diff --git a/onnxruntime/contrib_ops/cpu/quantization/quant_gemm.cc b/onnxruntime/contrib_ops/cpu/quantization/quant_gemm.cc index 8d83ab7f68..c0ab9ef7aa 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/quant_gemm.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/quant_gemm.cc @@ -104,7 +104,8 @@ class QGemm : protected GemmBase, public MatMulIntegerBase { std::vector output_scales = ComputeOutputScale(a_scale, b_scale, y_scale); std::unique_ptr scale_bias_proc_ptr; std::unique_ptr requant_proc_ptr; - SetPostProcessor(y_zp, N, output_scales, y, gemm_param, scale_bias_proc_ptr, requant_proc_ptr); + std::unique_ptr requant_param; + SetPostProcessor(y_zp, N, output_scales, y, gemm_param, scale_bias_proc_ptr, requant_proc_ptr, requant_param); MlasGemmBatch(gemm_shape, &gemm_param, 1, context->GetOperatorThreadPool()); return Status::OK(); @@ -181,17 +182,17 @@ class QGemm : protected GemmBase, public MatMulIntegerBase { Tensor* y, MLAS_GEMM_QUANT_DATA_PARAMS& gemm_param, std::unique_ptr& scale_bias_proc_ptr, - std::unique_ptr& requant_proc_ptr) { + std::unique_ptr& requant_proc_ptr, + std::unique_ptr& requant_param) { if (nullptr != y_zp) { bool is_y_signed = y->IsDataType(); int32_t y_zero_point = is_y_signed ? *y_zp->template Data() : *y_zp->template Data(); + requant_param = std::make_unique(output_scales.data(), output_scales.size(), y_zero_point); requant_proc_ptr = std::make_unique( y->MutableDataRaw(), out_lda, nullptr, - output_scales.data(), - output_scales.size() > 1, - y_zero_point, + requant_param.get(), is_y_signed); gemm_param.OutputProcessor = requant_proc_ptr.get(); } else { diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index 9d8cc96153..142425a2d5 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -535,7 +535,7 @@ private: /** * @brief Supply matrices shape and data type information to quantized gemm functions * - ** NOTE: AIsSigned == true is not supported on non-ARM devices for now. + ** NOTE: AIsSigned == true is not supported on non-ARM devices for now. ** AIsSigned == true is supported on ARM devices when BIsSigned is also true. * */ @@ -623,10 +623,10 @@ struct MLAS_SYMM_QGEMM_DATA_PARAMS { * @param [IN] Shape A single shape descriptor for all multiplicatons. Currently A and B must be signed, and accumulation mode not supported - * @param [IN] DataParams Array of data descriptors, one for each mutliplication + * @param [IN] DataParams Array of data descriptors, one for each multiplication * B must be prepacked * @param [IN] BatchN Number of multiplications - * @param [IN] ThreadPool + * @param [IN] ThreadPool */ void MLASCALL @@ -683,8 +683,8 @@ MlasGemmPackB( /** * @brief For symmetric quantized GEMM, returns size of the - * packing buffer needed for right hand side - * @param N Number of columns + * packing buffer needed for right hand side + * @param N Number of columns * @param K Number of rows * @param AIsSigned Whether left hand size is signed int8_t * @return size of the packing buffer, @@ -694,7 +694,7 @@ size_t MLASCALL MlasSymmQgemmPackBSize( size_t N, - size_t K, + size_t K, bool AIsSigned ); @@ -829,6 +829,95 @@ MlasConvSymFixupInputZeroPoint( bool InputIsSigned ); +inline +uint32_t +BitsOfFp32(float FloatValue) +{ + union { + uint32_t IntegerValue; + float FloatValue; + } u; + u.FloatValue = FloatValue; + return u.IntegerValue; +} + +inline +void +MlasFloatToFixedPoint( + float Scale, + int32_t& Multiplier, + int32_t& PreShift, + int32_t& PostShift) +{ + const uint32_t ScaleBits = BitsOfFp32(Scale); + + // Multiplier is in [0x40000000, 0x7FFFFF80] range. + Multiplier = (int32_t)(((ScaleBits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000)) << 7); + + // Shift is in [-8, 31] range. + const int32_t Shift = 127 + 31 - 32 - (ScaleBits >> 23); + + // Split shift into PreShift + PostShift, PostShift in [1, 31] range. + PostShift = Shift > 1 ? Shift : 1; + PreShift = Shift - PostShift; + + PreShift = -PreShift; + PostShift = -PostShift; +} + +inline +void +MlasFloatToFixedPoint( + const float* Scale, + int32_t* Multiplier, + int32_t* PreShift, + int32_t* PostShift, + size_t N) +{ + for(size_t i = 0; i < N; i++) { + MlasFloatToFixedPoint(Scale[i], Multiplier[i], PreShift[i], PostShift[i]); + } +} + +enum MLAS_ROUND_KIND { + MlasRoundHalfEven, + MlasRoundHalfUp, +}; + + +/** + * @brief Supply parameters to requantize a tensor. It supports 2 modes: + ** RequantRoundKind == MlasRoundHalfEven: Scale is used. + ** RequantRoundKind == MlasRoundHalfUp: Multiplier, PreShift and PostShift are used. + * +*/ +struct MLAS_REQUANT_PARAM { + MLAS_REQUANT_PARAM()=default; + + MLAS_REQUANT_PARAM(const float* Scale, + size_t Size, + int32_t ZeroPoint): + RequantRoundKind(MLAS_ROUND_KIND::MlasRoundHalfEven), + Scale(Scale), Size(Size), ZeroPoint(ZeroPoint){} + + MLAS_REQUANT_PARAM(const int32_t* Multiplier, + const int32_t* PreShift, + const int32_t* PostShift, + size_t Size, + int32_t ZeroPoint): + RequantRoundKind(MLAS_ROUND_KIND::MlasRoundHalfUp), + Multiplier(Multiplier), PreShift(PreShift), PostShift(PostShift), + Size(Size), ZeroPoint(ZeroPoint){} + + MLAS_ROUND_KIND RequantRoundKind; + const float* Scale; + const int32_t* Multiplier; + const int32_t* PreShift; + const int32_t* PostShift; + size_t Size; + int32_t ZeroPoint; +}; + struct MLAS_CONV_SYM_PARAMS { const void* InputDirect; const void* const* InputIndirection; @@ -839,9 +928,7 @@ struct MLAS_CONV_SYM_PARAMS { size_t OutputCount; size_t KernelSize; const int32_t* Bias; - const float* Scale; - bool PerChannelScale; - int32_t OutputZeroPoint; + const MLAS_REQUANT_PARAM* RequantParam; bool InputIsSigned; }; @@ -1137,9 +1224,7 @@ MlasQuantizeLinear( * @param OutputLeadingDimension Output matrix leading dimension * @param Bias Optional bias vector, to be added to the input before quantization - * @param Scale Quantization scale - * @param PerColumnScale true if scale is per-column - * @param ZeroPoint quantization zero point value + * @param MLAS_REQUANT_PARAM Requantization parameters * @param StartM * @param StartN * @param CountM @@ -1155,9 +1240,7 @@ MlasRequantizeOutput( OutputType* Output, size_t OutputLeadingDimension, const int32_t* Bias, - const float* Scale, - bool PerColumnScale, - OutputType ZeroPoint, + const MLAS_REQUANT_PARAM* RequantParam, size_t StartM, size_t StartN, size_t CountM, @@ -1171,16 +1254,12 @@ class MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR : public MLAS_QGEMM_OUTPUT_PROCESSOR void* Output, size_t OutputLeadingDimension, const int32_t* Bias, - const float* Scale, - bool PerColumnScale, - int32_t ZeroPoint, + const MLAS_REQUANT_PARAM* RequantParam, bool OutputIsSigned) : Output_(Output), OutputLeadingDimension_(OutputLeadingDimension), Bias_(Bias), - Scale_(Scale), - PerColumnScale_(PerColumnScale), - ZeroPoint_(ZeroPoint), + RequantParam_(RequantParam), OutputIsSigned_(OutputIsSigned) { } @@ -1194,12 +1273,10 @@ class MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR : public MLAS_QGEMM_OUTPUT_PROCESSOR { if(OutputIsSigned_){ MlasRequantizeOutput(C, ldc, reinterpret_cast(Output_), OutputLeadingDimension_, - Bias_, Scale_, PerColumnScale_, static_cast(ZeroPoint_), - StartM, StartN, CountM, CountN); + Bias_, RequantParam_, StartM, StartN, CountM, CountN); } else { MlasRequantizeOutput(C, ldc, reinterpret_cast(Output_), OutputLeadingDimension_, - Bias_, Scale_, PerColumnScale_, static_cast(ZeroPoint_), - StartM, StartN, CountM, CountN); + Bias_, RequantParam_, StartM, StartN, CountM, CountN); } } @@ -1208,9 +1285,7 @@ class MLAS_QGEMM_REQUANT_OUTPUT_PROCESSOR : public MLAS_QGEMM_OUTPUT_PROCESSOR void* Output_; size_t OutputLeadingDimension_; const int32_t* Bias_; - const float* Scale_; - bool PerColumnScale_; - int32_t ZeroPoint_; + const MLAS_REQUANT_PARAM* RequantParam_; bool OutputIsSigned_; }; diff --git a/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelDot.S b/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelDot.S index 30b7276340..77223c887f 100644 --- a/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelDot.S +++ b/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelDot.S @@ -19,20 +19,43 @@ Abstract: #include "AssembleDotProduct.h" .equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2 + .equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4 // // Stack frame layout for the symmetric convolution kernel. // d8-d15, x19-x30 need to be preserved if used // - .equ .LConvSymFrame_SavedRegisters, (6 * 8) - .equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters - .equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters - .equ .LConvSymPostProcessParams_Bias, 0 - .equ .LConvSymPostProcessParams_Scale, 8 - .equ .LConvSymPostProcessParams_Min, 16 - .equ .LConvSymPostProcessParams_Max, 20 - .equ .LConvSymPostProcessParams_ZeroPoint, 24 + .equ .LConvSymFrame_SavedRegisters_d8_d9, 0 + .equ .LConvSymFrame_SavedRegisters_d10_d11, 16 + .equ .LConvSymFrame_SavedRegisters_x19_x20, 32 + .equ .LConvSymFrame_SavedRegisters_x21_x22, 48 + .equ .LConvSymFrame_SavedRegisters, 64 + .equ .LConvSymFrame_SavedRegisters_Neg, -64 + .equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters + .equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters + +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ + .equ .LConvSymPostProcessParams_Bias, 0 + .equ .LConvSymPostProcessParams_Scale, 8 + .equ .LConvSymPostProcessParams_Min, 16 + .equ .LConvSymPostProcessParams_Max, 20 + .equ .LConvSymPostProcessParams_ZeroPoint, 24 + .equ .LConvSymPostProcessParams_Multiplier, 32 + .equ .LConvSymPostProcessParams_PreShift, 40 + .equ .LConvSymPostProcessParams_PostShift, 48 .text @@ -78,13 +101,13 @@ Return Value: --*/ FUNCTION_ENTRY MlasConvSymS8KernelDot - stp d8,d9,[sp,#-.LConvSymFrame_SavedRegisters]! + stp d8,d9,[sp,#.LConvSymFrame_SavedRegisters_Neg]! ldr x8,[sp,#.LConvSymFrame_PostProcessParams] - str d10,[sp,#16] cmp x7,2 // OutputCount < 2 ? - str d11,[sp,#24] add x16,x2,x5 // x16 -> C1 - str x19,[sp,#32] + stp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11] + stp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20] + stp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22] lsl x3,x3,#3 // KernelSize * sizeof(int8_t*) csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0 add x4,x4,3 // InputChannels align to 4 @@ -97,6 +120,9 @@ Return Value: add x5,x17,x5 // x5 -> C3 ldr x19,[x8,#.LConvSymPostProcessParams_Scale] csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2 + ldr x20,[x8,#.LConvSymPostProcessParams_PreShift] + ldr x21,[x8,#.LConvSymPostProcessParams_Multiplier] + ldr x22,[x8,#.LConvSymPostProcessParams_PostShift] // TODO!! tiptoe around loading biases if we need to support // output channels none divisible by 16 @@ -347,12 +373,15 @@ InChLoopEpilogue: b.hi KernelSizeLoop Requantize: + tst w10,#.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale +FloatPointScale: tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint] beq BroadcastScaleValue ldp q0,q1,[x19],32 // load scale vector ldp q2,q3,[x19],32 - b AccumulatorsToFloat + b ScaleWithFloatPoint BroadcastScaleValue: ld1r {v0.4s},[x19] // load scale Value @@ -360,7 +389,7 @@ BroadcastScaleValue: mov v2.16b, v0.16b mov v3.16b, v0.16b -AccumulatorsToFloat: +ScaleWithFloatPoint: scvtf v16.4s,v16.4s // convert to float scvtf v17.4s,v17.4s scvtf v18.4s,v18.4s @@ -409,7 +438,87 @@ AccumulatorsToFloat: fcvtns v29.4s,v29.4s fcvtns v30.4s,v30.4s fcvtns v31.4s,v31.4s + b Quantize +FixedPointScale: + tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq BroadcastFixedPointValue + ldp q0,q1,[x20],32 // load preshift vector + ldp q2,q3,[x20],32 + ldp q4,q5,[x21],32 // load multiplier vector + ldp q6,q7,[x21],32 + ldp q8,q9,[x22],32 // load postshift vector + ldp q10,q11,[x22],32 + b ScaleWithFixedPoint + +BroadcastFixedPointValue: + ld1r {v0.4s},[x20] // load preshift Value + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + + ld1r {v4.4s},[x21] // load multiplier Value + mov v5.16b, v4.16b + mov v6.16b, v4.16b + mov v7.16b, v4.16b + + ld1r {v8.4s},[x22] // load postshift Value + mov v9.16b, v8.16b + mov v10.16b, v8.16b + mov v11.16b, v8.16b + +ScaleWithFixedPoint: + sqshl v16.4s, v16.4s, v0.4s // preshift + sqshl v17.4s, v17.4s, v0.4s + sqshl v18.4s, v18.4s, v0.4s + sqshl v19.4s, v19.4s, v0.4s + sqshl v20.4s, v20.4s, v1.4s + sqshl v21.4s, v21.4s, v1.4s + sqshl v22.4s, v22.4s, v1.4s + sqshl v23.4s, v23.4s, v1.4s + sqshl v24.4s, v24.4s, v2.4s + sqshl v25.4s, v25.4s, v2.4s + sqshl v26.4s, v26.4s, v2.4s + sqshl v27.4s, v27.4s, v2.4s + sqshl v28.4s, v28.4s, v3.4s + sqshl v29.4s, v29.4s, v3.4s + sqshl v30.4s, v30.4s, v3.4s + sqshl v31.4s, v31.4s, v3.4s + sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale + sqdmulh v17.4s, v17.4s, v4.4s + sqdmulh v18.4s, v18.4s, v4.4s + sqdmulh v19.4s, v19.4s, v4.4s + sqdmulh v20.4s, v20.4s, v5.4s + sqdmulh v21.4s, v21.4s, v5.4s + sqdmulh v22.4s, v22.4s, v5.4s + sqdmulh v23.4s, v23.4s, v5.4s + sqdmulh v24.4s, v24.4s, v6.4s + sqdmulh v25.4s, v25.4s, v6.4s + sqdmulh v26.4s, v26.4s, v6.4s + sqdmulh v27.4s, v27.4s, v6.4s + sqdmulh v28.4s, v28.4s, v7.4s + sqdmulh v29.4s, v29.4s, v7.4s + sqdmulh v30.4s, v30.4s, v7.4s + sqdmulh v31.4s, v31.4s, v7.4s + srshl v16.4s,v16.4s, v8.4s // post shift + srshl v17.4s,v17.4s, v8.4s + srshl v18.4s,v18.4s, v8.4s + srshl v19.4s,v19.4s, v8.4s + srshl v20.4s,v20.4s, v9.4s + srshl v21.4s,v21.4s, v9.4s + srshl v22.4s,v22.4s, v9.4s + srshl v23.4s,v23.4s, v9.4s + srshl v24.4s,v24.4s, v10.4s + srshl v25.4s,v25.4s, v10.4s + srshl v26.4s,v26.4s, v10.4s + srshl v27.4s,v27.4s, v10.4s + srshl v28.4s,v28.4s, v11.4s + srshl v29.4s,v29.4s, v11.4s + srshl v30.4s,v30.4s, v11.4s + srshl v31.4s,v31.4s, v11.4s + +Quantize: sqxtn v16.4h,v16.4s sqxtn v17.4h,v17.4s sqxtn v18.4h,v18.4s @@ -454,8 +563,9 @@ AccumulatorsToFloat: b.hi OutputChannelLoop ExitKernel: - ldr x19,[sp,#32] - ldp d10,d11,[sp,#16] + ldp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22] + ldp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20] + ldp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11] ldp d8,d9,[sp],#.LConvSymFrame_SavedRegisters ret diff --git a/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelDotLd64.S b/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelDotLd64.S index 3e03ff7b42..db9e191489 100644 --- a/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelDotLd64.S +++ b/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelDotLd64.S @@ -18,22 +18,51 @@ Abstract: #include "asmmacro.h" #include "AssembleDotProduct.h" - .equ .LMLAS_CONV_SYM_FLAG_INPUT_DIRECT, 1 .equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2 + .equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4 // // Stack frame layout for the symmetric convolution kernel. // d8-d15, x19-x30 need to be preserved if used // - .equ .LConvSymFrame_SavedRegisters, (10 * 8) + .equ .LConvSymFrame_SavedRegisters_d8, 0 + .equ .LConvSymFrame_SavedRegisters_d9, 8 + .equ .LConvSymFrame_SavedRegisters_d10, 16 + .equ .LConvSymFrame_SavedRegisters_d11, 24 + .equ .LConvSymFrame_SavedRegisters_x19, 32 + .equ .LConvSymFrame_SavedRegisters_x20, 40 + .equ .LConvSymFrame_SavedRegisters_x21, 48 + .equ .LConvSymFrame_SavedRegisters_x22, 56 + .equ .LConvSymFrame_SavedRegisters_x23, 64 + .equ .LConvSymFrame_SavedRegisters_x24, 72 + .equ .LConvSymFrame_SavedRegisters_x25, 80 + .equ .LConvSymFrame_SavedRegisters_x26, 88 + .equ .LConvSymFrame_SavedRegisters, 96 + .equ .LConvSymFrame_SavedRegisters_Neg, -96 .equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters .equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ .equ .LConvSymPostProcessParams_Bias, 0 .equ .LConvSymPostProcessParams_Scale, 8 .equ .LConvSymPostProcessParams_Min, 16 .equ .LConvSymPostProcessParams_Max, 20 .equ .LConvSymPostProcessParams_ZeroPoint, 24 + .equ .LConvSymPostProcessParams_Multiplier,32 + .equ .LConvSymPostProcessParams_PreShift, 40 + .equ .LConvSymPostProcessParams_PostShift, 48 .text @@ -46,24 +75,17 @@ Routine Description: Arguments: - Input (x0) - Points to the input buffer. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points - directly at the input tensor. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an - indirection buffer. Every pointer in the indirection buffer points at a - InputChannels length vector (either from the input tensor or a vector of - padding values). These are grouped in batches of length KernelSize. - These batches are then repeated OutputCount times. + Input (x0) - Points to the indirection buffer. Every pointer in the indirection + buffer points at a InputChannels length vector (either from the input tensor + or a vector of padding values). These are grouped in batches of length + KernelSize. These batches are then repeated OutputCount times. Filter (x1) - Points to the filter buffer. Output (x2) - Points the output buffer. KernelSize (x3/x9) - Size of the kernel (most commonly. 3x3=9, 5x5=25). - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1. + Must be > 1 InputChannels (x4/x7) - Number of input channels. @@ -86,30 +108,35 @@ Return Value: --*/ FUNCTION_ENTRY MlasConvSymS8KernelDotLd64 - stp d8,d9,[sp,#-.LConvSymFrame_SavedRegisters]! + stp d8,d9,[sp,#.LConvSymFrame_SavedRegisters_Neg]! ldr x8,[sp,#.LConvSymFrame_PostProcessParams] - str d10,[sp,#16] + str d10,[sp,#.LConvSymFrame_SavedRegisters_d10] cmp x7,2 // OutputCount < 2 ? - str d11,[sp,#24] + str d11,[sp,#.LConvSymFrame_SavedRegisters_d11] add x16,x2,x5 // x16 -> C1 - str x19,[sp,#32] + str x19,[sp,#.LConvSymFrame_SavedRegisters_x19] lsl x3,x3,#3 // KernelSize * sizeof(int8_t*) - str x20,[sp,#40] + str x20,[sp,#.LConvSymFrame_SavedRegisters_x20] csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0 - str x21,[sp,#48] + str x21,[sp,#.LConvSymFrame_SavedRegisters_x21] add x4,x4,3 // InputChannels align to 4 - str x22,[sp,#56] + str x22,[sp,#.LConvSymFrame_SavedRegisters_x22] add x17,x16,x5 // x17 -> C2 - str x23,[sp,#64] + str x23,[sp,#.LConvSymFrame_SavedRegisters_x23] ldr x11,[x8,#.LConvSymPostProcessParams_Bias] + str x24,[sp,#.LConvSymFrame_SavedRegisters_x24] csel x17,x16,x17,ls // if OutputCount <= 2 x17/C2 -> C1 + str x25,[sp,#.LConvSymFrame_SavedRegisters_x25] bic x4,x4,3 + str x26,[sp,#.LConvSymFrame_SavedRegisters_x26] cmp x7,4 // OutputCount < 4 ? ldr w10,[sp,#.LConvSymFrame_KernelFlags] add x5,x17,x5 // x5 -> C3 ldr x19,[x8,#.LConvSymPostProcessParams_Scale] csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2 - + ldr x24,[x8,#.LConvSymPostProcessParams_PreShift] + ldr x25,[x8,#.LConvSymPostProcessParams_Multiplier] + ldr x26,[x8,#.LConvSymPostProcessParams_PostShift] // TODO!! tiptoe around loading biases if we need to support // output channels none divisible by 16 OutputChannelLoop: @@ -423,12 +450,15 @@ InChLoopEpilogue: b.hi KernelSizeLoop Requantize: + tst w10,#.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale +FloatPointScale: tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint] beq BroadcastScaleValue ldp q0,q1,[x19],32 // load scale vector ldp q2,q3,[x19],32 - b AccumulatorsToFloat + b ScaleWithFloatPoint BroadcastScaleValue: ld1r {v0.4s},[x19] // load scale Value @@ -436,7 +466,7 @@ BroadcastScaleValue: mov v2.16b, v0.16b mov v3.16b, v0.16b -AccumulatorsToFloat: +ScaleWithFloatPoint: scvtf v16.4s,v16.4s // convert to float scvtf v17.4s,v17.4s scvtf v18.4s,v18.4s @@ -485,7 +515,87 @@ AccumulatorsToFloat: fcvtns v29.4s,v29.4s fcvtns v30.4s,v30.4s fcvtns v31.4s,v31.4s + b Quantize +FixedPointScale: + tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq BroadcastFixedPointValue + ldp q0,q1,[x24],32 // load preshift vector + ldp q2,q3,[x24],32 + ldp q4,q5,[x25],32 // load multiplier vector + ldp q6,q7,[x25],32 + ldp q8,q9,[x26],32 // load postshift vector + ldp q10,q11,[x26],32 + b ScaleWithFixedPoint + +BroadcastFixedPointValue: + ld1r {v0.4s},[x24] // load preshift Value + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + + ld1r {v4.4s},[x25] // load multiplier Value + mov v5.16b, v4.16b + mov v6.16b, v4.16b + mov v7.16b, v4.16b + + ld1r {v8.4s},[x26] // load postshift Value + mov v9.16b, v8.16b + mov v10.16b, v8.16b + mov v11.16b, v8.16b + +ScaleWithFixedPoint: + sqshl v16.4s, v16.4s, v0.4s // preshift + sqshl v17.4s, v17.4s, v0.4s + sqshl v18.4s, v18.4s, v0.4s + sqshl v19.4s, v19.4s, v0.4s + sqshl v20.4s, v20.4s, v1.4s + sqshl v21.4s, v21.4s, v1.4s + sqshl v22.4s, v22.4s, v1.4s + sqshl v23.4s, v23.4s, v1.4s + sqshl v24.4s, v24.4s, v2.4s + sqshl v25.4s, v25.4s, v2.4s + sqshl v26.4s, v26.4s, v2.4s + sqshl v27.4s, v27.4s, v2.4s + sqshl v28.4s, v28.4s, v3.4s + sqshl v29.4s, v29.4s, v3.4s + sqshl v30.4s, v30.4s, v3.4s + sqshl v31.4s, v31.4s, v3.4s + sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale + sqdmulh v17.4s, v17.4s, v4.4s + sqdmulh v18.4s, v18.4s, v4.4s + sqdmulh v19.4s, v19.4s, v4.4s + sqdmulh v20.4s, v20.4s, v5.4s + sqdmulh v21.4s, v21.4s, v5.4s + sqdmulh v22.4s, v22.4s, v5.4s + sqdmulh v23.4s, v23.4s, v5.4s + sqdmulh v24.4s, v24.4s, v6.4s + sqdmulh v25.4s, v25.4s, v6.4s + sqdmulh v26.4s, v26.4s, v6.4s + sqdmulh v27.4s, v27.4s, v6.4s + sqdmulh v28.4s, v28.4s, v7.4s + sqdmulh v29.4s, v29.4s, v7.4s + sqdmulh v30.4s, v30.4s, v7.4s + sqdmulh v31.4s, v31.4s, v7.4s + srshl v16.4s,v16.4s, v8.4s // post shift + srshl v17.4s,v17.4s, v8.4s + srshl v18.4s,v18.4s, v8.4s + srshl v19.4s,v19.4s, v8.4s + srshl v20.4s,v20.4s, v9.4s + srshl v21.4s,v21.4s, v9.4s + srshl v22.4s,v22.4s, v9.4s + srshl v23.4s,v23.4s, v9.4s + srshl v24.4s,v24.4s, v10.4s + srshl v25.4s,v25.4s, v10.4s + srshl v26.4s,v26.4s, v10.4s + srshl v27.4s,v27.4s, v10.4s + srshl v28.4s,v28.4s, v11.4s + srshl v29.4s,v29.4s, v11.4s + srshl v30.4s,v30.4s, v11.4s + srshl v31.4s,v31.4s, v11.4s + +Quantize: sqxtn v16.4h,v16.4s sqxtn v17.4h,v17.4s sqxtn v18.4h,v18.4s @@ -530,10 +640,11 @@ AccumulatorsToFloat: b.hi OutputChannelLoop ExitKernel: - ldr x23,[sp,#64] - ldp x21,x22,[sp,#48] - ldp x19,x20,[sp,#32] - ldp d10,d11,[sp,#16] + ldp x25,x26,[sp,#.LConvSymFrame_SavedRegisters_x25] + ldp x23,x24,[sp,#.LConvSymFrame_SavedRegisters_x23] + ldp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21] + ldp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19] + ldp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10] ldp d8,d9,[sp],#.LConvSymFrame_SavedRegisters ret @@ -588,7 +699,7 @@ InChannels4: ldr s1,[x13],4 ldr s2,[x14],4 ldr s3,[x15],4 - ldr q5, [x1], 16 + ldr q5,[x1],16 SdotByElement 16, 4, 0,0 SdotByElement 17, 4, 1,0 ldp q6, q7, [x1], 32 diff --git a/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelNeon.S b/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelNeon.S index 9f623ee7b2..2819d48e3c 100644 --- a/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelNeon.S +++ b/onnxruntime/core/mlas/lib/aarch64/ConvSymS8KernelNeon.S @@ -18,21 +18,46 @@ Abstract: #include "asmmacro.h" .equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2 + .equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4 // // Stack frame layout for the symmetric convolution kernel. // d8-d15, x19-x30 need to be preserved if used // - .equ .LConvSymFrame_SavedNeonRegisters, (8 * 8) - .equ .LConvSymFrame_SavedRegisters, .LConvSymFrame_SavedNeonRegisters - .equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters - .equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters + .equ .LConvSymFrame_SavedRegisters_d8_d9, 0 + .equ .LConvSymFrame_SavedRegisters_d10_d11, 16 + .equ .LConvSymFrame_SavedRegisters_d12_d13, 32 + .equ .LConvSymFrame_SavedRegisters_d14_d15, 48 + .equ .LConvSymFrame_SavedNeonRegisters, 64 + .equ .LConvSymFrame_SavedRegisters_x19_x20, 64 + .equ .LConvSymFrame_SavedRegisters_x21_x22, 80 + .equ .LConvSymFrame_SavedRegisters, 96 + .equ .LConvSymFrame_SavedRegisters_Neg, -96 + .equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters + .equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters + +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ .equ .LConvSymPostProcessParams_Bias, 0 .equ .LConvSymPostProcessParams_Scale, 8 .equ .LConvSymPostProcessParams_Min, 16 .equ .LConvSymPostProcessParams_Max, 20 .equ .LConvSymPostProcessParams_ZeroPoint, 24 + .equ .LConvSymPostProcessParams_Multiplier,32 + .equ .LConvSymPostProcessParams_PreShift, 40 + .equ .LConvSymPostProcessParams_PostShift, 48 .text @@ -82,12 +107,15 @@ Return Value: --*/ FUNCTION_ENTRY MlasConvSymS8KernelNeon - stp d8,d9,[sp,#-.LConvSymFrame_SavedRegisters]! + stp d8,d9,[sp,#.LConvSymFrame_SavedRegisters_Neg]! ldr x8,[sp,#.LConvSymFrame_PostProcessParams] ldrb w10,[sp,#.LConvSymFrame_KernelFlags] - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] + stp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11] + stp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13] + stp d14,d15,[sp,#.LConvSymFrame_SavedRegisters_d14_d15] + stp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20] + stp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22] + mov x9,x3 // save kernel size ldr x11,[x8,#.LConvSymPostProcessParams_Bias] mov x16,x4 // save input channels @@ -96,6 +124,9 @@ Return Value: add x5,x2,x5 // c1 = c0 + ldc add x4,x4,7 // kc = (kc + 7) & ~7 csel x5,x2,x5,lo // if OutputCount < 2 c1 = c0 + ldr x19,[x8,#.LConvSymPostProcessParams_PreShift] + ldr x20,[x8,#.LConvSymPostProcessParams_Multiplier] + ldr x21,[x8,#.LConvSymPostProcessParams_PostShift] bic x4,x4,7 ldp s16,s18,[x11],8 // init accumulators with bias ldp s20,s22,[x11],8 @@ -296,17 +327,20 @@ Return Value: b.hi .LConvSym.KernelSizeLoop .LConvSym.Requantize: - ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint] + tst w10,#.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne .LConvSym.FixedPointScale +.LConvSym.FloatPointScale: tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint] beq .LConvSym.BroadcastScaleValue ld1 {v4.4s,v5.4s},[x12] // load scale vector - b .LConvSym.AccumulatorsToFloat + b .LConvSym.ScaleWithFloatPoint .LConvSym.BroadcastScaleValue: ld1r {v4.4s},[x12] // load scale Value mov v5.16b, v4.16b -.LConvSym.AccumulatorsToFloat: +.LConvSym.ScaleWithFloatPoint: addp v16.4s,v16.4s,v18.4s addp v20.4s,v20.4s,v22.4s addp v24.4s,v24.4s,v26.4s @@ -329,12 +363,57 @@ Return Value: fmul v3.4s,v3.4s,v5.4s fcvtns v0.4s,v0.4s // convert to int fcvtns v1.4s,v1.4s - dup v9.8h,w11 fcvtns v2.4s,v2.4s fcvtns v3.4s,v3.4s + b .LConvSym.Quantize + +.LConvSym.FixedPointScale: + tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint] + beq .LConvSym.BroadcastFixedPointValue + ld1 {v4.4s,v5.4s},[x19] // load preshift + ld1 {v6.4s,v7.4s},[x20] // load multiplier + ld1 {v8.4s,v9.4s},[x21] // load postshift + b .LConvSym.ScaleWithFixedPoint +.LConvSym.BroadcastFixedPointValue: + ld1r {v4.4s},[x19] // load preshift Value + mov v5.16b, v4.16b + ld1r {v6.4s},[x20] // load multiplier Value + mov v7.16b, v6.16b + ld1r {v8.4s},[x21] // load postshift Value + mov v9.16b, v8.16b + +.LConvSym.ScaleWithFixedPoint: + addp v16.4s,v16.4s,v18.4s + addp v20.4s,v20.4s,v22.4s + addp v24.4s,v24.4s,v26.4s + addp v28.4s,v28.4s,v30.4s + addp v17.4s,v17.4s,v19.4s + addp v21.4s,v21.4s,v23.4s + addp v25.4s,v25.4s,v27.4s + addp v29.4s,v29.4s,v31.4s + addp v0.4s,v16.4s,v20.4s + addp v1.4s,v24.4s,v28.4s + addp v2.4s,v17.4s,v21.4s + addp v3.4s,v25.4s,v29.4s + sqshl v0.4s, v0.4s, v4.4s // preshift + sqshl v1.4s, v1.4s, v5.4s + sqshl v2.4s, v2.4s, v4.4s + sqshl v3.4s, v3.4s, v5.4s + sqdmulh v0.4s, v0.4s, v6.4s // multiplier + sqdmulh v1.4s, v1.4s, v7.4s + sqdmulh v2.4s, v2.4s, v6.4s + sqdmulh v3.4s, v3.4s, v7.4s + srshl v0.4s,v0.4s, v8.4s // postshift + srshl v1.4s,v1.4s, v9.4s + srshl v2.4s,v2.4s, v8.4s + srshl v3.4s,v3.4s, v9.4s + +.LConvSym.Quantize: + dup v9.8h,w11 sqxtn v0.4h,v0.4s - sqxtn2 v0.8h,v1.4s sqxtn v2.4h,v2.4s + sqxtn2 v0.8h,v1.4s sqxtn2 v2.8h,v3.4s subs x6, x6, 8 sqadd v0.8h,v0.8h,v9.8h @@ -343,14 +422,16 @@ Return Value: sqxtn2 v0.16b,v2.8h b.lo .LConvSym.PartialStore - st1 {v0.d}[1],[x5] // full 2x8 store to c + st1 {v0.d}[1],[x5] // full 2x8 store to c st1 {v0.8b},[x2] .LConvSym.ExitKernel: - ldp d14,d15,[sp,#48] - ldp d12,d13,[sp,#32] - ldp d10,d11,[sp,#16] - ldp d8,d9,[sp],#64 + ldp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22] + ldp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20] + ldp d14,d15,[sp,#.LConvSymFrame_SavedRegisters_d14_d15] + ldp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13] + ldp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11] + ldp d8,d9,[sp],#.LConvSymFrame_SavedRegisters ret .LConvSym.8InputChannels: diff --git a/onnxruntime/core/mlas/lib/aarch64/ConvSymU8KernelDot.S b/onnxruntime/core/mlas/lib/aarch64/ConvSymU8KernelDot.S index dc11ad93c6..c060884d55 100644 --- a/onnxruntime/core/mlas/lib/aarch64/ConvSymU8KernelDot.S +++ b/onnxruntime/core/mlas/lib/aarch64/ConvSymU8KernelDot.S @@ -18,22 +18,44 @@ Abstract: #include "asmmacro.h" #include "AssembleDotProduct.h" - .equ .LMLAS_CONV_SYM_FLAG_INPUT_DIRECT, 1 .equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2 + .equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4 // // Stack frame layout for the symmetric convolution kernel. // d8-d15, x19-x30 need to be preserved if used // - .equ .LConvSymFrame_SavedRegisters, (8 * 8) - .equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters - .equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters + .equ .LConvSymFrame_SavedRegisters_d8_d9, 0 + .equ .LConvSymFrame_SavedRegisters_d10_d11, 16 + .equ .LConvSymFrame_SavedRegisters_d12_d13, 32 + .equ .LConvSymFrame_SavedRegisters_x19_x20, 48 + .equ .LConvSymFrame_SavedRegisters_x21_x22, 64 + .equ .LConvSymFrame_SavedRegisters, 80 + .equ .LConvSymFrame_SavedRegisters_Neg, -80 + .equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters + .equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ .equ .LConvSymPostProcessParams_Bias, 0 .equ .LConvSymPostProcessParams_Scale, 8 .equ .LConvSymPostProcessParams_Min, 16 .equ .LConvSymPostProcessParams_Max, 20 .equ .LConvSymPostProcessParams_ZeroPoint, 24 + .equ .LConvSymPostProcessParams_Multiplier,32 + .equ .LConvSymPostProcessParams_PreShift, 40 + .equ .LConvSymPostProcessParams_PostShift, 48 .text @@ -46,15 +68,10 @@ Routine Description: Arguments: - Input (x0) - Points to the input buffer. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points - directly at the input tensor. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an - indirection buffer. Every pointer in the indirection buffer points at a - InputChannels length vector (either from the input tensor or a vector of - padding values). These are grouped in batches of length KernelSize. + Input (x0) - Supplies the address of the indirect buffer. Every pointer in + the indirection buffer points at a InputChannels length vector (either + from the input tensor or a vector of padding values). These are grouped + in batches of length KernelSize. These batches are then repeated OutputCount times. Filter (x1) - Points to the filter buffer. @@ -63,8 +80,6 @@ Arguments: KernelSize (x3/x9) - Size of the kernel (most commonly. 3x3=9, 5x5=25). - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1. - InputChannels (x4/x7) - Number of input channels. OutputChannels (x5) - Number of output channels. @@ -86,18 +101,18 @@ Return Value: --*/ FUNCTION_ENTRY MlasConvSymU8KernelDot - stp d8,d9,[sp,#-.LConvSymFrame_SavedRegisters]! + stp d8,d9,[sp,#.LConvSymFrame_SavedRegisters_Neg]! ldr x8,[sp,#.LConvSymFrame_PostProcessParams] ldr w10,[sp,#.LConvSymFrame_KernelFlags] - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp x19,x20,[sp,#48] + stp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11] + stp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13] + stp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20] + stp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22] cmp x7,2 // OutputCount < 2 ? add x16,x2,x5 // x16 -> C1 lsl x3,x3,#3 // KernelSize * sizeof(int8_t*) csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0 - mov x20,x4 add x4,x4,3 // InputChannels align to 4 add x17,x16,x5 // x17 -> C2 ldr x11,[x8,#.LConvSymPostProcessParams_Bias] @@ -108,6 +123,9 @@ Return Value: ldr x19,[x8,#.LConvSymPostProcessParams_Scale] csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2 movi v12.16b,128 // for top bit flipping + ldr x20,[x8,#.LConvSymPostProcessParams_PreShift] + ldr x21,[x8,#.LConvSymPostProcessParams_Multiplier] + ldr x22,[x8,#.LConvSymPostProcessParams_PostShift] OutputChannelLoop: ldp q16,q20,[x11],32 // Init accumulators with biases @@ -127,23 +145,6 @@ OutputChannelLoop: mov x9,x3 // restore KernelSize * sizeof(int8_t*) KernelSizeLoop: - tst w10,#.LMLAS_CONV_SYM_FLAG_INPUT_DIRECT - beq InputIndirection - -InputDirect: - cmp x16,x2 - mov x12,x0 // x12 -> A0 - add x13,x0,x20 // x13 -> A1 = A0 + input channels - csel x13,x0,x13,eq - cmp x17,x16 - add x14,x0,x20,lsl#1 // x14 -> A2 - csel x14,x13,x14,eq - cmp x5,x17 - add x15,x13,x20,lsl#1 // x15 -> A3 - csel x15,x14,x15,eq - b FinishLoadAPtr - -InputIndirection: ldr x12,[x0] // x12 -> A0 cmp x16,x2 b.eq SkipLoadA1 // C1==C0 -> A0=A1=A2=A3 @@ -391,12 +392,15 @@ InChLoopEpilogue: b.hi KernelSizeLoop Requantize: + tst w10,#.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale +FloatPointScale: tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint] beq BroadcastScaleValue ldp q0,q1,[x19],32 // load scale vector ldp q2,q3,[x19],32 - b AccumulatorsToFloat + b ScaleWithFloatPoint BroadcastScaleValue: ld1r {v0.4s},[x19] // load scale Value @@ -404,7 +408,7 @@ BroadcastScaleValue: mov v2.16b, v0.16b mov v3.16b, v0.16b -AccumulatorsToFloat: +ScaleWithFloatPoint: scvtf v16.4s,v16.4s // convert to float scvtf v17.4s,v17.4s scvtf v18.4s,v18.4s @@ -453,7 +457,87 @@ AccumulatorsToFloat: fcvtns v29.4s,v29.4s fcvtns v30.4s,v30.4s fcvtns v31.4s,v31.4s + b Quantize +FixedPointScale: + tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w13,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq BroadcastFixedPointValue + ldp q0,q1,[x20],32 // load preshift vector + ldp q2,q3,[x20],32 + ldp q4,q5,[x21],32 // load multiplier vector + ldp q6,q7,[x21],32 + ldp q8,q9,[x22],32 // load postshift vector + ldp q10,q11,[x22],32 + b ScaleWithFixedPoint + +BroadcastFixedPointValue: + ld1r {v0.4s},[x20] // load preshift Value + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + + ld1r {v4.4s},[x21] // load multiplier Value + mov v5.16b, v4.16b + mov v6.16b, v4.16b + mov v7.16b, v4.16b + + ld1r {v8.4s},[x22] // load postshift Value + mov v9.16b, v8.16b + mov v10.16b, v8.16b + mov v11.16b, v8.16b + +ScaleWithFixedPoint: + sqshl v16.4s, v16.4s, v0.4s // preshift + sqshl v17.4s, v17.4s, v0.4s + sqshl v18.4s, v18.4s, v0.4s + sqshl v19.4s, v19.4s, v0.4s + sqshl v20.4s, v20.4s, v1.4s + sqshl v21.4s, v21.4s, v1.4s + sqshl v22.4s, v22.4s, v1.4s + sqshl v23.4s, v23.4s, v1.4s + sqshl v24.4s, v24.4s, v2.4s + sqshl v25.4s, v25.4s, v2.4s + sqshl v26.4s, v26.4s, v2.4s + sqshl v27.4s, v27.4s, v2.4s + sqshl v28.4s, v28.4s, v3.4s + sqshl v29.4s, v29.4s, v3.4s + sqshl v30.4s, v30.4s, v3.4s + sqshl v31.4s, v31.4s, v3.4s + sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale + sqdmulh v17.4s, v17.4s, v4.4s + sqdmulh v18.4s, v18.4s, v4.4s + sqdmulh v19.4s, v19.4s, v4.4s + sqdmulh v20.4s, v20.4s, v5.4s + sqdmulh v21.4s, v21.4s, v5.4s + sqdmulh v22.4s, v22.4s, v5.4s + sqdmulh v23.4s, v23.4s, v5.4s + sqdmulh v24.4s, v24.4s, v6.4s + sqdmulh v25.4s, v25.4s, v6.4s + sqdmulh v26.4s, v26.4s, v6.4s + sqdmulh v27.4s, v27.4s, v6.4s + sqdmulh v28.4s, v28.4s, v7.4s + sqdmulh v29.4s, v29.4s, v7.4s + sqdmulh v30.4s, v30.4s, v7.4s + sqdmulh v31.4s, v31.4s, v7.4s + srshl v16.4s,v16.4s, v8.4s // post shift + srshl v17.4s,v17.4s, v8.4s + srshl v18.4s,v18.4s, v8.4s + srshl v19.4s,v19.4s, v8.4s + srshl v20.4s,v20.4s, v9.4s + srshl v21.4s,v21.4s, v9.4s + srshl v22.4s,v22.4s, v9.4s + srshl v23.4s,v23.4s, v9.4s + srshl v24.4s,v24.4s, v10.4s + srshl v25.4s,v25.4s, v10.4s + srshl v26.4s,v26.4s, v10.4s + srshl v27.4s,v27.4s, v10.4s + srshl v28.4s,v28.4s, v11.4s + srshl v29.4s,v29.4s, v11.4s + srshl v30.4s,v30.4s, v11.4s + srshl v31.4s,v31.4s, v11.4s + +Quantize: sqxtn v16.4h,v16.4s sqxtn v17.4h,v17.4s sqxtn v18.4h,v18.4s @@ -498,9 +582,10 @@ AccumulatorsToFloat: b.hi OutputChannelLoop ExitKernel: - ldp x19,x20,[sp,#48] - ldp d12,d13,[sp,#32] - ldp d10,d11,[sp,#16] + ldp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22] + ldp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20] + ldp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13] + ldp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11] ldp d8,d9,[sp],#.LConvSymFrame_SavedRegisters ret diff --git a/onnxruntime/core/mlas/lib/aarch64/ConvSymU8KernelNeon.S b/onnxruntime/core/mlas/lib/aarch64/ConvSymU8KernelNeon.S index fd16b2cbae..c2d11479d1 100644 --- a/onnxruntime/core/mlas/lib/aarch64/ConvSymU8KernelNeon.S +++ b/onnxruntime/core/mlas/lib/aarch64/ConvSymU8KernelNeon.S @@ -17,23 +17,45 @@ Abstract: #include "asmmacro.h" - .equ .LMLAS_CONV_SYM_FLAG_INPUT_DIRECT, 1 .equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2 + .equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4 // // Stack frame layout for the symmetric convolution kernel. // d8-d15, x19-x30 need to be preserved if used // - .equ .LConvSymFrame_SavedNeonRegisters, (8 * 8) - .equ .LConvSymFrame_SavedRegisters, .LConvSymFrame_SavedNeonRegisters - .equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters - .equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters + .equ .LConvSymFrame_SavedRegisters_d8_d9, 0 + .equ .LConvSymFrame_SavedRegisters_d10_d11, 16 + .equ .LConvSymFrame_SavedRegisters_d12_d13, 32 + .equ .LConvSymFrame_SavedRegisters_d14_d15, 48 + .equ .LConvSymFrame_SavedRegisters_x19_x20, 64 + .equ .LConvSymFrame_SavedRegisters_x21_x22, 80 + .equ .LConvSymFrame_SavedRegisters, 96 + .equ .LConvSymFrame_SavedRegisters_Neg, -96 + .equ .LConvSymFrame_PostProcessParams, 0 + .LConvSymFrame_SavedRegisters + .equ .LConvSymFrame_KernelFlags, 8 + .LConvSymFrame_SavedRegisters +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ .equ .LConvSymPostProcessParams_Bias, 0 .equ .LConvSymPostProcessParams_Scale, 8 .equ .LConvSymPostProcessParams_Min, 16 .equ .LConvSymPostProcessParams_Max, 20 .equ .LConvSymPostProcessParams_ZeroPoint, 24 + .equ .LConvSymPostProcessParams_Multiplier,32 + .equ .LConvSymPostProcessParams_PreShift, 40 + .equ .LConvSymPostProcessParams_PostShift, 48 .text @@ -46,15 +68,10 @@ Routine Description: Arguments: - Input (x0) - Supplies the address of the input buffer. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points - directly at the input tensor. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an - indirection buffer. Every pointer in the indirection buffer points at a - InputChannels length vector (either from the input tensor or a vector of - padding values). These are grouped in batches of length KernelSize. + Input (x0) - Supplies the address of the indirect buffer. Every pointer in + the indirection buffer points at a InputChannels length vector (either + from the input tensor or a vector of padding values). These are grouped + in batches of length KernelSize. These batches are then repeated OutputCount times. Filter (x1) - Supplies the address of the filter buffer. @@ -63,8 +80,6 @@ Arguments: KernelSize (x3) - Supplies the size of the kernel. - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1. - InputChannels (x4) - Supplies the number of input channels. This implementation requires the count to be a multiple of 8. @@ -90,12 +105,14 @@ Return Value: --*/ FUNCTION_ENTRY MlasConvSymU8KernelNeon - stp d8,d9,[sp,#-64]! + stp d8,d9,[sp,#.LConvSymFrame_SavedRegisters_Neg]! ldr x8,[sp,#.LConvSymFrame_PostProcessParams] ldrb w10,[sp,#.LConvSymFrame_KernelFlags] - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] + stp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11] + stp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13] + stp d14,d15,[sp,#.LConvSymFrame_SavedRegisters_d14_d15] + stp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20] + stp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22] mov x9,x3 // save kernel size ldr x11,[x8,#.LConvSymPostProcessParams_Bias] mov x16,x4 // save input channels @@ -104,6 +121,9 @@ Return Value: add x5,x2,x5 // c1 = c0 + ldc add x4,x4,7 // kc = (kc + 7) & ~7 csel x5,x2,x5,lo // if OutputCount < 2 c1 = c0 + ldr x19,[x8,#.LConvSymPostProcessParams_PreShift] + ldr x20,[x8,#.LConvSymPostProcessParams_Multiplier] + ldr x21,[x8,#.LConvSymPostProcessParams_PostShift] bic x4,x4,7 ldp s16,s18,[x11],8 // init accumulators with bias ldp s20,s22,[x11],8 @@ -137,30 +157,18 @@ Return Value: // .LConvSym.KernelSizeLoop: - - # Load next 2 A pointers - tst w10,#.LMLAS_CONV_SYM_FLAG_INPUT_DIRECT - ldr d4,[x1] - ldr d5,[x1,8] - beq .LConvSym.InputIndirection - -.LConvSym.InputDirect: - mov x13,x0 // x13 -> A0 - add x15,x0,x16 // x15 -> A1 = A0 + input channels - b .LConvSym.BlockLoopPrologue - -.LConvSym.InputIndirection: cmp x7,2 // test if OutputCount < 2 ldr x13,[x0] // x13 -> A0 - blo .LConvSym.SkipLoadA1 + bhs .LConvSym.LoadA1 + ldr x15,[x0],#8 // x15 -> A0 + b .LConvSym.BlockLoopPrologue +.LConvSym.LoadA1: ldr x15,[x0,x3,lsl#3] // x15 -> A1 -.LConvSym.SkipLoadA1: - -.LConvSym.BlockLoopPrologue: - cmp x7,2 // test if OutputCount < 2 add x0,x0,8 // indirect A advance to next pointer, prepare for kernel size loop - csel x15,x13,x15,lo // if OutputCount < 2 x15 -> A0 +.LConvSym.BlockLoopPrologue: + ldr d4,[x1] subs x14,x4,16 // input channel - 16 + ldr d5,[x1,8] movi v12.8b,128 blo .LConvSym.8InputChannels // less than 16 deep, no unroll @@ -325,17 +333,20 @@ Return Value: b.hi .LConvSym.KernelSizeLoop .LConvSym.Requantize: - ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint] + tst w10,#.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne .LConvSym.FixedPointScale +.LConvSym.FloatPointScale: tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint] beq .LConvSym.BroadcastScaleValue ld1 {v4.4s,v5.4s},[x12] // load scale vector - b .LConvSym.AccumulatorsToFloat + b .LConvSym.ScaleWithFloatPoint .LConvSym.BroadcastScaleValue: ld1r {v4.4s},[x12] // load scale Value mov v5.16b, v4.16b -.LConvSym.AccumulatorsToFloat: +.LConvSym.ScaleWithFloatPoint: addp v16.4s,v16.4s,v18.4s addp v20.4s,v20.4s,v22.4s addp v24.4s,v24.4s,v26.4s @@ -358,12 +369,57 @@ Return Value: fmul v3.4s,v3.4s,v5.4s fcvtns v0.4s,v0.4s // convert to int fcvtns v1.4s,v1.4s - dup v9.8h,w11 fcvtns v2.4s,v2.4s fcvtns v3.4s,v3.4s + b .LConvSym.Quantize + +.LConvSym.FixedPointScale: + tst w10,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w11, [x8, #.LConvSymPostProcessParams_ZeroPoint] + beq .LConvSym.BroadcastFixedPointValue + ld1 {v4.4s,v5.4s},[x19] // load preshift + ld1 {v6.4s,v7.4s},[x20] // load multiplier + ld1 {v8.4s,v9.4s},[x21] // load postshift + b .LConvSym.ScaleWithFixedPoint +.LConvSym.BroadcastFixedPointValue: + ld1r {v4.4s},[x19] // load preshift Value + mov v5.16b, v4.16b + ld1r {v6.4s},[x20] // load multiplier Value + mov v7.16b, v6.16b + ld1r {v8.4s},[x21] // load postshift Value + mov v9.16b, v8.16b + +.LConvSym.ScaleWithFixedPoint: + addp v16.4s,v16.4s,v18.4s + addp v20.4s,v20.4s,v22.4s + addp v24.4s,v24.4s,v26.4s + addp v28.4s,v28.4s,v30.4s + addp v17.4s,v17.4s,v19.4s + addp v21.4s,v21.4s,v23.4s + addp v25.4s,v25.4s,v27.4s + addp v29.4s,v29.4s,v31.4s + addp v0.4s,v16.4s,v20.4s + addp v1.4s,v24.4s,v28.4s + addp v2.4s,v17.4s,v21.4s + addp v3.4s,v25.4s,v29.4s + sqshl v0.4s, v0.4s, v4.4s // preshift + sqshl v1.4s, v1.4s, v5.4s + sqshl v2.4s, v2.4s, v4.4s + sqshl v3.4s, v3.4s, v5.4s + sqdmulh v0.4s, v0.4s, v6.4s // multiplier + sqdmulh v1.4s, v1.4s, v7.4s + sqdmulh v2.4s, v2.4s, v6.4s + sqdmulh v3.4s, v3.4s, v7.4s + srshl v0.4s,v0.4s, v8.4s // postshift + srshl v1.4s,v1.4s, v9.4s + srshl v2.4s,v2.4s, v8.4s + srshl v3.4s,v3.4s, v9.4s + +.LConvSym.Quantize: + dup v9.8h,w11 sqxtn v0.4h,v0.4s - sqxtn2 v0.8h,v1.4s sqxtn v2.4h,v2.4s + sqxtn2 v0.8h,v1.4s sqxtn2 v2.8h,v3.4s subs x6, x6, 8 sqadd v0.8h,v0.8h,v9.8h @@ -372,14 +428,16 @@ Return Value: sqxtun2 v0.16b,v2.8h b.lo .LConvSym.PartialStore - st1 {v0.d}[1],[x5] // full 2x8 store to c + st1 {v0.d}[1],[x5] // full 2x8 store to c st1 {v0.8b},[x2] .LConvSym.ExitKernel: - ldp d14,d15,[sp,#48] - ldp d12,d13,[sp,#32] - ldp d10,d11,[sp,#16] - ldp d8,d9,[sp],#64 + ldp x21,x22,[sp,#.LConvSymFrame_SavedRegisters_x21_x22] + ldp x19,x20,[sp,#.LConvSymFrame_SavedRegisters_x19_x20] + ldp d14,d15,[sp,#.LConvSymFrame_SavedRegisters_d14_d15] + ldp d12,d13,[sp,#.LConvSymFrame_SavedRegisters_d12_d13] + ldp d10,d11,[sp,#.LConvSymFrame_SavedRegisters_d10_d11] + ldp d8,d9,[sp],#.LConvSymFrame_SavedRegisters ret .LConvSym.8InputChannels: diff --git a/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvKernelSize9Neon.S b/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvKernelSize9Neon.S index 2be4b17a17..b05f34e542 100644 --- a/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvKernelSize9Neon.S +++ b/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvKernelSize9Neon.S @@ -18,28 +18,47 @@ Abstract: #include "asmmacro.h" - - .equ .LConvSymDepthwisePostProcessParams_Bias, 0 - .equ .LConvSymDepthwisePostProcessParams_Scale, 8 - .equ .LConvSymDepthwisePostProcessParams_ZeroPoint, 24 - .equ .LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, 1 + .equ .LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4 // // Stack frame layout for the depthwise conv kernel. d8-d15, x19-x30 need save // - .equ .LMlasConvSymDepthwiseKernelSize9_backup_x19_x20, 0 - .equ .LMlasConvSymDepthwiseKernelSize9_backup_x21_x22, 16 - .equ .LMlasConvSymDepthwiseKernelSize9_backup_x23_x24, 32 - .equ .LMlasConvSymDepthwiseKernelSize9_backup_x25_x26, 48 - .equ .LMlasConvSymDepthwiseKernelSize9_backup_x27_x28, 64 - .equ .LMlasConvSymDepthwiseKernelSize9_backup_d8_d9, 80 - .equ .LMlasConvSymDepthwiseKernelSize9_backup_d10_d11, 96 - .equ .LMlasConvSymDepthwiseKernelSize9_backup_d12_d13, 112 - .equ .LMlasConvSymDepthwiseKernelSize9_backup_d14_d15, 128 - .equ .LMlasConvSymDepthwiseKernelSize9_SavedRegisters, 144 - .equ .LMlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg, -144 + .equ .LMlasConvSymDepthwiseKernelSize9_x19_x20, 0 + .equ .LMlasConvSymDepthwiseKernelSize9_x21_x22, 16 + .equ .LMlasConvSymDepthwiseKernelSize9_x23_x24, 32 + .equ .LMlasConvSymDepthwiseKernelSize9_x25_x26, 48 + .equ .LMlasConvSymDepthwiseKernelSize9_x27_x28, 64 + .equ .LMlasConvSymDepthwiseKernelSize9_d8_d9, 80 + .equ .LMlasConvSymDepthwiseKernelSize9_d10_d11, 96 + .equ .LMlasConvSymDepthwiseKernelSize9_d12_d13, 112 + .equ .LMlasConvSymDepthwiseKernelSize9_d14_d15, 128 + .equ .LMlasConvSymDepthwiseKernelSize9_x0_x4, 144 + .equ .LMlasConvSymDepthwiseKernelSize9_multiplier_postshift, 160 + .equ .LMlasConvSymDepthwiseKernelSize9_SavedRegisters, 176 + .equ .LMlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg, -176 +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ + .equ .LConvSymPostProcessParams_Bias, 0 + .equ .LConvSymPostProcessParams_Scale, 8 + .equ .LConvSymPostProcessParams_Min, 16 + .equ .LConvSymPostProcessParams_Max, 20 + .equ .LConvSymPostProcessParams_ZeroPoint, 24 + .equ .LConvSymPostProcessParams_Multiplier, 32 + .equ .LConvSymPostProcessParams_PreShift, 40 + .equ .LConvSymPostProcessParams_PostShift, 48 .text @@ -75,26 +94,32 @@ Return Value: FUNCTION_ENTRY MlasConvSymDepthwiseKernelSize9Arm64U8S8 stp x19, x20, [sp, #.LMlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg]! - stp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x21_x22] - stp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x23_x24] - stp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x25_x26] - stp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x27_x28] - stp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d8_d9] - stp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d10_d11] - stp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d12_d13] - stp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d14_d15] + stp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_x21_x22] + stp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_x23_x24] + stp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_x25_x26] + stp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_x27_x28] + stp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_d8_d9] + stp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_d10_d11] + stp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_d12_d13] + stp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_d14_d15] - ldr x9, [x5, #.LConvSymDepthwisePostProcessParams_Bias] - ldr x8, [x5, #.LConvSymDepthwisePostProcessParams_Scale] - add x5, x5, #.LConvSymDepthwisePostProcessParams_ZeroPoint + tst x6, #.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + ldr x11, [x5, #.LConvSymPostProcessParams_Multiplier] + ldr x12, [x5, #.LConvSymPostProcessParams_PostShift] + ldr x10, [x5, #.LConvSymPostProcessParams_PreShift] + ldr x9, [x5, #.LConvSymPostProcessParams_Bias] + ldr x8, [x5, #.LConvSymPostProcessParams_Scale] + add x5, x5, #.LConvSymPostProcessParams_ZeroPoint + stp x11, x12,[sp, #.LMlasConvSymDepthwiseKernelSize9_multiplier_postshift] + csel x8, x8, x10, eq ins v12.d[0], x1 // Filter ins v13.d[0], x9 // Bias - ins v13.d[1], x8 // Scale + ins v13.d[1], x8 // Scale/PreShift ld1r {v0.8h}, [x5] // zero point movi v5.16b, #0x80 // flip 0x80 tbnz x6, #.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LMlasConvSymDepthwiseKernelSize9_SkipPerTensorScaleInit - ld1r {v1.4s}, [x8] // load scale value + ld1r {v1.4s}, [x8] // load scale/PreShift value mov v2.16b, v1.16b mov v3.16b, v1.16b mov v4.16b, v1.16b @@ -113,7 +138,7 @@ Return Value: ldur x28, [x0, #-8] cbz x4, .LMlasConvSymDepthwiseKernelSize9_Dup_Inputs - ldp x10, x11, [x0], #72 // input ptrs for Output0 + ldp x10, x11, [x0], #72 // input ptrs for Output1 ldp x12, x13, [x0, #-56] sub x4, x4, #1 ldp x14, x15, [x0, #-40] @@ -123,7 +148,7 @@ Return Value: .LMlasConvSymDepthwiseKernelSize9_Dup_Inputs: mov x9, x3 // Output1 <-- Output0 - mov x10, x20 + mov x10, x20 // Input1 <-- Input0 mov x11, x21 mov x12, x22 mov x13, x23 @@ -135,10 +160,12 @@ Return Value: .LMlasConvSymDepthwiseKernelSize9_Loaded_Input: + stp x0, x4, [sp, #.LMlasConvSymDepthwiseKernelSize9_x0_x4] eor x8, x8, x8 // Processed channels umov x1, v12.D[0] // filter umov x5, v13.D[0] // bias - umov x7, v13.D[1] // scale + umov x7, v13.D[1] // scale/PreShift + ldp x0, x4, [sp, #.LMlasConvSymDepthwiseKernelSize9_multiplier_postshift] cmp x8, x2 // Save one register by not using count down to zero here bhs .LMlasConvSymDepthwiseKernelSize9_Finish_Channels16_Loop @@ -232,7 +259,7 @@ Return Value: ldr q23, [x17, x8] // out1 vi7 tbz x6, #.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LDonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9 - ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales 0-15 for outs + ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales/preshift 0-15 for outs .LDonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9: eor v20.16b, v20.16b, v5.16b @@ -288,6 +315,9 @@ Return Value: saddw v8.4s, v8.4s, v27.4h saddw2 v9.4s, v9.4s, v27.8h + tst x6, #.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne .LFixedPointScale_U8S8 +.LFloatPointScale_U8S8: scvtf v16.4s, v16.4s // Requantize scvtf v17.4s, v17.4s scvtf v18.4s, v18.4s @@ -314,7 +344,55 @@ Return Value: fcvtns v7.4s, v7.4s fcvtns v8.4s, v8.4s fcvtns v9.4s, v9.4s + b .LQuantize_U8S8 +.LFixedPointScale_U8S8: + tbz x6,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LBroadcastFixedPointValue_U8S8 + ldp q20,q21,[x0],32 // load multiplier vector + ldp q22,q23,[x0],32 + ldp q24,q25,[x4],32 // load postshift vector + ldp q26,q27,[x4],32 + b .LScaleWithFixedPoint_U8S8 + +.LBroadcastFixedPointValue_U8S8: + ld1r {v20.4s},[x0] // load multiplier Value + mov v21.16b, v20.16b + mov v22.16b, v20.16b + mov v23.16b, v20.16b + + ld1r {v24.4s},[x4] // load postshift Value + mov v25.16b, v24.16b + mov v26.16b, v24.16b + mov v27.16b, v24.16b +.LScaleWithFixedPoint_U8S8: + sqshl v6.4s, v6.4s, v1.4s // preshift + sqshl v7.4s, v7.4s, v2.4s + sqshl v8.4s, v8.4s, v3.4s + sqshl v9.4s, v9.4s, v4.4s + sqshl v16.4s, v16.4s, v1.4s + sqshl v17.4s, v17.4s, v2.4s + sqshl v18.4s, v18.4s, v3.4s + sqshl v19.4s, v19.4s, v4.4s + + sqdmulh v6.4s, v6.4s, v20.4s // multiply by multiplier + sqdmulh v7.4s, v7.4s, v21.4s + sqdmulh v8.4s, v8.4s, v22.4s + sqdmulh v9.4s, v9.4s, v23.4s + sqdmulh v16.4s, v16.4s, v20.4s + sqdmulh v17.4s, v17.4s, v21.4s + sqdmulh v18.4s, v18.4s, v22.4s + sqdmulh v19.4s, v19.4s, v23.4s + + srshl v6.4s,v6.4s, v24.4s // post shift + srshl v7.4s,v7.4s, v25.4s + srshl v8.4s,v8.4s, v26.4s + srshl v9.4s,v9.4s, v27.4s + srshl v16.4s,v16.4s, v24.4s + srshl v17.4s,v17.4s, v25.4s + srshl v18.4s,v18.4s, v26.4s + srshl v19.4s,v19.4s, v27.4s + +.LQuantize_U8S8: sqxtn v16.4h, v16.4s // +zp, narrow and combine sqxtn v18.4h, v18.4s sqxtn v6.4h, v6.4s @@ -341,19 +419,20 @@ Return Value: blo .LMlasConvSymDepthwiseKernelSize9_Channels16_Loop .LMlasConvSymDepthwiseKernelSize9_Finish_Channels16_Loop: + ldp x0, x4, [sp, #.LMlasConvSymDepthwiseKernelSize9_x0_x4] add x3, x3, x2, LSL #1 add x9, x9, x2, LSL #1 cbnz x4, .LMlasConvSymDepthwiseKernelSize9_OutputLoop .LMlasConvSymDepthwiseKernelSize9_Exit: - ldp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d14_d15] - ldp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d12_d13] - ldp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d10_d11] - ldp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d8_d9] - ldp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x27_x28] - ldp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x25_x26] - ldp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x23_x24] - ldp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x21_x22] + ldp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_d14_d15] + ldp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_d12_d13] + ldp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_d10_d11] + ldp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_d8_d9] + ldp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_x27_x28] + ldp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_x25_x26] + ldp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_x23_x24] + ldp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_x21_x22] ldp x19, x20, [sp], #.LMlasConvSymDepthwiseKernelSize9_SavedRegisters ret @@ -390,25 +469,32 @@ Return Value: FUNCTION_ENTRY MlasConvSymDepthwiseKernelSize9Arm64S8S8 stp x19, x20, [sp, #.LMlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg]! - stp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x21_x22] - stp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x23_x24] - stp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x25_x26] - stp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x27_x28] - stp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d8_d9] - stp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d10_d11] - stp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d12_d13] - stp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d14_d15] + stp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_x21_x22] + stp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_x23_x24] + stp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_x25_x26] + stp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_x27_x28] + stp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_d8_d9] + stp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_d10_d11] + stp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_d12_d13] + stp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_d14_d15] - ldr x9, [x5, #.LConvSymDepthwisePostProcessParams_Bias] - ldr x8, [x5, #.LConvSymDepthwisePostProcessParams_Scale] - add x5, x5, #.LConvSymDepthwisePostProcessParams_ZeroPoint + tst x6, #.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + ldr x9, [x5, #.LConvSymPostProcessParams_Bias] + ldr x8, [x5, #.LConvSymPostProcessParams_Scale] + ldr x10, [x5, #.LConvSymPostProcessParams_PreShift] + ldr x11, [x5, #.LConvSymPostProcessParams_Multiplier] + ldr x12, [x5, #.LConvSymPostProcessParams_PostShift] + add x5, x5, #.LConvSymPostProcessParams_ZeroPoint + csel x8, x8, x10, eq ins v12.d[0], x1 // Filter ins v13.d[0], x9 // Bias - ins v13.d[1], x8 // Scale + ins v13.d[1], x8 // Scale/PreShift + ins v5.d[0], x11 // Multiplier + ins v5.d[1], x12 // PostShift ld1r {v0.8h}, [x5] // zero point tbnz x6, #.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LMlasConvSymDepthwiseKernelSize9S8S8_SkipPerTensorScaleInit - ld1r {v1.4s}, [x8] // load scale value + ld1r {v1.4s}, [x8] // load scale/PreShift value mov v2.16b, v1.16b mov v3.16b, v1.16b mov v4.16b, v1.16b @@ -449,10 +535,13 @@ Return Value: .LMlasConvSymDepthwiseKernelSize9S8S8_Loaded_Input: + stp x0, x4, [sp, #.LMlasConvSymDepthwiseKernelSize9_x0_x4] eor x8, x8, x8 // Processed channels umov x1, v12.D[0] // filter umov x5, v13.D[0] // bias - umov x7, v13.D[1] // scale + umov x7, v13.D[1] // scale/PreShift + umov x0, v5.D[0] + umov x4, v5.D[1] cmp x8, x2 // Save one register by not using count down to zero here bhs .LMlasConvSymDepthwiseKernelSize9S8S8_Finish_Channels16_Loop @@ -534,7 +623,7 @@ Return Value: ldr q23, [x17, x8] // out1 vi7 tbz x6, #.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LDonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8 - ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales 0-15 for outs + ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales/preshift 0-15 for outs .LDonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8: ldr q10, [x1] // vk8 @@ -584,6 +673,9 @@ Return Value: saddw v8.4s, v8.4s, v27.4h saddw2 v9.4s, v9.4s, v27.8h + tst x6, #.LMLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale +FloatPointScale: scvtf v16.4s, v16.4s // Requantize scvtf v17.4s, v17.4s scvtf v18.4s, v18.4s @@ -610,7 +702,55 @@ Return Value: fcvtns v7.4s, v7.4s fcvtns v8.4s, v8.4s fcvtns v9.4s, v9.4s + b Quantize +FixedPointScale: + tbz x6,#.LMLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, .LBroadcastFixedPointValue + ldp q20,q21,[x0],32 // load multiplier vector + ldp q22,q23,[x0],32 + ldp q24,q25,[x4],32 // load postshift vector + ldp q26,q27,[x4],32 + b ScaleWithFixedPoint + +.LBroadcastFixedPointValue: + ld1r {v20.4s},[x0] // load multiplier Value + mov v21.16b, v20.16b + mov v22.16b, v20.16b + mov v23.16b, v20.16b + + ld1r {v24.4s},[x4] // load postshift Value + mov v25.16b, v24.16b + mov v26.16b, v24.16b + mov v27.16b, v24.16b +ScaleWithFixedPoint: + sqshl v6.4s, v6.4s, v1.4s // preshift + sqshl v7.4s, v7.4s, v2.4s + sqshl v8.4s, v8.4s, v3.4s + sqshl v9.4s, v9.4s, v4.4s + sqshl v16.4s, v16.4s, v1.4s + sqshl v17.4s, v17.4s, v2.4s + sqshl v18.4s, v18.4s, v3.4s + sqshl v19.4s, v19.4s, v4.4s + + sqdmulh v6.4s, v6.4s, v20.4s // multiply by multiplier + sqdmulh v7.4s, v7.4s, v21.4s + sqdmulh v8.4s, v8.4s, v22.4s + sqdmulh v9.4s, v9.4s, v23.4s + sqdmulh v16.4s, v16.4s, v20.4s + sqdmulh v17.4s, v17.4s, v21.4s + sqdmulh v18.4s, v18.4s, v22.4s + sqdmulh v19.4s, v19.4s, v23.4s + + srshl v6.4s,v6.4s, v24.4s // post shift + srshl v7.4s,v7.4s, v25.4s + srshl v8.4s,v8.4s, v26.4s + srshl v9.4s,v9.4s, v27.4s + srshl v16.4s,v16.4s, v24.4s + srshl v17.4s,v17.4s, v25.4s + srshl v18.4s,v18.4s, v26.4s + srshl v19.4s,v19.4s, v27.4s + +Quantize: sqxtn v16.4h, v16.4s // +zp, narrow and combine sqxtn v18.4h, v18.4s sqxtn v6.4h, v6.4s @@ -637,19 +777,20 @@ Return Value: blo .LMlasConvSymDepthwiseKernelSize9S8S8_Channels16_Loop .LMlasConvSymDepthwiseKernelSize9S8S8_Finish_Channels16_Loop: + ldp x0, x4, [sp, #.LMlasConvSymDepthwiseKernelSize9_x0_x4] add x3, x3, x2, LSL #1 add x9, x9, x2, LSL #1 cbnz x4, .LMlasConvSymDepthwiseKernelSize9S8S8_OutputLoop .LMlasConvSymDepthwiseKernelSize9S8S8_Exit: - ldp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d14_d15] - ldp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d12_d13] - ldp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d10_d11] - ldp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_d8_d9] - ldp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x27_x28] - ldp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x25_x26] - ldp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x23_x24] - ldp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_backup_x21_x22] + ldp d14, d15, [sp, #.LMlasConvSymDepthwiseKernelSize9_d14_d15] + ldp d12, d13, [sp, #.LMlasConvSymDepthwiseKernelSize9_d12_d13] + ldp d10, d11, [sp, #.LMlasConvSymDepthwiseKernelSize9_d10_d11] + ldp d8, d9, [sp, #.LMlasConvSymDepthwiseKernelSize9_d8_d9] + ldp x27, x28, [sp, #.LMlasConvSymDepthwiseKernelSize9_x27_x28] + ldp x25, x26, [sp, #.LMlasConvSymDepthwiseKernelSize9_x25_x26] + ldp x23, x24, [sp, #.LMlasConvSymDepthwiseKernelSize9_x23_x24] + ldp x21, x22, [sp, #.LMlasConvSymDepthwiseKernelSize9_x21_x22] ldp x19, x20, [sp], #.LMlasConvSymDepthwiseKernelSize9_SavedRegisters ret diff --git a/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvSymS8KernelNeon.S b/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvSymS8KernelNeon.S index 7a27b9c92a..ec256be0c8 100644 --- a/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvSymS8KernelNeon.S +++ b/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvSymS8KernelNeon.S @@ -17,23 +17,44 @@ Abstract: #include "asmmacro.h" + + .equ MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2 + .equ MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4 + // // Stack frame layout for the depthwise conv kernel. // d8-d15, x19-x30 need to be preserved if used // + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d8_d9, 0 + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11, 16 + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13, 32 + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15, 48 + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters, 64 + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters_Neg, -64 + .equ .LConvSymDepthwiseKernelFrame_PostProcessParams, 0 + .LConvSymDepthwiseKernelFrame_SavedRegisters + .equ .LConvSymDepthwiseKernelFrame_KernelFlags, 8 + .LConvSymDepthwiseKernelFrame_SavedRegisters - .equ .LConvSymDepthwiseKernelFrame_SavedRegisters, (4 * 8) - .equ .LConvSymDepthwiseKernelFrame_PostProcessParams, 0 + .LConvSymDepthwiseKernelFrame_SavedRegisters - .equ .LConvSymDepthwiseKernelFrame_KernelFlags, 8 + .LConvSymDepthwiseKernelFrame_SavedRegisters - - .equ .LConvSymDepthwisePostProcessParams_Bias, 0 - .equ .LConvSymDepthwisePostProcessParams_Scale, 8 - .equ .LConvSymDepthwisePostProcessParams_Min, 16 - .equ .LConvSymDepthwisePostProcessParams_Max, 20 - .equ .LConvSymDepthwisePostProcessParams_ZeroPoint, 24 - - .equ MLAS_CONV_SYM_FLAG_INPUT_DIRECT, 1 - .equ MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2 +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ + .equ .LConvSymPostProcessParams_Bias, 0 + .equ .LConvSymPostProcessParams_Scale, 8 + .equ .LConvSymPostProcessParams_Min, 16 + .equ .LConvSymPostProcessParams_Max, 20 + .equ .LConvSymPostProcessParams_ZeroPoint, 24 + .equ .LConvSymPostProcessParams_Multiplier, 32 + .equ .LConvSymPostProcessParams_PreShift, 40 + .equ .LConvSymPostProcessParams_PostShift, 48 .text @@ -47,28 +68,28 @@ Routine Description: Arguments: Input (x0) - Supplies the address of the indirection buffer. - + Filter (x1) - Supplies the address of the filter buffer. Output (x2) - Supplies the address of the output buffer. KernelSize (x3) - Supplies the size of the kernel. - + Channels (x4) - Supplies the number of input and output channels. - + ChannelOffset (x5) - Supplies the byte offset from the indirection buffer base address for this iteration. - + ChannelCount (x6) - Supplies the number of channels this iteration produces. - + This implementation requires the count to be 16 or 8 - + OutputCount (x7)- Supplies the number of output elements this iteration produces. - - This implementation requires the count to be in the range 1 to 2. - + + This implementation requires the count to be in the range 1 to 4. + PostProcessParams - Supplies the address of the post process parameter block. - + KernelFlags - Supplies additional flags controlling the operation. Return Value: @@ -79,14 +100,16 @@ Return Value: FUNCTION_ENTRY MlasConvSymDepthwiseS8KernelNeon - stp d12,d13,[sp,#-.LConvSymDepthwiseKernelFrame_SavedRegisters]! + stp d8,d9,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_Neg]! ldr x8,[sp,#.LConvSymDepthwiseKernelFrame_PostProcessParams] - stp d14,d15,[sp,#16] + stp d10,d11,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11] + stp d12,d13,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13] + stp d14,d15,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15] cmp x7,2 add x9,x0,x3,lsl#3 // x9 -> &A1 add x14,x0,x3,lsl#4 // x14 -> &A2 add x15,x9,x3,lsl#4 // x15 -> &A3 - ldr x16,[x8,#.LConvSymDepthwisePostProcessParams_Bias] + ldr x16,[x8,#.LConvSymPostProcessParams_Bias] csel x9,x0,x9,lo // x9 -> &A0 if OutputCount < 2 csel x14,x0,x14,ls // x14 -> &A0 if OutputCount <= 2 ldr x11,[x9],#8 // x11 -> A1 iter 0 @@ -161,7 +184,7 @@ Return Value: smlal v12.8h,v1.8b,v2.8b ldr x11,[x9],#8 // x11 -> A1 iter 2 smlal2 v13.8h,v1.16b,v2.16b - b.eq .LEpilogueC16P3 // 3 pixel remains + b.eq .LEpilogueC16P3 // 3 pixel remains ldr q2,[x12,x5] // A2 iter 1 smlal v14.8h,v1.8b,v3.8b ldr x12,[x0],#8 // x12 -> A0 iter 3 @@ -228,22 +251,15 @@ Return Value: saddw v30.4s,v30.4s,v15.4h saddw2 v31.4s,v31.4s,v15.8h ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags] - ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b smull2 v13.8h,v0.16b,v4.16b - ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b smull2 v15.8h,v0.16b,v6.16b smlal v12.8h,v1.8b,v2.8b smlal2 v13.8h,v1.16b,v2.16b smlal v14.8h,v1.8b,v3.8b smlal2 v15.8h,v1.16b,v3.16b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq .LSkipScaleVecLoad2 - ldp q4,q5,[x12],#32 // load scale vector if per channel - ldp q6,q3,[x12] -.LSkipScaleVecLoad2: + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v18.4s,v18.4s,v13.4h @@ -252,7 +268,7 @@ Return Value: saddw2 v21.4s,v21.4s,v14.8h saddw v22.4s,v22.4s,v15.4h saddw2 v23.4s,v23.4s,v15.8h - b .LDequantization + b .LRequantization .LProcC16P1: // @@ -327,10 +343,8 @@ Return Value: // Loop epilogue (process last single pixel) mixed with loading of dequantization params // ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags] - ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b smull2 v13.8h,v0.16b,v4.16b - ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b smull2 v15.8h,v0.16b,v6.16b saddw v24.4s,v24.4s,v12.4h @@ -345,12 +359,7 @@ Return Value: smull2 v13.8h,v0.16b,v2.16b smull v14.8h,v0.8b,v3.8b smull2 v15.8h,v0.16b,v3.16b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq .LSkipScaleVecLoad - ldp q4,q5,[x12],#32 // load scale vector if per channel - ldp q6,q3,[x12] -.LSkipScaleVecLoad: + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v18.4s,v18.4s,v13.4h @@ -360,7 +369,24 @@ Return Value: saddw v22.4s,v22.4s,v15.4h saddw2 v23.4s,v23.4s,v15.8h -.LDequantization: +.LRequantization: + b.ne .LFixedPointScale +.LFloatingPointScale: + ldr x12,[x8,#.LConvSymPostProcessParams_Scale] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq .LBroadcastScaleValue + ldp q4,q5,[x12],#32 // load scale vector if per channel + ldp q6,q3,[x12] + b .LScaleWithFloatPoint + +.LBroadcastScaleValue: + ld1r {v4.4s},[x12] // load scale val + mov v5.16b,v4.16b // broadcast scale val if not per channel + mov v6.16b,v4.16b + mov v3.16b,v4.16b + +.LScaleWithFloatPoint: scvtf v24.4s,v24.4s // convert to float scvtf v25.4s,v25.4s scvtf v26.4s,v26.4s @@ -377,11 +403,6 @@ Return Value: scvtf v21.4s,v21.4s scvtf v22.4s,v22.4s scvtf v23.4s,v23.4s - b.ne .LSkipScaleBroadcast - mov v5.16b,v4.16b // broadcast scale val if not per channel - mov v6.16b,v4.16b - mov v3.16b,v4.16b -.LSkipScaleBroadcast: fmul v24.4s,v24.4s,v4.4s // multiply by scale fmul v25.4s,v25.4s,v5.4s fmul v26.4s,v26.4s,v6.4s @@ -414,6 +435,90 @@ Return Value: fcvtns v21.4s,v21.4s fcvtns v22.4s,v22.4s fcvtns v23.4s,v23.4s + b .LQuantize + +.LFixedPointScale: + ldr x10,[x8,#.LConvSymPostProcessParams_PreShift] + ldr x11,[x8,#.LConvSymPostProcessParams_Multiplier] + ldr x12,[x8,#.LConvSymPostProcessParams_PostShift] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq .LBroadcastFixedPointValue + ldp q0,q1,[x10],32 // load preshift vector + ldp q2,q3,[x10],32 + ldp q4,q5,[x11],32 // load multiplier vector + ldp q6,q7,[x11],32 + ldp q8,q9,[x12],32 // load postshift vector + ldp q10,q11,[x12],32 + b .LScaleWithFixedPoint + +.LBroadcastFixedPointValue: + ld1r {v0.4s},[x10] // load preshift Value + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + + ld1r {v4.4s},[x11] // load multiplier Value + mov v5.16b, v4.16b + mov v6.16b, v4.16b + mov v7.16b, v4.16b + + ld1r {v8.4s},[x12] // load postshift Value + mov v9.16b, v8.16b + mov v10.16b, v8.16b + mov v11.16b, v8.16b + +.LScaleWithFixedPoint: + sqshl v24.4s, v24.4s, v0.4s // preshift + sqshl v25.4s, v25.4s, v1.4s + sqshl v26.4s, v26.4s, v2.4s + sqshl v27.4s, v27.4s, v3.4s + sqshl v28.4s, v28.4s, v0.4s + sqshl v29.4s, v29.4s, v1.4s + sqshl v30.4s, v30.4s, v2.4s + sqshl v31.4s, v31.4s, v3.4s + sqshl v16.4s, v16.4s, v0.4s + sqshl v17.4s, v17.4s, v1.4s + sqshl v18.4s, v18.4s, v2.4s + sqshl v19.4s, v19.4s, v3.4s + sqshl v20.4s, v20.4s, v0.4s + sqshl v21.4s, v21.4s, v1.4s + sqshl v22.4s, v22.4s, v2.4s + sqshl v23.4s, v23.4s, v3.4s + sqdmulh v24.4s, v24.4s, v4.4s // multiplier + sqdmulh v25.4s, v25.4s, v5.4s + sqdmulh v26.4s, v26.4s, v6.4s + sqdmulh v27.4s, v27.4s, v7.4s + sqdmulh v28.4s, v28.4s, v4.4s + sqdmulh v29.4s, v29.4s, v5.4s + sqdmulh v30.4s, v30.4s, v6.4s + sqdmulh v31.4s, v31.4s, v7.4s + sqdmulh v16.4s, v16.4s, v4.4s + sqdmulh v17.4s, v17.4s, v5.4s + sqdmulh v18.4s, v18.4s, v6.4s + sqdmulh v19.4s, v19.4s, v7.4s + sqdmulh v20.4s, v20.4s, v4.4s + sqdmulh v21.4s, v21.4s, v5.4s + sqdmulh v22.4s, v22.4s, v6.4s + sqdmulh v23.4s, v23.4s, v7.4s + srshl v24.4s, v24.4s, v8.4s // post shift + srshl v25.4s, v25.4s, v9.4s + srshl v26.4s, v26.4s, v10.4s + srshl v27.4s, v27.4s, v11.4s + srshl v28.4s, v28.4s, v8.4s + srshl v29.4s, v29.4s, v9.4s + srshl v30.4s, v30.4s, v10.4s + srshl v31.4s, v31.4s, v11.4s + srshl v16.4s, v16.4s, v8.4s + srshl v17.4s, v17.4s, v9.4s + srshl v18.4s, v18.4s, v10.4s + srshl v19.4s, v19.4s, v11.4s + srshl v20.4s, v20.4s, v8.4s + srshl v21.4s, v21.4s, v9.4s + srshl v22.4s, v22.4s, v10.4s + srshl v23.4s, v23.4s, v11.4s + +.LQuantize: sqxtn v24.4h,v24.4s // shorten to int16 sqxtn v26.4h,v26.4s sqxtn2 v24.8h,v25.4s @@ -458,8 +563,10 @@ Return Value: str q20,[x2] .LExitKernel: - ldp d14,d15,[sp,#16] - ldp d12,d13,[sp],#.LConvSymDepthwiseKernelFrame_SavedRegisters + ldp d14,d15,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15] + ldp d12,d13,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13] + ldp d10,d11,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11] + ldp d8,d9,[sp], #.LConvSymDepthwiseKernelFrame_SavedRegisters ret .LProcess8Channels: @@ -501,7 +608,7 @@ Return Value: ldr x11,[x9],#8 // x11 -> A1 iter 2 smlal v14.8h,v1.8b,v3.8b ldr d2,[x12,x5] // A2 iter 1 - b.eq .LEpilogueC8P3 // 3 pixel remains + b.eq .LEpilogueC8P3 // 3 pixel remains ldr d3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h ldr x12,[x0],#8 // x12 -> A0 iter 3 @@ -546,21 +653,15 @@ Return Value: saddw2 v29.4s,v29.4s,v14.8h ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags] smull v12.8h,v0.8b,v4.8b - ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale] smull v14.8h,v0.8b,v6.8b - ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint] smlal v12.8h,v1.8b,v2.8b smlal v14.8h,v1.8b,v3.8b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq .LSkipScaleVecLoad2C8 - ldp q4,q5,[x12],#32 // load scale vector if per channel -.LSkipScaleVecLoad2C8: + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v20.4s,v20.4s,v14.4h saddw2 v21.4s,v21.4s,v14.8h - b .LDequantC8 + b .LRequantizationC8 .LProcC8P1: // @@ -613,9 +714,7 @@ Return Value: // Loop epilogue (process last single pixel) mixed with loading of dequantization params // ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags] - ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b - ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h @@ -623,17 +722,27 @@ Return Value: saddw2 v29.4s,v29.4s,v14.8h smull v12.8h,v0.8b,v2.8b smull v14.8h,v0.8b,v3.8b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq .LSkipScaleVecLoadC8 - ldp q4,q5,[x12] // load scale vector if per channel -.LSkipScaleVecLoadC8: + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v20.4s,v20.4s,v14.4h saddw2 v21.4s,v21.4s,v14.8h -.LDequantC8: +.LRequantizationC8: + b.ne .LFixedPointScaleC8 +.LFloatingPointScaleC8: + ldr x12,[x8,#.LConvSymPostProcessParams_Scale] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq .LBroadcastScaleValueC8 + ldp q4,q5,[x12] // load scale vector if per channel + b .LScaleWithFloatPointC8 + +.LBroadcastScaleValueC8: + ld1r {v4.4s},[x12] // load scale val + mov v5.16b,v4.16b // broadcast scale val if not per channel + +.LScaleWithFloatPointC8: scvtf v24.4s,v24.4s // convert to float scvtf v25.4s,v25.4s scvtf v28.4s,v28.4s @@ -642,9 +751,6 @@ Return Value: scvtf v17.4s,v17.4s scvtf v20.4s,v20.4s scvtf v21.4s,v21.4s - b.ne .LSkipScaleBroadcastC8 - mov v5.16b,v4.16b // broadcast scale val if not per channel -.LSkipScaleBroadcastC8: fmul v24.4s,v24.4s,v4.4s // multiply by scale fmul v25.4s,v25.4s,v5.4s fmul v28.4s,v28.4s,v4.4s @@ -661,6 +767,57 @@ Return Value: fcvtns v17.4s,v17.4s fcvtns v20.4s,v20.4s fcvtns v21.4s,v21.4s + b .LQuantizeC8 + +.LFixedPointScaleC8: + ldr x10,[x8,#.LConvSymPostProcessParams_PreShift] + ldr x11,[x8,#.LConvSymPostProcessParams_Multiplier] + ldr x12,[x8,#.LConvSymPostProcessParams_PostShift] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq .LBroadcastFixedPointValueC8 + ldp q0,q1,[x10],32 // load preshift vector + ldp q4,q5,[x11],32 // load multiplier vector + ldp q8,q9,[x12],32 // load postshift vector + b .LScaleWithFixedPointC8 + +.LBroadcastFixedPointValueC8: + ld1r {v0.4s},[x10] // load preshift Value + mov v1.16b, v0.16b + + ld1r {v4.4s},[x11] // load multiplier Value + mov v5.16b, v4.16b + + ld1r {v8.4s},[x12] // load postshift Value + mov v9.16b, v8.16b + +.LScaleWithFixedPointC8: + sqshl v24.4s, v24.4s, v0.4s // preshift + sqshl v25.4s, v25.4s, v1.4s + sqshl v28.4s, v28.4s, v0.4s + sqshl v29.4s, v29.4s, v1.4s + sqshl v16.4s, v16.4s, v0.4s + sqshl v17.4s, v17.4s, v1.4s + sqshl v20.4s, v20.4s, v0.4s + sqshl v21.4s, v21.4s, v1.4s + sqdmulh v24.4s, v24.4s, v4.4s // multiplier + sqdmulh v25.4s, v25.4s, v5.4s + sqdmulh v28.4s, v28.4s, v4.4s + sqdmulh v29.4s, v29.4s, v5.4s + sqdmulh v16.4s, v16.4s, v4.4s + sqdmulh v17.4s, v17.4s, v5.4s + sqdmulh v20.4s, v20.4s, v4.4s + sqdmulh v21.4s, v21.4s, v5.4s + srshl v24.4s, v24.4s, v8.4s // post shift + srshl v25.4s, v25.4s, v9.4s + srshl v28.4s, v28.4s, v8.4s + srshl v29.4s, v29.4s, v9.4s + srshl v16.4s, v16.4s, v8.4s + srshl v17.4s, v17.4s, v9.4s + srshl v20.4s, v20.4s, v8.4s + srshl v21.4s, v21.4s, v9.4s + +.LQuantizeC8: dup v0.8h,w15 sqxtn v24.4h,v24.4s // shorten to int16 sqxtn2 v24.8h,v25.4s diff --git a/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvSymU8KernelNeon.S b/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvSymU8KernelNeon.S index 05a51253b0..b40d340ef4 100644 --- a/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvSymU8KernelNeon.S +++ b/onnxruntime/core/mlas/lib/aarch64/DepthwiseQConvSymU8KernelNeon.S @@ -17,24 +17,43 @@ Abstract: #include "asmmacro.h" + .equ MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2 + .equ MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE, 4 + // // Stack frame layout for the depthwise conv kernel. // d8-d15, x19-x30 need to be preserved if used // + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d8_d9, 0 + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11, 16 + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13, 32 + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15, 48 + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters, 64 + .equ .LConvSymDepthwiseKernelFrame_SavedRegisters_Neg, -64 + .equ .LConvSymDepthwiseKernelFrame_PostProcessParams, 0 + .LConvSymDepthwiseKernelFrame_SavedRegisters + .equ .LConvSymDepthwiseKernelFrame_KernelFlags, 8 + .LConvSymDepthwiseKernelFrame_SavedRegisters - .equ .LConvSymDepthwiseKernelFrame_SavedNeonRegisters, (8 * 8) - .equ .LConvSymDepthwiseKernelFrame_SavedRegisters, .LConvSymDepthwiseKernelFrame_SavedNeonRegisters - .equ .LConvSymDepthwiseKernelFrame_PostProcessParams, 0 + .LConvSymDepthwiseKernelFrame_SavedRegisters - .equ .LConvSymDepthwiseKernelFrame_KernelFlags, 8 + .LConvSymDepthwiseKernelFrame_SavedRegisters - - .equ .LConvSymDepthwisePostProcessParams_Bias, 0 - .equ .LConvSymDepthwisePostProcessParams_Scale, 8 - .equ .LConvSymDepthwisePostProcessParams_Min, 16 - .equ .LConvSymDepthwisePostProcessParams_Max, 20 - .equ .LConvSymDepthwisePostProcessParams_ZeroPoint, 24 - - .equ MLAS_CONV_SYM_FLAG_INPUT_DIRECT, 1 - .equ MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE, 2 +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ + .equ .LConvSymPostProcessParams_Bias, 0 + .equ .LConvSymPostProcessParams_Scale, 8 + .equ .LConvSymPostProcessParams_Min, 16 + .equ .LConvSymPostProcessParams_Max, 20 + .equ .LConvSymPostProcessParams_ZeroPoint, 24 + .equ .LConvSymPostProcessParams_Multiplier, 32 + .equ .LConvSymPostProcessParams_PreShift, 40 + .equ .LConvSymPostProcessParams_PostShift, 48 .text @@ -48,28 +67,28 @@ Routine Description: Arguments: Input (x0) - Supplies the address of the indirection buffer. - + Filter (x1) - Supplies the address of the filter buffer. Output (x2) - Supplies the address of the output buffer. KernelSize (x3) - Supplies the size of the kernel. - + Channels (x4) - Supplies the number of input and output channels. - + ChannelOffset (x5) - Supplies the byte offset from the indirection buffer base address for this iteration. - + ChannelCount (x6) - Supplies the number of channels this iteration produces. - + This implementation requires the count to be 16 or 8 - + OutputCount (x7)- Supplies the number of output elements this iteration produces. - + This implementation requires the count to be in the range 1 to 2. - + PostProcessParams - Supplies the address of the post process parameter block. - + KernelFlags - Supplies additional flags controlling the operation. Return Value: @@ -79,15 +98,14 @@ Return Value: --*/ FUNCTION_ENTRY MlasConvSymDepthwiseU8KernelNeon - - stp d8,d9,[sp,#-64]! + stp d8,d9,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_Neg]! ldr x8,[sp,#.LConvSymDepthwiseKernelFrame_PostProcessParams] mov w10,#0x80808080 - stp d10,d11,[sp,#16] - stp d12,d13,[sp,#32] - stp d14,d15,[sp,#48] + stp d10,d11,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11] + stp d12,d13,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13] + stp d14,d15,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15] dup v8.4s,w10 // bit flip vector - ldr x16,[x8,#.LConvSymDepthwisePostProcessParams_Bias] + ldr x16,[x8,#.LConvSymPostProcessParams_Bias] cmp x7,2 add x9,x0,x3,lsl#3 // x9 -> &A1 add x14,x0,x3,lsl#4 // x14 -> &A2 @@ -111,11 +129,11 @@ Return Value: // x0 -> x10 -> v4 // -> x12 -> v2 // unroll // x9 -> x11 -> v6 -// -> x13 -> v10 // unroll +// -> x13 -> v3 // unroll // x14 -> x10 -> v4 // -> x12 -> v2 // unroll // x15 -> x11 -> v6 -// -> x13 -> v10 // unroll +// -> x13 -> v3 // unroll // .LProcess16Channels: @@ -144,7 +162,7 @@ Return Value: mov v18.16b,v26.16b ldr x12,[x14],#8 // x12 -> A2 iter 1 mov v19.16b,v27.16b - ldr q10,[x13,x5] // A1 iter 1 + ldr q3,[x13,x5] // A1 iter 1 mov v22.16b,v26.16b ldr x13,[x15],#8 // x13 -> A3 iter 1 mov v23.16b,v27.16b @@ -169,13 +187,13 @@ Return Value: smlal v12.8h,v1.8b,v2.8b ldr x11,[x9],#8 // x11 -> A1 iter 2 smlal2 v13.8h,v1.16b,v2.16b - b.eq .LEpilogueC16P3 // 3 pixel remains - eor v10.16b,v10.16b,v8.16b + b.eq .LEpilogueC16P3 // 3 pixel remains + eor v3.16b,v3.16b,v8.16b ldr q2,[x12,x5] // A2 iter 1 - smlal v14.8h,v1.8b,v10.8b + smlal v14.8h,v1.8b,v3.8b ldr x12,[x0],#8 // x12 -> A0 iter 3 - smlal2 v15.8h,v1.16b,v10.16b - ldr q10,[x13,x5] // A3 iter 1 + smlal2 v15.8h,v1.16b,v3.16b + ldr q3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h ldr x13,[x9],#8 // x13 -> A1 iter 3 @@ -200,12 +218,12 @@ Return Value: smlal v12.8h,v1.8b,v2.8b ldr x11,[x15],#8 // x11 -> A3 iter 2 smlal2 v13.8h,v1.16b,v2.16b - eor v10.16b,v10.16b,v8.16b + eor v3.16b,v3.16b,v8.16b ldr q2,[x12,x5] // A0 iter 3 - smlal v14.8h,v1.8b,v10.8b + smlal v14.8h,v1.8b,v3.8b ldr x12,[x14],#8 // x12 -> A2 iter 3 - smlal2 v15.8h,v1.16b,v10.16b - ldr q10,[x13,x5] // A1 iter 3 + smlal2 v15.8h,v1.16b,v3.16b + ldr q3,[x13,x5] // A1 iter 3 saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h ld1 {v1.16b},[x1],x4 // filter iter 3 @@ -229,11 +247,11 @@ Return Value: eor v2.16b,v2.16b,v8.16b smlal v12.8h,v1.8b,v2.8b smlal2 v13.8h,v1.16b,v2.16b - eor v10.16b,v10.16b,v8.16b + eor v3.16b,v3.16b,v8.16b ldr q2,[x12,x5] // A2 iter 1 - smlal v14.8h,v1.8b,v10.8b - smlal2 v15.8h,v1.16b,v10.16b - ldr q10,[x13,x5] // A3 iter 1 + smlal v14.8h,v1.8b,v3.8b + smlal2 v15.8h,v1.16b,v3.16b + ldr q3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h saddw v26.4s,v26.4s,v13.4h @@ -244,25 +262,18 @@ Return Value: saddw2 v31.4s,v31.4s,v15.8h ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags] eor v4.16b,v4.16b,v8.16b - ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b smull2 v13.8h,v0.16b,v4.16b eor v6.16b,v6.16b,v8.16b - ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b smull2 v15.8h,v0.16b,v6.16b eor v2.16b,v2.16b,v8.16b smlal v12.8h,v1.8b,v2.8b smlal2 v13.8h,v1.16b,v2.16b - eor v10.16b,v10.16b,v8.16b - smlal v14.8h,v1.8b,v10.8b - smlal2 v15.8h,v1.16b,v10.16b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq .LSkipScaleVecLoad2 - ldp q4,q11,[x12],#32 // load scale vector if per channel - ldp q6,q9,[x12] -.LSkipScaleVecLoad2: + eor v3.16b,v3.16b,v8.16b + smlal v14.8h,v1.8b,v3.8b + smlal2 v15.8h,v1.16b,v3.16b + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v18.4s,v18.4s,v13.4h @@ -271,7 +282,7 @@ Return Value: saddw2 v21.4s,v21.4s,v14.8h saddw v22.4s,v22.4s,v15.4h saddw2 v23.4s,v23.4s,v15.8h - b .LDequantization + b .LRequantization .LProcC16P1: // @@ -295,7 +306,7 @@ Return Value: subs x3,x3,2 // decrement input blocks remaining mov v18.16b,v26.16b mov v19.16b,v27.16b - ldr q10,[x13,x5] + ldr q3,[x13,x5] mov v22.16b,v26.16b mov v23.16b,v27.16b b .LEpilogueC16P1 @@ -305,12 +316,12 @@ Return Value: // Loop epilogue (process last 2 pixels) mixed // with loading of dequantization params // - eor v10.16b,v10.16b,v8.16b + eor v3.16b,v3.16b,v8.16b ldr q2,[x12,x5] // A2 iter 1 - smlal v14.8h,v1.8b,v10.8b + smlal v14.8h,v1.8b,v3.8b ldr x12,[x14],#8 // x12 -> A2 iter 2 - smlal2 v15.8h,v1.16b,v10.16b - ldr q10,[x13,x5] // A3 iter 1 + smlal2 v15.8h,v1.16b,v3.16b + ldr q3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h ldr x13,[x15],#8 // x13 -> A3 iter 2 @@ -332,11 +343,11 @@ Return Value: eor v2.16b,v2.16b,v8.16b smlal v12.8h,v1.8b,v2.8b smlal2 v13.8h,v1.16b,v2.16b - eor v10.16b,v10.16b,v8.16b + eor v3.16b,v3.16b,v8.16b ldr q2,[x12,x5] // A2 iter 2 - smlal v14.8h,v1.8b,v10.8b - smlal2 v15.8h,v1.16b,v10.16b - ldr q10,[x13,x5] // A3 iter 2 + smlal v14.8h,v1.8b,v3.8b + smlal2 v15.8h,v1.16b,v3.16b + ldr q3,[x13,x5] // A3 iter 2 saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v18.4s,v18.4s,v13.4h @@ -352,11 +363,9 @@ Return Value: // ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags] eor v4.16b,v4.16b,v8.16b - ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b smull2 v13.8h,v0.16b,v4.16b eor v6.16b,v6.16b,v8.16b - ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b smull2 v15.8h,v0.16b,v6.16b saddw v24.4s,v24.4s,v12.4h @@ -370,15 +379,10 @@ Return Value: eor v2.16b,v2.16b,v8.16b smull v12.8h,v0.8b,v2.8b smull2 v13.8h,v0.16b,v2.16b - eor v10.16b,v10.16b,v8.16b - smull v14.8h,v0.8b,v10.8b - smull2 v15.8h,v0.16b,v10.16b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq .LSkipScaleVecLoad - ldp q4,q11,[x12],#32 // load scale vector if per channel - ldp q6,q9,[x12] -.LSkipScaleVecLoad: + eor v3.16b,v3.16b,v8.16b + smull v14.8h,v0.8b,v3.8b + smull2 v15.8h,v0.16b,v3.16b + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v18.4s,v18.4s,v13.4h @@ -388,7 +392,24 @@ Return Value: saddw v22.4s,v22.4s,v15.4h saddw2 v23.4s,v23.4s,v15.8h -.LDequantization: +.LRequantization: + b.ne .LFixedPointScale +.LFloatingPointScale: + ldr x12,[x8,#.LConvSymPostProcessParams_Scale] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq .LBroadcastScaleValue + ldp q4,q5,[x12],#32 // load scale vector if per channel + ldp q6,q3,[x12] + b .LScaleWithFloatPoint + +.LBroadcastScaleValue: + ld1r {v4.4s},[x12] // load scale val + mov v5.16b,v4.16b // broadcast scale val if not per channel + mov v6.16b,v4.16b + mov v3.16b,v4.16b + +.LScaleWithFloatPoint: scvtf v24.4s,v24.4s // convert to float scvtf v25.4s,v25.4s scvtf v26.4s,v26.4s @@ -405,27 +426,22 @@ Return Value: scvtf v21.4s,v21.4s scvtf v22.4s,v22.4s scvtf v23.4s,v23.4s - b.ne .LSkipScaleBroadcast - mov v11.16b,v4.16b // broadcast scale val if not per channel - mov v6.16b,v4.16b - mov v9.16b,v4.16b -.LSkipScaleBroadcast: fmul v24.4s,v24.4s,v4.4s // multiply by scale - fmul v25.4s,v25.4s,v11.4s + fmul v25.4s,v25.4s,v5.4s fmul v26.4s,v26.4s,v6.4s - fmul v27.4s,v27.4s,v9.4s + fmul v27.4s,v27.4s,v3.4s fmul v28.4s,v28.4s,v4.4s - fmul v29.4s,v29.4s,v11.4s + fmul v29.4s,v29.4s,v5.4s fmul v30.4s,v30.4s,v6.4s - fmul v31.4s,v31.4s,v9.4s + fmul v31.4s,v31.4s,v3.4s fmul v16.4s,v16.4s,v4.4s - fmul v17.4s,v17.4s,v11.4s + fmul v17.4s,v17.4s,v5.4s fmul v18.4s,v18.4s,v6.4s - fmul v19.4s,v19.4s,v9.4s + fmul v19.4s,v19.4s,v3.4s fmul v20.4s,v20.4s,v4.4s - fmul v21.4s,v21.4s,v11.4s + fmul v21.4s,v21.4s,v5.4s fmul v22.4s,v22.4s,v6.4s - fmul v23.4s,v23.4s,v9.4s + fmul v23.4s,v23.4s,v3.4s fcvtns v24.4s,v24.4s // convert to int fcvtns v25.4s,v25.4s fcvtns v26.4s,v26.4s @@ -442,6 +458,90 @@ Return Value: fcvtns v21.4s,v21.4s fcvtns v22.4s,v22.4s fcvtns v23.4s,v23.4s + b .LQuantize + +.LFixedPointScale: + ldr x10,[x8,#.LConvSymPostProcessParams_PreShift] + ldr x11,[x8,#.LConvSymPostProcessParams_Multiplier] + ldr x12,[x8,#.LConvSymPostProcessParams_PostShift] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq .LBroadcastFixedPointValue + ldp q0,q1,[x10],32 // load preshift vector + ldp q2,q3,[x10],32 + ldp q4,q5,[x11],32 // load multiplier vector + ldp q6,q7,[x11],32 + ldp q8,q9,[x12],32 // load postshift vector + ldp q10,q11,[x12],32 + b .LScaleWithFixedPoint + +.LBroadcastFixedPointValue: + ld1r {v0.4s},[x10] // load preshift Value + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + + ld1r {v4.4s},[x11] // load multiplier Value + mov v5.16b, v4.16b + mov v6.16b, v4.16b + mov v7.16b, v4.16b + + ld1r {v8.4s},[x12] // load postshift Value + mov v9.16b, v8.16b + mov v10.16b, v8.16b + mov v11.16b, v8.16b + +.LScaleWithFixedPoint: + sqshl v24.4s, v24.4s, v0.4s // preshift + sqshl v25.4s, v25.4s, v1.4s + sqshl v26.4s, v26.4s, v2.4s + sqshl v27.4s, v27.4s, v3.4s + sqshl v28.4s, v28.4s, v0.4s + sqshl v29.4s, v29.4s, v1.4s + sqshl v30.4s, v30.4s, v2.4s + sqshl v31.4s, v31.4s, v3.4s + sqshl v16.4s, v16.4s, v0.4s + sqshl v17.4s, v17.4s, v1.4s + sqshl v18.4s, v18.4s, v2.4s + sqshl v19.4s, v19.4s, v3.4s + sqshl v20.4s, v20.4s, v0.4s + sqshl v21.4s, v21.4s, v1.4s + sqshl v22.4s, v22.4s, v2.4s + sqshl v23.4s, v23.4s, v3.4s + sqdmulh v24.4s, v24.4s, v4.4s // multiplier + sqdmulh v25.4s, v25.4s, v5.4s + sqdmulh v26.4s, v26.4s, v6.4s + sqdmulh v27.4s, v27.4s, v7.4s + sqdmulh v28.4s, v28.4s, v4.4s + sqdmulh v29.4s, v29.4s, v5.4s + sqdmulh v30.4s, v30.4s, v6.4s + sqdmulh v31.4s, v31.4s, v7.4s + sqdmulh v16.4s, v16.4s, v4.4s + sqdmulh v17.4s, v17.4s, v5.4s + sqdmulh v18.4s, v18.4s, v6.4s + sqdmulh v19.4s, v19.4s, v7.4s + sqdmulh v20.4s, v20.4s, v4.4s + sqdmulh v21.4s, v21.4s, v5.4s + sqdmulh v22.4s, v22.4s, v6.4s + sqdmulh v23.4s, v23.4s, v7.4s + srshl v24.4s, v24.4s, v8.4s // post shift + srshl v25.4s, v25.4s, v9.4s + srshl v26.4s, v26.4s, v10.4s + srshl v27.4s, v27.4s, v11.4s + srshl v28.4s, v28.4s, v8.4s + srshl v29.4s, v29.4s, v9.4s + srshl v30.4s, v30.4s, v10.4s + srshl v31.4s, v31.4s, v11.4s + srshl v16.4s, v16.4s, v8.4s + srshl v17.4s, v17.4s, v9.4s + srshl v18.4s, v18.4s, v10.4s + srshl v19.4s, v19.4s, v11.4s + srshl v20.4s, v20.4s, v8.4s + srshl v21.4s, v21.4s, v9.4s + srshl v22.4s, v22.4s, v10.4s + srshl v23.4s, v23.4s, v11.4s + +.LQuantize: sqxtn v24.4h,v24.4s // shorten to int16 sqxtn v26.4h,v26.4s sqxtn2 v24.8h,v25.4s @@ -486,10 +586,10 @@ Return Value: str q20,[x2] .LExitKernel: - ldp d14,d15,[sp,#48] - ldp d12,d13,[sp,#32] - ldp d10,d11,[sp,#16] - ldp d8,d9,[sp],#64 + ldp d14,d15,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15] + ldp d12,d13,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13] + ldp d10,d11,[sp,#.LConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11] + ldp d8,d9,[sp], #.LConvSymDepthwiseKernelFrame_SavedRegisters ret .LProcess8Channels: @@ -511,7 +611,7 @@ Return Value: mov v17.16b,v25.16b ldr x12,[x14],#8 // x12 -> A2 iter 1 subs x3,x3,2 // decrement input blocks remaining - ldr d10,[x13,x5] // A1 iter 1 + ldr d3,[x13,x5] // A1 iter 1 mov v20.16b,v24.16b ldr x13,[x15],#8 // x13 -> A3 iter 1 mov v21.16b,v25.16b @@ -528,15 +628,15 @@ Return Value: b.eq .LEpilogueC8P2 ldr x10,[x0],#8 // x10 -> A0 iter 2 eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b ldr d6,[x11,x5] // A3 iter 0 cmp x3,1 smlal v12.8h,v1.8b,v2.8b ldr x11,[x9],#8 // x11 -> A1 iter 2 - smlal v14.8h,v1.8b,v10.8b + smlal v14.8h,v1.8b,v3.8b ldr d2,[x12,x5] // A2 iter 1 - b.eq .LEpilogueC8P3 // 3 pixel remains - ldr d10,[x13,x5] // A3 iter 1 + b.eq .LEpilogueC8P3 // 3 pixel remains + ldr d3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h ldr x12,[x0],#8 // x12 -> A0 iter 3 saddw2 v25.4s,v25.4s,v12.8h @@ -552,15 +652,15 @@ Return Value: ldr x10,[x14],#8 // x10 -> A2 iter 2 ldr d6,[x11,x5] // A1 iter 2 eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b ld1 {v0.8b},[x1],x4 // filter iter 2 smlal v12.8h,v1.8b,v2.8b ldr x11,[x15],#8 // x11 -> A3 iter 2 ldr d2,[x12,x5] // A0 iter 3 - smlal v14.8h,v1.8b,v10.8b + smlal v14.8h,v1.8b,v3.8b ldr x12,[x14],#8 // x12 -> A2 iter 3 saddw v16.4s,v16.4s,v12.4h - ldr d10,[x13,x5] // A1 iter 3 + ldr d3,[x13,x5] // A1 iter 3 saddw2 v17.4s,v17.4s,v12.8h ld1 {v1.8b},[x1],x4 // filter iter 3 saddw v20.4s,v20.4s,v14.4h @@ -575,11 +675,11 @@ Return Value: // ldr d6,[x11,x5] // A3 iter 0 eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b smlal v12.8h,v1.8b,v2.8b ldr d2,[x12,x5] // A2 iter 1 - smlal v14.8h,v1.8b,v10.8b - ldr d10,[x13,x5] // A3 iter 1 + smlal v14.8h,v1.8b,v3.8b + ldr d3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h saddw v28.4s,v28.4s,v14.4h @@ -588,23 +688,17 @@ Return Value: eor v4.8b,v4.8b,v8.8b eor v6.8b,v6.8b,v8.8b smull v12.8h,v0.8b,v4.8b - ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale] smull v14.8h,v0.8b,v6.8b - ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint] eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b smlal v12.8h,v1.8b,v2.8b - smlal v14.8h,v1.8b,v10.8b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq .LSkipScaleVecLoad2C8 - ldp q4,q11,[x12],#32 // load scale vector if per channel -.LSkipScaleVecLoad2C8: + smlal v14.8h,v1.8b,v3.8b + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v20.4s,v20.4s,v14.4h saddw2 v21.4s,v21.4s,v14.8h - b .LDequantC8 + b .LRequantizationC8 .LProcC8P1: // @@ -623,7 +717,7 @@ Return Value: mov v20.16b,v24.16b ldr d2,[x12,x5] subs x3,x3,2 // decrement input blocks remaining - ldr d10,[x13,x5] + ldr d3,[x13,x5] mov v21.16b,v25.16b b .LEpilogueC8P1 @@ -632,7 +726,7 @@ Return Value: // Loop epilogue (process 2 of last 3 pixels) // ldr x12,[x14],#8 // x12 -> A2 iter 2 - ldr d10,[x13,x5] // A3 iter 1 + ldr d3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h ldr x13,[x15],#8 // x13 -> A3 iter 2 @@ -645,12 +739,12 @@ Return Value: smull v14.8h,v0.8b,v6.8b ld1 {v0.8b},[x1] // filter iter 2 eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b ldr d6,[x11,x5] // A1 iter 2 smlal v12.8h,v1.8b,v2.8b ldr d2,[x12,x5] // A2 iter 2 - smlal v14.8h,v1.8b,v10.8b - ldr d10,[x13,x5] // A3 iter 2 + smlal v14.8h,v1.8b,v3.8b + ldr d3,[x13,x5] // A3 iter 2 saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v20.4s,v20.4s,v14.4h @@ -663,29 +757,37 @@ Return Value: ldr w9,[sp,#.LConvSymDepthwiseKernelFrame_KernelFlags] eor v4.8b,v4.8b,v8.8b eor v6.8b,v6.8b,v8.8b - ldr x12,[x8,#.LConvSymDepthwisePostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b - ldr w15,[x8,#.LConvSymDepthwisePostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h saddw v28.4s,v28.4s,v14.4h saddw2 v29.4s,v29.4s,v14.8h eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b smull v12.8h,v0.8b,v2.8b - smull v14.8h,v0.8b,v10.8b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq .LSkipScaleVecLoadC8 - ldp q4,q11,[x12] // load scale vector if per channel -.LSkipScaleVecLoadC8: + smull v14.8h,v0.8b,v3.8b + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v20.4s,v20.4s,v14.4h saddw2 v21.4s,v21.4s,v14.8h -.LDequantC8: +.LRequantizationC8: + b.ne .LFixedPointScaleC8 +.LFloatingPointScaleC8: + ldr x12,[x8,#.LConvSymPostProcessParams_Scale] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq .LBroadcastScaleValueC8 + ldp q4,q5,[x12] // load scale vector if per channel + b .LScaleWithFloatPointC8 + +.LBroadcastScaleValueC8: + ld1r {v4.4s},[x12] // load scale val + mov v5.16b,v4.16b // broadcast scale val if not per channel + +.LScaleWithFloatPointC8: scvtf v24.4s,v24.4s // convert to float scvtf v25.4s,v25.4s scvtf v28.4s,v28.4s @@ -694,17 +796,14 @@ Return Value: scvtf v17.4s,v17.4s scvtf v20.4s,v20.4s scvtf v21.4s,v21.4s - b.ne .LSkipScaleBroadcastC8 - mov v11.16b,v4.16b // broadcast scale val if not per channel -.LSkipScaleBroadcastC8: fmul v24.4s,v24.4s,v4.4s // multiply by scale - fmul v25.4s,v25.4s,v11.4s + fmul v25.4s,v25.4s,v5.4s fmul v28.4s,v28.4s,v4.4s - fmul v29.4s,v29.4s,v11.4s + fmul v29.4s,v29.4s,v5.4s fmul v16.4s,v16.4s,v4.4s - fmul v17.4s,v17.4s,v11.4s + fmul v17.4s,v17.4s,v5.4s fmul v20.4s,v20.4s,v4.4s - fmul v21.4s,v21.4s,v11.4s + fmul v21.4s,v21.4s,v5.4s fcvtns v24.4s,v24.4s // convert to int fcvtns v25.4s,v25.4s fcvtns v28.4s,v28.4s @@ -713,6 +812,57 @@ Return Value: fcvtns v17.4s,v17.4s fcvtns v20.4s,v20.4s fcvtns v21.4s,v21.4s + b .LQuantizeC8 + +.LFixedPointScaleC8: + ldr x10,[x8,#.LConvSymPostProcessParams_PreShift] + ldr x11,[x8,#.LConvSymPostProcessParams_Multiplier] + ldr x12,[x8,#.LConvSymPostProcessParams_PostShift] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#.LConvSymPostProcessParams_ZeroPoint] + beq .LBroadcastFixedPointValueC8 + ldp q0,q1,[x10],32 // load preshift vector + ldp q4,q5,[x11],32 // load multiplier vector + ldp q8,q9,[x12],32 // load postshift vector + b .LScaleWithFixedPointC8 + +.LBroadcastFixedPointValueC8: + ld1r {v0.4s},[x10] // load preshift Value + mov v1.16b, v0.16b + + ld1r {v4.4s},[x11] // load multiplier Value + mov v5.16b, v4.16b + + ld1r {v8.4s},[x12] // load postshift Value + mov v9.16b, v8.16b + +.LScaleWithFixedPointC8: + sqshl v24.4s, v24.4s, v0.4s // preshift + sqshl v25.4s, v25.4s, v1.4s + sqshl v28.4s, v28.4s, v0.4s + sqshl v29.4s, v29.4s, v1.4s + sqshl v16.4s, v16.4s, v0.4s + sqshl v17.4s, v17.4s, v1.4s + sqshl v20.4s, v20.4s, v0.4s + sqshl v21.4s, v21.4s, v1.4s + sqdmulh v24.4s, v24.4s, v4.4s // multiplier + sqdmulh v25.4s, v25.4s, v5.4s + sqdmulh v28.4s, v28.4s, v4.4s + sqdmulh v29.4s, v29.4s, v5.4s + sqdmulh v16.4s, v16.4s, v4.4s + sqdmulh v17.4s, v17.4s, v5.4s + sqdmulh v20.4s, v20.4s, v4.4s + sqdmulh v21.4s, v21.4s, v5.4s + srshl v24.4s, v24.4s, v8.4s // post shift + srshl v25.4s, v25.4s, v9.4s + srshl v28.4s, v28.4s, v8.4s + srshl v29.4s, v29.4s, v9.4s + srshl v16.4s, v16.4s, v8.4s + srshl v17.4s, v17.4s, v9.4s + srshl v20.4s, v20.4s, v8.4s + srshl v21.4s, v21.4s, v9.4s + +.LQuantizeC8: dup v0.8h,w15 sqxtn v24.4h,v24.4s // shorten to int16 sqxtn2 v24.8h,v25.4s diff --git a/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelDot.asm b/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelDot.asm index d9eafb8203..92d4f9ffd1 100644 --- a/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelDot.asm +++ b/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelDot.asm @@ -19,20 +19,43 @@ Abstract: #include "AssembleDotProduct.h" #define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2 +#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4 // // Stack frame layout for the symmetric convolution kernel. // d8-d15, x19-x30 need to be preserved if used // -#define ConvSymFrame_SavedRegisters (6 * 8) -#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters -#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters -#define ConvSymPostProcessParams_Bias 0 -#define ConvSymPostProcessParams_Scale 8 -#define ConvSymPostProcessParams_Min 16 -#define ConvSymPostProcessParams_Max 20 -#define ConvSymPostProcessParams_ZeroPoint 24 +#define ConvSymFrame_SavedRegisters_d8_d9 0 +#define ConvSymFrame_SavedRegisters_d10_d11 16 +#define ConvSymFrame_SavedRegisters_x19_x20 32 +#define ConvSymFrame_SavedRegisters_x21_x22 48 +#define ConvSymFrame_SavedRegisters 64 +#define ConvSymFrame_SavedRegisters_Neg -64 +#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters +#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters + +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ +#define ConvSymPostProcessParams_Bias 0 +#define ConvSymPostProcessParams_Scale 8 +#define ConvSymPostProcessParams_Min 16 +#define ConvSymPostProcessParams_Max 20 +#define ConvSymPostProcessParams_ZeroPoint 24 +#define ConvSymPostProcessParams_Multiplier 32 +#define ConvSymPostProcessParams_PreShift 40 +#define ConvSymPostProcessParams_PostShift 48 TEXTAREA @@ -78,13 +101,13 @@ Return Value: --*/ NESTED_ENTRY MlasConvSymS8KernelDot - PROLOG_SAVE_REG_PAIR d8,d9,#-ConvSymFrame_SavedRegisters! + PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters_Neg! PROLOG_NOP ldr x8,[sp,#ConvSymFrame_PostProcessParams] - PROLOG_SAVE_REG d10,#16 PROLOG_NOP cmp x7,2 // OutputCount < 2 ? - PROLOG_SAVE_REG d11,#24 PROLOG_NOP add x16,x2,x5 // x16 -> C1 - PROLOG_SAVE_REG x19,#32 + PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11 + PROLOG_SAVE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20 + PROLOG_SAVE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22 lsl x3,x3,#3 // KernelSize * sizeof(int8_t*) csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0 add x4,x4,3 // InputChannels align to 4 @@ -97,6 +120,9 @@ Return Value: add x5,x17,x5 // x5 -> C3 ldr x19,[x8,#ConvSymPostProcessParams_Scale] csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2 + ldr x20,[x8,#ConvSymPostProcessParams_PreShift] + ldr x21,[x8,#ConvSymPostProcessParams_Multiplier] + ldr x22,[x8,#ConvSymPostProcessParams_PostShift] // TODO!! tiptoe around loading biases if we need to support // output channels none divisible by 16 @@ -347,12 +373,15 @@ InChLoopEpilogue b.hi KernelSizeLoop Requantize + tst w10,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale +FloatPointScale tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint] beq BroadcastScaleValue ldp q0,q1,[x19],32 // load scale vector ldp q2,q3,[x19],32 - b AccumulatorsToFloat + b ScaleWithFloatPoint BroadcastScaleValue ld1r {v0.4s},[x19] // load scale Value @@ -360,7 +389,7 @@ BroadcastScaleValue mov v2.16b, v0.16b mov v3.16b, v0.16b -AccumulatorsToFloat +ScaleWithFloatPoint scvtf v16.4s,v16.4s // convert to float scvtf v17.4s,v17.4s scvtf v18.4s,v18.4s @@ -409,7 +438,87 @@ AccumulatorsToFloat fcvtns v29.4s,v29.4s fcvtns v30.4s,v30.4s fcvtns v31.4s,v31.4s + b Quantize +FixedPointScale + tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint] + beq BroadcastFixedPointValue + ldp q0,q1,[x20],32 // load preshift vector + ldp q2,q3,[x20],32 + ldp q4,q5,[x21],32 // load multiplier vector + ldp q6,q7,[x21],32 + ldp q8,q9,[x22],32 // load postshift vector + ldp q10,q11,[x22],32 + b ScaleWithFixedPoint + +BroadcastFixedPointValue + ld1r {v0.4s},[x20] // load preshift Value + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + + ld1r {v4.4s},[x21] // load multiplier Value + mov v5.16b, v4.16b + mov v6.16b, v4.16b + mov v7.16b, v4.16b + + ld1r {v8.4s},[x22] // load postshift Value + mov v9.16b, v8.16b + mov v10.16b, v8.16b + mov v11.16b, v8.16b + +ScaleWithFixedPoint + sqshl v16.4s, v16.4s, v0.4s // preshift + sqshl v17.4s, v17.4s, v0.4s + sqshl v18.4s, v18.4s, v0.4s + sqshl v19.4s, v19.4s, v0.4s + sqshl v20.4s, v20.4s, v1.4s + sqshl v21.4s, v21.4s, v1.4s + sqshl v22.4s, v22.4s, v1.4s + sqshl v23.4s, v23.4s, v1.4s + sqshl v24.4s, v24.4s, v2.4s + sqshl v25.4s, v25.4s, v2.4s + sqshl v26.4s, v26.4s, v2.4s + sqshl v27.4s, v27.4s, v2.4s + sqshl v28.4s, v28.4s, v3.4s + sqshl v29.4s, v29.4s, v3.4s + sqshl v30.4s, v30.4s, v3.4s + sqshl v31.4s, v31.4s, v3.4s + sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale + sqdmulh v17.4s, v17.4s, v4.4s + sqdmulh v18.4s, v18.4s, v4.4s + sqdmulh v19.4s, v19.4s, v4.4s + sqdmulh v20.4s, v20.4s, v5.4s + sqdmulh v21.4s, v21.4s, v5.4s + sqdmulh v22.4s, v22.4s, v5.4s + sqdmulh v23.4s, v23.4s, v5.4s + sqdmulh v24.4s, v24.4s, v6.4s + sqdmulh v25.4s, v25.4s, v6.4s + sqdmulh v26.4s, v26.4s, v6.4s + sqdmulh v27.4s, v27.4s, v6.4s + sqdmulh v28.4s, v28.4s, v7.4s + sqdmulh v29.4s, v29.4s, v7.4s + sqdmulh v30.4s, v30.4s, v7.4s + sqdmulh v31.4s, v31.4s, v7.4s + srshl v16.4s,v16.4s, v8.4s // post shift + srshl v17.4s,v17.4s, v8.4s + srshl v18.4s,v18.4s, v8.4s + srshl v19.4s,v19.4s, v8.4s + srshl v20.4s,v20.4s, v9.4s + srshl v21.4s,v21.4s, v9.4s + srshl v22.4s,v22.4s, v9.4s + srshl v23.4s,v23.4s, v9.4s + srshl v24.4s,v24.4s, v10.4s + srshl v25.4s,v25.4s, v10.4s + srshl v26.4s,v26.4s, v10.4s + srshl v27.4s,v27.4s, v10.4s + srshl v28.4s,v28.4s, v11.4s + srshl v29.4s,v29.4s, v11.4s + srshl v30.4s,v30.4s, v11.4s + srshl v31.4s,v31.4s, v11.4s + +Quantize sqxtn v16.4h,v16.4s sqxtn v17.4h,v17.4s sqxtn v18.4h,v18.4s @@ -454,8 +563,9 @@ AccumulatorsToFloat b.hi OutputChannelLoop ExitKernel - EPILOG_RESTORE_REG x19,#32 - EPILOG_RESTORE_REG_PAIR d10,d11,#16 + EPILOG_RESTORE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22 + EPILOG_RESTORE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20 + EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11 EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters! EPILOG_RETURN diff --git a/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelDotLd64.asm b/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelDotLd64.asm index d513c8ae5f..abc3362ac1 100644 --- a/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelDotLd64.asm +++ b/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelDotLd64.asm @@ -19,20 +19,50 @@ Abstract: #include "AssembleDotProduct.h" #define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2 +#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4 // // Stack frame layout for the symmetric convolution kernel. // d8-d15, x19-x30 need to be preserved if used // -#define ConvSymFrame_SavedRegisters (10 * 8) -#define ConvSymFrame_PostProcessParams (0 + ConvSymFrame_SavedRegisters) -#define ConvSymFrame_KernelFlags (8 + ConvSymFrame_SavedRegisters) +#define ConvSymFrame_SavedRegisters_d8 0 +#define ConvSymFrame_SavedRegisters_d9 8 +#define ConvSymFrame_SavedRegisters_d10 16 +#define ConvSymFrame_SavedRegisters_d11 24 +#define ConvSymFrame_SavedRegisters_x19 32 +#define ConvSymFrame_SavedRegisters_x20 40 +#define ConvSymFrame_SavedRegisters_x21 48 +#define ConvSymFrame_SavedRegisters_x22 56 +#define ConvSymFrame_SavedRegisters_x23 64 +#define ConvSymFrame_SavedRegisters_x24 72 +#define ConvSymFrame_SavedRegisters_x25 80 +#define ConvSymFrame_SavedRegisters_x26 88 +#define ConvSymFrame_SavedRegisters 96 +#define ConvSymFrame_SavedRegisters_Neg -96 +#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters +#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters -#define ConvSymPostProcessParams_Bias 0 -#define ConvSymPostProcessParams_Scale 8 -#define ConvSymPostProcessParams_Min 16 -#define ConvSymPostProcessParams_Max 20 -#define ConvSymPostProcessParams_ZeroPoint 24 +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ +#define ConvSymPostProcessParams_Bias 0 +#define ConvSymPostProcessParams_Scale 8 +#define ConvSymPostProcessParams_Min 16 +#define ConvSymPostProcessParams_Max 20 +#define ConvSymPostProcessParams_ZeroPoint 24 +#define ConvSymPostProcessParams_Multiplier 32 +#define ConvSymPostProcessParams_PreShift 40 +#define ConvSymPostProcessParams_PostShift 48 TEXTAREA @@ -45,13 +75,7 @@ Routine Description: Arguments: - Input (x0) - Points to the input buffer. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points - directly at the input tensor. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an - indirection buffer. Every pointer in the indirection buffer points at a + Input (x0) - Points to the input indirection buffer. Every pointer in the indirection buffer points at a InputChannels length vector (either from the input tensor or a vector of padding values). These are grouped in batches of length KernelSize. These batches are then repeated OutputCount times. @@ -61,8 +85,7 @@ Arguments: Output (x2) - Points the output buffer. KernelSize (x3/x9) - Size of the kernel (most commonly. 3x3=9, 5x5=25). - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1. + Must be > 1 InputChannels (x4/x7) - Number of input channels. @@ -85,29 +108,35 @@ Return Value: --*/ NESTED_ENTRY MlasConvSymS8KernelDotLd64 - PROLOG_SAVE_REG_PAIR d8,d9,#-ConvSymFrame_SavedRegisters! + PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters_Neg! PROLOG_NOP ldr x8,[sp,#ConvSymFrame_PostProcessParams] - PROLOG_SAVE_REG d10,#16 + PROLOG_SAVE_REG d10,#ConvSymFrame_SavedRegisters_d10 PROLOG_NOP cmp x7,2 // OutputCount < 2 ? - PROLOG_SAVE_REG d11,#24 + PROLOG_SAVE_REG d11,#ConvSymFrame_SavedRegisters_d11 PROLOG_NOP add x16,x2,x5 // x16 -> C1 - PROLOG_SAVE_REG x19,#32 + PROLOG_SAVE_REG x19,#ConvSymFrame_SavedRegisters_x19 PROLOG_NOP lsl x3,x3,#3 // KernelSize * sizeof(int8_t*) - PROLOG_SAVE_REG x20,#40 + PROLOG_SAVE_REG x20,#ConvSymFrame_SavedRegisters_x20 PROLOG_NOP csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0 - PROLOG_SAVE_REG x21,#48 + PROLOG_SAVE_REG x21,#ConvSymFrame_SavedRegisters_x21 PROLOG_NOP add x4,x4,3 // InputChannels align to 4 - PROLOG_SAVE_REG x22,#56 + PROLOG_SAVE_REG x22,#ConvSymFrame_SavedRegisters_x22 PROLOG_NOP add x17,x16,x5 // x17 -> C2 - PROLOG_SAVE_REG x23,#64 + PROLOG_SAVE_REG x23,#ConvSymFrame_SavedRegisters_x23 ldr x11,[x8,#ConvSymPostProcessParams_Bias] + PROLOG_SAVE_REG x24,#ConvSymFrame_SavedRegisters_x24 csel x17,x16,x17,ls // if OutputCount <= 2 x17/C2 -> C1 + PROLOG_SAVE_REG x25,#ConvSymFrame_SavedRegisters_x25 bic x4,x4,3 + PROLOG_SAVE_REG x26,#ConvSymFrame_SavedRegisters_x26 cmp x7,4 // OutputCount < 4 ? ldr w10,[sp,#ConvSymFrame_KernelFlags] add x5,x17,x5 // x5 -> C3 ldr x19,[x8,#ConvSymPostProcessParams_Scale] csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2 + ldr x24,[x8,#ConvSymPostProcessParams_PreShift] + ldr x25,[x8,#ConvSymPostProcessParams_Multiplier] + ldr x26,[x8,#ConvSymPostProcessParams_PostShift] // TODO!! tiptoe around loading biases if we need to support // output channels none divisible by 16 @@ -422,12 +451,15 @@ InChLoopEpilogue b.hi KernelSizeLoop Requantize + tst w10,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale +FloatPointScale tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint] beq BroadcastScaleValue ldp q0,q1,[x19],32 // load scale vector ldp q2,q3,[x19],32 - b AccumulatorsToFloat + b ScaleWithFloatPoint BroadcastScaleValue ld1r {v0.4s},[x19] // load scale Value @@ -435,7 +467,7 @@ BroadcastScaleValue mov v2.16b, v0.16b mov v3.16b, v0.16b -AccumulatorsToFloat +ScaleWithFloatPoint scvtf v16.4s,v16.4s // convert to float scvtf v17.4s,v17.4s scvtf v18.4s,v18.4s @@ -484,7 +516,87 @@ AccumulatorsToFloat fcvtns v29.4s,v29.4s fcvtns v30.4s,v30.4s fcvtns v31.4s,v31.4s + b Quantize +FixedPointScale + tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint] + beq BroadcastFixedPointValue + ldp q0,q1,[x24],32 // load preshift vector + ldp q2,q3,[x24],32 + ldp q4,q5,[x25],32 // load multiplier vector + ldp q6,q7,[x25],32 + ldp q8,q9,[x26],32 // load postshift vector + ldp q10,q11,[x26],32 + b ScaleWithFixedPoint + +BroadcastFixedPointValue + ld1r {v0.4s},[x24] // load preshift Value + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + + ld1r {v4.4s},[x25] // load multiplier Value + mov v5.16b, v4.16b + mov v6.16b, v4.16b + mov v7.16b, v4.16b + + ld1r {v8.4s},[x26] // load postshift Value + mov v9.16b, v8.16b + mov v10.16b, v8.16b + mov v11.16b, v8.16b + +ScaleWithFixedPoint + sqshl v16.4s, v16.4s, v0.4s // preshift + sqshl v17.4s, v17.4s, v0.4s + sqshl v18.4s, v18.4s, v0.4s + sqshl v19.4s, v19.4s, v0.4s + sqshl v20.4s, v20.4s, v1.4s + sqshl v21.4s, v21.4s, v1.4s + sqshl v22.4s, v22.4s, v1.4s + sqshl v23.4s, v23.4s, v1.4s + sqshl v24.4s, v24.4s, v2.4s + sqshl v25.4s, v25.4s, v2.4s + sqshl v26.4s, v26.4s, v2.4s + sqshl v27.4s, v27.4s, v2.4s + sqshl v28.4s, v28.4s, v3.4s + sqshl v29.4s, v29.4s, v3.4s + sqshl v30.4s, v30.4s, v3.4s + sqshl v31.4s, v31.4s, v3.4s + sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale + sqdmulh v17.4s, v17.4s, v4.4s + sqdmulh v18.4s, v18.4s, v4.4s + sqdmulh v19.4s, v19.4s, v4.4s + sqdmulh v20.4s, v20.4s, v5.4s + sqdmulh v21.4s, v21.4s, v5.4s + sqdmulh v22.4s, v22.4s, v5.4s + sqdmulh v23.4s, v23.4s, v5.4s + sqdmulh v24.4s, v24.4s, v6.4s + sqdmulh v25.4s, v25.4s, v6.4s + sqdmulh v26.4s, v26.4s, v6.4s + sqdmulh v27.4s, v27.4s, v6.4s + sqdmulh v28.4s, v28.4s, v7.4s + sqdmulh v29.4s, v29.4s, v7.4s + sqdmulh v30.4s, v30.4s, v7.4s + sqdmulh v31.4s, v31.4s, v7.4s + srshl v16.4s,v16.4s, v8.4s // post shift + srshl v17.4s,v17.4s, v8.4s + srshl v18.4s,v18.4s, v8.4s + srshl v19.4s,v19.4s, v8.4s + srshl v20.4s,v20.4s, v9.4s + srshl v21.4s,v21.4s, v9.4s + srshl v22.4s,v22.4s, v9.4s + srshl v23.4s,v23.4s, v9.4s + srshl v24.4s,v24.4s, v10.4s + srshl v25.4s,v25.4s, v10.4s + srshl v26.4s,v26.4s, v10.4s + srshl v27.4s,v27.4s, v10.4s + srshl v28.4s,v28.4s, v11.4s + srshl v29.4s,v29.4s, v11.4s + srshl v30.4s,v30.4s, v11.4s + srshl v31.4s,v31.4s, v11.4s + +Quantize sqxtn v16.4h,v16.4s sqxtn v17.4h,v17.4s sqxtn v18.4h,v18.4s @@ -529,10 +641,11 @@ AccumulatorsToFloat b.hi OutputChannelLoop ExitKernel - EPILOG_RESTORE_REG x23,#64 - EPILOG_RESTORE_REG_PAIR x21,x22,#48 - EPILOG_RESTORE_REG_PAIR x19,x20,#32 - EPILOG_RESTORE_REG_PAIR d10,d11,#16 + EPILOG_RESTORE_REG_PAIR x25,x26,#ConvSymFrame_SavedRegisters_x25 + EPILOG_RESTORE_REG_PAIR x23,x24,#ConvSymFrame_SavedRegisters_x23 + EPILOG_RESTORE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21 + EPILOG_RESTORE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19 + EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10 EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters! EPILOG_RETURN @@ -587,7 +700,7 @@ InChannels4 ldr s1,[x13],4 ldr s2,[x14],4 ldr s3,[x15],4 - ldr q5, [x1], 16 + ldr q5, [x1],16 SdotByElement 16, 4, 0,0 SdotByElement 17, 4, 1,0 ldp q6, q7, [x1], 32 diff --git a/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelNeon.asm b/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelNeon.asm index c227302313..6cea1e8d01 100644 --- a/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelNeon.asm +++ b/onnxruntime/core/mlas/lib/arm64/ConvSymS8KernelNeon.asm @@ -18,21 +18,45 @@ Abstract: #include "kxarm64.h" #define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2 +#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4 // // Stack frame layout for the symmetric convolution kernel. // d8-d15, x19-x30 need to be preserved if used // -#define ConvSymFrame_SavedNeonRegisters (8 * 8) -#define ConvSymFrame_SavedRegisters ConvSymFrame_SavedNeonRegisters -#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters -#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters -#define ConvSymPostProcessParams_Bias 0 -#define ConvSymPostProcessParams_Scale 8 -#define ConvSymPostProcessParams_Min 16 -#define ConvSymPostProcessParams_Max 20 -#define ConvSymPostProcessParams_ZeroPoint 24 +#define ConvSymFrame_SavedRegisters_d8_d9 0 +#define ConvSymFrame_SavedRegisters_d10_d11 16 +#define ConvSymFrame_SavedRegisters_d12_d13 32 +#define ConvSymFrame_SavedRegisters_d14_d15 48 +#define ConvSymFrame_SavedRegisters_x19_x20 64 +#define ConvSymFrame_SavedRegisters_x21_x22 80 +#define ConvSymFrame_SavedRegisters 96 +#define ConvSymFrame_SavedRegisters_Neg -96 +#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters +#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters + +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ +#define ConvSymPostProcessParams_Bias 0 +#define ConvSymPostProcessParams_Scale 8 +#define ConvSymPostProcessParams_Min 16 +#define ConvSymPostProcessParams_Max 20 +#define ConvSymPostProcessParams_ZeroPoint 24 +#define ConvSymPostProcessParams_Multiplier 32 +#define ConvSymPostProcessParams_PreShift 40 +#define ConvSymPostProcessParams_PostShift 48 TEXTAREA @@ -82,12 +106,14 @@ Return Value: --*/ NESTED_ENTRY MlasConvSymS8KernelNeon - PROLOG_SAVE_REG_PAIR d8,d9,#-ConvSymFrame_SavedRegisters! + PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters_Neg! PROLOG_NOP ldr x8,[sp,#ConvSymFrame_PostProcessParams] PROLOG_NOP ldrb w10,[sp,#ConvSymFrame_KernelFlags] - PROLOG_SAVE_REG_PAIR d10,d11,#16 - PROLOG_SAVE_REG_PAIR d12,d13,#32 - PROLOG_SAVE_REG_PAIR d14,d15,#48 + PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11 + PROLOG_SAVE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13 + PROLOG_SAVE_REG_PAIR d14,d15,#ConvSymFrame_SavedRegisters_d14_d15 + PROLOG_SAVE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20 + PROLOG_SAVE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22 mov x9,x3 // save kernel size ldr x11,[x8,#ConvSymPostProcessParams_Bias] mov x16,x4 // save input channels @@ -96,6 +122,9 @@ Return Value: add x5,x2,x5 // c1 = c0 + ldc add x4,x4,7 // kc = (kc + 7) & ~7 csel x5,x2,x5,lo // if OutputCount < 2 c1 = c0 + ldr x19,[x8,#ConvSymPostProcessParams_PreShift] + ldr x20,[x8,#ConvSymPostProcessParams_Multiplier] + ldr x21,[x8,#ConvSymPostProcessParams_PostShift] bic x4,x4,7 ldp s16,s18,[x11],8 // init accumulators with bias ldp s20,s22,[x11],8 @@ -296,17 +325,19 @@ BlockLoopEpilogue // remaining 16 input channels b.hi KernelSizeLoop Requantize - ldr w11,[x8,#ConvSymPostProcessParams_ZeroPoint] + tst w10,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale +FloatPointScale tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE beq BroadcastScaleValue ld1 {v4.4s,v5.4s},[x12] // load scale vector - b AccumulatorsToFloat + b ScaleWithFloatPoint BroadcastScaleValue ld1r {v4.4s},[x12] // load scale Value mov v5.16b, v4.16b -AccumulatorsToFloat +ScaleWithFloatPoint addp v16.4s,v16.4s,v18.4s addp v20.4s,v20.4s,v22.4s addp v24.4s,v24.4s,v26.4s @@ -329,25 +360,72 @@ AccumulatorsToFloat fmul v3.4s,v3.4s,v5.4s fcvtns v0.4s,v0.4s // convert to int fcvtns v1.4s,v1.4s - dup v9.8h,w11 fcvtns v2.4s,v2.4s fcvtns v3.4s,v3.4s + b Quantize + +FixedPointScale + tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w11, [x8, #ConvSymPostProcessParams_ZeroPoint] + beq BroadcastFixedPointValue + ld1 {v4.4s,v5.4s},[x19] // load preshift + ld1 {v6.4s,v7.4s},[x20] // load multiplier + ld1 {v8.4s,v9.4s},[x21] // load postshift + b ScaleWithFixedPoint +BroadcastFixedPointValue + ld1r {v4.4s},[x19] // load preshift Value + mov v5.16b, v4.16b + ld1r {v6.4s},[x20] // load multiplier Value + mov v7.16b, v6.16b + ld1r {v8.4s},[x21] // load postshift Value + mov v9.16b, v8.16b + +ScaleWithFixedPoint + addp v16.4s,v16.4s,v18.4s + addp v20.4s,v20.4s,v22.4s + addp v24.4s,v24.4s,v26.4s + addp v28.4s,v28.4s,v30.4s + addp v17.4s,v17.4s,v19.4s + addp v21.4s,v21.4s,v23.4s + addp v25.4s,v25.4s,v27.4s + addp v29.4s,v29.4s,v31.4s + addp v0.4s,v16.4s,v20.4s + addp v1.4s,v24.4s,v28.4s + addp v2.4s,v17.4s,v21.4s + addp v3.4s,v25.4s,v29.4s + sqshl v0.4s, v0.4s, v4.4s // preshift + sqshl v1.4s, v1.4s, v5.4s + sqshl v2.4s, v2.4s, v4.4s + sqshl v3.4s, v3.4s, v5.4s + sqdmulh v0.4s, v0.4s, v6.4s // multiplier + sqdmulh v1.4s, v1.4s, v7.4s + sqdmulh v2.4s, v2.4s, v6.4s + sqdmulh v3.4s, v3.4s, v7.4s + srshl v0.4s,v0.4s, v8.4s // postshift + srshl v1.4s,v1.4s, v9.4s + srshl v2.4s,v2.4s, v8.4s + srshl v3.4s,v3.4s, v9.4s + +Quantize + dup v9.8h,w11 sqxtn v0.4h,v0.4s - sqxtn2 v0.8h,v1.4s sqxtn v2.4h,v2.4s + sqxtn2 v0.8h,v1.4s sqxtn2 v2.8h,v3.4s sqadd v0.8h,v0.8h,v9.8h sqadd v2.8h,v2.8h,v9.8h sqxtn v0.8b,v0.8h // shorten to int8 sqxtn2 v0.16b,v2.8h - st1 {v0.d}[1],[x5] // full 2x8 store to c + st1 {v0.d}[1],[x5] // full 2x8 store to c st1 {v0.8b},[x2] ExitKernel - EPILOG_RESTORE_REG_PAIR d14,d15,#48 - EPILOG_RESTORE_REG_PAIR d12,d13,#32 - EPILOG_RESTORE_REG_PAIR d10,d11,#16 - EPILOG_RESTORE_REG_PAIR d8,d9,#64! + EPILOG_RESTORE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22 + EPILOG_RESTORE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20 + EPILOG_RESTORE_REG_PAIR d14,d15,#ConvSymFrame_SavedRegisters_d14_d15 + EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13 + EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11 + EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters! EPILOG_RETURN InputChannel8 diff --git a/onnxruntime/core/mlas/lib/arm64/ConvSymU8KernelDot.asm b/onnxruntime/core/mlas/lib/arm64/ConvSymU8KernelDot.asm index 7b0917b955..0a4bb22f14 100644 --- a/onnxruntime/core/mlas/lib/arm64/ConvSymU8KernelDot.asm +++ b/onnxruntime/core/mlas/lib/arm64/ConvSymU8KernelDot.asm @@ -17,23 +17,44 @@ Abstract: #include "kxarm64.h" -#define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 1 #define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2 +#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4 // // Stack frame layout for the symmetric convolution kernel. // d8-d15, x19-x30 need to be preserved if used // -#define ConvSymFrame_SavedNeonRegisters (8 * 8) -#define ConvSymFrame_SavedRegisters ConvSymFrame_SavedNeonRegisters -#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters -#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters +#define ConvSymFrame_SavedRegisters_d8_d9 0 +#define ConvSymFrame_SavedRegisters_d10_d11 16 +#define ConvSymFrame_SavedRegisters_d12_d13 32 +#define ConvSymFrame_SavedRegisters_x19_x20 48 +#define ConvSymFrame_SavedRegisters_x21_x22 64 +#define ConvSymFrame_SavedRegisters 80 +#define ConvSymFrame_SavedRegisters_Neg -80 +#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters +#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters -#define ConvSymPostProcessParams_Bias 0 -#define ConvSymPostProcessParams_Scale 8 -#define ConvSymPostProcessParams_Min 16 -#define ConvSymPostProcessParams_Max 20 -#define ConvSymPostProcessParams_ZeroPoint 24 +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ +#define ConvSymPostProcessParams_Bias 0 +#define ConvSymPostProcessParams_Scale 8 +#define ConvSymPostProcessParams_Min 16 +#define ConvSymPostProcessParams_Max 20 +#define ConvSymPostProcessParams_ZeroPoint 24 +#define ConvSymPostProcessParams_Multiplier 32 +#define ConvSymPostProcessParams_PreShift 40 +#define ConvSymPostProcessParams_PostShift 48 TEXTAREA @@ -46,15 +67,10 @@ Routine Description: Arguments: - Input (x0) - Points to the input buffer. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points - directly at the input tensor. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an - indirection buffer. Every pointer in the indirection buffer points at a - InputChannels length vector (either from the input tensor or a vector of - padding values). These are grouped in batches of length KernelSize. + Input (x0) - Supplies the address of the indirect buffer. Every pointer in + the indirection buffer points at a InputChannels length vector (either + from the input tensor or a vector of padding values). These are grouped + in batches of length KernelSize. These batches are then repeated OutputCount times. Filter (x1) - Points to the filter buffer. @@ -63,8 +79,6 @@ Arguments: KernelSize (x3/x9) - Size of the kernel (most commonly. 3x3=9, 5x5=25). - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1. - InputChannels (x4/x7) - Number of input channels. OutputChannels (x5) - Number of output channels. @@ -86,19 +100,19 @@ Return Value: --*/ NESTED_ENTRY MlasConvSymU8KernelDot - PROLOG_SAVE_REG_PAIR d8,d9,#-64! + PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters_Neg! PROLOG_NOP ldr x8,[sp,#ConvSymFrame_PostProcessParams] PROLOG_NOP ldr w10,[sp,#ConvSymFrame_KernelFlags] - PROLOG_SAVE_REG_PAIR d10,d11,#16 - PROLOG_SAVE_REG_PAIR d12,d13,#32 - PROLOG_SAVE_REG_PAIR x19,x20,#48 + PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11 + PROLOG_SAVE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13 + PROLOG_SAVE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20 + PROLOG_SAVE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22 // compute C pointers: x2, x16, x17, x5 cmp x7,2 // OutputCount < 2 ? add x16,x2,x5 // x16 -> C1 lsl x3,x3,#3 // KernelSize * sizeof(int8_t*) csel x16,x2,x16,lo // if OutputCount < 2 x16/C1 -> C0 - mov x20,x4 add x4,x4,3 // InputChannels align to 4 add x17,x16,x5 // x17 -> C2 ldr x11,[x8,#ConvSymPostProcessParams_Bias] @@ -109,6 +123,9 @@ Return Value: ldr x19,[x8,#ConvSymPostProcessParams_Scale] csel x5,x17,x5,lo // if OutputCount < 4 x5/C3 -> C2 movi v12.16b,128 // for top bit flipping + ldr x20,[x8,#ConvSymPostProcessParams_PreShift] + ldr x21,[x8,#ConvSymPostProcessParams_Multiplier] + ldr x22,[x8,#ConvSymPostProcessParams_PostShift] OutputChannelLoop ldp q16,q20,[x11],32 // Init accumulators with biases @@ -128,23 +145,6 @@ OutputChannelLoop mov x9,x3 // restore KernelSize * sizeof(int8_t*) KernelSizeLoop - tst w10,#MLAS_CONV_SYM_FLAG_INPUT_DIRECT - beq InputIndirection - -InputDirect - cmp x16,x2 - mov x12,x0 // x12 -> A0 - add x13,x0,x20 // x13 -> A1 = A0 + input channels - csel x13,x0,x13,eq - cmp x17,x16 - add x14,x0,x20,lsl#1 // x14 -> A2 - csel x14,x13,x14,eq - cmp x5,x17 - add x15,x13,x20,lsl#1 // x15 -> A3 - csel x15,x14,x15,eq - b FinishLoadAPtr - -InputIndirection ldr x12,[x0] // x12 -> A0 cmp x16,x2 b.eq SkipLoadA1 // C1==C0 -> A0=A1=A2=A3 @@ -392,12 +392,15 @@ InChLoopEpilogue b.hi KernelSizeLoop Requantize + tst w10,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale +FloatPointScale tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint] beq BroadcastScaleValue ldp q0,q1,[x19],32 // load scale vector ldp q2,q3,[x19],32 - b AccumulatorsToFloat + b ScaleWithFloatPoint BroadcastScaleValue ld1r {v0.4s},[x19] // load scale Value @@ -405,7 +408,7 @@ BroadcastScaleValue mov v2.16b, v0.16b mov v3.16b, v0.16b -AccumulatorsToFloat +ScaleWithFloatPoint scvtf v16.4s,v16.4s // convert to float scvtf v17.4s,v17.4s scvtf v18.4s,v18.4s @@ -454,7 +457,87 @@ AccumulatorsToFloat fcvtns v29.4s,v29.4s fcvtns v30.4s,v30.4s fcvtns v31.4s,v31.4s + b Quantize +FixedPointScale + tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w13,[x8,#ConvSymPostProcessParams_ZeroPoint] + beq BroadcastFixedPointValue + ldp q0,q1,[x20],32 // load preshift vector + ldp q2,q3,[x20],32 + ldp q4,q5,[x21],32 // load multiplier vector + ldp q6,q7,[x21],32 + ldp q8,q9,[x22],32 // load postshift vector + ldp q10,q11,[x22],32 + b ScaleWithFixedPoint + +BroadcastFixedPointValue + ld1r {v0.4s},[x20] // load preshift Value + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + + ld1r {v4.4s},[x21] // load multiplier Value + mov v5.16b, v4.16b + mov v6.16b, v4.16b + mov v7.16b, v4.16b + + ld1r {v8.4s},[x22] // load postshift Value + mov v9.16b, v8.16b + mov v10.16b, v8.16b + mov v11.16b, v8.16b + +ScaleWithFixedPoint + sqshl v16.4s, v16.4s, v0.4s // preshift + sqshl v17.4s, v17.4s, v0.4s + sqshl v18.4s, v18.4s, v0.4s + sqshl v19.4s, v19.4s, v0.4s + sqshl v20.4s, v20.4s, v1.4s + sqshl v21.4s, v21.4s, v1.4s + sqshl v22.4s, v22.4s, v1.4s + sqshl v23.4s, v23.4s, v1.4s + sqshl v24.4s, v24.4s, v2.4s + sqshl v25.4s, v25.4s, v2.4s + sqshl v26.4s, v26.4s, v2.4s + sqshl v27.4s, v27.4s, v2.4s + sqshl v28.4s, v28.4s, v3.4s + sqshl v29.4s, v29.4s, v3.4s + sqshl v30.4s, v30.4s, v3.4s + sqshl v31.4s, v31.4s, v3.4s + sqdmulh v16.4s, v16.4s, v4.4s // multiply by scale + sqdmulh v17.4s, v17.4s, v4.4s + sqdmulh v18.4s, v18.4s, v4.4s + sqdmulh v19.4s, v19.4s, v4.4s + sqdmulh v20.4s, v20.4s, v5.4s + sqdmulh v21.4s, v21.4s, v5.4s + sqdmulh v22.4s, v22.4s, v5.4s + sqdmulh v23.4s, v23.4s, v5.4s + sqdmulh v24.4s, v24.4s, v6.4s + sqdmulh v25.4s, v25.4s, v6.4s + sqdmulh v26.4s, v26.4s, v6.4s + sqdmulh v27.4s, v27.4s, v6.4s + sqdmulh v28.4s, v28.4s, v7.4s + sqdmulh v29.4s, v29.4s, v7.4s + sqdmulh v30.4s, v30.4s, v7.4s + sqdmulh v31.4s, v31.4s, v7.4s + srshl v16.4s,v16.4s, v8.4s // post shift + srshl v17.4s,v17.4s, v8.4s + srshl v18.4s,v18.4s, v8.4s + srshl v19.4s,v19.4s, v8.4s + srshl v20.4s,v20.4s, v9.4s + srshl v21.4s,v21.4s, v9.4s + srshl v22.4s,v22.4s, v9.4s + srshl v23.4s,v23.4s, v9.4s + srshl v24.4s,v24.4s, v10.4s + srshl v25.4s,v25.4s, v10.4s + srshl v26.4s,v26.4s, v10.4s + srshl v27.4s,v27.4s, v10.4s + srshl v28.4s,v28.4s, v11.4s + srshl v29.4s,v29.4s, v11.4s + srshl v30.4s,v30.4s, v11.4s + srshl v31.4s,v31.4s, v11.4s + +Quantize sqxtn v16.4h,v16.4s sqxtn v17.4h,v17.4s sqxtn v18.4h,v18.4s @@ -499,10 +582,11 @@ AccumulatorsToFloat b.hi OutputChannelLoop ExitKernel - EPILOG_RESTORE_REG_PAIR x19,x20,#48 - EPILOG_RESTORE_REG_PAIR d12,d13,#32 - EPILOG_RESTORE_REG_PAIR d10,d11,#16 - EPILOG_RESTORE_REG_PAIR d8,d9,#64! + EPILOG_RESTORE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22 + EPILOG_RESTORE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20 + EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13 + EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11 + EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters! EPILOG_RETURN InChannels8 diff --git a/onnxruntime/core/mlas/lib/arm64/ConvSymU8KernelNeon.asm b/onnxruntime/core/mlas/lib/arm64/ConvSymU8KernelNeon.asm index a8a11fe620..ca6ca46481 100644 --- a/onnxruntime/core/mlas/lib/arm64/ConvSymU8KernelNeon.asm +++ b/onnxruntime/core/mlas/lib/arm64/ConvSymU8KernelNeon.asm @@ -17,23 +17,45 @@ Abstract: #include "kxarm64.h" -#define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 1 #define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2 +#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4 // // Stack frame layout for the symmetric convolution kernel. // d8-d15, x19-x30 need to be preserved if used // -#define ConvSymFrame_SavedNeonRegisters (8 * 8) -#define ConvSymFrame_SavedRegisters ConvSymFrame_SavedNeonRegisters -#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters -#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters +#define ConvSymFrame_SavedRegisters_d8_d9 0 +#define ConvSymFrame_SavedRegisters_d10_d11 16 +#define ConvSymFrame_SavedRegisters_d12_d13 32 +#define ConvSymFrame_SavedRegisters_d14_d15 48 +#define ConvSymFrame_SavedRegisters_x19_x20 64 +#define ConvSymFrame_SavedRegisters_x21_x22 80 +#define ConvSymFrame_SavedRegisters 96 +#define ConvSymFrame_SavedRegisters_Neg -96 +#define ConvSymFrame_PostProcessParams 0 + ConvSymFrame_SavedRegisters +#define ConvSymFrame_KernelFlags 8 + ConvSymFrame_SavedRegisters -#define ConvSymPostProcessParams_Bias 0 -#define ConvSymPostProcessParams_Scale 8 -#define ConvSymPostProcessParams_Min 16 -#define ConvSymPostProcessParams_Max 20 -#define ConvSymPostProcessParams_ZeroPoint 24 +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ +#define ConvSymPostProcessParams_Bias 0 +#define ConvSymPostProcessParams_Scale 8 +#define ConvSymPostProcessParams_Min 16 +#define ConvSymPostProcessParams_Max 20 +#define ConvSymPostProcessParams_ZeroPoint 24 +#define ConvSymPostProcessParams_Multiplier 32 +#define ConvSymPostProcessParams_PreShift 40 +#define ConvSymPostProcessParams_PostShift 48 TEXTAREA @@ -46,16 +68,10 @@ Routine Description: Arguments: - Input (x0) - Supplies the address of the input buffer. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then the input buffer points - directly at the input tensor. - - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is clear, then the input buffer is an - indirection buffer. Every pointer in the indirection buffer points at a - InputChannels length vector (either from the input tensor or a vector of - padding values). These are grouped in batches of length KernelSize. - These batches are then repeated OutputCount times. + Input (x0) - Supplies the address of the indirect buffer. Every pointer in + the indirection buffer points at a InputChannels length vector (either + from the input tensor or a vector of padding values). These are grouped + in batches of length KernelSize. Filter (x1) - Supplies the address of the filter buffer. @@ -63,8 +79,6 @@ Arguments: KernelSize (x3) - Supplies the size of the kernel. - If MLAS_CONV_SYM_FLAG_INPUT_DIRECT is set, then kernel size should be 1. - InputChannels (x4) - Supplies the number of input channels. This implementation requires the count to be a multiple of 8. @@ -90,12 +104,14 @@ Return Value: --*/ NESTED_ENTRY MlasConvSymU8KernelNeon - PROLOG_SAVE_REG_PAIR d8,d9,#-64! + PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters_Neg! PROLOG_NOP ldr x8,[sp,#ConvSymFrame_PostProcessParams] PROLOG_NOP ldrb w10,[sp,#ConvSymFrame_KernelFlags] - PROLOG_SAVE_REG_PAIR d10,d11,#16 - PROLOG_SAVE_REG_PAIR d12,d13,#32 - PROLOG_SAVE_REG_PAIR d14,d15,#48 + PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11 + PROLOG_SAVE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13 + PROLOG_SAVE_REG_PAIR d14,d15,#ConvSymFrame_SavedRegisters_d14_d15 + PROLOG_SAVE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20 + PROLOG_SAVE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22 mov x9,x3 // save kernel size ldr x11,[x8,#ConvSymPostProcessParams_Bias] mov x16,x4 // save input channels @@ -104,6 +120,9 @@ Return Value: add x5,x2,x5 // c1 = c0 + ldc add x4,x4,7 // kc = (kc + 7) & ~7 csel x5,x2,x5,lo // if OutputCount < 2 c1 = c0 + ldr x19,[x8,#ConvSymPostProcessParams_PreShift] + ldr x20,[x8,#ConvSymPostProcessParams_Multiplier] + ldr x21,[x8,#ConvSymPostProcessParams_PostShift] bic x4,x4,7 ldp s16,s18,[x11],8 // init accumulators with bias ldp s20,s22,[x11],8 @@ -137,30 +156,18 @@ Return Value: // KernelSizeLoop - - // Load next 2 A pointers - tst w10,#MLAS_CONV_SYM_FLAG_INPUT_DIRECT - ldr d4,[x1] - ldr d5,[x1,8] - beq InputIndirection - -InputDirect - mov x13,x0 // x13 -> A0 - add x15,x0,x16 // x15 -> A1 = A0 + input channels - b BlockLoopPrologue - -InputIndirection cmp x7,2 // test if OutputCount < 2 ldr x13,[x0] // x13 -> A0 - blo SkipLoadA1 + bhs LoadA1 + ldr x15,[x0],#8 // x15 -> A0 + b BlockLoopPrologue +LoadA1 ldr x15,[x0,x3,lsl#3] // x15 -> A1 -SkipLoadA1 - -BlockLoopPrologue - cmp x7,2 // test if OutputCount < 2 add x0,x0,8 // indirect A advance to next pointer, prepare for kernel size loop - csel x15,x13,x15,lo // if OutputCount < 2 x15 -> A0 +BlockLoopPrologue + ldr d4,[x1] subs x14,x4,16 // input channel - 16 + ldr d5,[x1,8] movi v12.8b,128 blo InputChannel8 // less than 16 deep, no unroll @@ -325,17 +332,20 @@ BlockLoopEpilogue // remaining 16 input channels b.hi KernelSizeLoop Requantize - ldr w11,[x8,#ConvSymPostProcessParams_ZeroPoint] + tst w10,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale +FloatPointScale tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w11,[x8,#ConvSymPostProcessParams_ZeroPoint] beq BroadcastScaleValue ld1 {v4.4s,v5.4s},[x12] // load scale vector - b AccumulatorsToFloat + b ScaleWithFloatPoint BroadcastScaleValue ld1r {v4.4s},[x12] // load scale Value mov v5.16b, v4.16b -AccumulatorsToFloat +ScaleWithFloatPoint addp v16.4s,v16.4s,v18.4s addp v20.4s,v20.4s,v22.4s addp v24.4s,v24.4s,v26.4s @@ -358,25 +368,72 @@ AccumulatorsToFloat fmul v3.4s,v3.4s,v5.4s fcvtns v0.4s,v0.4s // convert to int fcvtns v1.4s,v1.4s - dup v9.8h,w11 fcvtns v2.4s,v2.4s fcvtns v3.4s,v3.4s + b Quantize + +FixedPointScale + tst w10,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w11, [x8, #ConvSymPostProcessParams_ZeroPoint] + beq BroadcastFixedPointValue + ld1 {v4.4s,v5.4s},[x19] // load preshift + ld1 {v6.4s,v7.4s},[x20] // load multiplier + ld1 {v8.4s,v9.4s},[x21] // load postshift + b ScaleWithFixedPoint +BroadcastFixedPointValue + ld1r {v4.4s},[x19] // load preshift Value + mov v5.16b, v4.16b + ld1r {v6.4s},[x20] // load multiplier Value + mov v7.16b, v6.16b + ld1r {v8.4s},[x21] // load postshift Value + mov v9.16b, v8.16b + +ScaleWithFixedPoint + addp v16.4s,v16.4s,v18.4s + addp v20.4s,v20.4s,v22.4s + addp v24.4s,v24.4s,v26.4s + addp v28.4s,v28.4s,v30.4s + addp v17.4s,v17.4s,v19.4s + addp v21.4s,v21.4s,v23.4s + addp v25.4s,v25.4s,v27.4s + addp v29.4s,v29.4s,v31.4s + addp v0.4s,v16.4s,v20.4s + addp v1.4s,v24.4s,v28.4s + addp v2.4s,v17.4s,v21.4s + addp v3.4s,v25.4s,v29.4s + sqshl v0.4s, v0.4s, v4.4s // preshift + sqshl v1.4s, v1.4s, v5.4s + sqshl v2.4s, v2.4s, v4.4s + sqshl v3.4s, v3.4s, v5.4s + sqdmulh v0.4s, v0.4s, v6.4s // multiplier + sqdmulh v1.4s, v1.4s, v7.4s + sqdmulh v2.4s, v2.4s, v6.4s + sqdmulh v3.4s, v3.4s, v7.4s + srshl v0.4s,v0.4s, v8.4s // postshift + srshl v1.4s,v1.4s, v9.4s + srshl v2.4s,v2.4s, v8.4s + srshl v3.4s,v3.4s, v9.4s + +Quantize + dup v9.8h,w11 sqxtn v0.4h,v0.4s - sqxtn2 v0.8h,v1.4s sqxtn v2.4h,v2.4s + sqxtn2 v0.8h,v1.4s sqxtn2 v2.8h,v3.4s sqadd v0.8h,v0.8h,v9.8h sqadd v2.8h,v2.8h,v9.8h sqxtun v0.8b,v0.8h // shorten to int8 sqxtun2 v0.16b,v2.8h - st1 {v0.d}[1],[x5] // full 2x8 store to c + st1 {v0.d}[1],[x5] // full 2x8 store to c st1 {v0.8b},[x2] ExitKernel - EPILOG_RESTORE_REG_PAIR d14,d15,#48 - EPILOG_RESTORE_REG_PAIR d12,d13,#32 - EPILOG_RESTORE_REG_PAIR d10,d11,#16 - EPILOG_RESTORE_REG_PAIR d8,d9,#64! + EPILOG_RESTORE_REG_PAIR x21,x22,#ConvSymFrame_SavedRegisters_x21_x22 + EPILOG_RESTORE_REG_PAIR x19,x20,#ConvSymFrame_SavedRegisters_x19_x20 + EPILOG_RESTORE_REG_PAIR d14,d15,#ConvSymFrame_SavedRegisters_d14_d15 + EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymFrame_SavedRegisters_d12_d13 + EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymFrame_SavedRegisters_d10_d11 + EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymFrame_SavedRegisters! EPILOG_RETURN InputChannel8 diff --git a/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvKernelSize9Neon.asm b/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvKernelSize9Neon.asm index 335e11be1e..8540e198b2 100644 --- a/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvKernelSize9Neon.asm +++ b/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvKernelSize9Neon.asm @@ -18,29 +18,48 @@ Abstract: #include "kxarm64.h" +#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX 1 +#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4 + // // Stack frame layout for the depthwise conv kernel. // d8-d15, x19-x30 need to be preserved if used // +#define MlasConvSymDepthwiseKernelSize9_x19_x20 0 +#define MlasConvSymDepthwiseKernelSize9_x21_x22 16 +#define MlasConvSymDepthwiseKernelSize9_x23_x24 32 +#define MlasConvSymDepthwiseKernelSize9_x25_x26 48 +#define MlasConvSymDepthwiseKernelSize9_x27_x28 64 +#define MlasConvSymDepthwiseKernelSize9_d8_d9 80 +#define MlasConvSymDepthwiseKernelSize9_d10_d11 96 +#define MlasConvSymDepthwiseKernelSize9_d12_d13 112 +#define MlasConvSymDepthwiseKernelSize9_d14_d15 128 +#define MlasConvSymDepthwiseKernelSize9_x0_x4 144 +#define MlasConvSymDepthwiseKernelSize9_multiplier_postshift 160 +#define MlasConvSymDepthwiseKernelSize9_SavedRegisters 176 +#define MlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg -176 -#define ConvSymDepthwisePostProcessParams_Bias 0 -#define ConvSymDepthwisePostProcessParams_Scale 8 -#define ConvSymDepthwisePostProcessParams_ZeroPoint 24 - -#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX 1 - -#define MlasConvSymDepthwiseKernelSize9_backup_x19_x20 0 -#define MlasConvSymDepthwiseKernelSize9_backup_x21_x22 16 -#define MlasConvSymDepthwiseKernelSize9_backup_x23_x24 32 -#define MlasConvSymDepthwiseKernelSize9_backup_x25_x26 48 -#define MlasConvSymDepthwiseKernelSize9_backup_x27_x28 64 -#define MlasConvSymDepthwiseKernelSize9_backup_d8_d9 80 -#define MlasConvSymDepthwiseKernelSize9_backup_d10_d11 96 -#define MlasConvSymDepthwiseKernelSize9_backup_d12_d13 112 -#define MlasConvSymDepthwiseKernelSize9_backup_d14_d15 128 -#define MlasConvSymDepthwiseKernelSize9_SavedRegisters 144 -#define MlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg -144 - +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ +#define ConvSymDepthwisePostProcessParams_Bias 0 +#define ConvSymDepthwisePostProcessParams_Scale 8 +#define ConvSymDepthwisePostProcessParams_Min 16 +#define ConvSymDepthwisePostProcessParams_Max 20 +#define ConvSymDepthwisePostProcessParams_ZeroPoint 24 +#define ConvSymPostProcessParams_Multiplier 32 +#define ConvSymPostProcessParams_PreShift 40 +#define ConvSymPostProcessParams_PostShift 48 TEXTAREA @@ -76,26 +95,32 @@ Return Value: LEAF_ENTRY MlasConvSymDepthwiseKernelSize9Arm64U8S8 PROLOG_SAVE_REG_PAIR x19, x20, #MlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg ! - PROLOG_SAVE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_backup_x21_x22 - PROLOG_SAVE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_backup_x23_x24 - PROLOG_SAVE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_backup_x25_x26 - PROLOG_SAVE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_backup_x27_x28 - PROLOG_SAVE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_backup_d8_d9 - PROLOG_SAVE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_backup_d10_d11 - PROLOG_SAVE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_backup_d12_d13 - PROLOG_SAVE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_backup_d14_d15 + PROLOG_SAVE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_x21_x22 + PROLOG_SAVE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_x23_x24 + PROLOG_SAVE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_x25_x26 + PROLOG_SAVE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_x27_x28 + PROLOG_SAVE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_d8_d9 + PROLOG_SAVE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_d10_d11 + PROLOG_SAVE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_d12_d13 + PROLOG_SAVE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_d14_d15 + tst x6, #MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + ldr x11, [x5, #ConvSymPostProcessParams_Multiplier] + ldr x12, [x5, #ConvSymPostProcessParams_PostShift] + ldr x10, [x5, #ConvSymPostProcessParams_PreShift] ldr x9, [x5, #ConvSymDepthwisePostProcessParams_Bias] ldr x8, [x5, #ConvSymDepthwisePostProcessParams_Scale] add x5, x5, #ConvSymDepthwisePostProcessParams_ZeroPoint + stp x11, x12,[sp, #MlasConvSymDepthwiseKernelSize9_multiplier_postshift] + csel x8, x8, x10, eq ins v12.d[0], x1 // Filter ins v13.d[0], x9 // Bias - ins v13.d[1], x8 // Scale + ins v13.d[1], x8 // Scale/PreShift ld1r {v0.8h}, [x5] // v0.8h <--- vector for output zero point movi v5.16b, #0x80 tbnz x6, #MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, MlasConvSymDepthwiseKernelSize9_SkipPerTensorScaleInit - ld1r {v1.4s}, [x8] // load and dup scale value + ld1r {v1.4s}, [x8] // load and dup scale/PreShift value mov v2.16b, v1.16b mov v3.16b, v1.16b mov v4.16b, v1.16b @@ -114,7 +139,7 @@ MlasConvSymDepthwiseKernelSize9_OutputLoop ldur x28, [x0, #-8] cbz x4, MlasConvSymDepthwiseKernelSize9_Dup_Inputs - ldp x10, x11, [x0], #72 // input ptrs for Output0 + ldp x10, x11, [x0], #72 // input ptrs for Output1 ldp x12, x13, [x0, #-56] sub x4, x4, #1 ldp x14, x15, [x0, #-40] @@ -124,7 +149,7 @@ MlasConvSymDepthwiseKernelSize9_OutputLoop MlasConvSymDepthwiseKernelSize9_Dup_Inputs mov x9, x3 // Output1 <-- Output0 - mov x10, x20 + mov x10, x20 // Input1 <-- Input0 mov x11, x21 mov x12, x22 mov x13, x23 @@ -136,10 +161,12 @@ MlasConvSymDepthwiseKernelSize9_Dup_Inputs MlasConvSymDepthwiseKernelSize9_Loaded_Input + stp x0, x4, [sp, #MlasConvSymDepthwiseKernelSize9_x0_x4] eor x8, x8, x8 // Processed channels umov x1, v12.d[0] // filter umov x5, v13.d[0] // bias - umov x7, v13.d[1] // scale + umov x7, v13.d[1] // scale/PreShift + ldp x0, x4, [sp, #MlasConvSymDepthwiseKernelSize9_multiplier_postshift] cmp x8, x2 // Save one register by not using count down to zero here bhs MlasConvSymDepthwiseKernelSize9_Finish_Channels16_Loop @@ -232,7 +259,7 @@ MlasConvSymDepthwiseKernelSize9_Channels16_Loop ldr q22, [x27, x8] // out0 vi7 ldr q23, [x17, x8] // out1 vi7 tbz x6, #MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9 - ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales 0-15 for outs + ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales/preshift 0-15 for outs DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9 eor v20.16b, v20.16b, v5.16b @@ -289,6 +316,9 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9 saddw v8.4s, v8.4s, v27.4h saddw2 v9.4s, v9.4s, v27.8h + tst x6, #MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale_U8S8 +FloatPointScale_U8S8 scvtf v16.4s, v16.4s // Requantize scvtf v17.4s, v17.4s scvtf v18.4s, v18.4s @@ -315,7 +345,55 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9 fcvtns v7.4s, v7.4s fcvtns v8.4s, v8.4s fcvtns v9.4s, v9.4s + b Quantize_U8S8 +FixedPointScale_U8S8 + tbz x6,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, BroadcastFixedPointValue_U8S8 + ldp q20,q21,[x0],32 // load multiplier vector + ldp q22,q23,[x0],32 + ldp q24,q25,[x4],32 // load postshift vector + ldp q26,q27,[x4],32 + b ScaleWithFixedPoint_U8S8 + +BroadcastFixedPointValue_U8S8 + ld1r {v20.4s},[x0] // load multiplier Value + mov v21.16b, v20.16b + mov v22.16b, v20.16b + mov v23.16b, v20.16b + + ld1r {v24.4s},[x4] // load postshift Value + mov v25.16b, v24.16b + mov v26.16b, v24.16b + mov v27.16b, v24.16b +ScaleWithFixedPoint_U8S8 + sqshl v6.4s, v6.4s, v1.4s // preshift + sqshl v7.4s, v7.4s, v2.4s + sqshl v8.4s, v8.4s, v3.4s + sqshl v9.4s, v9.4s, v4.4s + sqshl v16.4s, v16.4s, v1.4s + sqshl v17.4s, v17.4s, v2.4s + sqshl v18.4s, v18.4s, v3.4s + sqshl v19.4s, v19.4s, v4.4s + + sqdmulh v6.4s, v6.4s, v20.4s // multiply by multiplier + sqdmulh v7.4s, v7.4s, v21.4s + sqdmulh v8.4s, v8.4s, v22.4s + sqdmulh v9.4s, v9.4s, v23.4s + sqdmulh v16.4s, v16.4s, v20.4s + sqdmulh v17.4s, v17.4s, v21.4s + sqdmulh v18.4s, v18.4s, v22.4s + sqdmulh v19.4s, v19.4s, v23.4s + + srshl v6.4s,v6.4s, v24.4s // post shift + srshl v7.4s,v7.4s, v25.4s + srshl v8.4s,v8.4s, v26.4s + srshl v9.4s,v9.4s, v27.4s + srshl v16.4s,v16.4s, v24.4s + srshl v17.4s,v17.4s, v25.4s + srshl v18.4s,v18.4s, v26.4s + srshl v19.4s,v19.4s, v27.4s + +Quantize_U8S8 sqxtn v16.4h, v16.4s // +zp, narrow and combine sqxtn v18.4h, v18.4s sqxtn v6.4h, v6.4s @@ -342,19 +420,20 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9 blo MlasConvSymDepthwiseKernelSize9_Channels16_Loop MlasConvSymDepthwiseKernelSize9_Finish_Channels16_Loop + ldp x0, x4, [sp, #MlasConvSymDepthwiseKernelSize9_x0_x4] add x3, x3, x2, LSL #1 add x9, x9, x2, LSL #1 cbnz x4, MlasConvSymDepthwiseKernelSize9_OutputLoop MlasConvSymDepthwiseKernelSize9_Exit - EPILOG_RESTORE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_backup_d14_d15 - EPILOG_RESTORE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_backup_d12_d13 - EPILOG_RESTORE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_backup_d10_d11 - EPILOG_RESTORE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_backup_d8_d9 - EPILOG_RESTORE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_backup_x27_x28 - EPILOG_RESTORE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_backup_x25_x26 - EPILOG_RESTORE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_backup_x23_x24 - EPILOG_RESTORE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_backup_x21_x22 + EPILOG_RESTORE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_d14_d15 + EPILOG_RESTORE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_d12_d13 + EPILOG_RESTORE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_d10_d11 + EPILOG_RESTORE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_d8_d9 + EPILOG_RESTORE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_x27_x28 + EPILOG_RESTORE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_x25_x26 + EPILOG_RESTORE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_x23_x24 + EPILOG_RESTORE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_x21_x22 EPILOG_RESTORE_REG_PAIR x19, x20, #MlasConvSymDepthwiseKernelSize9_SavedRegisters ! EPILOG_RETURN @@ -394,25 +473,32 @@ Return Value: LEAF_ENTRY MlasConvSymDepthwiseKernelSize9Arm64S8S8 PROLOG_SAVE_REG_PAIR x19, x20, #MlasConvSymDepthwiseKernelSize9_SavedRegisters_Neg ! - PROLOG_SAVE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_backup_x21_x22 - PROLOG_SAVE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_backup_x23_x24 - PROLOG_SAVE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_backup_x25_x26 - PROLOG_SAVE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_backup_x27_x28 - PROLOG_SAVE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_backup_d8_d9 - PROLOG_SAVE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_backup_d10_d11 - PROLOG_SAVE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_backup_d12_d13 - PROLOG_SAVE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_backup_d14_d15 + PROLOG_SAVE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_x21_x22 + PROLOG_SAVE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_x23_x24 + PROLOG_SAVE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_x25_x26 + PROLOG_SAVE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_x27_x28 + PROLOG_SAVE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_d8_d9 + PROLOG_SAVE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_d10_d11 + PROLOG_SAVE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_d12_d13 + PROLOG_SAVE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_d14_d15 + tst x6, #MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE ldr x9, [x5, #ConvSymDepthwisePostProcessParams_Bias] ldr x8, [x5, #ConvSymDepthwisePostProcessParams_Scale] + ldr x10, [x5, #ConvSymPostProcessParams_PreShift] + ldr x11, [x5, #ConvSymPostProcessParams_Multiplier] + ldr x12, [x5, #ConvSymPostProcessParams_PostShift] add x5, x5, #ConvSymDepthwisePostProcessParams_ZeroPoint + csel x8, x8, x10, eq ins v12.d[0], x1 // Filter ins v13.d[0], x9 // Bias - ins v13.d[1], x8 // Scale + ins v13.d[1], x8 // Scale/PreShift + ins v5.d[0], x11 // Multiplier + ins v5.d[1], x12 // PostShift ld1r {v0.8h}, [x5] // v0.8h <--- vector for output zero point tbnz x6, #MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, MlasConvSymDepthwiseKernelSize9S8S8_SkipPerTensorScaleInit - ld1r {v1.4s}, [x8] // load and dup scale value + ld1r {v1.4s}, [x8] // load and dup scale/PreShift value mov v2.16b, v1.16b mov v3.16b, v1.16b mov v4.16b, v1.16b @@ -453,10 +539,13 @@ MlasConvSymDepthwiseKernelSize9S8S8_Dup_Inputs MlasConvSymDepthwiseKernelSize9S8S8_Loaded_Input + stp x0, x4, [sp, #MlasConvSymDepthwiseKernelSize9_x0_x4] eor x8, x8, x8 // Processed channels umov x1, v12.d[0] // filter umov x5, v13.d[0] // bias - umov x7, v13.d[1] // scale + umov x7, v13.d[1] // scale/PreShift + umov x0, v5.D[0] + umov x4, v5.D[1] cmp x8, x2 // Save one register by not using count down to zero here bhs MlasConvSymDepthwiseKernelSize9S8S8_Finish_Channels16_Loop @@ -537,7 +626,7 @@ MlasConvSymDepthwiseKernelSize9S8S8_Channels16_Loop ldr q22, [x27, x8] // out0 vi7 ldr q23, [x17, x8] // out1 vi7 tbz x6, #MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8 - ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales 0-15 for outs + ld1 {v1.4s, v2.4s, v3.4s, v4.4s}, [x7], #64 // scales/preshift 0-15 for outs DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8 ldr q10, [x1] // vk8 @@ -588,6 +677,9 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8 saddw v8.4s, v8.4s, v27.4h saddw2 v9.4s, v9.4s, v27.8h + tst x6, #MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE + bne FixedPointScale +FloatPointScale scvtf v16.4s, v16.4s // Requantize scvtf v17.4s, v17.4s scvtf v18.4s, v18.4s @@ -614,7 +706,55 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8 fcvtns v7.4s, v7.4s fcvtns v8.4s, v8.4s fcvtns v9.4s, v9.4s + b Quantize +FixedPointScale + tbz x6,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE_BIT_INDEX, BroadcastFixedPointValue + ldp q20,q21,[x0],32 // load multiplier vector + ldp q22,q23,[x0],32 + ldp q24,q25,[x4],32 // load postshift vector + ldp q26,q27,[x4],32 + b ScaleWithFixedPoint + +BroadcastFixedPointValue + ld1r {v20.4s},[x0] // load multiplier Value + mov v21.16b, v20.16b + mov v22.16b, v20.16b + mov v23.16b, v20.16b + + ld1r {v24.4s},[x4] // load postshift Value + mov v25.16b, v24.16b + mov v26.16b, v24.16b + mov v27.16b, v24.16b +ScaleWithFixedPoint + sqshl v6.4s, v6.4s, v1.4s // preshift + sqshl v7.4s, v7.4s, v2.4s + sqshl v8.4s, v8.4s, v3.4s + sqshl v9.4s, v9.4s, v4.4s + sqshl v16.4s, v16.4s, v1.4s + sqshl v17.4s, v17.4s, v2.4s + sqshl v18.4s, v18.4s, v3.4s + sqshl v19.4s, v19.4s, v4.4s + + sqdmulh v6.4s, v6.4s, v20.4s // multiply by multiplier + sqdmulh v7.4s, v7.4s, v21.4s + sqdmulh v8.4s, v8.4s, v22.4s + sqdmulh v9.4s, v9.4s, v23.4s + sqdmulh v16.4s, v16.4s, v20.4s + sqdmulh v17.4s, v17.4s, v21.4s + sqdmulh v18.4s, v18.4s, v22.4s + sqdmulh v19.4s, v19.4s, v23.4s + + srshl v6.4s,v6.4s, v24.4s // post shift + srshl v7.4s,v7.4s, v25.4s + srshl v8.4s,v8.4s, v26.4s + srshl v9.4s,v9.4s, v27.4s + srshl v16.4s,v16.4s, v24.4s + srshl v17.4s,v17.4s, v25.4s + srshl v18.4s,v18.4s, v26.4s + srshl v19.4s,v19.4s, v27.4s + +Quantize sqxtn v16.4h, v16.4s // +zp, narrow and combine sqxtn v18.4h, v18.4s sqxtn v6.4h, v6.4s @@ -641,19 +781,20 @@ DonePerChannelScaleLoad_MlasConvSymDepthwiseKernelSize9S8S8 blo MlasConvSymDepthwiseKernelSize9S8S8_Channels16_Loop MlasConvSymDepthwiseKernelSize9S8S8_Finish_Channels16_Loop + ldp x0, x4, [sp, #MlasConvSymDepthwiseKernelSize9_x0_x4] add x3, x3, x2, LSL #1 add x9, x9, x2, LSL #1 cbnz x4, MlasConvSymDepthwiseKernelSize9S8S8_OutputLoop MlasConvSymDepthwiseKernelSize9S8S8_Exit - EPILOG_RESTORE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_backup_d14_d15 - EPILOG_RESTORE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_backup_d12_d13 - EPILOG_RESTORE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_backup_d10_d11 - EPILOG_RESTORE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_backup_d8_d9 - EPILOG_RESTORE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_backup_x27_x28 - EPILOG_RESTORE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_backup_x25_x26 - EPILOG_RESTORE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_backup_x23_x24 - EPILOG_RESTORE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_backup_x21_x22 + EPILOG_RESTORE_REG_PAIR d14, d15, #MlasConvSymDepthwiseKernelSize9_d14_d15 + EPILOG_RESTORE_REG_PAIR d12, d13, #MlasConvSymDepthwiseKernelSize9_d12_d13 + EPILOG_RESTORE_REG_PAIR d10, d11, #MlasConvSymDepthwiseKernelSize9_d10_d11 + EPILOG_RESTORE_REG_PAIR d8, d9, #MlasConvSymDepthwiseKernelSize9_d8_d9 + EPILOG_RESTORE_REG_PAIR x27, x28, #MlasConvSymDepthwiseKernelSize9_x27_x28 + EPILOG_RESTORE_REG_PAIR x25, x26, #MlasConvSymDepthwiseKernelSize9_x25_x26 + EPILOG_RESTORE_REG_PAIR x23, x24, #MlasConvSymDepthwiseKernelSize9_x23_x24 + EPILOG_RESTORE_REG_PAIR x21, x22, #MlasConvSymDepthwiseKernelSize9_x21_x22 EPILOG_RESTORE_REG_PAIR x19, x20, #MlasConvSymDepthwiseKernelSize9_SavedRegisters ! EPILOG_RETURN diff --git a/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvSymS8KernelNeon.asm b/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvSymS8KernelNeon.asm index ba565ca587..b565c54336 100644 --- a/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvSymS8KernelNeon.asm +++ b/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvSymS8KernelNeon.asm @@ -17,23 +17,45 @@ Abstract: #include "kxarm64.h" +#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2 +#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4 + // // Stack frame layout for the depthwise conv kernel. // d8-d15, x19-x30 need to be preserved if used // -#define ConvSymDepthwiseKernelFrame_SavedRegisters (4 * 8) -#define ConvSymDepthwiseKernelFrame_PostProcessParams 0 + ConvSymDepthwiseKernelFrame_SavedRegisters -#define ConvSymDepthwiseKernelFrame_KernelFlags 8 + ConvSymDepthwiseKernelFrame_SavedRegisters +#define ConvSymDepthwiseKernelFrame_SavedRegisters_d8_d9 0 +#define ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11 16 +#define ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13 32 +#define ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15 48 +#define ConvSymDepthwiseKernelFrame_SavedRegisters 64 +#define ConvSymDepthwiseKernelFrame_SavedRegisters_Neg -64 +#define ConvSymDepthwiseKernelFrame_PostProcessParams 0 + ConvSymDepthwiseKernelFrame_SavedRegisters +#define ConvSymDepthwiseKernelFrame_KernelFlags 8 + ConvSymDepthwiseKernelFrame_SavedRegisters -#define ConvSymDepthwisePostProcessParams_Bias 0 -#define ConvSymDepthwisePostProcessParams_Scale 8 -#define ConvSymDepthwisePostProcessParams_Min 16 -#define ConvSymDepthwisePostProcessParams_Max 20 -#define ConvSymDepthwisePostProcessParams_ZeroPoint 24 +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ +#define ConvSymDepthwisePostProcessParams_Bias 0 +#define ConvSymDepthwisePostProcessParams_Scale 8 +#define ConvSymDepthwisePostProcessParams_Min 16 +#define ConvSymDepthwisePostProcessParams_Max 20 +#define ConvSymDepthwisePostProcessParams_ZeroPoint 24 +#define ConvSymDepthwisePostProcessParams_Multiplier 32 +#define ConvSymDepthwisePostProcessParams_PreShift 40 +#define ConvSymDepthwisePostProcessParams_PostShift 48 -#define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 1 -#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2 TEXTAREA @@ -47,28 +69,28 @@ Routine Description: Arguments: Input (x0) - Supplies the address of the indirection buffer. - + Filter (x1) - Supplies the address of the filter buffer. Output (x2) - Supplies the address of the output buffer. KernelSize (x3) - Supplies the size of the kernel. - + Channels (x4) - Supplies the number of input and output channels. - + ChannelOffset (x5) - Supplies the byte offset from the indirection buffer base address for this iteration. - + ChannelCount (x6) - Supplies the number of channels this iteration produces. - + This implementation requires the count to be 16 or 8 - + OutputCount (x7)- Supplies the number of output elements this iteration produces. - - This implementation requires the count to be in the range 1 to 2. - + + This implementation requires the count to be in the range 1 to 4. + PostProcessParams - Supplies the address of the post process parameter block. - + KernelFlags - Supplies additional flags controlling the operation. Return Value: @@ -79,9 +101,11 @@ Return Value: NESTED_ENTRY MlasConvSymDepthwiseS8KernelNeon - PROLOG_SAVE_REG_PAIR d12,d13,#-ConvSymDepthwiseKernelFrame_SavedRegisters! + PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymDepthwiseKernelFrame_SavedRegisters_Neg! PROLOG_NOP ldr x8,[sp,#ConvSymDepthwiseKernelFrame_PostProcessParams] - PROLOG_SAVE_REG_PAIR d14,d15,#16 + PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11 + PROLOG_SAVE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13 + PROLOG_SAVE_REG_PAIR d14,d15,#ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15 cmp x7,2 add x9,x0,x3,lsl#3 // x9 -> &A1 add x14,x0,x3,lsl#4 // x14 -> &A2 @@ -161,7 +185,7 @@ BlockLoopC16 smlal v12.8h,v1.8b,v2.8b ldr x11,[x9],#8 // x11 -> A1 iter 2 smlal2 v13.8h,v1.16b,v2.16b - b.eq EpilogueC16P3 // 3 pixel remains + b.eq EpilogueC16P3 // 3 pixel remains ldr q2,[x12,x5] // A2 iter 1 smlal v14.8h,v1.8b,v3.8b ldr x12,[x0],#8 // x12 -> A0 iter 3 @@ -228,22 +252,15 @@ EpilogueC16P2 saddw v30.4s,v30.4s,v15.4h saddw2 v31.4s,v31.4s,v15.8h ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags] - ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b smull2 v13.8h,v0.16b,v4.16b - ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b smull2 v15.8h,v0.16b,v6.16b smlal v12.8h,v1.8b,v2.8b smlal2 v13.8h,v1.16b,v2.16b smlal v14.8h,v1.8b,v3.8b smlal2 v15.8h,v1.16b,v3.16b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq SkipScaleVecLoad2 - ldp q4,q5,[x12],#32 // load scale vector if per channel - ldp q6,q3,[x12] -SkipScaleVecLoad2 + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v18.4s,v18.4s,v13.4h @@ -252,7 +269,7 @@ SkipScaleVecLoad2 saddw2 v21.4s,v21.4s,v14.8h saddw v22.4s,v22.4s,v15.4h saddw2 v23.4s,v23.4s,v15.8h - b Dequantization + b Requantization ProcC16P1 // @@ -327,10 +344,8 @@ EpilogueC16P1 // Loop epilogue (process last single pixel) mixed with loading of dequantization params // ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags] - ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b smull2 v13.8h,v0.16b,v4.16b - ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b smull2 v15.8h,v0.16b,v6.16b saddw v24.4s,v24.4s,v12.4h @@ -345,12 +360,7 @@ EpilogueC16P1 smull2 v13.8h,v0.16b,v2.16b smull v14.8h,v0.8b,v3.8b smull2 v15.8h,v0.16b,v3.16b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq SkipScaleVecLoad - ldp q4,q5,[x12],#32 // load scale vector if per channel - ldp q6,q3,[x12] -SkipScaleVecLoad + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v18.4s,v18.4s,v13.4h @@ -360,7 +370,24 @@ SkipScaleVecLoad saddw v22.4s,v22.4s,v15.4h saddw2 v23.4s,v23.4s,v15.8h -Dequantization +Requantization + b.ne FixedPointScale +FloatingPointScale + ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] + beq BroadcastScaleValue + ldp q4,q5,[x12],#32 // load scale vector if per channel + ldp q6,q3,[x12] + b ScaleWithFloatPoint + +BroadcastScaleValue + ld1r {v4.4s},[x12] // load scale val + mov v5.16b,v4.16b // broadcast scale val if not per channel + mov v6.16b,v4.16b + mov v3.16b,v4.16b + +ScaleWithFloatPoint scvtf v24.4s,v24.4s // convert to float scvtf v25.4s,v25.4s scvtf v26.4s,v26.4s @@ -377,11 +404,6 @@ Dequantization scvtf v21.4s,v21.4s scvtf v22.4s,v22.4s scvtf v23.4s,v23.4s - b.ne SkipScaleBroadcast - mov v5.16b,v4.16b // broadcast scale val if not per channel - mov v6.16b,v4.16b - mov v3.16b,v4.16b -SkipScaleBroadcast fmul v24.4s,v24.4s,v4.4s // multiply by scale fmul v25.4s,v25.4s,v5.4s fmul v26.4s,v26.4s,v6.4s @@ -414,6 +436,90 @@ SkipScaleBroadcast fcvtns v21.4s,v21.4s fcvtns v22.4s,v22.4s fcvtns v23.4s,v23.4s + b Quantize + +FixedPointScale + ldr x10,[x8,#ConvSymDepthwisePostProcessParams_PreShift] + ldr x11,[x8,#ConvSymDepthwisePostProcessParams_Multiplier] + ldr x12,[x8,#ConvSymDepthwisePostProcessParams_PostShift] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] + beq BroadcastFixedPointValue + ldp q0,q1,[x10],32 // load preshift vector + ldp q2,q3,[x10],32 + ldp q4,q5,[x11],32 // load multiplier vector + ldp q6,q7,[x11],32 + ldp q8,q9,[x12],32 // load postshift vector + ldp q10,q11,[x12],32 + b ScaleWithFixedPoint + +BroadcastFixedPointValue + ld1r {v0.4s},[x10] // load preshift Value + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + + ld1r {v4.4s},[x11] // load multiplier Value + mov v5.16b, v4.16b + mov v6.16b, v4.16b + mov v7.16b, v4.16b + + ld1r {v8.4s},[x12] // load postshift Value + mov v9.16b, v8.16b + mov v10.16b, v8.16b + mov v11.16b, v8.16b + +ScaleWithFixedPoint + sqshl v24.4s, v24.4s, v0.4s // preshift + sqshl v25.4s, v25.4s, v1.4s + sqshl v26.4s, v26.4s, v2.4s + sqshl v27.4s, v27.4s, v3.4s + sqshl v28.4s, v28.4s, v0.4s + sqshl v29.4s, v29.4s, v1.4s + sqshl v30.4s, v30.4s, v2.4s + sqshl v31.4s, v31.4s, v3.4s + sqshl v16.4s, v16.4s, v0.4s + sqshl v17.4s, v17.4s, v1.4s + sqshl v18.4s, v18.4s, v2.4s + sqshl v19.4s, v19.4s, v3.4s + sqshl v20.4s, v20.4s, v0.4s + sqshl v21.4s, v21.4s, v1.4s + sqshl v22.4s, v22.4s, v2.4s + sqshl v23.4s, v23.4s, v3.4s + sqdmulh v24.4s, v24.4s, v4.4s // multiplier + sqdmulh v25.4s, v25.4s, v5.4s + sqdmulh v26.4s, v26.4s, v6.4s + sqdmulh v27.4s, v27.4s, v7.4s + sqdmulh v28.4s, v28.4s, v4.4s + sqdmulh v29.4s, v29.4s, v5.4s + sqdmulh v30.4s, v30.4s, v6.4s + sqdmulh v31.4s, v31.4s, v7.4s + sqdmulh v16.4s, v16.4s, v4.4s + sqdmulh v17.4s, v17.4s, v5.4s + sqdmulh v18.4s, v18.4s, v6.4s + sqdmulh v19.4s, v19.4s, v7.4s + sqdmulh v20.4s, v20.4s, v4.4s + sqdmulh v21.4s, v21.4s, v5.4s + sqdmulh v22.4s, v22.4s, v6.4s + sqdmulh v23.4s, v23.4s, v7.4s + srshl v24.4s, v24.4s, v8.4s // post shift + srshl v25.4s, v25.4s, v9.4s + srshl v26.4s, v26.4s, v10.4s + srshl v27.4s, v27.4s, v11.4s + srshl v28.4s, v28.4s, v8.4s + srshl v29.4s, v29.4s, v9.4s + srshl v30.4s, v30.4s, v10.4s + srshl v31.4s, v31.4s, v11.4s + srshl v16.4s, v16.4s, v8.4s + srshl v17.4s, v17.4s, v9.4s + srshl v18.4s, v18.4s, v10.4s + srshl v19.4s, v19.4s, v11.4s + srshl v20.4s, v20.4s, v8.4s + srshl v21.4s, v21.4s, v9.4s + srshl v22.4s, v22.4s, v10.4s + srshl v23.4s, v23.4s, v11.4s + +Quantize sqxtn v24.4h,v24.4s // shorten to int16 sqxtn v26.4h,v26.4s sqxtn2 v24.8h,v25.4s @@ -458,8 +564,10 @@ SkipScaleBroadcast str q20,[x2] ExitKernel - EPILOG_RESTORE_REG_PAIR d14,d15,#16 - EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters! + EPILOG_RESTORE_REG_PAIR d14,d15,#ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15 + EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13 + EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11 + EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymDepthwiseKernelFrame_SavedRegisters! EPILOG_RETURN Process8Channels @@ -501,7 +609,7 @@ BlockLoopC8 ldr x11,[x9],#8 // x11 -> A1 iter 2 smlal v14.8h,v1.8b,v3.8b ldr d2,[x12,x5] // A2 iter 1 - b.eq EpilogueC8P3 // 3 pixel remains + b.eq EpilogueC8P3 // 3 pixel remains ldr d3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h ldr x12,[x0],#8 // x12 -> A0 iter 3 @@ -546,21 +654,15 @@ EpilogueC8P2 saddw2 v29.4s,v29.4s,v14.8h ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags] smull v12.8h,v0.8b,v4.8b - ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale] smull v14.8h,v0.8b,v6.8b - ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] smlal v12.8h,v1.8b,v2.8b smlal v14.8h,v1.8b,v3.8b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq SkipScaleVecLoad2C8 - ldp q4,q5,[x12],#32 // load scale vector if per channel -SkipScaleVecLoad2C8 + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v20.4s,v20.4s,v14.4h saddw2 v21.4s,v21.4s,v14.8h - b DequantC8 + b RequantizationC8 ProcC8P1 // @@ -613,9 +715,7 @@ EpilogueC8P1 // Loop epilogue (process last single pixel) mixed with loading of dequantization params // ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags] - ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b - ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h @@ -623,17 +723,27 @@ EpilogueC8P1 saddw2 v29.4s,v29.4s,v14.8h smull v12.8h,v0.8b,v2.8b smull v14.8h,v0.8b,v3.8b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq SkipScaleVecLoadC8 - ldp q4,q5,[x12] // load scale vector if per channel -SkipScaleVecLoadC8 + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v20.4s,v20.4s,v14.4h saddw2 v21.4s,v21.4s,v14.8h -DequantC8 +RequantizationC8 + b.ne FixedPointScaleC8 +FloatingPointScaleC8: + ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] + beq BroadcastScaleValueC8 + ldp q4,q5,[x12] // load scale vector if per channel + b ScaleWithFloatPointC8 + +BroadcastScaleValueC8 + ld1r {v4.4s},[x12] // load scale val + mov v5.16b,v4.16b // broadcast scale val if not per channel + +ScaleWithFloatPointC8 scvtf v24.4s,v24.4s // convert to float scvtf v25.4s,v25.4s scvtf v28.4s,v28.4s @@ -642,9 +752,6 @@ DequantC8 scvtf v17.4s,v17.4s scvtf v20.4s,v20.4s scvtf v21.4s,v21.4s - b.ne SkipScaleBroadcastC8 - mov v5.16b,v4.16b // broadcast scale val if not per channel -SkipScaleBroadcastC8 fmul v24.4s,v24.4s,v4.4s // multiply by scale fmul v25.4s,v25.4s,v5.4s fmul v28.4s,v28.4s,v4.4s @@ -661,6 +768,57 @@ SkipScaleBroadcastC8 fcvtns v17.4s,v17.4s fcvtns v20.4s,v20.4s fcvtns v21.4s,v21.4s + b QuantizeC8 + +FixedPointScaleC8 + ldr x10,[x8,#ConvSymDepthwisePostProcessParams_PreShift] + ldr x11,[x8,#ConvSymDepthwisePostProcessParams_Multiplier] + ldr x12,[x8,#ConvSymDepthwisePostProcessParams_PostShift] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] + beq BroadcastFixedPointValueC8 + ldp q0,q1,[x10],32 // load preshift vector + ldp q4,q5,[x11],32 // load multiplier vector + ldp q8,q9,[x12],32 // load postshift vector + b ScaleWithFixedPointC8 + +BroadcastFixedPointValueC8 + ld1r {v0.4s},[x10] // load preshift Value + mov v1.16b, v0.16b + + ld1r {v4.4s},[x11] // load multiplier Value + mov v5.16b, v4.16b + + ld1r {v8.4s},[x12] // load postshift Value + mov v9.16b, v8.16b + +ScaleWithFixedPointC8 + sqshl v24.4s, v24.4s, v0.4s // preshift + sqshl v25.4s, v25.4s, v1.4s + sqshl v28.4s, v28.4s, v0.4s + sqshl v29.4s, v29.4s, v1.4s + sqshl v16.4s, v16.4s, v0.4s + sqshl v17.4s, v17.4s, v1.4s + sqshl v20.4s, v20.4s, v0.4s + sqshl v21.4s, v21.4s, v1.4s + sqdmulh v24.4s, v24.4s, v4.4s // multiplier + sqdmulh v25.4s, v25.4s, v5.4s + sqdmulh v28.4s, v28.4s, v4.4s + sqdmulh v29.4s, v29.4s, v5.4s + sqdmulh v16.4s, v16.4s, v4.4s + sqdmulh v17.4s, v17.4s, v5.4s + sqdmulh v20.4s, v20.4s, v4.4s + sqdmulh v21.4s, v21.4s, v5.4s + srshl v24.4s, v24.4s, v8.4s // post shift + srshl v25.4s, v25.4s, v9.4s + srshl v28.4s, v28.4s, v8.4s + srshl v29.4s, v29.4s, v9.4s + srshl v16.4s, v16.4s, v8.4s + srshl v17.4s, v17.4s, v9.4s + srshl v20.4s, v20.4s, v8.4s + srshl v21.4s, v21.4s, v9.4s + +QuantizeC8 dup v0.8h,w15 sqxtn v24.4h,v24.4s // shorten to int16 sqxtn2 v24.8h,v25.4s diff --git a/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvSymU8KernelNeon.asm b/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvSymU8KernelNeon.asm index e9f75f1be5..8560fcf70a 100644 --- a/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvSymU8KernelNeon.asm +++ b/onnxruntime/core/mlas/lib/arm64/DepthwiseQConvSymU8KernelNeon.asm @@ -17,24 +17,45 @@ Abstract: #include "kxarm64.h" +#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2 +#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 4 + // // Stack frame layout for the depthwise conv kernel. // d8-d15, x19-x30 need to be preserved if used // -#define ConvSymDepthwiseKernelFrame_SavedNeonRegisters (8 * 8) -#define ConvSymDepthwiseKernelFrame_SavedRegisters ConvSymDepthwiseKernelFrame_SavedNeonRegisters -#define ConvSymDepthwiseKernelFrame_PostProcessParams 0 + ConvSymDepthwiseKernelFrame_SavedRegisters -#define ConvSymDepthwiseKernelFrame_KernelFlags 8 + ConvSymDepthwiseKernelFrame_SavedRegisters +#define ConvSymDepthwiseKernelFrame_SavedRegisters_d8_d9 0 +#define ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11 16 +#define ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13 32 +#define ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15 48 +#define ConvSymDepthwiseKernelFrame_SavedRegisters 64 +#define ConvSymDepthwiseKernelFrame_SavedRegisters_Neg -64 +#define ConvSymDepthwiseKernelFrame_PostProcessParams 0 + ConvSymDepthwiseKernelFrame_SavedRegisters +#define ConvSymDepthwiseKernelFrame_KernelFlags 8 + ConvSymDepthwiseKernelFrame_SavedRegisters -#define ConvSymDepthwisePostProcessParams_Bias 0 -#define ConvSymDepthwisePostProcessParams_Scale 8 -#define ConvSymDepthwisePostProcessParams_Min 16 -#define ConvSymDepthwisePostProcessParams_Max 20 -#define ConvSymDepthwisePostProcessParams_ZeroPoint 24 +/* +struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; + float MinimumValue; + float MaximumValue; + int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; +}; +*/ +#define ConvSymPostProcessParams_Bias 0 +#define ConvSymPostProcessParams_Scale 8 +#define ConvSymPostProcessParams_Min 16 +#define ConvSymPostProcessParams_Max 20 +#define ConvSymPostProcessParams_ZeroPoint 24 +#define ConvSymPostProcessParams_Multiplier 32 +#define ConvSymPostProcessParams_PreShift 40 +#define ConvSymPostProcessParams_PostShift 48 -#define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 1 -#define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 2 TEXTAREA @@ -48,28 +69,28 @@ Routine Description: Arguments: Input (x0) - Supplies the address of the indirection buffer. - + Filter (x1) - Supplies the address of the filter buffer. Output (x2) - Supplies the address of the output buffer. KernelSize (x3) - Supplies the size of the kernel. - + Channels (x4) - Supplies the number of input and output channels. - + ChannelOffset (x5) - Supplies the byte offset from the indirection buffer base address for this iteration. - + ChannelCount (x6) - Supplies the number of channels this iteration produces. - + This implementation requires the count to be 16 or 8 - + OutputCount (x7)- Supplies the number of output elements this iteration produces. - + This implementation requires the count to be in the range 1 to 2. - + PostProcessParams - Supplies the address of the post process parameter block. - + KernelFlags - Supplies additional flags controlling the operation. Return Value: @@ -80,14 +101,14 @@ Return Value: NESTED_ENTRY MlasConvSymDepthwiseU8KernelNeon - PROLOG_SAVE_REG_PAIR d8,d9,#-64! + PROLOG_SAVE_REG_PAIR d8,d9,#ConvSymDepthwiseKernelFrame_SavedRegisters_Neg! PROLOG_NOP ldr x8,[sp,#ConvSymDepthwiseKernelFrame_PostProcessParams] PROLOG_NOP mov w10,#0x80808080 - PROLOG_SAVE_REG_PAIR d10,d11,#16 - PROLOG_SAVE_REG_PAIR d12,d13,#32 - PROLOG_SAVE_REG_PAIR d14,d15,#48 + PROLOG_SAVE_REG_PAIR d10,d11,#ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11 + PROLOG_SAVE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13 + PROLOG_SAVE_REG_PAIR d14,d15,#ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15 dup v8.4s,w10 // bit flip vector - ldr x16,[x8,#ConvSymDepthwisePostProcessParams_Bias] + ldr x16,[x8,#ConvSymPostProcessParams_Bias] cmp x7,2 add x9,x0,x3,lsl#3 // x9 -> &A1 add x14,x0,x3,lsl#4 // x14 -> &A2 @@ -111,11 +132,11 @@ Return Value: // x0 -> x10 -> v4 // -> x12 -> v2 // unroll // x9 -> x11 -> v6 -// -> x13 -> v10 // unroll +// -> x13 -> v3 // unroll // x14 -> x10 -> v4 // -> x12 -> v2 // unroll // x15 -> x11 -> v6 -// -> x13 -> v10 // unroll +// -> x13 -> v3 // unroll // Process16Channels @@ -144,7 +165,7 @@ Process16Channels mov v18.16b,v26.16b ldr x12,[x14],#8 // x12 -> A2 iter 1 mov v19.16b,v27.16b - ldr q10,[x13,x5] // A1 iter 1 + ldr q3,[x13,x5] // A1 iter 1 mov v22.16b,v26.16b ldr x13,[x15],#8 // x13 -> A3 iter 1 mov v23.16b,v27.16b @@ -169,13 +190,13 @@ BlockLoopC16 smlal v12.8h,v1.8b,v2.8b ldr x11,[x9],#8 // x11 -> A1 iter 2 smlal2 v13.8h,v1.16b,v2.16b - b.eq EpilogueC16P3 // 3 pixel remains - eor v10.16b,v10.16b,v8.16b + b.eq EpilogueC16P3 // 3 pixel remains + eor v3.16b,v3.16b,v8.16b ldr q2,[x12,x5] // A2 iter 1 - smlal v14.8h,v1.8b,v10.8b + smlal v14.8h,v1.8b,v3.8b ldr x12,[x0],#8 // x12 -> A0 iter 3 - smlal2 v15.8h,v1.16b,v10.16b - ldr q10,[x13,x5] // A3 iter 1 + smlal2 v15.8h,v1.16b,v3.16b + ldr q3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h ldr x13,[x9],#8 // x13 -> A1 iter 3 @@ -200,12 +221,12 @@ BlockLoopC16 smlal v12.8h,v1.8b,v2.8b ldr x11,[x15],#8 // x11 -> A3 iter 2 smlal2 v13.8h,v1.16b,v2.16b - eor v10.16b,v10.16b,v8.16b + eor v3.16b,v3.16b,v8.16b ldr q2,[x12,x5] // A0 iter 3 - smlal v14.8h,v1.8b,v10.8b + smlal v14.8h,v1.8b,v3.8b ldr x12,[x14],#8 // x12 -> A2 iter 3 - smlal2 v15.8h,v1.16b,v10.16b - ldr q10,[x13,x5] // A1 iter 3 + smlal2 v15.8h,v1.16b,v3.16b + ldr q3,[x13,x5] // A1 iter 3 saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h ld1 {v1.16b},[x1],x4 // filter iter 3 @@ -229,11 +250,11 @@ EpilogueC16P2 eor v2.16b,v2.16b,v8.16b smlal v12.8h,v1.8b,v2.8b smlal2 v13.8h,v1.16b,v2.16b - eor v10.16b,v10.16b,v8.16b + eor v3.16b,v3.16b,v8.16b ldr q2,[x12,x5] // A2 iter 1 - smlal v14.8h,v1.8b,v10.8b - smlal2 v15.8h,v1.16b,v10.16b - ldr q10,[x13,x5] // A3 iter 1 + smlal v14.8h,v1.8b,v3.8b + smlal2 v15.8h,v1.16b,v3.16b + ldr q3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h saddw v26.4s,v26.4s,v13.4h @@ -244,25 +265,20 @@ EpilogueC16P2 saddw2 v31.4s,v31.4s,v15.8h ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags] eor v4.16b,v4.16b,v8.16b - ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale] + ldr x12,[x8,#ConvSymPostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b smull2 v13.8h,v0.16b,v4.16b eor v6.16b,v6.16b,v8.16b - ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] + ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b smull2 v15.8h,v0.16b,v6.16b eor v2.16b,v2.16b,v8.16b smlal v12.8h,v1.8b,v2.8b smlal2 v13.8h,v1.16b,v2.16b - eor v10.16b,v10.16b,v8.16b - smlal v14.8h,v1.8b,v10.8b - smlal2 v15.8h,v1.16b,v10.16b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq SkipScaleVecLoad2 - ldp q4,q11,[x12],#32 // load scale vector if per channel - ldp q6,q9,[x12] -SkipScaleVecLoad2 + eor v3.16b,v3.16b,v8.16b + smlal v14.8h,v1.8b,v3.8b + smlal2 v15.8h,v1.16b,v3.16b + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v18.4s,v18.4s,v13.4h @@ -271,7 +287,7 @@ SkipScaleVecLoad2 saddw2 v21.4s,v21.4s,v14.8h saddw v22.4s,v22.4s,v15.4h saddw2 v23.4s,v23.4s,v15.8h - b Dequantization + b Requantization ProcC16P1 // @@ -295,7 +311,7 @@ ProcC16P1 subs x3,x3,2 // decrement input blocks remaining mov v18.16b,v26.16b mov v19.16b,v27.16b - ldr q10,[x13,x5] + ldr q3,[x13,x5] mov v22.16b,v26.16b mov v23.16b,v27.16b b EpilogueC16P1 @@ -305,12 +321,12 @@ EpilogueC16P3 // Loop epilogue (process last 2 pixels) mixed // with loading of dequantization params // - eor v10.16b,v10.16b,v8.16b + eor v3.16b,v3.16b,v8.16b ldr q2,[x12,x5] // A2 iter 1 - smlal v14.8h,v1.8b,v10.8b + smlal v14.8h,v1.8b,v3.8b ldr x12,[x14],#8 // x12 -> A2 iter 2 - smlal2 v15.8h,v1.16b,v10.16b - ldr q10,[x13,x5] // A3 iter 1 + smlal2 v15.8h,v1.16b,v3.16b + ldr q3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h ldr x13,[x15],#8 // x13 -> A3 iter 2 @@ -332,11 +348,11 @@ EpilogueC16P3 eor v2.16b,v2.16b,v8.16b smlal v12.8h,v1.8b,v2.8b smlal2 v13.8h,v1.16b,v2.16b - eor v10.16b,v10.16b,v8.16b + eor v3.16b,v3.16b,v8.16b ldr q2,[x12,x5] // A2 iter 2 - smlal v14.8h,v1.8b,v10.8b - smlal2 v15.8h,v1.16b,v10.16b - ldr q10,[x13,x5] // A3 iter 2 + smlal v14.8h,v1.8b,v3.8b + smlal2 v15.8h,v1.16b,v3.16b + ldr q3,[x13,x5] // A3 iter 2 saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v18.4s,v18.4s,v13.4h @@ -352,11 +368,9 @@ EpilogueC16P1 // ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags] eor v4.16b,v4.16b,v8.16b - ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b smull2 v13.8h,v0.16b,v4.16b eor v6.16b,v6.16b,v8.16b - ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b smull2 v15.8h,v0.16b,v6.16b saddw v24.4s,v24.4s,v12.4h @@ -370,15 +384,10 @@ EpilogueC16P1 eor v2.16b,v2.16b,v8.16b smull v12.8h,v0.8b,v2.8b smull2 v13.8h,v0.16b,v2.16b - eor v10.16b,v10.16b,v8.16b - smull v14.8h,v0.8b,v10.8b - smull2 v15.8h,v0.16b,v10.16b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq SkipScaleVecLoad - ldp q4,q11,[x12],#32 // load scale vector if per channel - ldp q6,q9,[x12] -SkipScaleVecLoad + eor v3.16b,v3.16b,v8.16b + smull v14.8h,v0.8b,v3.8b + smull2 v15.8h,v0.16b,v3.16b + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v18.4s,v18.4s,v13.4h @@ -388,7 +397,24 @@ SkipScaleVecLoad saddw v22.4s,v22.4s,v15.4h saddw2 v23.4s,v23.4s,v15.8h -Dequantization +Requantization + b.ne FixedPointScale +FloatingPointScale + ldr x12,[x8,#ConvSymPostProcessParams_Scale] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint] + beq BroadcastScaleValue + ldp q4,q5,[x12],#32 // load scale vector if per channel + ldp q6,q3,[x12] + b ScaleWithFloatPoint + +BroadcastScaleValue + ld1r {v4.4s},[x12] // load scale val + mov v5.16b,v4.16b // broadcast scale val if not per channel + mov v6.16b,v4.16b + mov v3.16b,v4.16b + +ScaleWithFloatPoint scvtf v24.4s,v24.4s // convert to float scvtf v25.4s,v25.4s scvtf v26.4s,v26.4s @@ -405,27 +431,22 @@ Dequantization scvtf v21.4s,v21.4s scvtf v22.4s,v22.4s scvtf v23.4s,v23.4s - b.ne SkipScaleBroadcast - mov v11.16b,v4.16b // broadcast scale val if not per channel - mov v6.16b,v4.16b - mov v9.16b,v4.16b -SkipScaleBroadcast fmul v24.4s,v24.4s,v4.4s // multiply by scale - fmul v25.4s,v25.4s,v11.4s + fmul v25.4s,v25.4s,v5.4s fmul v26.4s,v26.4s,v6.4s - fmul v27.4s,v27.4s,v9.4s + fmul v27.4s,v27.4s,v3.4s fmul v28.4s,v28.4s,v4.4s - fmul v29.4s,v29.4s,v11.4s + fmul v29.4s,v29.4s,v5.4s fmul v30.4s,v30.4s,v6.4s - fmul v31.4s,v31.4s,v9.4s + fmul v31.4s,v31.4s,v3.4s fmul v16.4s,v16.4s,v4.4s - fmul v17.4s,v17.4s,v11.4s + fmul v17.4s,v17.4s,v5.4s fmul v18.4s,v18.4s,v6.4s - fmul v19.4s,v19.4s,v9.4s + fmul v19.4s,v19.4s,v3.4s fmul v20.4s,v20.4s,v4.4s - fmul v21.4s,v21.4s,v11.4s + fmul v21.4s,v21.4s,v5.4s fmul v22.4s,v22.4s,v6.4s - fmul v23.4s,v23.4s,v9.4s + fmul v23.4s,v23.4s,v3.4s fcvtns v24.4s,v24.4s // convert to int fcvtns v25.4s,v25.4s fcvtns v26.4s,v26.4s @@ -442,6 +463,90 @@ SkipScaleBroadcast fcvtns v21.4s,v21.4s fcvtns v22.4s,v22.4s fcvtns v23.4s,v23.4s + b Quantize + +FixedPointScale + ldr x10,[x8,#ConvSymPostProcessParams_PreShift] + ldr x11,[x8,#ConvSymPostProcessParams_Multiplier] + ldr x12,[x8,#ConvSymPostProcessParams_PostShift] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint] + beq BroadcastFixedPointValue + ldp q0,q1,[x10],32 // load preshift vector + ldp q2,q3,[x10],32 + ldp q4,q5,[x11],32 // load multiplier vector + ldp q6,q7,[x11],32 + ldp q8,q9,[x12],32 // load postshift vector + ldp q10,q11,[x12],32 + b ScaleWithFixedPoint + +BroadcastFixedPointValue + ld1r {v0.4s},[x10] // load preshift Value + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + + ld1r {v4.4s},[x11] // load multiplier Value + mov v5.16b, v4.16b + mov v6.16b, v4.16b + mov v7.16b, v4.16b + + ld1r {v8.4s},[x12] // load postshift Value + mov v9.16b, v8.16b + mov v10.16b, v8.16b + mov v11.16b, v8.16b + +ScaleWithFixedPoint + sqshl v24.4s, v24.4s, v0.4s // preshift + sqshl v25.4s, v25.4s, v1.4s + sqshl v26.4s, v26.4s, v2.4s + sqshl v27.4s, v27.4s, v3.4s + sqshl v28.4s, v28.4s, v0.4s + sqshl v29.4s, v29.4s, v1.4s + sqshl v30.4s, v30.4s, v2.4s + sqshl v31.4s, v31.4s, v3.4s + sqshl v16.4s, v16.4s, v0.4s + sqshl v17.4s, v17.4s, v1.4s + sqshl v18.4s, v18.4s, v2.4s + sqshl v19.4s, v19.4s, v3.4s + sqshl v20.4s, v20.4s, v0.4s + sqshl v21.4s, v21.4s, v1.4s + sqshl v22.4s, v22.4s, v2.4s + sqshl v23.4s, v23.4s, v3.4s + sqdmulh v24.4s, v24.4s, v4.4s // multiplier + sqdmulh v25.4s, v25.4s, v5.4s + sqdmulh v26.4s, v26.4s, v6.4s + sqdmulh v27.4s, v27.4s, v7.4s + sqdmulh v28.4s, v28.4s, v4.4s + sqdmulh v29.4s, v29.4s, v5.4s + sqdmulh v30.4s, v30.4s, v6.4s + sqdmulh v31.4s, v31.4s, v7.4s + sqdmulh v16.4s, v16.4s, v4.4s + sqdmulh v17.4s, v17.4s, v5.4s + sqdmulh v18.4s, v18.4s, v6.4s + sqdmulh v19.4s, v19.4s, v7.4s + sqdmulh v20.4s, v20.4s, v4.4s + sqdmulh v21.4s, v21.4s, v5.4s + sqdmulh v22.4s, v22.4s, v6.4s + sqdmulh v23.4s, v23.4s, v7.4s + srshl v24.4s, v24.4s, v8.4s // post shift + srshl v25.4s, v25.4s, v9.4s + srshl v26.4s, v26.4s, v10.4s + srshl v27.4s, v27.4s, v11.4s + srshl v28.4s, v28.4s, v8.4s + srshl v29.4s, v29.4s, v9.4s + srshl v30.4s, v30.4s, v10.4s + srshl v31.4s, v31.4s, v11.4s + srshl v16.4s, v16.4s, v8.4s + srshl v17.4s, v17.4s, v9.4s + srshl v18.4s, v18.4s, v10.4s + srshl v19.4s, v19.4s, v11.4s + srshl v20.4s, v20.4s, v8.4s + srshl v21.4s, v21.4s, v9.4s + srshl v22.4s, v22.4s, v10.4s + srshl v23.4s, v23.4s, v11.4s + +Quantize sqxtn v24.4h,v24.4s // shorten to int16 sqxtn v26.4h,v26.4s sqxtn2 v24.8h,v25.4s @@ -486,10 +591,10 @@ SkipScaleBroadcast str q20,[x2] ExitKernel - EPILOG_RESTORE_REG_PAIR d14,d15,#48 - EPILOG_RESTORE_REG_PAIR d12,d13,#32 - EPILOG_RESTORE_REG_PAIR d10,d11,#16 - EPILOG_RESTORE_REG_PAIR d8,d9,#64! + EPILOG_RESTORE_REG_PAIR d14,d15,#ConvSymDepthwiseKernelFrame_SavedRegisters_d14_d15 + EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters_d12_d13 + EPILOG_RESTORE_REG_PAIR d10,d11,#ConvSymDepthwiseKernelFrame_SavedRegisters_d10_d11 + EPILOG_RESTORE_REG_PAIR d8,d9,#ConvSymDepthwiseKernelFrame_SavedRegisters! EPILOG_RETURN Process8Channels @@ -511,7 +616,7 @@ Process8Channels mov v17.16b,v25.16b ldr x12,[x14],#8 // x12 -> A2 iter 1 subs x3,x3,2 // decrement input blocks remaining - ldr d10,[x13,x5] // A1 iter 1 + ldr d3,[x13,x5] // A1 iter 1 mov v20.16b,v24.16b ldr x13,[x15],#8 // x13 -> A3 iter 1 mov v21.16b,v25.16b @@ -528,15 +633,15 @@ BlockLoopC8 b.eq EpilogueC8P2 ldr x10,[x0],#8 // x10 -> A0 iter 2 eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b ldr d6,[x11,x5] // A3 iter 0 cmp x3,1 smlal v12.8h,v1.8b,v2.8b ldr x11,[x9],#8 // x11 -> A1 iter 2 - smlal v14.8h,v1.8b,v10.8b + smlal v14.8h,v1.8b,v3.8b ldr d2,[x12,x5] // A2 iter 1 - b.eq EpilogueC8P3 // 3 pixel remains - ldr d10,[x13,x5] // A3 iter 1 + b.eq EpilogueC8P3 // 3 pixel remains + ldr d3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h ldr x12,[x0],#8 // x12 -> A0 iter 3 saddw2 v25.4s,v25.4s,v12.8h @@ -552,15 +657,15 @@ BlockLoopC8 ldr x10,[x14],#8 // x10 -> A2 iter 2 ldr d6,[x11,x5] // A1 iter 2 eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b ld1 {v0.8b},[x1],x4 // filter iter 2 smlal v12.8h,v1.8b,v2.8b ldr x11,[x15],#8 // x11 -> A3 iter 2 ldr d2,[x12,x5] // A0 iter 3 - smlal v14.8h,v1.8b,v10.8b + smlal v14.8h,v1.8b,v3.8b ldr x12,[x14],#8 // x12 -> A2 iter 3 saddw v16.4s,v16.4s,v12.4h - ldr d10,[x13,x5] // A1 iter 3 + ldr d3,[x13,x5] // A1 iter 3 saddw2 v17.4s,v17.4s,v12.8h ld1 {v1.8b},[x1],x4 // filter iter 3 saddw v20.4s,v20.4s,v14.4h @@ -575,11 +680,11 @@ EpilogueC8P2 // ldr d6,[x11,x5] // A3 iter 0 eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b smlal v12.8h,v1.8b,v2.8b ldr d2,[x12,x5] // A2 iter 1 - smlal v14.8h,v1.8b,v10.8b - ldr d10,[x13,x5] // A3 iter 1 + smlal v14.8h,v1.8b,v3.8b + ldr d3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h saddw v28.4s,v28.4s,v14.4h @@ -588,23 +693,19 @@ EpilogueC8P2 eor v4.8b,v4.8b,v8.8b eor v6.8b,v6.8b,v8.8b smull v12.8h,v0.8b,v4.8b - ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale] + ldr x12,[x8,#ConvSymPostProcessParams_Scale] smull v14.8h,v0.8b,v6.8b - ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] + ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint] eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b smlal v12.8h,v1.8b,v2.8b - smlal v14.8h,v1.8b,v10.8b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq SkipScaleVecLoad2C8 - ldp q4,q11,[x12],#32 // load scale vector if per channel -SkipScaleVecLoad2C8 + smlal v14.8h,v1.8b,v3.8b + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v20.4s,v20.4s,v14.4h saddw2 v21.4s,v21.4s,v14.8h - b DequantC8 + b RequantizationC8 ProcC8P1 // @@ -623,7 +724,7 @@ ProcC8P1 mov v20.16b,v24.16b ldr d2,[x12,x5] subs x3,x3,2 // decrement input blocks remaining - ldr d10,[x13,x5] + ldr d3,[x13,x5] mov v21.16b,v25.16b b EpilogueC8P1 @@ -632,7 +733,7 @@ EpilogueC8P3 // Loop epilogue (process 2 of last 3 pixels) // ldr x12,[x14],#8 // x12 -> A2 iter 2 - ldr d10,[x13,x5] // A3 iter 1 + ldr d3,[x13,x5] // A3 iter 1 saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h ldr x13,[x15],#8 // x13 -> A3 iter 2 @@ -645,12 +746,12 @@ EpilogueC8P3 smull v14.8h,v0.8b,v6.8b ld1 {v0.8b},[x1] // filter iter 2 eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b ldr d6,[x11,x5] // A1 iter 2 smlal v12.8h,v1.8b,v2.8b ldr d2,[x12,x5] // A2 iter 2 - smlal v14.8h,v1.8b,v10.8b - ldr d10,[x13,x5] // A3 iter 2 + smlal v14.8h,v1.8b,v3.8b + ldr d3,[x13,x5] // A3 iter 2 saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v20.4s,v20.4s,v14.4h @@ -663,29 +764,39 @@ EpilogueC8P1 ldr w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags] eor v4.8b,v4.8b,v8.8b eor v6.8b,v6.8b,v8.8b - ldr x12,[x8,#ConvSymDepthwisePostProcessParams_Scale] + ldr x12,[x8,#ConvSymPostProcessParams_Scale] smull v12.8h,v0.8b,v4.8b - ldr w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint] + ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint] smull v14.8h,v0.8b,v6.8b saddw v24.4s,v24.4s,v12.4h saddw2 v25.4s,v25.4s,v12.8h saddw v28.4s,v28.4s,v14.4h saddw2 v29.4s,v29.4s,v14.8h eor v2.8b,v2.8b,v8.8b - eor v10.8b,v10.8b,v8.8b + eor v3.8b,v3.8b,v8.8b smull v12.8h,v0.8b,v2.8b - smull v14.8h,v0.8b,v10.8b - tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE - ld1r {v4.4s},[x12] // load scale val - b.eq SkipScaleVecLoadC8 - ldp q4,q11,[x12] // load scale vector if per channel -SkipScaleVecLoadC8 + smull v14.8h,v0.8b,v3.8b + tst w9,#MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE saddw v16.4s,v16.4s,v12.4h saddw2 v17.4s,v17.4s,v12.8h saddw v20.4s,v20.4s,v14.4h saddw2 v21.4s,v21.4s,v14.8h -DequantC8 +RequantizationC8 + b.ne FixedPointScaleC8 +FloatingPointScaleC8 + ldr x12,[x8,#ConvSymPostProcessParams_Scale] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint] + beq BroadcastScaleValueC8 + ldp q4,q5,[x12] // load scale vector if per channel + b ScaleWithFloatPointC8 + +BroadcastScaleValueC8 + ld1r {v4.4s},[x12] // load scale val + mov v5.16b,v4.16b // broadcast scale val if not per channel + +ScaleWithFloatPointC8 scvtf v24.4s,v24.4s // convert to float scvtf v25.4s,v25.4s scvtf v28.4s,v28.4s @@ -694,17 +805,14 @@ DequantC8 scvtf v17.4s,v17.4s scvtf v20.4s,v20.4s scvtf v21.4s,v21.4s - b.ne SkipScaleBroadcastC8 - mov v11.16b,v4.16b // broadcast scale val if not per channel -SkipScaleBroadcastC8 fmul v24.4s,v24.4s,v4.4s // multiply by scale - fmul v25.4s,v25.4s,v11.4s + fmul v25.4s,v25.4s,v5.4s fmul v28.4s,v28.4s,v4.4s - fmul v29.4s,v29.4s,v11.4s + fmul v29.4s,v29.4s,v5.4s fmul v16.4s,v16.4s,v4.4s - fmul v17.4s,v17.4s,v11.4s + fmul v17.4s,v17.4s,v5.4s fmul v20.4s,v20.4s,v4.4s - fmul v21.4s,v21.4s,v11.4s + fmul v21.4s,v21.4s,v5.4s fcvtns v24.4s,v24.4s // convert to int fcvtns v25.4s,v25.4s fcvtns v28.4s,v28.4s @@ -713,6 +821,57 @@ SkipScaleBroadcastC8 fcvtns v17.4s,v17.4s fcvtns v20.4s,v20.4s fcvtns v21.4s,v21.4s + b QuantizeC8 + +FixedPointScaleC8 + ldr x10,[x8,#ConvSymPostProcessParams_PreShift] + ldr x11,[x8,#ConvSymPostProcessParams_Multiplier] + ldr x12,[x8,#ConvSymPostProcessParams_PostShift] + tst w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE + ldr w15,[x8,#ConvSymPostProcessParams_ZeroPoint] + beq BroadcastFixedPointValueC8 + ldp q0,q1,[x10],32 // load preshift vector + ldp q4,q5,[x11],32 // load multiplier vector + ldp q8,q9,[x12],32 // load postshift vector + b ScaleWithFixedPointC8 + +BroadcastFixedPointValueC8 + ld1r {v0.4s},[x10] // load preshift Value + mov v1.16b, v0.16b + + ld1r {v4.4s},[x11] // load multiplier Value + mov v5.16b, v4.16b + + ld1r {v8.4s},[x12] // load postshift Value + mov v9.16b, v8.16b + +ScaleWithFixedPointC8 + sqshl v24.4s, v24.4s, v0.4s // preshift + sqshl v25.4s, v25.4s, v1.4s + sqshl v28.4s, v28.4s, v0.4s + sqshl v29.4s, v29.4s, v1.4s + sqshl v16.4s, v16.4s, v0.4s + sqshl v17.4s, v17.4s, v1.4s + sqshl v20.4s, v20.4s, v0.4s + sqshl v21.4s, v21.4s, v1.4s + sqdmulh v24.4s, v24.4s, v4.4s // multiplier + sqdmulh v25.4s, v25.4s, v5.4s + sqdmulh v28.4s, v28.4s, v4.4s + sqdmulh v29.4s, v29.4s, v5.4s + sqdmulh v16.4s, v16.4s, v4.4s + sqdmulh v17.4s, v17.4s, v5.4s + sqdmulh v20.4s, v20.4s, v4.4s + sqdmulh v21.4s, v21.4s, v5.4s + srshl v24.4s, v24.4s, v8.4s // post shift + srshl v25.4s, v25.4s, v9.4s + srshl v28.4s, v28.4s, v8.4s + srshl v29.4s, v29.4s, v9.4s + srshl v16.4s, v16.4s, v8.4s + srshl v17.4s, v17.4s, v9.4s + srshl v20.4s, v20.4s, v8.4s + srshl v21.4s, v21.4s, v9.4s + +QuantizeC8 dup v0.8h,w15 sqxtn v24.4h,v24.4s // shorten to int16 sqxtn2 v24.8h,v25.4s diff --git a/onnxruntime/core/mlas/lib/convsym.cpp b/onnxruntime/core/mlas/lib/convsym.cpp index a667fe4048..f473eb9747 100644 --- a/onnxruntime/core/mlas/lib/convsym.cpp +++ b/onnxruntime/core/mlas/lib/convsym.cpp @@ -478,7 +478,7 @@ MlasConvSym( { const MLAS_CONV_SYM_DISPATCH* ConvSymDispatch = GetConvSymDispatch(Params.InputIsSigned); - // Pick the suitable kernel for current core. Currently we only have specialized core for + // Pick the suitable kernel for current core. Currently we only have specialized core for // s8s8 under ARM64 #if defined(MLAS_TARGET_ARM64) const auto Kernel = @@ -490,8 +490,16 @@ MlasConvSym( #endif int32_t KernelFlags = 0; + const MLAS_REQUANT_PARAM* RequantParam = Params.RequantParam; + bool PerChannelScale = RequantParam->Size > 1; + int32_t OutputZeroPoint = RequantParam->ZeroPoint; + MLAS_ROUND_KIND RequantRoundKind = RequantParam->RequantRoundKind; - if (Params.PerChannelScale) { + if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfUp) { + KernelFlags |= MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE; + } + + if (PerChannelScale) { KernelFlags |= MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE; } @@ -500,8 +508,7 @@ MlasConvSym( } MLAS_CONV_SYM_POST_PROCESS_PARAMS PostProcessParams = {}; - - MlasConvSymSetOutputZeroPoint(PostProcessParams, Params.OutputZeroPoint, Params.InputIsSigned); + MlasConvSymSetOutputZeroPoint(PostProcessParams, OutputZeroPoint, Params.InputIsSigned); const size_t KernelChannelCount = (ConvSymDispatch->KernelChannelCount == 0) ? std::numeric_limits::max() @@ -523,7 +530,14 @@ MlasConvSym( void* conv_out = static_cast(Params.Output) + (oc_outside * OutputChannels) + co; PostProcessParams.Bias = Params.Bias + co; - PostProcessParams.Scale = Params.Scale + (Params.PerChannelScale ? co : 0); + + if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfEven) { + PostProcessParams.Scale = RequantParam->Scale + (PerChannelScale ? co : 0); + } else { + PostProcessParams.Multiplier = RequantParam->Multiplier + (PerChannelScale ? co : 0); + PostProcessParams.PreShift = RequantParam->PreShift + (PerChannelScale ? co : 0); + PostProcessParams.PostShift = RequantParam->PostShift + (PerChannelScale ? co : 0); + } for (size_t oc = 0; oc < oc_outside_block_size;) { @@ -567,23 +581,38 @@ MlasConvSymDepthwise( unsigned KernelFlags = 0; - if (Params.PerChannelScale) { + const MLAS_REQUANT_PARAM* RequantParam = Params.RequantParam; + bool PerChannelScale = RequantParam->Size > 1; + int32_t OutputZeroPoint = RequantParam->ZeroPoint; + MLAS_ROUND_KIND RequantRoundKind = RequantParam->RequantRoundKind; + if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfUp) { + KernelFlags |= MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE; + } + + if (PerChannelScale) { KernelFlags |= MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE; } MLAS_CONV_SYM_POST_PROCESS_PARAMS PostProcessParams = {}; - - MlasConvSymSetOutputZeroPoint(PostProcessParams, Params.OutputZeroPoint, Params.InputIsSigned); + MlasConvSymSetOutputZeroPoint(PostProcessParams, OutputZeroPoint, Params.InputIsSigned); if ((Params.OutputChannels & 15) == 0) { PostProcessParams.Bias = Params.Bias; - PostProcessParams.Scale = Params.Scale; + if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfEven) { + PostProcessParams.Scale = RequantParam->Scale; + } else { + PostProcessParams.Multiplier = RequantParam->Multiplier; + PostProcessParams.PreShift = RequantParam->PreShift; + PostProcessParams.PostShift = RequantParam->PostShift; + } + if (ConvSymDispatch->Depthwise3x3Proc && Params.KernelSize == 9) { ConvSymDispatch->Depthwise3x3Proc(Params.InputIndirection, (int8_t const*)Params.Filter, Params.OutputChannels, Params.Output, Params.OutputCount, &PostProcessParams, KernelFlags); return; } + if (ConvSymDispatch->Depthwise5x5Proc && Params.KernelSize == 25) { ConvSymDispatch->Depthwise5x5Proc(Params.InputIndirection, (int8_t const*)Params.Filter, Params.OutputChannels, Params.Output, @@ -610,7 +639,13 @@ MlasConvSymDepthwise( const size_t ChannelCount = std::min(OutputChannels - ChannelOffset, KernelChannelCount); PostProcessParams.Bias = Params.Bias + ChannelOffset; - PostProcessParams.Scale = Params.Scale + (Params.PerChannelScale ? ChannelOffset : 0); + if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfEven) { + PostProcessParams.Scale = RequantParam->Scale + (PerChannelScale ? ChannelOffset : 0); + } else { + PostProcessParams.Multiplier = RequantParam->Multiplier + (PerChannelScale ? ChannelOffset : 0); + PostProcessParams.PreShift = RequantParam->PreShift + (PerChannelScale ? ChannelOffset : 0); + PostProcessParams.PostShift = RequantParam->PostShift + (PerChannelScale ? ChannelOffset : 0); + } ConvSymDispatch->DepthwiseKernel( InputIndirection, diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index df00121c36..7483bb390c 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -804,17 +804,22 @@ MlasConvDepthwiseKernelAvx2( #define MLAS_CONV_SYM_FLAG_INPUT_DIRECT 0x00000001 #define MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE 0x00000002 +#define MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE 0x00000004 // Only available on ARM64 // // Define the post-processing parameters for conv sym: bias and re-quant params // struct MLAS_CONV_SYM_POST_PROCESS_PARAMS { - const int32_t* Bias; - const float* Scale; + const int32_t* Bias{nullptr}; + const float* Scale{nullptr}; float MinimumValue; float MaximumValue; int32_t OutputZeroPoint; + int32_t Padding; + const int32_t* Multiplier{nullptr}; + const int32_t* PreShift{nullptr}; + const int32_t* PostShift{nullptr}; }; // @@ -988,7 +993,7 @@ MlasPartitionWork( #if defined(_MSC_VER) && !defined(__clang__) #pragma warning(push) // VC++ suggests we can attempt to make 'MlasBitsOfFp32' constexpr, but it is not valid. - #pragma warning(disable:26497) + #pragma warning(disable:26497) #endif MLAS_FORCEINLINE diff --git a/onnxruntime/core/mlas/lib/qdwconv_kernelsize.cpp b/onnxruntime/core/mlas/lib/qdwconv_kernelsize.cpp index 4985f91b64..13aafa4682 100644 --- a/onnxruntime/core/mlas/lib/qdwconv_kernelsize.cpp +++ b/onnxruntime/core/mlas/lib/qdwconv_kernelsize.cpp @@ -43,13 +43,12 @@ Abstract: #include "mlasi.h" -extern "C" { - #if defined(MLAS_TARGET_ARM64) +template void MLASCALL -MlasConvSymDepthwiseKernelSize25ArmU8S8( +MlasConvSymDepthwiseKernelSize25ArmU8S8Impl( void const* const* InputIndirection, int8_t const* Filter, size_t Channels, @@ -64,10 +63,44 @@ MlasConvSymDepthwiseKernelSize25ArmU8S8( const uint8x16_t vu128 = vdupq_n_u8(128); const int16x8_t voutput_zero_point = vld1q_dup_s16((int16_t const*)&PostProcessParams->OutputZeroPoint); float32x4_t vscale_0123, vscale_4567, vscale_89AB, vscale_CDEF; + int32x4_t vpreshift_0123, vpreshift_4567, vpreshift_89AB, vpreshift_CDEF; + int32x4_t vmultiplier_0123, vmultiplier_4567, vmultiplier_89AB, vmultiplier_CDEF; + int32x4_t vpostshift_0123, vpostshift_4567, vpostshift_89AB, vpostshift_CDEF; const bool is_per_channel = ((KernelFlags & MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE) != 0); + // Init them anyway due to some compiler will generate uninitialized warnings. - vscale_0123 = vscale_4567 = vscale_89AB = vscale_CDEF = vld1q_dup_f32(PostProcessParams->Scale); + if constexpr (IsFixedPoint) { + + vpreshift_0123 = vpreshift_4567 = vpreshift_89AB = vpreshift_CDEF = vld1q_dup_s32(PostProcessParams->PreShift); + vmultiplier_0123 = vmultiplier_4567 = vmultiplier_89AB = vmultiplier_CDEF = vld1q_dup_s32(PostProcessParams->Multiplier); + vpostshift_0123 = vpostshift_4567 = vpostshift_89AB = vpostshift_CDEF = vld1q_dup_s32(PostProcessParams->PostShift); + + MLAS_UNREFERENCED_PARAMETER(vscale_0123); + MLAS_UNREFERENCED_PARAMETER(vscale_4567); + MLAS_UNREFERENCED_PARAMETER(vscale_89AB); + MLAS_UNREFERENCED_PARAMETER(vscale_CDEF); + + } else { + + vscale_0123 = vscale_4567 = vscale_89AB = vscale_CDEF = vld1q_dup_f32(PostProcessParams->Scale); + + MLAS_UNREFERENCED_PARAMETER(vpreshift_0123); + MLAS_UNREFERENCED_PARAMETER(vpreshift_4567); + MLAS_UNREFERENCED_PARAMETER(vpreshift_89AB); + MLAS_UNREFERENCED_PARAMETER(vpreshift_CDEF); + MLAS_UNREFERENCED_PARAMETER(vmultiplier_0123); + MLAS_UNREFERENCED_PARAMETER(vmultiplier_4567); + MLAS_UNREFERENCED_PARAMETER(vmultiplier_89AB); + MLAS_UNREFERENCED_PARAMETER(vmultiplier_CDEF); + MLAS_UNREFERENCED_PARAMETER(vpostshift_0123); + MLAS_UNREFERENCED_PARAMETER(vpostshift_4567); + MLAS_UNREFERENCED_PARAMETER(vpostshift_89AB); + MLAS_UNREFERENCED_PARAMETER(vpostshift_CDEF); + + } + while (OutputCount-- > 0) { + const uint8_t* i00 = IndirectBuf[0]; const uint8_t* i01 = IndirectBuf[1]; const uint8_t* i02 = IndirectBuf[2]; @@ -99,7 +132,11 @@ MlasConvSymDepthwiseKernelSize25ArmU8S8( IndirectBuf += 25; int32_t const* bias = PostProcessParams->Bias; float const* scale = PostProcessParams->Scale; + int32_t const* preshift = PostProcessParams->PreShift; + int32_t const* multiplier = PostProcessParams->Multiplier; + int32_t const* postshift = PostProcessParams->PostShift; for (size_t c = 0; c < Channels; c += 16) { + int8_t const* w = Filter + c; int32x4_t vacc_0123 = vld1q_s32(bias); bias += 4; int32x4_t vacc_4567 = vld1q_s32(bias); bias += 4; @@ -309,18 +346,50 @@ MlasConvSymDepthwiseKernelSize25ArmU8S8( vacc_89AB = vaddw_s16(vacc_89AB, vget_low_s16(vprod_89ABCDEF)); vacc_CDEF = vaddw_s16(vacc_CDEF, vget_high_s16(vprod_89ABCDEF)); - if (is_per_channel) { - vscale_0123 = vld1q_f32(scale); scale += 4; - vscale_4567 = vld1q_f32(scale); scale += 4; - vscale_89AB = vld1q_f32(scale); scale += 4; - vscale_CDEF = vld1q_f32(scale); scale += 4; - } + if constexpr (IsFixedPoint) { - // requantize - vacc_0123 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_0123), vscale_0123)); - vacc_4567 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_4567), vscale_4567)); - vacc_89AB = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_89AB), vscale_89AB)); - vacc_CDEF = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_CDEF), vscale_CDEF)); + if (is_per_channel) { + + vpreshift_0123 = vld1q_s32(preshift); preshift += 4; + vpreshift_4567 = vld1q_s32(preshift); preshift += 4; + vpreshift_89AB = vld1q_s32(preshift); preshift += 4; + vpreshift_CDEF = vld1q_s32(preshift); preshift += 4; + + vmultiplier_0123 = vld1q_s32(multiplier); multiplier += 4; + vmultiplier_4567 = vld1q_s32(multiplier); multiplier += 4; + vmultiplier_89AB = vld1q_s32(multiplier); multiplier += 4; + vmultiplier_CDEF = vld1q_s32(multiplier); multiplier += 4; + + vpostshift_0123 = vld1q_s32(postshift); postshift += 4; + vpostshift_4567 = vld1q_s32(postshift); postshift += 4; + vpostshift_89AB = vld1q_s32(postshift); postshift += 4; + vpostshift_CDEF = vld1q_s32(postshift); postshift += 4; + + } + + vacc_0123 = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_0123, vpreshift_0123), vmultiplier_0123), vpostshift_0123); + vacc_4567 = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_4567, vpreshift_4567), vmultiplier_4567), vpostshift_4567); + vacc_89AB = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_89AB, vpreshift_89AB), vmultiplier_89AB), vpostshift_89AB); + vacc_CDEF = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_CDEF, vpreshift_CDEF), vmultiplier_CDEF), vpostshift_CDEF); + + } else { + + if (is_per_channel) { + + vscale_0123 = vld1q_f32(scale); scale += 4; + vscale_4567 = vld1q_f32(scale); scale += 4; + vscale_89AB = vld1q_f32(scale); scale += 4; + vscale_CDEF = vld1q_f32(scale); scale += 4; + + } + + // requantize + vacc_0123 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_0123), vscale_0123)); + vacc_4567 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_4567), vscale_4567)); + vacc_89AB = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_89AB), vscale_89AB)); + vacc_CDEF = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_CDEF), vscale_CDEF)); + + } const int16x8_t vacc_01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc_0123), vacc_4567), voutput_zero_point); const int16x8_t vacc_89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc_89AB), vacc_CDEF), voutput_zero_point); @@ -332,9 +401,11 @@ MlasConvSymDepthwiseKernelSize25ArmU8S8( } } + +template void MLASCALL -MlasConvSymDepthwiseKernelSize25ArmS8S8( +MlasConvSymDepthwiseKernelSize25ArmS8S8Impl( void const* const* InputIndirection, int8_t const* Filter, size_t Channels, @@ -349,10 +420,43 @@ MlasConvSymDepthwiseKernelSize25ArmS8S8( const int16x8_t voutput_zero_point = vld1q_dup_s16((int16_t const*)&PostProcessParams->OutputZeroPoint); float32x4_t vscale_0123, vscale_4567, vscale_89AB, vscale_CDEF; + int32x4_t vpreshift_0123, vpreshift_4567, vpreshift_89AB, vpreshift_CDEF; + int32x4_t vmultiplier_0123, vmultiplier_4567, vmultiplier_89AB, vmultiplier_CDEF; + int32x4_t vpostshift_0123, vpostshift_4567, vpostshift_89AB, vpostshift_CDEF; const bool is_per_channel = ((KernelFlags & MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE) != 0); // Init them anyway due to some compiler will generate uninitialized warnings. - vscale_0123 = vscale_4567 = vscale_89AB = vscale_CDEF = vld1q_dup_f32(PostProcessParams->Scale); + if constexpr (IsFixedPoint) { + + vpreshift_0123 = vpreshift_4567 = vpreshift_89AB = vpreshift_CDEF = vld1q_dup_s32(PostProcessParams->PreShift); + vmultiplier_0123 = vmultiplier_4567 = vmultiplier_89AB = vmultiplier_CDEF = vld1q_dup_s32(PostProcessParams->Multiplier); + vpostshift_0123 = vpostshift_4567 = vpostshift_89AB = vpostshift_CDEF = vld1q_dup_s32(PostProcessParams->PostShift); + + MLAS_UNREFERENCED_PARAMETER(vscale_0123); + MLAS_UNREFERENCED_PARAMETER(vscale_4567); + MLAS_UNREFERENCED_PARAMETER(vscale_89AB); + MLAS_UNREFERENCED_PARAMETER(vscale_CDEF); + + } else { + + vscale_0123 = vscale_4567 = vscale_89AB = vscale_CDEF = vld1q_dup_f32(PostProcessParams->Scale); + + MLAS_UNREFERENCED_PARAMETER(vpreshift_0123); + MLAS_UNREFERENCED_PARAMETER(vpreshift_4567); + MLAS_UNREFERENCED_PARAMETER(vpreshift_89AB); + MLAS_UNREFERENCED_PARAMETER(vpreshift_CDEF); + MLAS_UNREFERENCED_PARAMETER(vmultiplier_0123); + MLAS_UNREFERENCED_PARAMETER(vmultiplier_4567); + MLAS_UNREFERENCED_PARAMETER(vmultiplier_89AB); + MLAS_UNREFERENCED_PARAMETER(vmultiplier_CDEF); + MLAS_UNREFERENCED_PARAMETER(vpostshift_0123); + MLAS_UNREFERENCED_PARAMETER(vpostshift_4567); + MLAS_UNREFERENCED_PARAMETER(vpostshift_89AB); + MLAS_UNREFERENCED_PARAMETER(vpostshift_CDEF); + + } + while (OutputCount-- > 0) { + const int8_t* i00 = IndirectBuf[0]; const int8_t* i01 = IndirectBuf[1]; const int8_t* i02 = IndirectBuf[2]; @@ -383,8 +487,12 @@ MlasConvSymDepthwiseKernelSize25ArmS8S8( IndirectBuf += 25; int32_t const* bias = PostProcessParams->Bias; - float const* scale = PostProcessParams->Scale; + const float* scale = PostProcessParams->Scale; + const int32_t* preshift = PostProcessParams->PreShift; + const int32_t* multiplier = PostProcessParams->Multiplier; + const int32_t* postshift = PostProcessParams->PostShift; for (size_t c = 0; c < Channels; c += 16) { + int8_t const* w = Filter + c; int32x4_t vacc_0123 = vld1q_s32(bias); bias += 4; int32x4_t vacc_4567 = vld1q_s32(bias); bias += 4; @@ -594,18 +702,50 @@ MlasConvSymDepthwiseKernelSize25ArmS8S8( vacc_89AB = vaddw_s16(vacc_89AB, vget_low_s16(vprod_89ABCDEF)); vacc_CDEF = vaddw_s16(vacc_CDEF, vget_high_s16(vprod_89ABCDEF)); - if (is_per_channel) { - vscale_0123 = vld1q_f32(scale); scale += 4; - vscale_4567 = vld1q_f32(scale); scale += 4; - vscale_89AB = vld1q_f32(scale); scale += 4; - vscale_CDEF = vld1q_f32(scale); scale += 4; - } + if constexpr (IsFixedPoint) { - // requantize - vacc_0123 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_0123), vscale_0123)); - vacc_4567 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_4567), vscale_4567)); - vacc_89AB = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_89AB), vscale_89AB)); - vacc_CDEF = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_CDEF), vscale_CDEF)); + if (is_per_channel) { + + vpreshift_0123 = vld1q_s32(preshift); preshift += 4; + vpreshift_4567 = vld1q_s32(preshift); preshift += 4; + vpreshift_89AB = vld1q_s32(preshift); preshift += 4; + vpreshift_CDEF = vld1q_s32(preshift); preshift += 4; + + vmultiplier_0123 = vld1q_s32(multiplier); multiplier += 4; + vmultiplier_4567 = vld1q_s32(multiplier); multiplier += 4; + vmultiplier_89AB = vld1q_s32(multiplier); multiplier += 4; + vmultiplier_CDEF = vld1q_s32(multiplier); multiplier += 4; + + vpostshift_0123 = vld1q_s32(postshift); postshift += 4; + vpostshift_4567 = vld1q_s32(postshift); postshift += 4; + vpostshift_89AB = vld1q_s32(postshift); postshift += 4; + vpostshift_CDEF = vld1q_s32(postshift); postshift += 4; + + } + + vacc_0123 = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_0123, vpreshift_0123), vmultiplier_0123), vpostshift_0123); + vacc_4567 = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_4567, vpreshift_4567), vmultiplier_4567), vpostshift_4567); + vacc_89AB = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_89AB, vpreshift_89AB), vmultiplier_89AB), vpostshift_89AB); + vacc_CDEF = vrshlq_s32(vqdmulhq_s32(vqshlq_s32(vacc_CDEF, vpreshift_CDEF), vmultiplier_CDEF), vpostshift_CDEF); + + } else { + + if (is_per_channel) { + + vscale_0123 = vld1q_f32(scale); scale += 4; + vscale_4567 = vld1q_f32(scale); scale += 4; + vscale_89AB = vld1q_f32(scale); scale += 4; + vscale_CDEF = vld1q_f32(scale); scale += 4; + + } + + // requantize + vacc_0123 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_0123), vscale_0123)); + vacc_4567 = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_4567), vscale_4567)); + vacc_89AB = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_89AB), vscale_89AB)); + vacc_CDEF = vcvtnq_s32_f32(vmulq_f32(vcvtq_f32_s32(vacc_CDEF), vscale_CDEF)); + + } const int16x8_t vacc_01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc_0123), vacc_4567), voutput_zero_point); const int16x8_t vacc_89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc_89AB), vacc_CDEF), voutput_zero_point); @@ -617,5 +757,72 @@ MlasConvSymDepthwiseKernelSize25ArmS8S8( } } +extern "C" { + +void +MLASCALL +MlasConvSymDepthwiseKernelSize25ArmU8S8( + void const* const* InputIndirection, + int8_t const* Filter, + size_t Channels, + void* Output, + size_t OutputCount, + MLAS_CONV_SYM_POST_PROCESS_PARAMS const* PostProcessParams, + unsigned KernelFlags + ) +{ + if((KernelFlags & MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE) != 0) { + MlasConvSymDepthwiseKernelSize25ArmU8S8Impl( + InputIndirection, + Filter, + Channels, + Output, + OutputCount, + PostProcessParams, + KernelFlags); + } else { + MlasConvSymDepthwiseKernelSize25ArmU8S8Impl( + InputIndirection, + Filter, + Channels, + Output, + OutputCount, + PostProcessParams, + KernelFlags); + } +} + +void +MLASCALL +MlasConvSymDepthwiseKernelSize25ArmS8S8( + void const* const* InputIndirection, + int8_t const* Filter, + size_t Channels, + void* Output, + size_t OutputCount, + MLAS_CONV_SYM_POST_PROCESS_PARAMS const* PostProcessParams, + unsigned KernelFlags + ) { + if((KernelFlags & MLAS_CONV_SYM_FLAG_FIXED_POINT_SCALE) != 0) { + MlasConvSymDepthwiseKernelSize25ArmS8S8Impl( + InputIndirection, + Filter, + Channels, + Output, + OutputCount, + PostProcessParams, + KernelFlags); + } else { + MlasConvSymDepthwiseKernelSize25ArmS8S8Impl( + InputIndirection, + Filter, + Channels, + Output, + OutputCount, + PostProcessParams, + KernelFlags); + } +} + +} #endif -} \ No newline at end of file diff --git a/onnxruntime/core/mlas/lib/qlgavgpool.cpp b/onnxruntime/core/mlas/lib/qlgavgpool.cpp index d1c15d315a..a36e53b014 100644 --- a/onnxruntime/core/mlas/lib/qlgavgpool.cpp +++ b/onnxruntime/core/mlas/lib/qlgavgpool.cpp @@ -153,8 +153,8 @@ MlasQLinearGlobalAveragePoolNchw( *sum_buffer++ = vget_lane_s32(vpadd_s32(vacc, vacc), 0); } - MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false, - static_cast(ZeroPointOutput), 0, 0, 1, Channels); + MLAS_REQUANT_PARAM RequantParam(&scale, 1, ZeroPointOutput); + MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &RequantParam, 0, 0, 1, Channels); } template @@ -321,8 +321,9 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( vst1q_s32(acc + 4, vacc_hi); } } - MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &Scale, false, - Output_zero_point, 0, 0, 1, Channels); + + MLAS_REQUANT_PARAM RequantParam(&Scale, 1, Output_zero_point); + MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &RequantParam, 0, 0, 1, Channels); } #elif defined(MLAS_SSE2_INTRINSICS) @@ -427,8 +428,8 @@ MlasQLinearGlobalAveragePoolNchw( *sum_buffer++ = _mm_cvtsi128_si32(vsums); } - MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false, - static_cast(ZeroPointOutput), 0, 0, 1, Channels); + MLAS_REQUANT_PARAM RequantParam(&scale, 1, ZeroPointOutput); + MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &RequantParam, 0, 0, 1, Channels); } template @@ -697,8 +698,9 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( _mm_storeu_si128(((__m128i*)acc) + 1, vacc_hi); } } - MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &Scale, false, - Output_zero_point, 0, 0, 1, Channels); + + MLAS_REQUANT_PARAM RequantParam(&Scale, 1, Output_zero_point); + MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &RequantParam, 0, 0, 1, Channels); } #else diff --git a/onnxruntime/core/mlas/lib/quantize.cpp b/onnxruntime/core/mlas/lib/quantize.cpp index e7db3a2672..794e6e01cf 100644 --- a/onnxruntime/core/mlas/lib/quantize.cpp +++ b/onnxruntime/core/mlas/lib/quantize.cpp @@ -396,15 +396,27 @@ MlasRequantizeOutput( OutputType* Output, size_t OutputLeadingDimension, const int32_t* Bias, - const float* Scale, - bool PerColumnScale, - OutputType ZeroPoint, + const MLAS_REQUANT_PARAM* RequantParam, size_t StartM, size_t StartN, size_t CountM, size_t CountN ) { + +MLAS_ROUND_KIND RequantRoundKind = RequantParam->RequantRoundKind; + if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfUp) { +#ifdef MLAS_NO_EXCEPTION + abort(); +#else + throw std::invalid_argument("Requantization rounding to nearest and tie to up is only supported on ARM64."); +#endif + } + + const float* Scale = RequantParam->Scale; + bool PerColumnScale = RequantParam->Size > 1; + OutputType ZeroPoint = static_cast(RequantParam->ZeroPoint); + const __m128 PerMatrixScaleVector = PerColumnScale ? _mm_setzero_ps() : _mm_load1_ps(Scale); const __m128 MinimumValueVector = _mm_set1_ps(float(std::numeric_limits::lowest() - ZeroPoint)); const __m128 MaximumValueVector = _mm_set1_ps(float(std::numeric_limits::max() - ZeroPoint)); @@ -631,21 +643,301 @@ MlasRequantizeOutput( template void MLASCALL -MlasRequantizeOutput( +MlasRequantizeOutputRoundNearestUp( const int32_t* Input, size_t InputLeadingDimension, OutputType* Output, size_t OutputLeadingDimension, const int32_t* Bias, - const float* Scale, - bool PerColumnScale, - OutputType ZeroPoint, + const MLAS_REQUANT_PARAM* RequantParam, size_t StartM, size_t StartN, size_t CountM, size_t CountN ) { + bool PerColumnScale = RequantParam->Size > 1; + const int32_t* Multiplier = RequantParam->Multiplier; + const int32_t* PreShift = RequantParam->PreShift; + const int32_t* PostShift = RequantParam->PostShift; + OutputType ZeroPoint = static_cast(RequantParam->ZeroPoint); + + const int32x4_t PerTensorMultiplierVector = PerColumnScale ? vdupq_n_s32(0) : vld1q_dup_s32(Multiplier); + const int32x4_t PerTensorPreShiftVector = PerColumnScale ? vdupq_n_s32(0) : vld1q_dup_s32(PreShift); + const int32x4_t PerTensorPostShiftVector = PerColumnScale ? vdupq_n_s32(0) : vld1q_dup_s32(PostShift); + + const int16x8_t ZeroPointVector = vdupq_n_s16(ZeroPoint); + + if (nullptr != Bias) { + Bias += StartN; + } + if (PerColumnScale) { + + Multiplier += StartN; + PreShift += StartN; + PostShift += StartN; + + } + + Input += StartM * InputLeadingDimension + StartN; + Output += StartM * OutputLeadingDimension + StartN; + + // + // Step through each row of the output matrix. + // + + while (CountM-- > 0) { + + const int32_t* bias = Bias; + const int32_t* multiplier = PerColumnScale ? Multiplier : nullptr; + const int32_t* preShift = PerColumnScale ? PreShift : nullptr; + const int32_t* postShift = PerColumnScale ? PostShift : nullptr; + size_t n = CountN; + + auto* RowInput = Input; + auto* RowOutput = Output; + + // + // Process 16 columns of the matrices at a time. + // + + while (n >= 16) { + + // + // Load the input data and optionally add the per-column bias. + // + + int32x4x4_t IntegerVector; + + IntegerVector.val[0] = vld1q_s32(&RowInput[0]); + IntegerVector.val[1] = vld1q_s32(&RowInput[4]); + IntegerVector.val[2] = vld1q_s32(&RowInput[8]); + IntegerVector.val[3] = vld1q_s32(&RowInput[12]); + RowInput += 16; + + if (bias != nullptr) { + + IntegerVector.val[0] = vaddq_s32(IntegerVector.val[0], vld1q_s32(&bias[0])); + IntegerVector.val[1] = vaddq_s32(IntegerVector.val[1], vld1q_s32(&bias[4])); + IntegerVector.val[2] = vaddq_s32(IntegerVector.val[2], vld1q_s32(&bias[8])); + IntegerVector.val[3] = vaddq_s32(IntegerVector.val[3], vld1q_s32(&bias[12])); + bias += 16; + } + + if (PerColumnScale) { + + int32x4x4_t PerColumnPreShiftVector; + PerColumnPreShiftVector.val[0] = vld1q_s32(&preShift[0]); + PerColumnPreShiftVector.val[1] = vld1q_s32(&preShift[4]); + PerColumnPreShiftVector.val[2] = vld1q_s32(&preShift[8]); + PerColumnPreShiftVector.val[3] = vld1q_s32(&preShift[12]); + preShift += 16; + + IntegerVector.val[0] = vqshlq_s32(IntegerVector.val[0], PerColumnPreShiftVector.val[0]); + IntegerVector.val[1] = vqshlq_s32(IntegerVector.val[1], PerColumnPreShiftVector.val[1]); + IntegerVector.val[2] = vqshlq_s32(IntegerVector.val[2], PerColumnPreShiftVector.val[2]); + IntegerVector.val[3] = vqshlq_s32(IntegerVector.val[3], PerColumnPreShiftVector.val[3]); + + int32x4x4_t PerColumnMultiplierVector; + PerColumnMultiplierVector.val[0] = vld1q_s32(&multiplier[0]); + PerColumnMultiplierVector.val[1] = vld1q_s32(&multiplier[4]); + PerColumnMultiplierVector.val[2] = vld1q_s32(&multiplier[8]); + PerColumnMultiplierVector.val[3] = vld1q_s32(&multiplier[12]); + multiplier += 16; + + IntegerVector.val[0] = vqdmulhq_s32(IntegerVector.val[0], PerColumnMultiplierVector.val[0]); + IntegerVector.val[1] = vqdmulhq_s32(IntegerVector.val[1], PerColumnMultiplierVector.val[1]); + IntegerVector.val[2] = vqdmulhq_s32(IntegerVector.val[2], PerColumnMultiplierVector.val[2]); + IntegerVector.val[3] = vqdmulhq_s32(IntegerVector.val[3], PerColumnMultiplierVector.val[3]); + + + int32x4x4_t PerColumnPostShiftVector; + PerColumnPostShiftVector.val[0] = vld1q_s32(&postShift[0]); + PerColumnPostShiftVector.val[1] = vld1q_s32(&postShift[4]); + PerColumnPostShiftVector.val[2] = vld1q_s32(&postShift[8]); + PerColumnPostShiftVector.val[3] = vld1q_s32(&postShift[12]); + postShift += 16; + + IntegerVector.val[0] = vrshlq_s32(IntegerVector.val[0], PerColumnPostShiftVector.val[0]); + IntegerVector.val[1] = vrshlq_s32(IntegerVector.val[1], PerColumnPostShiftVector.val[1]); + IntegerVector.val[2] = vrshlq_s32(IntegerVector.val[2], PerColumnPostShiftVector.val[2]); + IntegerVector.val[3] = vrshlq_s32(IntegerVector.val[3], PerColumnPostShiftVector.val[3]); + } else { + + IntegerVector.val[0] = vqshlq_s32(IntegerVector.val[0], PerTensorPreShiftVector); + IntegerVector.val[1] = vqshlq_s32(IntegerVector.val[1], PerTensorPreShiftVector); + IntegerVector.val[2] = vqshlq_s32(IntegerVector.val[2], PerTensorPreShiftVector); + IntegerVector.val[3] = vqshlq_s32(IntegerVector.val[3], PerTensorPreShiftVector); + + IntegerVector.val[0] = vqdmulhq_s32(IntegerVector.val[0], PerTensorMultiplierVector); + IntegerVector.val[1] = vqdmulhq_s32(IntegerVector.val[1], PerTensorMultiplierVector); + IntegerVector.val[2] = vqdmulhq_s32(IntegerVector.val[2], PerTensorMultiplierVector); + IntegerVector.val[3] = vqdmulhq_s32(IntegerVector.val[3], PerTensorMultiplierVector); + + IntegerVector.val[0] = vrshlq_s32(IntegerVector.val[0], PerTensorPostShiftVector); + IntegerVector.val[1] = vrshlq_s32(IntegerVector.val[1], PerTensorPostShiftVector); + IntegerVector.val[2] = vrshlq_s32(IntegerVector.val[2], PerTensorPostShiftVector); + IntegerVector.val[3] = vrshlq_s32(IntegerVector.val[3], PerTensorPostShiftVector); + } + + // + // Pack the integers with saturation to 16-bit values and shift by + // the zero point, then pack the integers again to bytes. + // + + int16x8x2_t WordVector; + + WordVector.val[0] = vqmovn_high_s32(vqmovn_s32(IntegerVector.val[0]), IntegerVector.val[1]); + WordVector.val[1] = vqmovn_high_s32(vqmovn_s32(IntegerVector.val[2]), IntegerVector.val[3]); + + WordVector.val[0] = vqaddq_s16(WordVector.val[0], ZeroPointVector); + WordVector.val[1] = vqaddq_s16(WordVector.val[1], ZeroPointVector); + + if (std::is_signed::value) { + vst1q_s8(reinterpret_cast(RowOutput), + vqmovn_high_s16(vqmovn_s16(WordVector.val[0]), WordVector.val[1])); + } else { + vst1q_u8(reinterpret_cast(RowOutput), + vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1])); + } + RowOutput += 16; + + n -= 16; + } + + // + // Process the remaining columns of the matrices. + // + + while (n > 0) { + + // + // Load the input data and optionally add the per-column bias. + // + + int32x4_t IntegerVector; + + if (n >= 4) { + + IntegerVector = vld1q_s32(&RowInput[0]); + RowInput += 4; + + if (bias != nullptr) { + + IntegerVector = vaddq_s32(IntegerVector, vld1q_s32(&bias[0])); + bias += 4; + } + + } else { + + IntegerVector = vld1q_dup_s32(RowInput); + RowInput += 1; + + if (bias != nullptr) { + + IntegerVector = vaddq_s32(IntegerVector, vld1q_dup_s32(bias)); + bias += 1; + } + } + + // + // Convert to integer values to float and apply the per-tensor or + // per-column scaling. + // + + int32x4_t MultiplierVector; + int32x4_t PreShiftVector; + int32x4_t PostShiftVector; + + if (PerColumnScale) { + + if (n >= 4) { + + MultiplierVector = vld1q_s32(multiplier); + PreShiftVector = vld1q_s32(preShift); + PostShiftVector = vld1q_s32(postShift); + multiplier += 4; + preShift += 4; + postShift += 4; + } else { + + MultiplierVector = vld1q_dup_s32(multiplier); + PreShiftVector = vld1q_dup_s32(preShift); + PostShiftVector = vld1q_dup_s32(postShift); + multiplier += 1; + preShift += 1; + postShift += 1; + } + + IntegerVector = vqshlq_s32(IntegerVector, PreShiftVector); + IntegerVector = vqdmulhq_s32(IntegerVector, MultiplierVector); + IntegerVector = vrshlq_s32(IntegerVector, PostShiftVector); + + } else { + + IntegerVector = vqshlq_s32(IntegerVector, PerTensorPreShiftVector); + IntegerVector = vqdmulhq_s32(IntegerVector, PerTensorMultiplierVector); + IntegerVector = vrshlq_s32(IntegerVector, PerTensorPostShiftVector); + } + + // + // Pack the integers with saturation to 16-bit values and shift by + // the zero point, then pack the integers again to unsigned bytes. + // + + int16x8_t WordVector = vcombine_s16(vqmovn_s32(IntegerVector), vdup_n_s16(0)); + WordVector = vqaddq_s16(WordVector, ZeroPointVector); + + uint8x16_t ByteVector; + + if (std::is_signed::value) { + ByteVector = vcombine_u8(vreinterpret_u8_s8(vqmovn_s16(WordVector)), vdup_n_u8(0)); + } else { + ByteVector = vcombine_u8(vqmovun_s16(WordVector), vdup_n_u8(0)); + } + + if (n >= 4) { + + vst1q_lane_u32(reinterpret_cast(RowOutput), + vreinterpretq_u32_u8(ByteVector), 0); + RowOutput += 4; + + n -= 4; + + } else { + + vst1q_lane_u8(reinterpret_cast(RowOutput), ByteVector, 0); + RowOutput += 1; + + n -= 1; + } + } + + // Next Row + Input += InputLeadingDimension; + Output += OutputLeadingDimension; + } +} + +template +void +MLASCALL +MlasRequantizeOutputRoundNearestEven( + const int32_t* Input, + size_t InputLeadingDimension, + OutputType* Output, + size_t OutputLeadingDimension, + const int32_t* Bias, + const MLAS_REQUANT_PARAM* RequantParam, + size_t StartM, + size_t StartN, + size_t CountM, + size_t CountN + ) +{ + const float* Scale = RequantParam->Scale; + bool PerColumnScale = RequantParam->Size > 1; + OutputType ZeroPoint = static_cast(RequantParam->ZeroPoint); const float32x4_t PerMatrixScaleVector = PerColumnScale ? vdupq_n_f32(0) : vld1q_dup_f32(Scale); const int16x8_t ZeroPointVector = vdupq_n_s16(ZeroPoint); @@ -871,6 +1163,32 @@ MlasRequantizeOutput( } } +template +void +MLASCALL +MlasRequantizeOutput( + const int32_t* Input, + size_t InputLeadingDimension, + OutputType* Output, + size_t OutputLeadingDimension, + const int32_t* Bias, + const MLAS_REQUANT_PARAM* RequantParam, + size_t StartM, + size_t StartN, + size_t CountM, + size_t CountN + ) +{ + if(RequantParam->RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfEven) + { + return MlasRequantizeOutputRoundNearestEven(Input, InputLeadingDimension, Output, OutputLeadingDimension, Bias, + RequantParam, StartM, StartN, CountM, CountN); + } else { + return MlasRequantizeOutputRoundNearestUp(Input, InputLeadingDimension, Output, OutputLeadingDimension, Bias, + RequantParam, StartM, StartN, CountM, CountN); + } +} + #else template @@ -882,15 +1200,26 @@ MlasRequantizeOutput( OutputType* Output, size_t OutputLeadingDimension, const int32_t* Bias, - const float* Scale, - bool PerColumnScale, - OutputType ZeroPoint, + const MLAS_REQUANT_PARAM* RequantParam, size_t StartM, size_t StartN, size_t CountM, size_t CountN ) { + MLAS_ROUND_KIND RequantRoundKind = RequantParam->RequantRoundKind; + if(RequantRoundKind == MLAS_ROUND_KIND::MlasRoundHalfUp) { +#ifdef MLAS_NO_EXCEPTION + abort(); +#else + throw std::invalid_argument("Requantization rounding to nearest and tie to up is only supported on ARM64."); +#endif + } + + const float* Scale = RequantParam->Scale; + bool PerColumnScale = RequantParam->Size > 1; + OutputType ZeroPoint = static_cast(RequantParam->ZeroPoint); + const float PerMatrixScaleValue = PerColumnScale ? 0.0f : *Scale; const float MinimumValue = float(std::numeric_limits::lowest() - ZeroPoint); const float MaximumValue = float(std::numeric_limits::max() - ZeroPoint); @@ -966,9 +1295,7 @@ MlasRequantizeOutput( int8_t* Output, size_t OutputLeadingDimension, const int32_t* Bias, - const float* Scale, - bool PerColumnScale, - int8_t ZeroPoint, + const MLAS_REQUANT_PARAM* RequantParam, size_t StartM, size_t StartN, size_t CountM, @@ -984,9 +1311,7 @@ MlasRequantizeOutput( uint8_t* Output, size_t OutputLeadingDimension, const int32_t* Bias, - const float* Scale, - bool PerColumnScale, - uint8_t ZeroPoint, + const MLAS_REQUANT_PARAM* RequantParam, size_t StartM, size_t StartN, size_t CountM, diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.h b/onnxruntime/core/providers/cpu/cpu_execution_provider.h index 9eb11cdbf2..e855cd286d 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.h +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.h @@ -12,9 +12,11 @@ namespace onnxruntime { // Information needed to construct CPU execution providers. struct CPUExecutionProviderInfo { bool create_arena{true}; + bool use_fixed_point_requant_on_arm64{false}; - explicit CPUExecutionProviderInfo(bool use_arena) - : create_arena(use_arena) {} + explicit CPUExecutionProviderInfo(bool use_arena, bool use_fixed_point_requant_on_arm64) + : create_arena(use_arena), + use_fixed_point_requant_on_arm64(use_fixed_point_requant_on_arm64) {} CPUExecutionProviderInfo() = default; }; @@ -41,12 +43,23 @@ class CPUExecutionProvider : public IExecutionProvider { 0, create_arena}; InsertAllocator(CreateAllocator(device_info)); + +#if defined(__aarch64__) || defined(_M_ARM64) + use_fixed_point_requant_on_arm64_ = info.use_fixed_point_requant_on_arm64; +#else + use_fixed_point_requant_on_arm64_ = false; +#endif } std::shared_ptr GetKernelRegistry() const override; std::unique_ptr GetDataTransfer() const override; + bool UseFixedPointRequantOnARM64() const { + return use_fixed_point_requant_on_arm64_; + } + private: + bool use_fixed_point_requant_on_arm64_; std::vector fuse_rules_; }; diff --git a/onnxruntime/core/providers/cpu/cpu_provider_factory.cc b/onnxruntime/core/providers/cpu/cpu_provider_factory.cc index 00222f5642..cf977a962c 100644 --- a/onnxruntime/core/providers/cpu/cpu_provider_factory.cc +++ b/onnxruntime/core/providers/cpu/cpu_provider_factory.cc @@ -8,33 +8,36 @@ #include "core/providers/cpu/cpu_execution_provider.h" #include "core/session/abi_session_options_impl.h" +#include "core/session/onnxruntime_session_options_config_keys.h" #include "core/session/ort_apis.h" namespace onnxruntime { struct CpuProviderFactory : IExecutionProviderFactory { - CpuProviderFactory(bool create_arena) : create_arena_(create_arena) {} + CpuProviderFactory(const CPUExecutionProviderInfo& info) : info(info) {} ~CpuProviderFactory() override = default; std::unique_ptr CreateProvider() override; private: - bool create_arena_; + CPUExecutionProviderInfo info; }; std::unique_ptr CpuProviderFactory::CreateProvider() { - CPUExecutionProviderInfo info; - info.create_arena = create_arena_; return std::make_unique(info); } -std::shared_ptr CreateExecutionProviderFactory_CPU(int use_arena) { - return std::make_shared(use_arena != 0); +std::shared_ptr CreateExecutionProviderFactory_CPU(const CPUExecutionProviderInfo& info) { + return std::make_shared(info); } } // namespace onnxruntime ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CPU, _In_ OrtSessionOptions* options, int use_arena) { - options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CPU(use_arena)); + const bool use_fixed_point_requant_on_arm64 = + options->value.config_options.GetConfigOrDefault(kOrtSessionOptionsConfigFixedPointRequantOnARM64, "0") == "1"; + options->provider_factories.push_back( + onnxruntime::CreateExecutionProviderFactory_CPU( + onnxruntime::CPUExecutionProviderInfo(use_arena, use_fixed_point_requant_on_arm64))); return nullptr; } #if defined(_MSC_VER) && !defined(__clang__) diff --git a/onnxruntime/core/providers/cpu/cpu_provider_factory_creator.h b/onnxruntime/core/providers/cpu/cpu_provider_factory_creator.h index 637cecbf8e..55d88d8640 100644 --- a/onnxruntime/core/providers/cpu/cpu_provider_factory_creator.h +++ b/onnxruntime/core/providers/cpu/cpu_provider_factory_creator.h @@ -5,10 +5,11 @@ #include +#include "core/providers/cpu/cpu_execution_provider.h" #include "core/providers/providers.h" namespace onnxruntime { -std::shared_ptr CreateExecutionProviderFactory_CPU(int use_arena); +std::shared_ptr CreateExecutionProviderFactory_CPU(const CPUExecutionProviderInfo& info); } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc b/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc index 5fa044402d..c33253afde 100644 --- a/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc +++ b/onnxruntime/core/providers/cpu/quantization/qlinearconv.cc @@ -3,6 +3,7 @@ #include "core/framework/op_kernel.h" #include "core/providers/cpu/nn/conv_attributes.h" +#include "core/providers/cpu/cpu_execution_provider.h" #include "core/common/cpuid_info.h" #include "core/common/safeint.h" #include "core/providers/common.h" @@ -10,6 +11,7 @@ #include "core/util/math_cpuonly.h" #include "core/util/qmath.h" #include "core/mlas/inc/mlas.h" +#include "core/quantization/quantization.h" #if defined(_M_ARM64) || defined(__aarch64__) @@ -36,6 +38,8 @@ class QLinearConv : public OpKernel { public: explicit QLinearConv(const OpKernelInfo& info) : OpKernel(info), conv_attrs_(info) { channels_last_ = (info.GetAttrOrDefault("channels_last", static_cast(0)) != 0); + const CPUExecutionProvider* ep = static_cast(info.GetExecutionProvider()); + use_fixed_point_requant_ = ep->UseFixedPointRequantOnARM64(); } Status Compute(OpKernelContext* context) const override; @@ -283,6 +287,7 @@ class QLinearConv : public OpKernel { bool is_symmetric_gemm_{false}; bool channels_last_{false}; std::vector column_sums_; + bool use_fixed_point_requant_{false}; }; // uint8_t kernel supports weight being either uint8_t or int8_t @@ -511,6 +516,26 @@ Status QLinearConv::Compute(OpKernelContext* context) const { uint8_t W_zero_point_value; ComputeOffset(context, M, X_zero_point_value, Y_zero_point_value, W_zero_point_value); std::vector output_scales = ComputeOutputScale(context, M); + MLAS_REQUANT_PARAM RequantParam(output_scales.data(), output_scales.size(), Y_zero_point_value); +#if defined(_M_ARM64) || defined(__aarch64__) + std::vector pre_shifts; + std::vector multipliers; + std::vector post_shifts; + if (use_fixed_point_requant_) { + pre_shifts.resize(output_scales.size()); + multipliers.resize(output_scales.size()); + post_shifts.resize(output_scales.size()); + MlasFloatToFixedPoint(output_scales.data(), + multipliers.data(), + pre_shifts.data(), + post_shifts.data(), + output_scales.size()); + RequantParam.RequantRoundKind = MLAS_ROUND_KIND::MlasRoundHalfUp; + RequantParam.PreShift = pre_shifts.data(); + RequantParam.Multiplier = multipliers.data(); + RequantParam.PostShift = post_shifts.data(); + } +#endif const Tensor* B = context->Input(InputTensors::IN_BIAS); @@ -656,7 +681,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { concurrency::ThreadPool* thread_pool = context->GetOperatorThreadPool(); #if defined(_M_ARM64) || defined(__aarch64__) - int32_t task_count = (output_image_size + (GEMM_KERNEL_STRIDE_M - 1)) / GEMM_KERNEL_STRIDE_M; + int32_t task_count = static_cast((output_image_size + (GEMM_KERNEL_STRIDE_M - 1)) / GEMM_KERNEL_STRIDE_M); #else int32_t task_count = ComputeTaskCount(output_image_size, group_output_channels, kernel_dim); task_count = std::min(task_count, concurrency::ThreadPool::DegreeOfParallelism(thread_pool)); @@ -735,6 +760,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { } else { conv_params.InputDirect = input_data + output_start * C; } + conv_params.Filter = packed_W_buffer_.get(); conv_params.Output = worker_output; conv_params.InputChannels = static_cast(C); @@ -742,9 +768,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { conv_params.OutputCount = static_cast(output_count); conv_params.KernelSize = static_cast(kernel_size); conv_params.Bias = column_sums_.data(); - conv_params.Scale = output_scales.data(); - conv_params.PerChannelScale = output_scales.size() > 1; - conv_params.OutputZeroPoint = Y_zero_point_value; + conv_params.RequantParam = &RequantParam; conv_params.InputIsSigned = std::is_signed::value; if (is_depthwise_conv) { @@ -871,9 +895,7 @@ Status QLinearConv::Compute(OpKernelContext* context) const { worker_output, static_cast(M), Bdata, - output_scales.data(), - output_scales.size() > 1, - Y_zero_point_value, + &RequantParam, 0, 0, static_cast(output_count), diff --git a/onnxruntime/core/providers/cpu/quantization/quantize_linear_matmul.cc b/onnxruntime/core/providers/cpu/quantization/quantize_linear_matmul.cc index f08bade90a..fc559a11ed 100644 --- a/onnxruntime/core/providers/cpu/quantization/quantize_linear_matmul.cc +++ b/onnxruntime/core/providers/cpu/quantization/quantize_linear_matmul.cc @@ -95,6 +95,22 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const { output_scales[i] = (a_scale_data * b_scale_data[i] / y_scale_data); } +#if defined(_M_ARM64) || defined(__aarch64__) + std::vector pre_shifts; + std::vector multipliers; + std::vector post_shifts; + if (use_fixed_point_requant_) { + pre_shifts.resize(output_scales.size()); + multipliers.resize(output_scales.size()); + post_shifts.resize(output_scales.size()); + MlasFloatToFixedPoint(output_scales.data(), + multipliers.data(), + pre_shifts.data(), + post_shifts.data(), + output_scales.size()); + } +#endif + const size_t num_gemms = helper.OutputOffsets().size(); MLAS_GEMM_QUANT_SHAPE_PARAMS gemm_shape; gemm_shape.M = static_cast(helper.M()); @@ -111,6 +127,7 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const { auto* gemm_output = static_cast(gemm_output_buffer.get()); std::vector gemm_params(num_gemms); + std::vector requant_params(num_gemms); std::vector requant_procs; requant_procs.reserve(num_gemms); @@ -133,12 +150,26 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const { gemm_params[i].PerColumnZeroPoints = !IsScalarOr1ElementVector(b_offset); + requant_params[i].Size = output_scales.size(); + requant_params[i].ZeroPoint = output_offset; +#if defined(_M_ARM64) || defined(__aarch64__) + if (use_fixed_point_requant_) { + requant_params[i].RequantRoundKind = MLAS_ROUND_KIND::MlasRoundHalfUp; + requant_params[i].PreShift = pre_shifts.data() + helper.RightScaleOffsets()[i]; + requant_params[i].Multiplier = multipliers.data() + helper.RightScaleOffsets()[i]; + requant_params[i].PostShift = post_shifts.data() + helper.RightScaleOffsets()[i]; + } else { + requant_params[i].RequantRoundKind = MLAS_ROUND_KIND::MlasRoundHalfEven; + requant_params[i].Scale = output_scales.data() + helper.RightScaleOffsets()[i]; + } +#else + requant_params[i].RequantRoundKind = MLAS_ROUND_KIND::MlasRoundHalfEven; + requant_params[i].Scale = output_scales.data() + helper.RightScaleOffsets()[i]; +#endif requant_procs.emplace_back(static_cast(y->MutableDataRaw()) + helper.OutputOffsets()[i], static_cast(helper.N()), nullptr, - output_scales.data() + helper.RightScaleOffsets()[i], - output_scales.size() > 1, - output_offset, + &requant_params[i], is_output_signed); gemm_params[i].OutputProcessor = &(requant_procs[i]); } diff --git a/onnxruntime/core/providers/cpu/quantization/quantize_linear_matmul.h b/onnxruntime/core/providers/cpu/quantization/quantize_linear_matmul.h index 569b334c09..fa127a3a67 100644 --- a/onnxruntime/core/providers/cpu/quantization/quantize_linear_matmul.h +++ b/onnxruntime/core/providers/cpu/quantization/quantize_linear_matmul.h @@ -3,13 +3,17 @@ #include "matmul_integer_base.h" +#include "core/providers/cpu/cpu_execution_provider.h" namespace onnxruntime { // Allow subclassing for test only class QLinearMatMul : public MatMulIntegerBase { public: - QLinearMatMul(const OpKernelInfo& info) : MatMulIntegerBase(info) {} + QLinearMatMul(const OpKernelInfo& info) : MatMulIntegerBase(info) { + const CPUExecutionProvider* ep = static_cast(info.GetExecutionProvider()); + use_fixed_point_requant_ = ep->UseFixedPointRequantOnARM64(); + } Status Compute(OpKernelContext* context) const override; @@ -36,6 +40,8 @@ class QLinearMatMul : public MatMulIntegerBase { int GetBIdx() const override { return IN_B; } + + bool use_fixed_point_requant_{false}; }; } // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index f1ca906a9a..7bb87cb8d5 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -1272,7 +1272,9 @@ common::Status InferenceSession::Initialize() { // RegisterExecutionProvider locks the session_mutex_ so we can't be holding it when we call that if (!have_cpu_ep) { LOGS(*session_logger_, INFO) << "Adding default CPU execution provider."; - CPUExecutionProviderInfo epi{session_options_.enable_cpu_mem_arena}; + const bool use_fixed_point_requant_on_arm64 = + session_options_.config_options.GetConfigOrDefault(kOrtSessionOptionsConfigFixedPointRequantOnARM64, "0") == "1"; + CPUExecutionProviderInfo epi(session_options_.enable_cpu_mem_arena, use_fixed_point_requant_on_arm64); auto p_cpu_exec_provider = std::make_unique(epi); ORT_RETURN_IF_ERROR_SESSIONID_(RegisterExecutionProvider(std::move(p_cpu_exec_provider))); } diff --git a/onnxruntime/python/onnxruntime_pybind_schema.cc b/onnxruntime/python/onnxruntime_pybind_schema.cc index b5b8ef2a29..dc6ed72d5d 100644 --- a/onnxruntime/python/onnxruntime_pybind_schema.cc +++ b/onnxruntime/python/onnxruntime_pybind_schema.cc @@ -22,7 +22,7 @@ void addGlobalSchemaFunctions(pybind11::module& m) { std::vector result; std::vector> factories = { - onnxruntime::CreateExecutionProviderFactory_CPU(0), + onnxruntime::CreateExecutionProviderFactory_CPU(onnxruntime::CPUExecutionProviderInfo(false, false)), #ifdef USE_CUDA []() { OrtCUDAProviderOptions provider_options{}; diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index a93539f2aa..bf2693ca80 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -365,8 +365,10 @@ std::unique_ptr CreateExecutionProviderInstance( const std::string& type, const ProviderOptionsMap& provider_options_map) { if (type == kCpuExecutionProvider) { + const bool use_fixed_point_requant_on_arm64 = + session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsConfigFixedPointRequantOnARM64, "0") == "1"; return onnxruntime::CreateExecutionProviderFactory_CPU( - session_options.enable_cpu_mem_arena) + onnxruntime::CPUExecutionProviderInfo(session_options.enable_cpu_mem_arena, use_fixed_point_requant_on_arm64)) ->CreateProvider(); } else if (type == kTensorrtExecutionProvider) { #ifdef USE_TENSORRT @@ -1236,7 +1238,7 @@ Applies to session load, initialization, etc. Default is 0.)pbdoc") const OrtValue* ml_value = ml_value_pyobject.attr(PYTHON_ORTVALUE_NATIVE_OBJECT_ATTR).cast(); ORT_THROW_IF_ERROR(options->AddInitializer(name, ml_value)); }) - .def("add_external_initializers", [](PySessionOptions* options, py::list& names, + .def("add_external_initializers", [](PySessionOptions* options, py::list& names, const py::list& ort_values) -> void { #if !defined(ORT_MINIMAL_BUILD) && !defined(DISABLE_EXTERNAL_INITIALIZERS) const auto init_num = ort_values.size(); diff --git a/onnxruntime/test/contrib_ops/fastgelu_op_test.cc b/onnxruntime/test/contrib_ops/fastgelu_op_test.cc index 302226e107..a28bf007b5 100644 --- a/onnxruntime/test/contrib_ops/fastgelu_op_test.cc +++ b/onnxruntime/test/contrib_ops/fastgelu_op_test.cc @@ -254,7 +254,7 @@ TEST(FastGeluTest, FastGeluWithBias_BFloat16) { execution_providers.push_back(DefaultCudaExecutionProvider()); #elif USE_ROCM execution_providers.push_back(DefaultRocmExecutionProvider()); -#endif +#endif tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } #endif diff --git a/onnxruntime/test/eager/ort_invoker_test.cc b/onnxruntime/test/eager/ort_invoker_test.cc index ce437acc8a..ae53bc9918 100644 --- a/onnxruntime/test/eager/ort_invoker_test.cc +++ b/onnxruntime/test/eager/ort_invoker_test.cc @@ -16,13 +16,14 @@ namespace onnxruntime { namespace test { TEST(InvokerTest, Basic) { - std::unique_ptr cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false)); + std::unique_ptr cpu_execution_provider = + std::make_unique(CPUExecutionProviderInfo(false, false)); const std::string logger_id{"InvokerTest"}; auto logging_manager = std::make_unique( std::unique_ptr{new logging::CLogSink{}}, logging::Severity::kVERBOSE, false, logging::LoggingManager::InstanceType::Default, - &logger_id); + &logger_id); std::unique_ptr env; ASSERT_STATUS_OK(Environment::Create(std::move(logging_manager), env)); IOnnxRuntimeOpSchemaRegistryList tmp_op_registry = {}; @@ -49,13 +50,13 @@ TEST(InvokerTest, Basic) { } TEST(InvokerTest, Inplace) { - std::unique_ptr cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false)); + std::unique_ptr cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false, false)); const std::string logger_id{"InvokerTest"}; auto logging_manager = std::make_unique( std::unique_ptr{new logging::CLogSink{}}, logging::Severity::kVERBOSE, false, logging::LoggingManager::InstanceType::Default, - &logger_id); + &logger_id); std::unique_ptr env; ASSERT_STATUS_OK(Environment::Create(std::move(logging_manager), env)); IOnnxRuntimeOpSchemaRegistryList tmp_op_registry = {}; @@ -93,26 +94,22 @@ Status CreateTestKernel(FuncManager&, const OpKernelInfo& info, std::unique_ptr< } TEST(InvokerTest, CustomOp) { - std::unique_ptr cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false)); + std::unique_ptr cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false, false)); // register "Test" kernel to "FakeDomain" auto kernel_registry = cpu_execution_provider->GetKernelRegistry(); auto kernel_def = KernelDefBuilder() - .MayInplace(0, 0) - .TypeConstraint("T", DataTypeImpl::GetTensorType()) - .SetName("Test") - .SetDomain("FakeDomain") - .SinceVersion(1) - .Provider(kCpuExecutionProvider) - .Build(); + .MayInplace(0, 0) + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .SetName("Test") + .SetDomain("FakeDomain") + .SinceVersion(1) + .Provider(kCpuExecutionProvider) + .Build(); ASSERT_STATUS_OK(kernel_registry->Register(KernelCreateInfo(std::move(kernel_def), CreateTestKernel))); // create custom op schema for "Test" op std::shared_ptr schema_registry = std::make_shared(); std::vector schema = { - ONNX_NAMESPACE::OpSchema().SetName("Test") - .Input(0, "X", "A N-D input tensor that is to be processed.", "T") - .Output(0, "Y", "desc", "T") - .TypeConstraint("T", ONNX_NAMESPACE::OpSchema::all_tensor_types(), "Constrain input and output types to any tensor type.") - .SetDomain("FakeDomain")}; + ONNX_NAMESPACE::OpSchema().SetName("Test").Input(0, "X", "A N-D input tensor that is to be processed.", "T").Output(0, "Y", "desc", "T").TypeConstraint("T", ONNX_NAMESPACE::OpSchema::all_tensor_types(), "Constrain input and output types to any tensor type.").SetDomain("FakeDomain")}; ASSERT_STATUS_OK(schema_registry->RegisterOpSet(schema, "FakeDomain", 0, 1)); std::list> regs = {schema_registry}; @@ -121,7 +118,7 @@ TEST(InvokerTest, CustomOp) { std::unique_ptr{new logging::CLogSink{}}, logging::Severity::kVERBOSE, false, logging::LoggingManager::InstanceType::Default, - &logger_id); + &logger_id); std::unique_ptr env; ASSERT_STATUS_OK(Environment::Create(std::move(logging_manager), env)); ORTInvoker kernel_invoker(std::move(cpu_execution_provider), env->GetLoggingManager()->DefaultLogger(), regs); diff --git a/onnxruntime/test/framework/parallel_executor_test.cc b/onnxruntime/test/framework/parallel_executor_test.cc index 8dcaad10c5..cf0327987a 100644 --- a/onnxruntime/test/framework/parallel_executor_test.cc +++ b/onnxruntime/test/framework/parallel_executor_test.cc @@ -93,8 +93,10 @@ TEST(ParallelExecutor, TestStatusPropagation) { tester.AddInput("action", {1}, {/*success*/ 0}); tester.AddOutput("action_out", {1}, {0}); // TensorRT doesn't handle a custom op. Possibly it should, but that would be a separate PR - tester.Run(OpTester::ExpectResult::kExpectSuccess, {}, {kTensorrtExecutionProvider}, nullptr, nullptr, - ExecutionMode::ORT_PARALLEL); + SessionOptions so; + tester.SetUpDefaultSessionOptions(so); + so.execution_mode = ExecutionMode::ORT_PARALLEL; + tester.Run(so, OpTester::ExpectResult::kExpectSuccess, {}, {kTensorrtExecutionProvider}); } { // test failure @@ -103,8 +105,11 @@ TEST(ParallelExecutor, TestStatusPropagation) { tester.AddInput("action", {1}, {/*failure*/ 1}); tester.AddOutput("action_out", {1}, {0}); - tester.Run(OpTester::ExpectResult::kExpectFailure, "Action was 1", {kTensorrtExecutionProvider}, nullptr, nullptr, - ExecutionMode::ORT_PARALLEL); + + SessionOptions so; + tester.SetUpDefaultSessionOptions(so); + so.execution_mode = ExecutionMode::ORT_PARALLEL; + tester.Run(so, OpTester::ExpectResult::kExpectFailure, "Action was 1", {kTensorrtExecutionProvider}); } { // test exception @@ -113,7 +118,11 @@ TEST(ParallelExecutor, TestStatusPropagation) { tester.AddInput("action", {1}, {/*exception*/ 2}); tester.AddOutput("action_out", {1}, {0}); - tester.Run(OpTester::ExpectResult::kExpectFailure, "Throwing as action was 2", {kTensorrtExecutionProvider}, nullptr, nullptr, ExecutionMode::ORT_PARALLEL); + + SessionOptions so; + tester.SetUpDefaultSessionOptions(so); + so.execution_mode = ExecutionMode::ORT_PARALLEL; + tester.Run(so, OpTester::ExpectResult::kExpectFailure, "Throwing as action was 2", {kTensorrtExecutionProvider}); } } diff --git a/onnxruntime/test/framework/session_state_test.cc b/onnxruntime/test/framework/session_state_test.cc index 95cf065c28..3b3ee05d4f 100644 --- a/onnxruntime/test/framework/session_state_test.cc +++ b/onnxruntime/test/framework/session_state_test.cc @@ -55,7 +55,7 @@ TEST_P(SessionStateAddGetKernelTest, AddGetKernelTest) { auto& graph = model.MainGraph(); ExecutionProviders execution_providers; - auto tmp_cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false)); + auto tmp_cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false, false)); auto* cpu_execution_provider = tmp_cpu_execution_provider.get(); ASSERT_STATUS_OK(execution_providers.Add(kCpuExecutionProvider, std::move(tmp_cpu_execution_provider))); @@ -128,7 +128,7 @@ TEST_P(SessionStateTestP, TestInitializerProcessing) { InitializedTensorSet initializers = graph.GetAllInitializedTensors(); ExecutionProviders execution_providers; - CPUExecutionProviderInfo epi{false}; + CPUExecutionProviderInfo epi{false, false}; status = execution_providers.Add(onnxruntime::kCpuExecutionProvider, std::make_unique(epi)); ASSERT_TRUE(status.IsOK()) << status; @@ -193,7 +193,7 @@ TEST(SessionStateTest, TestInitializerMemoryAllocatedUsingNonArenaMemory) { Graph& graph = model->MainGraph(); ExecutionProviders execution_providers; - CPUExecutionProviderInfo epi{true}; // use an arena-based allocator for this EP + CPUExecutionProviderInfo epi{true, false}; // use an arena-based allocator for this EP status = execution_providers.Add(onnxruntime::kCpuExecutionProvider, std::make_unique(epi)); ASSERT_TRUE(status.IsOK()) << status; @@ -243,7 +243,7 @@ TEST(SessionStateTest, TestInitializerMemoryAllocatedUsingNonArenaMemory) { Graph& graph = model->MainGraph(); ExecutionProviders execution_providers; - CPUExecutionProviderInfo epi{true}; // use an arena-based allocator for this EP + CPUExecutionProviderInfo epi{true, false}; // use an arena-based allocator for this EP status = execution_providers.Add(onnxruntime::kCpuExecutionProvider, std::make_unique(epi)); ASSERT_TRUE(status.IsOK()) << status; @@ -480,7 +480,7 @@ TEST_P(SessionStatePrepackingTest, PrePackingTest) { .Output(0, "output_0", "docstr for output_0.", "tensor(float)"); ExecutionProviders execution_providers; - auto cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false)); + auto cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false, false)); ASSERT_STATUS_OK(execution_providers.Add(kCpuExecutionProvider, std::move(cpu_execution_provider))); DataTransferManager dtm; @@ -541,7 +541,7 @@ TEST(SessionStateTest, SharedInitalizersWithPrePackingTest) { .Output(0, "output_0", "docstr for output_0.", "tensor(float)"); ExecutionProviders execution_providers; - auto cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false)); + auto cpu_execution_provider = std::make_unique(CPUExecutionProviderInfo(false, false)); ASSERT_STATUS_OK(execution_providers.Add(kCpuExecutionProvider, std::move(cpu_execution_provider))); DataTransferManager dtm; diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 104287f50d..36de4b28bf 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -56,6 +56,7 @@ namespace perftest { "\t-q: [CUDA only] use separate stream for copy. \n" "\t-z: Set denormal as zero. When turning on this option reduces latency dramatically, a model may have denormals.\n" "\t-i: Specify EP specific runtime options as key value pairs. Different runtime options available are: \n" + "\t-R: Use fixed point based requantization on ARM64." "\t [OpenVINO only] [device_type]: Overrides the accelerator hardware type and precision with these values at runtime.\n" "\t [OpenVINO only] [device_id]: Selects a particular hardware device for inference.\n" "\t [OpenVINO only] [enable_vpu_fast_compile]: Optionally enabled to speeds up the model's compilation on VPU device targets.\n" @@ -114,7 +115,7 @@ static bool ParseDimensionOverride(std::basic_string& dim_identifier, /*static*/ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]) { int ch; - while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:y:c:d:o:u:i:f:F:AMPIvhsqz"))) != -1) { + while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:y:c:d:o:u:i:f:F:AMPIvhsqzR"))) != -1) { switch (ch) { case 'f': { std::basic_string dim_name; @@ -272,6 +273,9 @@ static bool ParseDimensionOverride(std::basic_string& dim_identifier, case 'i': test_config.run_config.ep_runtime_config_string = optarg; break; + case 'R': + test_config.run_config.use_fixed_point_requant = true; + break; case '?': case 'h': default: diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 487f9ad80a..928e7745fd 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -460,6 +460,10 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device } } + if (performance_test_config.run_config.use_fixed_point_requant) { + session_options.AddConfigEntry(kOrtSessionOptionsConfigFixedPointRequantOnARM64, "1"); + } + session_ = Ort::Session(env, performance_test_config.model_info.model_file_path.c_str(), session_options); size_t output_count = session_.GetOutputCount(); diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h index 892db39850..af1c385c57 100644 --- a/onnxruntime/test/perftest/test_configuration.h +++ b/onnxruntime/test/perftest/test_configuration.h @@ -54,6 +54,7 @@ struct RunConfig { int cudnn_conv_algo{0}; bool do_cuda_copy_in_separate_stream{false}; bool set_denormal_as_zero{false}; + bool use_fixed_point_requant{false}; std::basic_string ep_runtime_config_string; std::map, int64_t> free_dim_name_overrides; std::map, int64_t> free_dim_denotation_overrides; diff --git a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc index 5177fcbfa9..c67e6203b3 100644 --- a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc @@ -685,8 +685,9 @@ TEST(Loop, SubgraphTypeOverride) { Graph::ResolveOptions options; options.override_types = true; test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kTensorrtExecutionProvider}, &session_run_options, nullptr, - ExecutionMode::ORT_SEQUENTIAL, options); + {kTensorrtExecutionProvider}, + &session_run_options, nullptr, + options); } // Regression test that a subgraph input overrides an outer scope value of the same name. diff --git a/onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc b/onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc index 0cb078b456..002012189a 100644 --- a/onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc +++ b/onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc @@ -6,12 +6,68 @@ #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" +#include "core/session/onnxruntime_session_options_config_keys.h" namespace onnxruntime { namespace test { +class OpTesterRequant : public OpTester { + public: + explicit OpTesterRequant( + const char* op, + int opset_version = 7, + const char* domain = onnxruntime::kOnnxDomain, + bool verify_output = true) : OpTester(op, opset_version, domain, verify_output) {} + + void Run(ExpectResult expect_result = ExpectResult::kExpectSuccess, const std::string& expected_failure_string = "", + const std::unordered_set& excluded_provider_types = {}, + const RunOptions* run_options = nullptr, + std::vector>* execution_providers = nullptr, + const Graph::ResolveOptions& resolve_options = {}) { + SessionOptions session_options; + SetUpDefaultSessionOptions(session_options); + for (auto use_fixed_point : {"0", "1"}) { + ASSERT_STATUS_OK(session_options.config_options.AddConfigEntry( + kOrtSessionOptionsConfigFixedPointRequantOnARM64, + use_fixed_point)); + OpTester::Run(session_options, + expect_result, + expected_failure_string, + excluded_provider_types, + run_options, + execution_providers, + resolve_options); + } + } + + void Run(SessionOptions session_options, + ExpectResult expect_result = ExpectResult::kExpectSuccess, + const std::string& expected_failure_string = "", + const std::unordered_set& excluded_provider_types = {}, + const RunOptions* run_options = nullptr, + std::vector>* execution_providers = nullptr, + const Graph::ResolveOptions& resolve_options = {}, + /*out*/ size_t* number_of_pre_packed_weights_counter = nullptr, + /*out*/ size_t* number_of_shared_pre_packed_weights_counter = nullptr) { + for (auto use_fixed_point : {"0", "1"}) { + ASSERT_STATUS_OK(session_options.config_options.AddConfigEntry( + kOrtSessionOptionsConfigFixedPointRequantOnARM64, + use_fixed_point)); + OpTester::Run(session_options, + expect_result, + expected_failure_string, + excluded_provider_types, + run_options, + execution_providers, + resolve_options, + number_of_pre_packed_weights_counter, + number_of_shared_pre_packed_weights_counter); + } + } +}; + TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_U8U8) { - OpTester test("QLinearMatMul", 10); + OpTesterRequant test("QLinearMatMul", 10); test.AddInput("T1", {2, 2, 4}, {208, 236, 0, 238, 3, 214, 255, 29, @@ -49,7 +105,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_U8U8) { } TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_U8S8) { - OpTester test("QLinearMatMul", 10); + OpTesterRequant test("QLinearMatMul", 10); test.AddInput("T1", {2, 2, 4}, {208, 126, 0, 238, 3, 214, 255, 29, @@ -87,7 +143,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_U8S8) { } TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_S8S8) { - OpTester test("QLinearMatMul", 10); + OpTesterRequant test("QLinearMatMul", 10); test.AddInput("T1", {2, 2, 4}, {80, -2, -128, 110, -125, 86, 127, -99, @@ -126,7 +182,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D_S8S8) { TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_U8U8) { auto run_test = [](bool only_t1_not_initializer) { - OpTester test("QLinearMatMul", 10); + OpTesterRequant test("QLinearMatMul", 10); test.AddInput("T1", {2, 4}, {208, 236, 0, 238, 3, 214, 255, 29}); @@ -161,7 +217,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_U8U8) { TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_U8S8) { auto run_test = [](bool only_t1_not_initializer) { - OpTester test("QLinearMatMul", 10); + OpTesterRequant test("QLinearMatMul", 10); test.AddInput("T1", {2, 4}, {208, 126, 0, 238, 3, 214, 255, 29}); @@ -196,7 +252,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_U8S8) { TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_S8S8) { auto run_test = [](bool only_t1_not_initializer) { - OpTester test("QLinearMatMul", 10); + OpTesterRequant test("QLinearMatMul", 10); test.AddInput("T1", {2, 4}, {80, -2, -128, 110, -125, 86, 127, -99}); @@ -231,7 +287,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_S8S8) { static void QLinearMatMul2DTest(bool only_t1_not_initializer) { // Test non-empty inputs - OpTester test_non_empty("QLinearMatMul", 10); + OpTesterRequant test_non_empty("QLinearMatMul", 10); test_non_empty.AddInput("T1", {2, 4}, {208, 236, 0, 238, 3, 214, 255, 29}); test_non_empty.AddInput("a_scale", {1}, {0.0066f}, only_t1_not_initializer); test_non_empty.AddInput("a_zero_point", {1}, {113}, only_t1_not_initializer); @@ -244,7 +300,7 @@ static void QLinearMatMul2DTest(bool only_t1_not_initializer) { test_non_empty.Run(); // Test with an empty input - OpTester test_empty("QLinearMatMul", 10); + OpTesterRequant test_empty("QLinearMatMul", 10); test_empty.AddInput("T1", {0, 4}, {}); test_empty.AddInput("a_scale", {1}, {0.0066f}, only_t1_not_initializer); test_empty.AddInput("a_zero_point", {1}, {113}, only_t1_not_initializer); @@ -269,7 +325,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMulAllInputExceptT1AreInitializers) { } TEST(QuantizeLinearMatmulOpTest, PerColumn_2D) { - OpTester test("QLinearMatMul", 10); + OpTesterRequant test("QLinearMatMul", 10); test.AddInput("a", {2, 4}, {125, 135, 133, 122, @@ -299,7 +355,7 @@ TEST(QuantizeLinearMatmulOpTest, PerColumn_2D) { } TEST(QuantizeLinearMatmulOpTest, PerColumn_2D_S8S8) { - OpTester test("QLinearMatMul", 10); + OpTesterRequant test("QLinearMatMul", 10); test.AddInput("a", {2, 4}, {-3, 7, 5, -6, @@ -329,7 +385,7 @@ TEST(QuantizeLinearMatmulOpTest, PerColumn_2D_S8S8) { } TEST(QuantizeLinearMatmulOpTest, PerColumn_ND) { - OpTester test("QLinearMatMul", 10); + OpTesterRequant test("QLinearMatMul", 10); test.AddInput("a", {2, 2, 4}, {125, 135, 133, 122, @@ -372,7 +428,7 @@ TEST(QuantizeLinearMatmulOpTest, PerColumn_ND) { } TEST(QuantizeLinearMatmulOpTest, PerColumn_ND_S8S8) { - OpTester test("QLinearMatMul", 10); + OpTesterRequant test("QLinearMatMul", 10); test.AddInput("a", {2, 2, 4}, {-3, 7, 5, -6, @@ -491,7 +547,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMulPrePack) { auto kernel_def = PrePackTestOp::KernelDef(); ASSERT_TRUE((status = registry->RegisterCustomKernel(kernel_def, kernel_create_fn)).IsOK()) << status; - OpTester test_non_empty(PrePackTestOp::OpName, 10, PrePackTestOp::OpDomain); + OpTesterRequant test_non_empty(PrePackTestOp::OpName, 10, PrePackTestOp::OpDomain); test_non_empty.AddCustomOpRegistry(registry); test_non_empty.AddInput("T1", {2, 4}, {208, 236, 0, 238, 3, 214, 255, 29}); diff --git a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc index 01f3c0e027..5a5750f913 100644 --- a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc @@ -9,6 +9,7 @@ #include "default_providers.h" #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" +#include "core/session/onnxruntime_session_options_config_keys.h" namespace onnxruntime { namespace test { @@ -318,21 +319,49 @@ class QLinearConvOpTester { float zero_point_; }; - inline float RoundHalfToEven(float input) { - if (!std::isfinite(input)) { - return input; - } - // std::remainder returns x - n, where n is the integral value nearest to x. When |x - n| = 0.5, n is chosen to be even - return input - std::remainderf(input, 1.f); - } + template + struct RequantizeOutput {}; template - T RequantizeOutput(int32_t sum, float scale, RequantizeValues& requantize_values) { - float f = static_cast(sum) * scale; - f = std::min(f, requantize_values.max_value_); - f = std::max(f, requantize_values.min_value_); - return static_cast(RoundHalfToEven(f) + requantize_values.zero_point_); - } + struct RequantizeOutput { + T operator()(int32_t sum, + float scale, + RequantizeValues& requantize_values) { + float f = static_cast(sum) * scale; + f = std::min(f, requantize_values.max_value_); + f = std::max(f, requantize_values.min_value_); + + if (std::isfinite(f)) { + f = f - std::remainderf(f, 1.f); // std::remainder returns x - n, where n is the integral value nearest to x. + // When |x - n| = 0.5, n is chosen to be even + } + + return static_cast(f + requantize_values.zero_point_); + } + }; + + // Compute RoundUp with double float precision to avoid corner case like: + // sum: 1140, scale: 0.00394736836. + // with single float precision, it is 4.50000000 and rounds to 5. + // with double float precision, it is 4.4999999303999996 and rounds to 4. + // Fixed point computation in arm64 is equivalent to double float precision, + // so use double float precision to get same result as fixed point. + template + struct RequantizeOutput { + T operator()(int32_t sum, + float scale, + RequantizeValues& requantize_values) { + double d = static_cast(sum) * static_cast(scale); + d = std::min(d, static_cast(requantize_values.max_value_)); + d = std::max(d, static_cast(requantize_values.min_value_)); + + if (std::isfinite(d)) { + d = std::round(d); + } + + return static_cast(d + requantize_values.zero_point_); + } + }; static bool NextPosition(int64_t N, const int64_t* shape, int64_t* dims) { // Loop over spatial axes in reverse order to choose an index, like counting. @@ -351,7 +380,9 @@ class QLinearConvOpTester { return incremented; } - void ComputeExpectedOutput(std::vector& Y_data, std::vector& Y_shape) { + void ComputeExpectedOutput(std::vector& Y_data, + std::vector& Y_shape, + bool requant_with_fixed_point_on_arm64) { ORT_ENFORCE(W_.shape_.size() > 2); ORT_ENFORCE(X_.shape_.size() == W_.shape_.size()); @@ -444,7 +475,17 @@ class QLinearConvOpTester { input_image += input_image_size; } - *Ydata++ = RequantizeOutput(sum, requantize_scale, requantize_values); + if (requant_with_fixed_point_on_arm64) { + *Ydata++ = RequantizeOutput()( + sum, + requantize_scale, + requantize_values); + } else { + *Ydata++ = RequantizeOutput()( + sum, + requantize_scale, + requantize_values); + } } while (NextPosition(kernel_rank, output_shape, d_output.data())); @@ -457,12 +498,12 @@ class QLinearConvOpTester { } } - void Run(bool all_input_initializer_except_x) { + void Run(bool all_input_initializer_except_x, bool requant_with_fixed_point_on_arm64) { OpTester test("QLinearConv", 10); std::vector Y_data; std::vector Y_shape; - ComputeExpectedOutput(Y_data, Y_shape); + ComputeExpectedOutput(Y_data, Y_shape, requant_with_fixed_point_on_arm64); test.AddInput("x", X_.shape_, X_.data_); test.AddInput("x_scale", {}, X_.scale_, all_input_initializer_except_x); @@ -512,7 +553,11 @@ class QLinearConvOpTester { test.AddAttribute("group", groups_); } - test.Run(OpTester::ExpectResult::kExpectSuccess, ""); + SessionOptions so; + ASSERT_STATUS_OK(so.config_options.AddConfigEntry( + kOrtSessionOptionsConfigFixedPointRequantOnARM64, + requant_with_fixed_point_on_arm64 ? "1" : "0")); + test.Run(so, OpTester::ExpectResult::kExpectSuccess, ""); } public: @@ -569,11 +614,16 @@ class QLinearConvOpTester { } void Run() { - for (bool all_input_initializer_except_x : std::initializer_list{false, true}) { - Run(all_input_initializer_except_x); - } +#if defined(_M_ARM64) || defined(__aarch64__) + for (bool requant_with_fixed_point_on_arm64 : std::initializer_list{false, true}) +#else + bool requant_with_fixed_point_on_arm64 = false; +#endif + for (bool all_input_initializer_except_x : std::initializer_list{false, true}) { + Run(all_input_initializer_except_x, requant_with_fixed_point_on_arm64); + } } -}; +}; // namespace TEST(QLinearConvTest, Conv1D_U8S8) { QLinearConvOpTester test; @@ -637,7 +687,7 @@ TEST(QLinearConvTest, Conv2D_U8S8_Sym_M32_C32_Bias_Pads) { TEST(QLinearConvTest, Conv2D_U8S8_Sym_M8_C8) { // Targeting code processing 8 channels, with odd number - // of output pixels + // of output pixels QLinearConvOpTester test; test.GenerateRandomInput({1, 8, 3, 5}, .85f, 4); test.GenerateRandomWeights({8, 8, 3, 3}, .125f, 0); @@ -1039,7 +1089,6 @@ TEST(QLinearConvTest, Conv2D_S8S8_Sym_M64_C64) { test.Run(); } - TEST(QLinearConvTest, Conv2D_S8S8_Sym_M16_C4_Bias) { QLinearConvOpTester test; test.GenerateRandomInput({1, 4, 3, 3}, .05f, 4); diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 1ec7a4c78c..d49d167415 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -5,6 +5,7 @@ #include "core/common/logging/logging.h" #include "core/common/logging/sinks/clog_sink.h" +#include "core/common/type_utils.h" #include "core/framework/tensorprotoutils.h" #include "core/framework/data_types_internal.h" #include "core/session/inference_session.h" @@ -57,7 +58,7 @@ void sort_expected_and_actual_buffers(std::vector& expected, // The default implementation compares for equality, specialized versions for // other types are below -template +template struct TensorCheck { void operator()(const Tensor& expected_tensor, const Tensor& output_tensor, const std::string& provider_type, const CheckParams& params) const { @@ -87,8 +88,8 @@ struct TensorCheck { } }; -template <> -struct TensorCheck { +template +struct TensorCheck::value>::type> { void operator()(const Tensor& expected_tensor, const Tensor& output_tensor, const std::string& provider_type, const CheckParams& params) const { @@ -96,8 +97,8 @@ struct TensorCheck { const bool has_rel_err = params.relative_error_.has_value(); Tensor expected_sorted, output_sorted; - const uint8_t* expected; - const uint8_t* output; + const T* expected; + const T* output; const auto size = output_tensor.Shape().Size(); if (params.sort_output_) { // if order can be jumbled in the output of an operator, sort both the @@ -106,15 +107,15 @@ struct TensorCheck { // requirement for the few ops that do require this // support without investing in a more sophisticated infrastructure for the // same - sort_expected_and_actual_buffers(expected_tensor, expected_sorted, output_tensor, output_sorted); - expected = expected_sorted.Data(); - output = output_sorted.Data(); + sort_expected_and_actual_buffers(expected_tensor, expected_sorted, output_tensor, output_sorted); + expected = expected_sorted.Data(); + output = output_sorted.Data(); } else { - expected = expected_tensor.template Data(); - output = output_tensor.template Data(); + expected = expected_tensor.template Data(); + output = output_tensor.template Data(); } - // For uint8_t results, we only allow NNAPI EP to have an error tolerance, see below for the reason + // For int8_t/uint8_t results, we only allow NNAPI EP to have an error tolerance, see below for the reason // For any other EPs, we still expect an exact match for the results if (provider_type == kNnapiExecutionProvider && (has_abs_err || has_rel_err)) { double threshold = has_abs_err @@ -897,15 +898,9 @@ void OpTester::Run( const std::unordered_set& excluded_provider_types, const RunOptions* run_options, std::vector>* execution_providers, - ExecutionMode execution_mode, const Graph::ResolveOptions& options) { SessionOptions so; - so.use_per_session_threads = false; - so.session_logid = op_; - so.session_log_verbosity_level = 1; - so.execution_mode = execution_mode; - so.use_deterministic_compute = use_determinism_; - so.graph_optimization_level = TransformerLevel::Default; // 'Default' == off + SetUpDefaultSessionOptions(so); Run(so, expect_result, expected_failure_string, excluded_provider_types, run_options, execution_providers, options); } @@ -1088,30 +1083,35 @@ void OpTester::Run( ASSERT_PROVIDER_STATUS_OK(session_object.RegisterCustomRegistry(custom_session_registry)); std::unique_ptr execution_provider; - if (provider_type == onnxruntime::kCpuExecutionProvider) - execution_provider = DefaultCpuExecutionProvider(); - else if (provider_type == onnxruntime::kCudaExecutionProvider) + if (provider_type == onnxruntime::kCpuExecutionProvider) { + const bool use_fixed_point_requant_on_arm64 = + so.config_options.GetConfigOrDefault(kOrtSessionOptionsConfigFixedPointRequantOnARM64, "0") == "1"; + execution_provider = DefaultCpuExecutionProvider( + CPUExecutionProviderInfo(so.enable_cpu_mem_arena, + use_fixed_point_requant_on_arm64)); + } else if (provider_type == onnxruntime::kCudaExecutionProvider) { execution_provider = DefaultCudaExecutionProvider(); - else if (provider_type == onnxruntime::kDnnlExecutionProvider) + } else if (provider_type == onnxruntime::kDnnlExecutionProvider) { execution_provider = DefaultDnnlExecutionProvider(); - else if (provider_type == onnxruntime::kOpenVINOExecutionProvider) + } else if (provider_type == onnxruntime::kOpenVINOExecutionProvider) { execution_provider = DefaultOpenVINOExecutionProvider(); - else if (provider_type == onnxruntime::kNupharExecutionProvider) + } else if (provider_type == onnxruntime::kNupharExecutionProvider) { execution_provider = DefaultNupharExecutionProvider(); - else if (provider_type == onnxruntime::kTensorrtExecutionProvider) + } else if (provider_type == onnxruntime::kTensorrtExecutionProvider) { execution_provider = DefaultTensorrtExecutionProvider(); - else if (provider_type == onnxruntime::kNnapiExecutionProvider) + } else if (provider_type == onnxruntime::kNnapiExecutionProvider) { execution_provider = DefaultNnapiExecutionProvider(); - else if (provider_type == onnxruntime::kRknpuExecutionProvider) + } else if (provider_type == onnxruntime::kRknpuExecutionProvider) { execution_provider = DefaultRknpuExecutionProvider(); - else if (provider_type == onnxruntime::kAclExecutionProvider) + } else if (provider_type == onnxruntime::kAclExecutionProvider) { execution_provider = DefaultAclExecutionProvider(); - else if (provider_type == onnxruntime::kArmNNExecutionProvider) + } else if (provider_type == onnxruntime::kArmNNExecutionProvider) { execution_provider = DefaultArmNNExecutionProvider(); - else if (provider_type == onnxruntime::kRocmExecutionProvider) + } else if (provider_type == onnxruntime::kRocmExecutionProvider) { execution_provider = DefaultRocmExecutionProvider(); - else if (provider_type == onnxruntime::kCoreMLExecutionProvider) + } else if (provider_type == onnxruntime::kCoreMLExecutionProvider) { execution_provider = DefaultCoreMLExecutionProvider(); + } // skip if execution provider is disabled if (execution_provider == nullptr) continue; diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h index 6d50bc2db6..62328c663b 100644 --- a/onnxruntime/test/providers/provider_test_utils.h +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -731,7 +731,6 @@ class OpTester { const std::unordered_set& excluded_provider_types = {}, const RunOptions* run_options = nullptr, std::vector>* execution_providers = nullptr, - ExecutionMode execution_mode = ExecutionMode::ORT_SEQUENTIAL, const Graph::ResolveOptions& resolve_options = {}); void Run(SessionOptions session_options, @@ -803,6 +802,15 @@ class OpTester { return prepacked_weights_container_.GetNumberOfElements(); } + void SetUpDefaultSessionOptions(SessionOptions& so) { + so.use_per_session_threads = false; + so.session_logid = op_; + so.session_log_verbosity_level = 1; + so.execution_mode = ExecutionMode::ORT_SEQUENTIAL; + so.use_deterministic_compute = use_determinism_; + so.graph_optimization_level = TransformerLevel::Default; // 'Default' == off + } + bool test_allow_released_onnx_opset_only_ = true; protected: diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 1a906a4860..8ca81238fd 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -5,6 +5,7 @@ #include "default_providers.h" #include "providers.h" #include "core/providers/cpu/cpu_provider_factory_creator.h" +#include "core/providers/cpu/cpu_execution_provider.h" #ifdef USE_COREML #include "core/providers/coreml/coreml_provider_factory.h" #endif @@ -13,8 +14,8 @@ namespace onnxruntime { namespace test { -std::unique_ptr DefaultCpuExecutionProvider(bool enable_arena) { - return CreateExecutionProviderFactory_CPU(enable_arena)->CreateProvider(); +std::unique_ptr DefaultCpuExecutionProvider(const CPUExecutionProviderInfo& info ) { + return CreateExecutionProviderFactory_CPU(info)->CreateProvider(); } std::unique_ptr DefaultTensorrtExecutionProvider() { diff --git a/onnxruntime/test/util/include/default_providers.h b/onnxruntime/test/util/include/default_providers.h index 655387cfbf..1aa7505ee2 100644 --- a/onnxruntime/test/util/include/default_providers.h +++ b/onnxruntime/test/util/include/default_providers.h @@ -2,8 +2,9 @@ // Licensed under the MIT License. #pragma once #include "core/common/optional.h" -#include "core/providers/providers.h" #include "core/framework/execution_provider.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/providers/providers.h" namespace onnxruntime { @@ -17,7 +18,7 @@ std::shared_ptr CreateExecutionProviderFactory_MIGrap std::shared_ptr CreateExecutionProviderFactory_Nnapi( uint32_t flags, const optional& partitioning_stop_ops_list); std::shared_ptr CreateExecutionProviderFactory_Nuphar(bool, const char*); -//std::shared_ptr CreateExecutionProviderFactory_Tvm(const char*); +// std::shared_ptr CreateExecutionProviderFactory_Tvm(const char*); std::shared_ptr CreateExecutionProviderFactory_OpenVINO( const char* device_type, bool enable_vpu_fast_compile, const char* device_id, size_t num_of_threads, bool use_compiled_network, const char* blob_dump_path); std::shared_ptr CreateExecutionProviderFactory_OpenVINO(const OrtOpenVINOProviderOptions* params); @@ -32,11 +33,12 @@ std::shared_ptr CreateExecutionProviderFactory_Intern namespace test { // unique_ptr providers with default values for session registration -std::unique_ptr DefaultCpuExecutionProvider(bool enable_arena = true); +std::unique_ptr DefaultCpuExecutionProvider(const CPUExecutionProviderInfo& info = + CPUExecutionProviderInfo(true, false)); std::unique_ptr DefaultCudaExecutionProvider(); std::unique_ptr DefaultDnnlExecutionProvider(bool enable_arena = true); std::unique_ptr DefaultNupharExecutionProvider(bool allow_unaligned_buffers = true); -//std::unique_ptr DefaultTvmExecutionProvider(); +// std::unique_ptr DefaultTvmExecutionProvider(); std::unique_ptr DefaultTensorrtExecutionProvider(); std::unique_ptr TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptions* params); std::unique_ptr TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptionsV2* params); diff --git a/orttraining/orttraining/test/gradient/allreduce_op_test.cc b/orttraining/orttraining/test/gradient/allreduce_op_test.cc index 9378c751a3..1af50d2bca 100644 --- a/orttraining/orttraining/test/gradient/allreduce_op_test.cc +++ b/orttraining/orttraining/test/gradient/allreduce_op_test.cc @@ -50,7 +50,6 @@ TEST(AllreduceTest, CPUAdasumAllreduceTestReduceTwoTensors) { allreduce_test.Run(OpTester::ExpectResult::kExpectSuccess /*expect_result*/, "" /*expected_failure_string*/, {} /*excluded_provider_types*/, nullptr /*run_options*/, &providers /*execution_providers*/, - ExecutionMode::ORT_SEQUENTIAL /*execution_mode*/, {} /*resolve_options*/); } @@ -93,7 +92,6 @@ TEST(AllreduceTest, CPUAdasumAllreduceTestReduceTwoTensorsFP16) { allreduce_test.Run(OpTester::ExpectResult::kExpectSuccess /*expect_result*/, "" /*expected_failure_string*/, {} /*excluded_provider_types*/, nullptr /*run_options*/, &providers /*execution_providers*/, - ExecutionMode::ORT_SEQUENTIAL /*execution_mode*/, {} /*resolve_options*/); } @@ -119,7 +117,6 @@ TEST(AllreduceTest, CPUAdasumAllreduceTestFailTensorCountMismatch) { allreduce_test.Run(OpTester::ExpectResult::kExpectFailure /*expect_result*/, "" /*expected_failure_string*/, {} /*excluded_provider_types*/, nullptr /*run_options*/, &providers /*execution_providers*/, - ExecutionMode::ORT_SEQUENTIAL /*execution_mode*/, {} /*resolve_options*/); } diff --git a/orttraining/orttraining/test/training_ops/cpu/nn/broadcast_grad_args_test.cc b/orttraining/orttraining/test/training_ops/cpu/nn/broadcast_grad_args_test.cc index 6a4d3ed3f5..24a78e00b7 100644 --- a/orttraining/orttraining/test/training_ops/cpu/nn/broadcast_grad_args_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/nn/broadcast_grad_args_test.cc @@ -40,9 +40,9 @@ void RunBroadcastGradientArgsTest(const char* op, std::vector> execution_providers; execution_providers.push_back(DefaultCpuExecutionProvider()); if (fail) - t.Run(OpTester::ExpectResult::kExpectFailure, "", {}, nullptr, &execution_providers, ExecutionMode::ORT_SEQUENTIAL); + t.Run(OpTester::ExpectResult::kExpectFailure, "", {}, nullptr, &execution_providers); else - t.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers, ExecutionMode::ORT_SEQUENTIAL); + t.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } } // namespace