diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index 329f32fbbd..f1d33b3bdd 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -1020,13 +1020,14 @@ MlasComputeSoftmax( MLAS_THREADPOOL* ThreadPool ); +template void MLASCALL MlasComputeSoftcap( - const MLAS_FP16* Input, - MLAS_FP16* Output, + const T* Input, + T* Output, size_t N, - MLAS_FP16 cap + T cap ); template diff --git a/onnxruntime/core/mlas/lib/activate_fp16.cpp b/onnxruntime/core/mlas/lib/activate_fp16.cpp index 776ec67fcc..0b1397db77 100644 --- a/onnxruntime/core/mlas/lib/activate_fp16.cpp +++ b/onnxruntime/core/mlas/lib/activate_fp16.cpp @@ -51,12 +51,12 @@ struct MLAS_HALF_ACTIVATION_FUNCTION MLAS_FLOAT16X8 Activate(MLAS_FLOAT16X8 Value) { - return MlasMaximumFloat16x8(ZeroVec, Value); + return MlasMaximum(ZeroVec, Value); } MLAS_FLOAT16X4 Activate(MLAS_FLOAT16X4 Value) { - return MlasMaximumFloat16x4(MlasToLowHalfFloat16x4(ZeroVec), Value); + return MlasMaximum(MlasToLowHalfFloat16x4(ZeroVec), Value); } }; @@ -75,7 +75,7 @@ struct MLAS_HALF_ACTIVATION_FUNCTION MLAS_FLOAT16X8 Activate(MLAS_FLOAT16X8 Value) { - MLAS_FLOAT16X8 ValueTimesAlpha = MlasMultiplyFloat16x8(Value, AlphaBroadcast); + MLAS_FLOAT16X8 ValueTimesAlpha = MlasMultiply(Value, AlphaBroadcast); return MlasBitwiseSelectFloat16x8(MlasCmpLessEqualFloat16x8(Value, ZeroVec), ValueTimesAlpha, Value); } @@ -83,7 +83,7 @@ struct MLAS_HALF_ACTIVATION_FUNCTION MLAS_FLOAT16X4 Activate(MLAS_FLOAT16X4 Value) { MLAS_FLOAT16X4 ValueTimesAlpha = - MlasMultiplyFloat16x4(Value, MlasToLowHalfFloat16x4(AlphaBroadcast)); + MlasMultiply(Value, MlasToLowHalfFloat16x4(AlphaBroadcast)); return MlasBitwiseSelectFloat16x4( MlasCmpLessEqualFloat16x4(Value, MlasToLowHalfFloat16x4(ZeroVec)), ValueTimesAlpha, Value); @@ -539,16 +539,16 @@ struct MLAS_HALF_ACTIVATION_FUNCTION { MLAS_FLOAT16X8 Activate(MLAS_FLOAT16X8 Value) { - Value = MlasMaximumFloat16x8(MinimumBroadcast, Value); - Value = MlasMinimumFloat16x8(MaximumBroadcast, Value); + Value = MlasMaximum(MinimumBroadcast, Value); + Value = MlasMinimum(MaximumBroadcast, Value); return Value; } MLAS_FLOAT16X4 Activate(MLAS_FLOAT16X4 Value) { - Value = MlasMaximumFloat16x4(MlasToLowHalfFloat16x4(MinimumBroadcast), Value); - Value = MlasMinimumFloat16x4(MlasToLowHalfFloat16x4(MaximumBroadcast), Value); + Value = MlasMaximum(MlasToLowHalfFloat16x4(MinimumBroadcast), Value); + Value = MlasMinimum(MlasToLowHalfFloat16x4(MaximumBroadcast), Value); return Value; } }; @@ -573,19 +573,19 @@ struct MLAS_HALF_ACTIVATION_FUNCTION MLAS_FLOAT16X8 Activate(MLAS_FLOAT16X8 Value) { - Value = MlasMultiplyAddFloat16x8(Value, AlphaBroadcast, BetaBroadcast); - Value = MlasMinimumFloat16x8(MaximumBroadcast, Value); - Value = MlasMaximumFloat16x8(MinimumBroadcast, Value); + Value = MlasMultiplyAdd(Value, AlphaBroadcast, BetaBroadcast); + Value = MlasMinimum(MaximumBroadcast, Value); + Value = MlasMaximum(MinimumBroadcast, Value); return Value; } MLAS_FLOAT16X4 Activate(MLAS_FLOAT16X4 Value) { - Value = MlasMultiplyAddFloat16x4(Value, MlasToLowHalfFloat16x4(AlphaBroadcast), + Value = MlasMultiplyAdd(Value, MlasToLowHalfFloat16x4(AlphaBroadcast), MlasToLowHalfFloat16x4(BetaBroadcast)); - Value = MlasMinimumFloat16x4(MlasToLowHalfFloat16x4(MaximumBroadcast), Value); - Value = MlasMaximumFloat16x4(MlasToLowHalfFloat16x4(MinimumBroadcast), Value); + Value = MlasMinimum(MlasToLowHalfFloat16x4(MaximumBroadcast), Value); + Value = MlasMaximum(MlasToLowHalfFloat16x4(MinimumBroadcast), Value); return Value; } @@ -692,7 +692,7 @@ MlasActivationKernel( MLAS_FLOAT16X8 AVec = MlasLoadFloat16x8(addsrc); MLAS_FLOAT16X8 Vector = MlasLoadFloat16x8(buffer); addsrc += 8; - Vector = MlasAddFloat16x8(Vector, AVec); + Vector = MlasAdd(Vector, AVec); Vector = ActivationFunction.Activate(Vector); MlasStoreFloat16x8(buffer, Vector); buffer += 8; @@ -703,7 +703,7 @@ MlasActivationKernel( MLAS_FLOAT16X4 AVec = MlasLoadFloat16x4(addsrc); MLAS_FLOAT16X4 Vector = MlasLoadFloat16x4(buffer); addsrc += 4; - Vector = MlasAddFloat16x4(Vector, AVec); + Vector = MlasAdd(Vector, AVec); Vector = ActivationFunction.Activate(Vector); MlasStoreFloat16x4(buffer, Vector); buffer += 4; @@ -715,7 +715,7 @@ MlasActivationKernel( MLAS_FLOAT16X4 buf; std::memcpy(&addbuf, addsrc, n * sizeof(_mlas_fp16_)); std::memcpy(&buf, buffer, n * sizeof(_mlas_fp16_)); - buf = MlasAddFloat16x4(buf, addbuf); + buf = MlasAdd(buf, addbuf); buf = ActivationFunction.Activate(buf); MlasStorePartialFloat16x4(buffer, buf, n); } diff --git a/onnxruntime/core/mlas/lib/dwconv.cpp b/onnxruntime/core/mlas/lib/dwconv.cpp index d48d9cbb17..febc2b15b2 100644 --- a/onnxruntime/core/mlas/lib/dwconv.cpp +++ b/onnxruntime/core/mlas/lib/dwconv.cpp @@ -43,7 +43,7 @@ MlasConvDepthwiseKernel( MLAS_FLOAT16X8 InputVector = MlasLoadFloat16x8(&Input[k][ChannelOffset]); MLAS_FLOAT16X8 FilterVector = MlasLoadFloat16x8(&Filter[ChannelKernelOffset]); - Accumulator = MlasMultiplyAddFloat16x8(InputVector, FilterVector, Accumulator); + Accumulator = MlasMultiplyAdd(InputVector, FilterVector, Accumulator); ChannelKernelOffset += Channels; } MlasStoreFloat16x8(Output, Accumulator); @@ -61,7 +61,7 @@ MlasConvDepthwiseKernel( MLAS_FLOAT16X4 InputVector = MlasLoadFloat16x4(&Input[k][ChannelOffset]); MLAS_FLOAT16X4 FilterVector = MlasLoadFloat16x4(&Filter[ChannelKernelOffset]); - Accumulator = MlasMultiplyAddFloat16x4(InputVector, FilterVector, Accumulator); + Accumulator = MlasMultiplyAdd(InputVector, FilterVector, Accumulator); ChannelKernelOffset += Channels; } MlasStoreFloat16x4(Output, Accumulator); @@ -80,7 +80,7 @@ MlasConvDepthwiseKernel( MLAS_FLOAT16X4 InputValue = MlasLoadFloat16x4(&Input[k][ChannelOffset]); MLAS_FLOAT16X4 FilterValue = MlasLoadFloat16x4(&Filter[ChannelKernelOffset]); - Accumulator = MlasMultiplyAddFloat16x4(InputValue, FilterValue, Accumulator); + Accumulator = MlasMultiplyAdd(InputValue, FilterValue, Accumulator); ChannelKernelOffset += Channels; } MlasStorePartialFloat16x4(Output, Accumulator, c); diff --git a/onnxruntime/core/mlas/lib/fp16_common.h b/onnxruntime/core/mlas/lib/fp16_common.h index acee567162..a76e22dbc8 100644 --- a/onnxruntime/core/mlas/lib/fp16_common.h +++ b/onnxruntime/core/mlas/lib/fp16_common.h @@ -27,10 +27,28 @@ typedef float16x8_t MLAS_FLOAT16X8; typedef float16x4_t MLAS_FLOAT16X4; typedef uint16x8_t MLAS_UINT16X8; typedef uint16x4_t MLAS_UINT16X4; +typedef int16x8_t MLAS_INT16X8; +typedef int16x4_t MLAS_INT16X4; MLAS_FORCEINLINE MLAS_FLOAT16X8 -MlasReinterpretAsFloat16x8(MLAS_INT32X4 Vector) { return vreinterpretq_f16_s32(Vector); } +MlasReinterpretAsFloat16(MLAS_INT32X4 Vector) { return vreinterpretq_f16_s32(Vector); } + +MLAS_FORCEINLINE +MLAS_FLOAT16X8 +MlasReinterpretAsFloat16(MLAS_INT16X8 Vector) { return vreinterpretq_f16_s16(Vector); } + +MLAS_FORCEINLINE +MLAS_FLOAT16X4 +MlasReinterpretAsFloat16(MLAS_INT16X4 Vector) { return vreinterpret_f16_s16(Vector); } + +MLAS_FORCEINLINE +MLAS_INT16X8 +MlasReinterpretAsInt16(MLAS_FLOAT16X8 Vector) { return vreinterpretq_s16_f16(Vector); } + +MLAS_FORCEINLINE +MLAS_INT16X4 +MlasReinterpretAsInt16(MLAS_FLOAT16X4 Vector) { return vreinterpret_s16_f16(Vector); } MLAS_FORCEINLINE MLAS_FLOAT16X8 @@ -142,42 +160,70 @@ MlasToLowHalfFloat16x4(MLAS_FLOAT16X8 V) MLAS_FORCEINLINE MLAS_FLOAT16X8 -MlasAddFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2) +MlasAdd(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2) { return vaddq_f16(Vector1, Vector2); } MLAS_FORCEINLINE MLAS_FLOAT16X4 -MlasAddFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) +MlasAdd(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) { return vadd_f16(Vector1, Vector2); } +MLAS_FORCEINLINE +MLAS_INT16X8 +MlasAdd(MLAS_INT16X8 Vector1, MLAS_INT16X8 Vector2) +{ + return vaddq_s16(Vector1, Vector2); +} + +MLAS_FORCEINLINE +MLAS_INT16X4 +MlasAdd(MLAS_INT16X4 Vector1, MLAS_INT16X4 Vector2) +{ + return vadd_s16(Vector1, Vector2); +} + MLAS_FORCEINLINE MLAS_FLOAT16X8 -MlasSubtractFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2) +MlasSubtract(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2) { return vsubq_f16(Vector1, Vector2); } MLAS_FORCEINLINE MLAS_FLOAT16X4 -MlasSubtractFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) +MlasSubtract(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) { return vsub_f16(Vector1, Vector2); } +MLAS_FORCEINLINE +MLAS_INT16X8 +MlasSubtract(MLAS_INT16X8 Vector1, MLAS_INT16X8 Vector2) +{ + return vsubq_s16(Vector1, Vector2); +} + +MLAS_FORCEINLINE +MLAS_INT16X4 +MlasSubtract(MLAS_INT16X4 Vector1, MLAS_INT16X4 Vector2) +{ + return vsub_s16(Vector1, Vector2); +} + MLAS_FORCEINLINE MLAS_FLOAT16X8 -MlasMultiplyFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2) +MlasMultiply(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2) { return vmulq_f16(Vector1, Vector2); } MLAS_FORCEINLINE MLAS_FLOAT16X4 -MlasMultiplyFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) +MlasMultiply(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) { return vmul_f16(Vector1, Vector2); } @@ -198,31 +244,30 @@ MlasDivFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) MLAS_FORCEINLINE MLAS_FLOAT16X8 -MlasMultiplyAddFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2, MLAS_FLOAT16X8 Vector3) +MlasMultiplyAdd(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2, MLAS_FLOAT16X8 Dest) { - return vfmaq_f16(Vector3, Vector1, Vector2); + return vfmaq_f16(Dest, Vector1, Vector2); } MLAS_FORCEINLINE MLAS_FLOAT16X4 -MlasMultiplyAddFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2, MLAS_FLOAT16X4 Vector3) +MlasMultiplyAdd(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2, MLAS_FLOAT16X4 Dest) { - return vfma_f16(Vector3, Vector1, Vector2); + return vfma_f16(Dest, Vector1, Vector2); } - MLAS_FORCEINLINE void MlasMultiplyAddFloat16x8(MLAS_FLOAT16X8 Vector1, _mlas_fp16_ Scalar2, MLAS_FLOAT16X8 Vector3) { - MlasMultiplyAddFloat16x8(Vector1, MlasBroadcastFloat16x8(Scalar2), Vector3); + MlasMultiplyAdd(Vector1, MlasBroadcastFloat16x8(Scalar2), Vector3); } MLAS_FORCEINLINE void MlasMultiplyAddFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2, _mlas_fp16_ Scalar3) { - MlasMultiplyAddFloat16x8(Vector1, Vector2, MlasBroadcastFloat16x8(Scalar3)); + MlasMultiplyAdd(Vector1, Vector2, MlasBroadcastFloat16x8(Scalar3)); } MLAS_FORCEINLINE @@ -277,50 +322,98 @@ MlasBlendFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2, MLAS_FLOAT16X MLAS_FORCEINLINE MLAS_FLOAT16X8 -MlasMaximumFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2) +MlasMaximum(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2) { return vmaxq_f16(Vector1, Vector2); } MLAS_FORCEINLINE MLAS_FLOAT16X4 -MlasMaximumFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) +MlasMaximum(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) { return vmax_f16(Vector1, Vector2); } +MLAS_FORCEINLINE +MLAS_INT16X8 +MlasMaximum(MLAS_INT16X8 Vector1, MLAS_INT16X8 Vector2) +{ + return vmaxq_s16(Vector1, Vector2); +} + +MLAS_FORCEINLINE +MLAS_INT16X4 +MlasMaximum(MLAS_INT16X4 Vector1, MLAS_INT16X4 Vector2) +{ + return vmax_s16(Vector1, Vector2); +} + MLAS_FORCEINLINE MLAS_FLOAT16X8 -MlasMinimumFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2) +MlasMinimum(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2) { return vminq_f16(Vector1, Vector2); } MLAS_FORCEINLINE MLAS_FLOAT16X4 -MlasMinimumFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) +MlasMinimum(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2) { return vmin_f16(Vector1, Vector2); } +MLAS_FORCEINLINE +MLAS_INT16X8 +MlasMinimum(MLAS_INT16X8 Vector1, MLAS_INT16X8 Vector2) +{ + return vminq_s16(Vector1, Vector2); +} + +MLAS_FORCEINLINE +MLAS_INT16X4 +MlasMinimum(MLAS_INT16X4 Vector1, MLAS_INT16X4 Vector2) +{ + return vmin_s16(Vector1, Vector2); +} + MLAS_FORCEINLINE MLAS_FLOAT16X8 MlasClampFloat16x8(MLAS_FLOAT16X8 Value, _mlas_fp16_ LowerRange, _mlas_fp16_ UpperRange) { - Value = MlasMaximumFloat16x8(MlasBroadcastFloat16x8(LowerRange), Value); - Value = MlasMinimumFloat16x8(MlasBroadcastFloat16x8(UpperRange), Value); + Value = MlasMaximum(MlasBroadcastFloat16x8(LowerRange), Value); + Value = MlasMinimum(MlasBroadcastFloat16x8(UpperRange), Value); + return Value; +} + +template +MLAS_FORCEINLINE +T +MlasClamp(T Value, T LowerRange, T UpperRange) +{ + Value = MlasMaximum(LowerRange, Value); + Value = MlasMinimum(UpperRange, Value); return Value; } MLAS_FORCEINLINE _mlas_fp16_ -MlasReduceAddFloat16x8(MLAS_FLOAT16X8 Vector) +MlasReduceAdd(MLAS_FLOAT16X8 Vector) { + Vector = vpaddq_f16(Vector, Vector); Vector = vpaddq_f16(Vector, Vector); Vector = vpaddq_f16(Vector, Vector); return vgetq_lane_u16(vreinterpretq_u16_f16(Vector), 0); } +MLAS_FORCEINLINE +_mlas_fp16_ +MlasReduceAdd(MLAS_FLOAT16X4 Vector) +{ + Vector = vpadd_f16(Vector, Vector); + Vector = vpadd_f16(Vector, Vector); + return vget_lane_u16(vreinterpret_u16_f16(Vector), 0); +} + MLAS_FORCEINLINE MLAS_UINT16X8 MlasCmpLessEqualFloat16x8(MLAS_FLOAT16X8 left, MLAS_FLOAT16X8 right) @@ -448,4 +541,20 @@ Transpose4x4(MLAS_FLOAT16X4& v0, MLAS_FLOAT16X4& v1, MLAS_FLOAT16X4& v2, MLAS_FL v3 = vreinterpret_f16_f32(vtrn2_f32(vreinterpret_f32_f16(t01.val[1]), vreinterpret_f32_f16(t23.val[1]))); } +template +MLAS_FORCEINLINE +MLAS_INT16X8 +MlasShiftLeft(MLAS_INT16X8 Vector) +{ + return vshlq_n_s16(Vector, ShiftCount); +} + +template +MLAS_FORCEINLINE +MLAS_INT16X4 +MlasShiftLeft(MLAS_INT16X4 Vector) +{ + return vshl_n_s16(Vector, ShiftCount); +} + #endif // fp16 vector intrinsic supported diff --git a/onnxruntime/core/mlas/lib/pooling_fp16.cpp b/onnxruntime/core/mlas/lib/pooling_fp16.cpp index 98e84736cb..685f3eb22a 100644 --- a/onnxruntime/core/mlas/lib/pooling_fp16.cpp +++ b/onnxruntime/core/mlas/lib/pooling_fp16.cpp @@ -80,23 +80,23 @@ PoolInit16x4() } template<> -MLAS_FORCEINLINE +MLAS_FORCEINLINE MLAS_FLOAT16X8 PoolAggregate16x8(MLAS_FLOAT16X8 agg, MLAS_FLOAT16X8 element) { - return MlasMaximumFloat16x8(agg, element); + return MlasMaximum(agg, element); } template<> -MLAS_FORCEINLINE +MLAS_FORCEINLINE MLAS_FLOAT16X4 PoolAggregate16x4(MLAS_FLOAT16X4 agg, MLAS_FLOAT16X4 element) { - return MlasMaximumFloat16x4(agg, element); + return MlasMaximum(agg, element); } template<> -MLAS_FORCEINLINE +MLAS_FORCEINLINE MLAS_FLOAT16X8 PoolSummary16x8(MLAS_FLOAT16X8 agg, size_t size) { @@ -105,7 +105,7 @@ PoolSummary16x8(MLAS_FLOAT16X8 agg, size_t size) } template<> -MLAS_FORCEINLINE +MLAS_FORCEINLINE MLAS_FLOAT16X4 PoolSummary16x4(MLAS_FLOAT16X4 agg, size_t size) { @@ -144,14 +144,14 @@ template <> MLAS_FORCEINLINE MLAS_FLOAT16X8 PoolAggregate16x8(MLAS_FLOAT16X8 agg, MLAS_FLOAT16X8 element) { - return MlasAddFloat16x8(agg, element); + return MlasAdd(agg, element); } template <> MLAS_FORCEINLINE MLAS_FLOAT16X4 PoolAggregate16x4(MLAS_FLOAT16X4 agg, MLAS_FLOAT16X4 element) { - return MlasAddFloat16x4(agg, element); + return MlasAdd(agg, element); } template <> diff --git a/onnxruntime/core/mlas/lib/softmax.h b/onnxruntime/core/mlas/lib/softmax.h index 6c9bbd706b..d1c5c39496 100644 --- a/onnxruntime/core/mlas/lib/softmax.h +++ b/onnxruntime/core/mlas/lib/softmax.h @@ -78,7 +78,7 @@ struct MLAS_SOFTMAX_DISPATCH { /** * @brief Compute the expotential function for each element of the input array and returnt he sum. It has smaller - * dynamic range for the input than Exp_Fp16_Fn. + * dynamic range for the input than Exp_Fp16_Fn thus is faster. * @param Input Address of the input array * @param Output Address of the output array. Could be the same as the input array or nullptr. * @param N Number of elements in the input array diff --git a/onnxruntime/core/mlas/lib/softmax_kernel_neon_fp16.cpp b/onnxruntime/core/mlas/lib/softmax_kernel_neon_fp16.cpp index 113ec53e91..6e41fb411a 100644 --- a/onnxruntime/core/mlas/lib/softmax_kernel_neon_fp16.cpp +++ b/onnxruntime/core/mlas/lib/softmax_kernel_neon_fp16.cpp @@ -22,7 +22,453 @@ Abstract: namespace softmax_neon { -// tanh kernel for fp16. Output and input can be the same buffer. +template +struct MlasExpConstants { + T LowerRange; + T UpperRange; + T LowerRangeSumExp; + T UpperRangeSumExp; + T RoundingBias; + T Log2Reciprocal; + T Log2High; + T Log2Mid; + T Log2Low; + T poly_0; + T poly_1; + T poly_2; + T poly_3; + T poly_4; + T poly_56; + T MinimumExponent; + T MaximumExponent; +}; + +const MlasExpConstants<_mlas_fp16_> ExpConstantsFp16 = { + 0xcc55, // -25 * ln2 + 0x498c, // 16 * ln2 + 0xc95f, // -15.5 * ln2 + 0x495f, // 15.5 * ln2 + 0x6600, // 1.5 * 2^10 + 0x3dc5, // 1/ln2 + 0xb98b, // -6.9287109375e-1f16 + 0x8c85, // -2.758502960205078e-4f16 + 0x8004, // -2.384185791015625e-7f16 + 0x15b0, // 1/6! + 0x2044, // 1/5! + 0x2955, // 1/4! + 0x3155, // 1/3! + 0x3800, // 1/2! + 0x3c00, // 1/1! + 0xC800, // -14 + 0x3C00, // 15 +}; + +const MlasExpConstants ExpConstantsFp16x8 = { + MlasBroadcastFloat16x8(ExpConstantsFp16.LowerRange), + MlasBroadcastFloat16x8(ExpConstantsFp16.UpperRange), + MlasBroadcastFloat16x8(ExpConstantsFp16.LowerRangeSumExp), + MlasBroadcastFloat16x8(ExpConstantsFp16.UpperRangeSumExp), + MlasBroadcastFloat16x8(ExpConstantsFp16.RoundingBias), + MlasBroadcastFloat16x8(ExpConstantsFp16.Log2Reciprocal), + MlasBroadcastFloat16x8(ExpConstantsFp16.Log2High), + MlasBroadcastFloat16x8(ExpConstantsFp16.Log2Mid), + MlasBroadcastFloat16x8(ExpConstantsFp16.Log2Low), + MlasBroadcastFloat16x8(ExpConstantsFp16.poly_0), + MlasBroadcastFloat16x8(ExpConstantsFp16.poly_1), + MlasBroadcastFloat16x8(ExpConstantsFp16.poly_2), + MlasBroadcastFloat16x8(ExpConstantsFp16.poly_3), + MlasBroadcastFloat16x8(ExpConstantsFp16.poly_4), + MlasBroadcastFloat16x8(ExpConstantsFp16.poly_56), + MlasBroadcastFloat16x8(ExpConstantsFp16.MinimumExponent), + MlasBroadcastFloat16x8(ExpConstantsFp16.MaximumExponent), +}; + +const MlasExpConstants ExpConstantsFp16x4 = { + MlasBroadcastFloat16x4(ExpConstantsFp16.LowerRange), + MlasBroadcastFloat16x4(ExpConstantsFp16.UpperRange), + MlasBroadcastFloat16x4(ExpConstantsFp16.LowerRangeSumExp), + MlasBroadcastFloat16x4(ExpConstantsFp16.UpperRangeSumExp), + MlasBroadcastFloat16x4(ExpConstantsFp16.RoundingBias), + MlasBroadcastFloat16x4(ExpConstantsFp16.Log2Reciprocal), + MlasBroadcastFloat16x4(ExpConstantsFp16.Log2High), + MlasBroadcastFloat16x4(ExpConstantsFp16.Log2Mid), + MlasBroadcastFloat16x4(ExpConstantsFp16.Log2Low), + MlasBroadcastFloat16x4(ExpConstantsFp16.poly_0), + MlasBroadcastFloat16x4(ExpConstantsFp16.poly_1), + MlasBroadcastFloat16x4(ExpConstantsFp16.poly_2), + MlasBroadcastFloat16x4(ExpConstantsFp16.poly_3), + MlasBroadcastFloat16x4(ExpConstantsFp16.poly_4), + MlasBroadcastFloat16x4(ExpConstantsFp16.poly_56), + MlasBroadcastFloat16x4(ExpConstantsFp16.MinimumExponent), + MlasBroadcastFloat16x4(ExpConstantsFp16.MaximumExponent), +}; + +template +MLAS_FORCEINLINE +MlasExpConstants Get_Exp_Constants(); + +template <> +MLAS_FORCEINLINE +MlasExpConstants Get_Exp_Constants() { + return ExpConstantsFp16x8; +} + +template <> +MLAS_FORCEINLINE +MlasExpConstants Get_Exp_Constants() { + return ExpConstantsFp16x4; +} + +// Range reduction + polynomial approximation. Refer algorithm details to MlasComputeExpVector. +template +MLAS_FORCEINLINE +T Exp_Vector_Fp16(T x) { + const auto constants = Get_Exp_Constants(); + auto clamped_x = MlasClamp(x, constants.LowerRange, constants.UpperRange); + + // integral + auto biased = MlasMultiplyAdd(clamped_x, constants.Log2Reciprocal, constants.RoundingBias); + auto m = MlasSubtract(biased, constants.RoundingBias); + + // residual + auto r = MlasMultiplyAdd(m, constants.Log2High, clamped_x); + r = MlasMultiplyAdd(m, constants.Log2Mid, r); + r = MlasMultiplyAdd(m, constants.Log2Low, r); + + // handle overflow + auto overflow = MlasShiftLeft<10>(MlasReinterpretAsInt16(biased)); + auto normal = overflow; + + auto minimum_exponent = MlasReinterpretAsInt16(constants.MinimumExponent); + auto maximum_exponent = MlasReinterpretAsInt16(constants.MaximumExponent); + normal = MlasClamp(normal, minimum_exponent, maximum_exponent); + + overflow = MlasSubtract(overflow, normal); + overflow = MlasAdd(overflow, maximum_exponent); + normal = MlasAdd(normal, maximum_exponent); + + // polynomial approximation + auto p = MlasMultiplyAdd(constants.poly_0, r, constants.poly_1); + p = MlasMultiplyAdd(p, r, constants.poly_2); + p = MlasMultiplyAdd(p, r, constants.poly_3); + p = MlasMultiplyAdd(p, r, constants.poly_4); + p = MlasMultiplyAdd(p, r, constants.poly_56); + + auto overflow_f = MlasReinterpretAsFloat16(overflow); + r = MlasMultiply(r, overflow_f); + p = MlasMultiplyAdd(p, r, overflow_f); + p = MlasMultiply(p, MlasReinterpretAsFloat16(normal)); + + return p; +} + +void Exp_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N) { + const auto* input = reinterpret_cast(Input); + auto* output = reinterpret_cast<_mlas_fp16_*>(Output); + + while (N >= 32) { + auto v0 = MlasLoadFloat16x8(input); + auto v1 = MlasLoadFloat16x8(input + 8); + auto v2 = MlasLoadFloat16x8(input + 16); + auto v3 = MlasLoadFloat16x8(input + 24); + + auto r0 = Exp_Vector_Fp16(v0); + auto r1 = Exp_Vector_Fp16(v1); + auto r2 = Exp_Vector_Fp16(v2); + auto r3 = Exp_Vector_Fp16(v3); + + MlasStoreFloat16x8(output, r0); + MlasStoreFloat16x8(output + 8, r1); + MlasStoreFloat16x8(output + 16, r2); + MlasStoreFloat16x8(output + 24, r3); + + input += 32; + output += 32; + N -= 32; + } + + if (N & 16) { + auto v0 = MlasLoadFloat16x8(input); + auto v1 = MlasLoadFloat16x8(input + 8); + + auto r0 = Exp_Vector_Fp16(v0); + auto r1 = Exp_Vector_Fp16(v1); + + MlasStoreFloat16x8(output, r0); + MlasStoreFloat16x8(output + 8, r1); + + input += 16; + output += 16; + N -= 16; + } + + if (N & 8) { + auto v0 = MlasLoadFloat16x8(input); + auto r0 = Exp_Vector_Fp16(v0); + MlasStoreFloat16x8(output, r0); + + input += 8; + output += 8; + N -= 8; + } + + if (N & 4) { + auto v0 = MlasLoadFloat16x4(input); + auto r0 = Exp_Vector_Fp16(v0); + MlasStoreFloat16x4(output, r0); + + input += 4; + output += 4; + N -= 4; + } + + if (N == 3) { + auto v0 = MlasLoadPartialFloat16x4(input, 3); + auto r0 = Exp_Vector_Fp16(v0); + MlasStorePartialFloat16x4(output, r0, 3); + } else if (N == 2) { + auto v0 = MlasLoadPartialFloat16x4(input, 2); + auto r0 = Exp_Vector_Fp16(v0); + MlasStorePartialFloat16x4(output, r0, 2); + } else if (N == 1) { + auto v0 = MlasLoadPartialFloat16x4(input, 1); + auto r0 = Exp_Vector_Fp16(v0); + MlasStorePartialFloat16x4(output, r0, 1); + } +} + +// assume no overflow +template +MLAS_FORCEINLINE +T SumExp_Vector_Fp16(T x, T negative_maximum) { + const auto constants = Get_Exp_Constants(); + auto clamped_x = MlasMaximum(MlasAdd(x, negative_maximum), constants.LowerRangeSumExp); + + // integral + auto biased = MlasMultiplyAdd(clamped_x, constants.Log2Reciprocal, constants.RoundingBias); + auto m = MlasSubtract(biased, constants.RoundingBias); + + // residual + auto r = MlasMultiplyAdd(m, constants.Log2High, clamped_x); + r = MlasMultiplyAdd(m, constants.Log2Mid, r); + r = MlasMultiplyAdd(m, constants.Log2Low, r); + + // 2^m + auto normal = MlasShiftLeft<10>(MlasReinterpretAsInt16(biased)); + normal = MlasAdd(normal, MlasReinterpretAsInt16(constants.MaximumExponent)); + + // polynomial approximation + auto p = MlasMultiplyAdd(constants.poly_0, r, constants.poly_1); + p = MlasMultiplyAdd(p, r, constants.poly_2); + p = MlasMultiplyAdd(p, r, constants.poly_3); + p = MlasMultiplyAdd(p, r, constants.poly_4); + p = MlasMultiplyAdd(p, r, constants.poly_56); + p = MlasMultiplyAdd(p, r, constants.poly_56); + + p = MlasMultiply(p, MlasReinterpretAsFloat16(normal)); + + return p; +} + +MLAS_FORCEINLINE +float16x8_t AddUp(float16x8_t v0, float16x8_t v1, float16x8_t v2, float16x8_t v3, float16x8_t v4) { + v0 = MlasAdd(v0, v1); + v2 = MlasAdd(v2, v3); + return MlasAdd(MlasAdd(v0, v2), v4); +} + +MLAS_FORCEINLINE +float16x8_t AddUp(float16x8_t v0, float16x8_t v1, float16x8_t v2) { + return MlasAdd(MlasAdd(v0, v1), v2); +} + +MLAS_FP16 SumExp_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N, const MLAS_FP16 NegativeMaximum) { + const auto* input = reinterpret_cast(Input); + auto* output = reinterpret_cast<_mlas_fp16_*>(Output); + float16x8_t negative_maximum8 = MlasBroadcastFloat16x8(NegativeMaximum.val); + float16x4_t negative_maximum4 = MlasBroadcastFloat16x4(NegativeMaximum.val); + const bool store_output = Output != nullptr; + float16x8_t accumulator8 = MlasZeroFloat16x8(); + float16x4_t accumulator4 = MlasZeroFloat16x4(); + _mlas_fp16_ buffer[4] = {0}; + + while (N >= 32) { + auto v0 = MlasLoadFloat16x8(input); + auto v1 = MlasLoadFloat16x8(input + 8); + auto v2 = MlasLoadFloat16x8(input + 16); + auto v3 = MlasLoadFloat16x8(input + 24); + + auto r0 = SumExp_Vector_Fp16(v0, negative_maximum8); + auto r1 = SumExp_Vector_Fp16(v1, negative_maximum8); + auto r2 = SumExp_Vector_Fp16(v2, negative_maximum8); + auto r3 = SumExp_Vector_Fp16(v3, negative_maximum8); + + accumulator8 = AddUp(r0, r1, r2, r3, accumulator8); + + if (store_output) { + MlasStoreFloat16x8(output, r0); + MlasStoreFloat16x8(output + 8, r1); + MlasStoreFloat16x8(output + 16, r2); + MlasStoreFloat16x8(output + 24, r3); + output += 32; + } + + input += 32; + N -= 32; + } + + if (N & 16) { + auto v0 = MlasLoadFloat16x8(input); + auto v1 = MlasLoadFloat16x8(input + 8); + + auto r0 = SumExp_Vector_Fp16(v0, negative_maximum8); + auto r1 = SumExp_Vector_Fp16(v1, negative_maximum8); + + accumulator8 = AddUp(r0, r1, accumulator8); + + if (store_output) { + MlasStoreFloat16x8(output, r0); + MlasStoreFloat16x8(output + 8, r1); + output += 16; + } + + input += 16; + N -= 16; + } + + if (N & 8) { + auto v0 = MlasLoadFloat16x8(input); + auto r0 = SumExp_Vector_Fp16(v0, negative_maximum8); + accumulator8 = MlasAdd(r0, accumulator8); + + if (store_output) { + MlasStoreFloat16x8(output, r0); + output += 8; + } + + input += 8; + N -= 8; + } + + if (N & 4) { + auto v0 = MlasLoadFloat16x4(input); + auto r0 = SumExp_Vector_Fp16(v0, negative_maximum4); + accumulator4 = MlasAdd(r0, accumulator4); + + if (store_output) { + MlasStoreFloat16x4(output, r0); + output += 4; + } + + input += 4; + N -= 4; + } + + if (N == 3) { + auto v0 = MlasLoadPartialFloat16x4(input, 3); + auto r0 = SumExp_Vector_Fp16(v0, negative_maximum4); + + if (store_output) { + MlasStorePartialFloat16x4(output, r0, 3); + } + + r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 3)); + accumulator4 = MlasAdd(r0, accumulator4); + } else if (N == 2) { + auto v0 = MlasLoadPartialFloat16x4(input, 2); + auto r0 = SumExp_Vector_Fp16(v0, negative_maximum4); + + if (store_output) { + MlasStorePartialFloat16x4(output, r0, 2); + } + + r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 3)); + r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 2)); + accumulator4 = MlasAdd(r0, accumulator4); + } else if (N == 1) { + auto v0 = MlasLoadPartialFloat16x4(input, 1); + auto r0 = SumExp_Vector_Fp16(v0, negative_maximum4); + + if (store_output) { + MlasStorePartialFloat16x4(output, r0, 1); + } + + r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 3)); + r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 2)); + r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 1)); + accumulator4 = MlasAdd(r0, accumulator4); + } + + auto t = MlasAdd(vget_low_f16(accumulator8), vget_high_f16(accumulator8)); + t = MlasAdd(t, accumulator4); + _mlas_fp16_ result = MlasReduceAdd(t); + return MLAS_FP16::FromBits(result); +} + +const struct { + _mlas_fp16_ LowerRange; + _mlas_fp16_ UpperRange; + _mlas_fp16_ alpha_9; + _mlas_fp16_ alpha_7; + _mlas_fp16_ alpha_5; + _mlas_fp16_ alpha_3; + _mlas_fp16_ alpha_1; + _mlas_fp16_ beta_10; + _mlas_fp16_ beta_8; + _mlas_fp16_ beta_6; + _mlas_fp16_ beta_4; + _mlas_fp16_ beta_2; + _mlas_fp16_ beta_0; +} MlasTanh16Constants = { + 0xc500, // -5.0f16 + 0x4500, // 5.0f16 + 0x002e, // 1/9! + 0x0a80, // 1/7! + 0x2044, // 1/5! + 0x3155, // 1/3! + 0x3c00, // 1 + 0x0005, // 1/10! + 0x01a0, // 1/8! + 0x15b0, // 1/6! + 0x2955, // 1/4! + 0x3800, // 1/2! + 0x3c00, // 1 +}; + +// _Float16 my_tanh(_Float16 Value) { +// _Float16 v_tmp; +// v_tmp = (Value < MlasTanh16Constants.LowerRange) ? MlasTanh16Constants.LowerRange : Value; +// Value = (v_tmp > MlasTanh16Constants.UpperRange) ? MlasTanh16Constants.UpperRange : v_tmp; + +// _Float16 ValueSquared = Value * Value; + +// _Float16 p = MlasTanh16Constants.alpha_9; +// p = p * ValueSquared + MlasTanh16Constants.alpha_7; +// p = p * ValueSquared + MlasTanh16Constants.alpha_5; +// p = p * ValueSquared + MlasTanh16Constants.alpha_3; +// p = p * ValueSquared + MlasTanh16Constants.alpha_1; +// p = p * Value; + +// _Float16 q = MlasTanh16Constants.beta_10; +// q = q * ValueSquared + MlasTanh16Constants.beta_8; +// q = q * ValueSquared + MlasTanh16Constants.beta_6; +// q = q * ValueSquared + MlasTanh16Constants.beta_4; +// q = q * ValueSquared + MlasTanh16Constants.beta_2; +// q = q * ValueSquared + MlasTanh16Constants.beta_0; + +// return (p / q); +// } + + +// _Float16 my_tanh_no_overflow(_Float16 Value) { +// if (Value > 0.5f16) { +// _Float16 exp = my_exp(Value); +// return (exp - 1.0f16/exp) / (exp + 1.0f16/exp); +// } else { +// return my_tanh(Value); +// } +// } + void Tanh_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N) { } @@ -31,20 +477,10 @@ void Softcap_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N, co } -// exp kernel for fp16. Output and input can be the same buffer. -void Exp_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N) { - -} - -// reduce max kernel for fp16 MLAS_FP16 ReduceMax_Kernel_Fp16(const MLAS_FP16* Input, size_t N) { return MLAS_FP16::FromBits(0); } -MLAS_FP16 SumExp_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N, const MLAS_FP16 NegativeMaximum) { - return MLAS_FP16::FromBits(0); -} - void Softmax_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N, const MLAS_FP16 scale) { } diff --git a/onnxruntime/core/mlas/lib/tanh.cpp b/onnxruntime/core/mlas/lib/tanh.cpp index 0a13b88918..af1dccf1c5 100644 --- a/onnxruntime/core/mlas/lib/tanh.cpp +++ b/onnxruntime/core/mlas/lib/tanh.cpp @@ -200,9 +200,10 @@ MlasComputeTanh( dispatch->Tanh_Fp16(Input, Output, N); } +template <> void MLASCALL -MlasComputeSoftcap( +MlasComputeSoftcap( const MLAS_FP16* Input, MLAS_FP16* Output, size_t N, diff --git a/onnxruntime/test/mlas/unittest/test_exp.cpp b/onnxruntime/test/mlas/unittest/test_exp.cpp index 178e42fc3f..d25ad071bd 100644 --- a/onnxruntime/test/mlas/unittest/test_exp.cpp +++ b/onnxruntime/test/mlas/unittest/test_exp.cpp @@ -111,12 +111,12 @@ const struct { int16_t MinimumExponent; int16_t MaximumExponent; } MlasExp16Constants = { - -17.328679513f16, // -25 * ln2 - 11.090354888f16, // 16 * ln2 - -10.743781298f16, - 10.743781298f16, // 15.5 * ln2 - 1536.f16, // 1.5 * 2^10 - 1.4423828125f16, + -17.328679513f16, // -25 * ln2 cc55 + 11.090354888f16, // 16 * ln2 498c + -10.743781298f16, // -15.5 * ln2 c95f + 10.743781298f16, // 15.5 * ln2 495f + 1536.f16, // 1.5 * 2^10 6600 + 1.4423828125f16, // 1/ln2, 3dc5 -6.9287109375e-1f16, // 0xb98b -2.758502960205078e-4f16, // 0x8c85 -2.384185791015625e-7f16, // 0x8004 @@ -125,7 +125,7 @@ const struct { 4.1666666666666664e-2f16, // 1/4! 0x2955 1.6666666e-1f16, // 1/3! 0x3155 0.5f16, // 1/2! 0x3800 - 1.0f16, + 1.0f16, // 1/1! 0x3c00 int16_t(0xC800), // -14 int16_t(0x3C00), // 15 }; @@ -338,8 +338,8 @@ const struct { _Float16 beta_2; _Float16 beta_0; } MlasTanh16Constants = { - -5.0f16, - 5.0f16, + -5.0f16, // c500 + 5.0f16, // 4500 2.755731922398589e-06f16, // 0x002e 0.00019841269841269839f16, // 0xa80 0.008333333333333333f16, // 0x2044 @@ -409,9 +409,9 @@ const struct { } _Float16 my_tanh_no_overflow(_Float16 Value) { - if (Value > 0.45f16) { - _Float16 exp = my_exp(2.f16 * Value); - return 1 - 2.f16 / (1 + exp); + if (Value > 0.5f16) { + _Float16 exp = my_exp(Value); + return (exp - 1.0f16/exp) / (exp + 1.0f16/exp); } else { return my_tanh(Value); } @@ -433,7 +433,7 @@ const struct { void test_tanh_no_overflow(_Float16 x) { float ref = std::tanh((float)x); - float out = fast_tanh(x); + float out = my_tanh_no_overflow(x); float diff = std::abs(out - ref); std::cout << "x " << (float)x << ", result: " << out << ", expecting: " << ref << " diff " << diff / ref << std::endl; } @@ -445,6 +445,12 @@ const struct { } void ExecuteShort(void) override { + // print_hex("lower range ", MlasExp16Constants.LowerRange); + // print_hex("upper range ", MlasExp16Constants.UpperRange); + // print_hex("lower range sum exp ", MlasExp16Constants.LowerRangeSumExp); + // print_hex("upper range sum exp ", MlasExp16Constants.UpperRangeSumExp); + // print_hex("rounding bias ", MlasExp16Constants.RoundingBias); + // print_hex("log2 reciprocal ", MlasExp16Constants.Log2Reciprocal); // print_hex("h ", (_Float16)MlasExp16Constants.Log2High); // print_hex("l ", (_Float16)MlasExp16Constants.Log2Low); // print_hex("ll ", MlasExp16Constants.Log2Lowest); @@ -455,20 +461,20 @@ const struct { // print_hex("poly3 ", (_Float16)MlasExp16Constants.poly_3); // print_hex("poly4 ", (_Float16)MlasExp16Constants.poly_4); // Test(.01f16); - // print_hex("alpha_13 ", MlasTanh16Constants.alpha_13); - // print_hex("alpha_11 ", MlasTanh16Constants.alpha_11); - // print_hex("alpha_9 ", MlasTanh16Constants.alpha_9); - // print_hex("alpha_7 ", MlasTanh16Constants.alpha_7); - // print_hex("alpha_5 ", MlasTanh16Constants.alpha_5); - // print_hex("alpha_3 ", MlasTanh16Constants.alpha_3); - // print_hex("alpha_1 ", MlasTanh16Constants.alpha_1); - // print_hex("beta_10 ", MlasTanh16Constants.beta_10); - // print_hex("beta_8 ", MlasTanh16Constants.beta_8); - // print_hex("beta_6 ", MlasTanh16Constants.beta_6); - // print_hex("beta_4 ", MlasTanh16Constants.beta_4); - // print_hex("beta_2 ", MlasTanh16Constants.beta_2); - // print_hex("beta_0 ", MlasTanh16Constants.beta_0); - for (_Float16 x = 0.f16; x <= 5.f16; x += 0.005f16) { + print_hex("lower range ", MlasTanh16Constants.LowerRange); + print_hex("upper range ", MlasTanh16Constants.UpperRange); + print_hex("alpha_9 ", MlasTanh16Constants.alpha_9); + print_hex("alpha_7 ", MlasTanh16Constants.alpha_7); + print_hex("alpha_5 ", MlasTanh16Constants.alpha_5); + print_hex("alpha_3 ", MlasTanh16Constants.alpha_3); + print_hex("alpha_1 ", MlasTanh16Constants.alpha_1); + print_hex("beta_10 ", MlasTanh16Constants.beta_10); + print_hex("beta_8 ", MlasTanh16Constants.beta_8); + print_hex("beta_6 ", MlasTanh16Constants.beta_6); + print_hex("beta_4 ", MlasTanh16Constants.beta_4); + print_hex("beta_2 ", MlasTanh16Constants.beta_2); + print_hex("beta_0 ", MlasTanh16Constants.beta_0); + for (_Float16 x = 0.f16; x <= 9.f16; x += 0.005f16) { test_tanh_no_overflow(x); } }