mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
finished exp and sum exp kernels
This commit is contained in:
Jing Fang
parent
7d7b535349
commit
7dd6ceede1
9 changed files with 645 additions and 92 deletions
|
|
@ -1020,13 +1020,14 @@ MlasComputeSoftmax(
|
|||
MLAS_THREADPOOL* ThreadPool
|
||||
);
|
||||
|
||||
template <typename T>
|
||||
void
|
||||
MLASCALL
|
||||
MlasComputeSoftcap(
|
||||
const MLAS_FP16* Input,
|
||||
MLAS_FP16* Output,
|
||||
const T* Input,
|
||||
T* Output,
|
||||
size_t N,
|
||||
MLAS_FP16 cap
|
||||
T cap
|
||||
);
|
||||
|
||||
template<typename T>
|
||||
|
|
|
|||
|
|
@ -51,12 +51,12 @@ struct MLAS_HALF_ACTIVATION_FUNCTION<MlasReluActivation>
|
|||
|
||||
MLAS_FLOAT16X8 Activate(MLAS_FLOAT16X8 Value)
|
||||
{
|
||||
return MlasMaximumFloat16x8(ZeroVec, Value);
|
||||
return MlasMaximum(ZeroVec, Value);
|
||||
}
|
||||
|
||||
MLAS_FLOAT16X4 Activate(MLAS_FLOAT16X4 Value)
|
||||
{
|
||||
return MlasMaximumFloat16x4(MlasToLowHalfFloat16x4(ZeroVec), Value);
|
||||
return MlasMaximum(MlasToLowHalfFloat16x4(ZeroVec), Value);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -75,7 +75,7 @@ struct MLAS_HALF_ACTIVATION_FUNCTION<MlasLeakyReluActivation>
|
|||
|
||||
MLAS_FLOAT16X8 Activate(MLAS_FLOAT16X8 Value)
|
||||
{
|
||||
MLAS_FLOAT16X8 ValueTimesAlpha = MlasMultiplyFloat16x8(Value, AlphaBroadcast);
|
||||
MLAS_FLOAT16X8 ValueTimesAlpha = MlasMultiply(Value, AlphaBroadcast);
|
||||
return MlasBitwiseSelectFloat16x8(MlasCmpLessEqualFloat16x8(Value, ZeroVec),
|
||||
ValueTimesAlpha, Value);
|
||||
}
|
||||
|
|
@ -83,7 +83,7 @@ struct MLAS_HALF_ACTIVATION_FUNCTION<MlasLeakyReluActivation>
|
|||
MLAS_FLOAT16X4 Activate(MLAS_FLOAT16X4 Value)
|
||||
{
|
||||
MLAS_FLOAT16X4 ValueTimesAlpha =
|
||||
MlasMultiplyFloat16x4(Value, MlasToLowHalfFloat16x4(AlphaBroadcast));
|
||||
MlasMultiply(Value, MlasToLowHalfFloat16x4(AlphaBroadcast));
|
||||
return MlasBitwiseSelectFloat16x4(
|
||||
MlasCmpLessEqualFloat16x4(Value, MlasToLowHalfFloat16x4(ZeroVec)), ValueTimesAlpha,
|
||||
Value);
|
||||
|
|
@ -539,16 +539,16 @@ struct MLAS_HALF_ACTIVATION_FUNCTION<MlasClipActivation> {
|
|||
|
||||
MLAS_FLOAT16X8 Activate(MLAS_FLOAT16X8 Value)
|
||||
{
|
||||
Value = MlasMaximumFloat16x8(MinimumBroadcast, Value);
|
||||
Value = MlasMinimumFloat16x8(MaximumBroadcast, Value);
|
||||
Value = MlasMaximum(MinimumBroadcast, Value);
|
||||
Value = MlasMinimum(MaximumBroadcast, Value);
|
||||
|
||||
return Value;
|
||||
}
|
||||
|
||||
MLAS_FLOAT16X4 Activate(MLAS_FLOAT16X4 Value)
|
||||
{
|
||||
Value = MlasMaximumFloat16x4(MlasToLowHalfFloat16x4(MinimumBroadcast), Value);
|
||||
Value = MlasMinimumFloat16x4(MlasToLowHalfFloat16x4(MaximumBroadcast), Value);
|
||||
Value = MlasMaximum(MlasToLowHalfFloat16x4(MinimumBroadcast), Value);
|
||||
Value = MlasMinimum(MlasToLowHalfFloat16x4(MaximumBroadcast), Value);
|
||||
return Value;
|
||||
}
|
||||
};
|
||||
|
|
@ -573,19 +573,19 @@ struct MLAS_HALF_ACTIVATION_FUNCTION<MlasHardSigmoidActivation>
|
|||
|
||||
MLAS_FLOAT16X8 Activate(MLAS_FLOAT16X8 Value)
|
||||
{
|
||||
Value = MlasMultiplyAddFloat16x8(Value, AlphaBroadcast, BetaBroadcast);
|
||||
Value = MlasMinimumFloat16x8(MaximumBroadcast, Value);
|
||||
Value = MlasMaximumFloat16x8(MinimumBroadcast, Value);
|
||||
Value = MlasMultiplyAdd(Value, AlphaBroadcast, BetaBroadcast);
|
||||
Value = MlasMinimum(MaximumBroadcast, Value);
|
||||
Value = MlasMaximum(MinimumBroadcast, Value);
|
||||
|
||||
return Value;
|
||||
}
|
||||
|
||||
MLAS_FLOAT16X4 Activate(MLAS_FLOAT16X4 Value)
|
||||
{
|
||||
Value = MlasMultiplyAddFloat16x4(Value, MlasToLowHalfFloat16x4(AlphaBroadcast),
|
||||
Value = MlasMultiplyAdd(Value, MlasToLowHalfFloat16x4(AlphaBroadcast),
|
||||
MlasToLowHalfFloat16x4(BetaBroadcast));
|
||||
Value = MlasMinimumFloat16x4(MlasToLowHalfFloat16x4(MaximumBroadcast), Value);
|
||||
Value = MlasMaximumFloat16x4(MlasToLowHalfFloat16x4(MinimumBroadcast), Value);
|
||||
Value = MlasMinimum(MlasToLowHalfFloat16x4(MaximumBroadcast), Value);
|
||||
Value = MlasMaximum(MlasToLowHalfFloat16x4(MinimumBroadcast), Value);
|
||||
|
||||
return Value;
|
||||
}
|
||||
|
|
@ -692,7 +692,7 @@ MlasActivationKernel(
|
|||
MLAS_FLOAT16X8 AVec = MlasLoadFloat16x8(addsrc);
|
||||
MLAS_FLOAT16X8 Vector = MlasLoadFloat16x8(buffer);
|
||||
addsrc += 8;
|
||||
Vector = MlasAddFloat16x8(Vector, AVec);
|
||||
Vector = MlasAdd(Vector, AVec);
|
||||
Vector = ActivationFunction.Activate(Vector);
|
||||
MlasStoreFloat16x8(buffer, Vector);
|
||||
buffer += 8;
|
||||
|
|
@ -703,7 +703,7 @@ MlasActivationKernel(
|
|||
MLAS_FLOAT16X4 AVec = MlasLoadFloat16x4(addsrc);
|
||||
MLAS_FLOAT16X4 Vector = MlasLoadFloat16x4(buffer);
|
||||
addsrc += 4;
|
||||
Vector = MlasAddFloat16x4(Vector, AVec);
|
||||
Vector = MlasAdd(Vector, AVec);
|
||||
Vector = ActivationFunction.Activate(Vector);
|
||||
MlasStoreFloat16x4(buffer, Vector);
|
||||
buffer += 4;
|
||||
|
|
@ -715,7 +715,7 @@ MlasActivationKernel(
|
|||
MLAS_FLOAT16X4 buf;
|
||||
std::memcpy(&addbuf, addsrc, n * sizeof(_mlas_fp16_));
|
||||
std::memcpy(&buf, buffer, n * sizeof(_mlas_fp16_));
|
||||
buf = MlasAddFloat16x4(buf, addbuf);
|
||||
buf = MlasAdd(buf, addbuf);
|
||||
buf = ActivationFunction.Activate(buf);
|
||||
MlasStorePartialFloat16x4(buffer, buf, n);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ MlasConvDepthwiseKernel(
|
|||
MLAS_FLOAT16X8 InputVector = MlasLoadFloat16x8(&Input[k][ChannelOffset]);
|
||||
MLAS_FLOAT16X8 FilterVector = MlasLoadFloat16x8(&Filter[ChannelKernelOffset]);
|
||||
|
||||
Accumulator = MlasMultiplyAddFloat16x8(InputVector, FilterVector, Accumulator);
|
||||
Accumulator = MlasMultiplyAdd(InputVector, FilterVector, Accumulator);
|
||||
ChannelKernelOffset += Channels;
|
||||
}
|
||||
MlasStoreFloat16x8(Output, Accumulator);
|
||||
|
|
@ -61,7 +61,7 @@ MlasConvDepthwiseKernel(
|
|||
MLAS_FLOAT16X4 InputVector = MlasLoadFloat16x4(&Input[k][ChannelOffset]);
|
||||
MLAS_FLOAT16X4 FilterVector = MlasLoadFloat16x4(&Filter[ChannelKernelOffset]);
|
||||
|
||||
Accumulator = MlasMultiplyAddFloat16x4(InputVector, FilterVector, Accumulator);
|
||||
Accumulator = MlasMultiplyAdd(InputVector, FilterVector, Accumulator);
|
||||
ChannelKernelOffset += Channels;
|
||||
}
|
||||
MlasStoreFloat16x4(Output, Accumulator);
|
||||
|
|
@ -80,7 +80,7 @@ MlasConvDepthwiseKernel(
|
|||
MLAS_FLOAT16X4 InputValue = MlasLoadFloat16x4(&Input[k][ChannelOffset]);
|
||||
MLAS_FLOAT16X4 FilterValue = MlasLoadFloat16x4(&Filter[ChannelKernelOffset]);
|
||||
|
||||
Accumulator = MlasMultiplyAddFloat16x4(InputValue, FilterValue, Accumulator);
|
||||
Accumulator = MlasMultiplyAdd(InputValue, FilterValue, Accumulator);
|
||||
ChannelKernelOffset += Channels;
|
||||
}
|
||||
MlasStorePartialFloat16x4(Output, Accumulator, c);
|
||||
|
|
|
|||
|
|
@ -27,10 +27,28 @@ typedef float16x8_t MLAS_FLOAT16X8;
|
|||
typedef float16x4_t MLAS_FLOAT16X4;
|
||||
typedef uint16x8_t MLAS_UINT16X8;
|
||||
typedef uint16x4_t MLAS_UINT16X4;
|
||||
typedef int16x8_t MLAS_INT16X8;
|
||||
typedef int16x4_t MLAS_INT16X4;
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
MlasReinterpretAsFloat16x8(MLAS_INT32X4 Vector) { return vreinterpretq_f16_s32(Vector); }
|
||||
MlasReinterpretAsFloat16(MLAS_INT32X4 Vector) { return vreinterpretq_f16_s32(Vector); }
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
MlasReinterpretAsFloat16(MLAS_INT16X8 Vector) { return vreinterpretq_f16_s16(Vector); }
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X4
|
||||
MlasReinterpretAsFloat16(MLAS_INT16X4 Vector) { return vreinterpret_f16_s16(Vector); }
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X8
|
||||
MlasReinterpretAsInt16(MLAS_FLOAT16X8 Vector) { return vreinterpretq_s16_f16(Vector); }
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X4
|
||||
MlasReinterpretAsInt16(MLAS_FLOAT16X4 Vector) { return vreinterpret_s16_f16(Vector); }
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
|
|
@ -142,42 +160,70 @@ MlasToLowHalfFloat16x4(MLAS_FLOAT16X8 V)
|
|||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
MlasAddFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2)
|
||||
MlasAdd(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2)
|
||||
{
|
||||
return vaddq_f16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X4
|
||||
MlasAddFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
||||
MlasAdd(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
||||
{
|
||||
return vadd_f16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X8
|
||||
MlasAdd(MLAS_INT16X8 Vector1, MLAS_INT16X8 Vector2)
|
||||
{
|
||||
return vaddq_s16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X4
|
||||
MlasAdd(MLAS_INT16X4 Vector1, MLAS_INT16X4 Vector2)
|
||||
{
|
||||
return vadd_s16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
MlasSubtractFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2)
|
||||
MlasSubtract(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2)
|
||||
{
|
||||
return vsubq_f16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X4
|
||||
MlasSubtractFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
||||
MlasSubtract(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
||||
{
|
||||
return vsub_f16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X8
|
||||
MlasSubtract(MLAS_INT16X8 Vector1, MLAS_INT16X8 Vector2)
|
||||
{
|
||||
return vsubq_s16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X4
|
||||
MlasSubtract(MLAS_INT16X4 Vector1, MLAS_INT16X4 Vector2)
|
||||
{
|
||||
return vsub_s16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
MlasMultiplyFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2)
|
||||
MlasMultiply(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2)
|
||||
{
|
||||
return vmulq_f16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X4
|
||||
MlasMultiplyFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
||||
MlasMultiply(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
||||
{
|
||||
return vmul_f16(Vector1, Vector2);
|
||||
}
|
||||
|
|
@ -198,31 +244,30 @@ MlasDivFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
|||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
MlasMultiplyAddFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2, MLAS_FLOAT16X8 Vector3)
|
||||
MlasMultiplyAdd(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2, MLAS_FLOAT16X8 Dest)
|
||||
{
|
||||
return vfmaq_f16(Vector3, Vector1, Vector2);
|
||||
return vfmaq_f16(Dest, Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X4
|
||||
MlasMultiplyAddFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2, MLAS_FLOAT16X4 Vector3)
|
||||
MlasMultiplyAdd(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2, MLAS_FLOAT16X4 Dest)
|
||||
{
|
||||
return vfma_f16(Vector3, Vector1, Vector2);
|
||||
return vfma_f16(Dest, Vector1, Vector2);
|
||||
}
|
||||
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
void
|
||||
MlasMultiplyAddFloat16x8(MLAS_FLOAT16X8 Vector1, _mlas_fp16_ Scalar2, MLAS_FLOAT16X8 Vector3)
|
||||
{
|
||||
MlasMultiplyAddFloat16x8(Vector1, MlasBroadcastFloat16x8(Scalar2), Vector3);
|
||||
MlasMultiplyAdd(Vector1, MlasBroadcastFloat16x8(Scalar2), Vector3);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
void
|
||||
MlasMultiplyAddFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2, _mlas_fp16_ Scalar3)
|
||||
{
|
||||
MlasMultiplyAddFloat16x8(Vector1, Vector2, MlasBroadcastFloat16x8(Scalar3));
|
||||
MlasMultiplyAdd(Vector1, Vector2, MlasBroadcastFloat16x8(Scalar3));
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
|
|
@ -277,50 +322,98 @@ MlasBlendFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2, MLAS_FLOAT16X
|
|||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
MlasMaximumFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2)
|
||||
MlasMaximum(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2)
|
||||
{
|
||||
return vmaxq_f16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X4
|
||||
MlasMaximumFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
||||
MlasMaximum(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
||||
{
|
||||
return vmax_f16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X8
|
||||
MlasMaximum(MLAS_INT16X8 Vector1, MLAS_INT16X8 Vector2)
|
||||
{
|
||||
return vmaxq_s16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X4
|
||||
MlasMaximum(MLAS_INT16X4 Vector1, MLAS_INT16X4 Vector2)
|
||||
{
|
||||
return vmax_s16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
MlasMinimumFloat16x8(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2)
|
||||
MlasMinimum(MLAS_FLOAT16X8 Vector1, MLAS_FLOAT16X8 Vector2)
|
||||
{
|
||||
return vminq_f16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X4
|
||||
MlasMinimumFloat16x4(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
||||
MlasMinimum(MLAS_FLOAT16X4 Vector1, MLAS_FLOAT16X4 Vector2)
|
||||
{
|
||||
return vmin_f16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X8
|
||||
MlasMinimum(MLAS_INT16X8 Vector1, MLAS_INT16X8 Vector2)
|
||||
{
|
||||
return vminq_s16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X4
|
||||
MlasMinimum(MLAS_INT16X4 Vector1, MLAS_INT16X4 Vector2)
|
||||
{
|
||||
return vmin_s16(Vector1, Vector2);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
MlasClampFloat16x8(MLAS_FLOAT16X8 Value, _mlas_fp16_ LowerRange, _mlas_fp16_ UpperRange)
|
||||
{
|
||||
Value = MlasMaximumFloat16x8(MlasBroadcastFloat16x8(LowerRange), Value);
|
||||
Value = MlasMinimumFloat16x8(MlasBroadcastFloat16x8(UpperRange), Value);
|
||||
Value = MlasMaximum(MlasBroadcastFloat16x8(LowerRange), Value);
|
||||
Value = MlasMinimum(MlasBroadcastFloat16x8(UpperRange), Value);
|
||||
return Value;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
MLAS_FORCEINLINE
|
||||
T
|
||||
MlasClamp(T Value, T LowerRange, T UpperRange)
|
||||
{
|
||||
Value = MlasMaximum(LowerRange, Value);
|
||||
Value = MlasMinimum(UpperRange, Value);
|
||||
return Value;
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
_mlas_fp16_
|
||||
MlasReduceAddFloat16x8(MLAS_FLOAT16X8 Vector)
|
||||
MlasReduceAdd(MLAS_FLOAT16X8 Vector)
|
||||
{
|
||||
Vector = vpaddq_f16(Vector, Vector);
|
||||
Vector = vpaddq_f16(Vector, Vector);
|
||||
Vector = vpaddq_f16(Vector, Vector);
|
||||
return vgetq_lane_u16(vreinterpretq_u16_f16(Vector), 0);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
_mlas_fp16_
|
||||
MlasReduceAdd(MLAS_FLOAT16X4 Vector)
|
||||
{
|
||||
Vector = vpadd_f16(Vector, Vector);
|
||||
Vector = vpadd_f16(Vector, Vector);
|
||||
return vget_lane_u16(vreinterpret_u16_f16(Vector), 0);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_UINT16X8
|
||||
MlasCmpLessEqualFloat16x8(MLAS_FLOAT16X8 left, MLAS_FLOAT16X8 right)
|
||||
|
|
@ -448,4 +541,20 @@ Transpose4x4(MLAS_FLOAT16X4& v0, MLAS_FLOAT16X4& v1, MLAS_FLOAT16X4& v2, MLAS_FL
|
|||
v3 = vreinterpret_f16_f32(vtrn2_f32(vreinterpret_f32_f16(t01.val[1]), vreinterpret_f32_f16(t23.val[1])));
|
||||
}
|
||||
|
||||
template<unsigned ShiftCount>
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X8
|
||||
MlasShiftLeft(MLAS_INT16X8 Vector)
|
||||
{
|
||||
return vshlq_n_s16(Vector, ShiftCount);
|
||||
}
|
||||
|
||||
template<unsigned ShiftCount>
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT16X4
|
||||
MlasShiftLeft(MLAS_INT16X4 Vector)
|
||||
{
|
||||
return vshl_n_s16(Vector, ShiftCount);
|
||||
}
|
||||
|
||||
#endif // fp16 vector intrinsic supported
|
||||
|
|
|
|||
|
|
@ -80,23 +80,23 @@ PoolInit16x4<MaxPoolAggregation>()
|
|||
}
|
||||
|
||||
template<>
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
PoolAggregate16x8<MaxPoolAggregation>(MLAS_FLOAT16X8 agg, MLAS_FLOAT16X8 element)
|
||||
{
|
||||
return MlasMaximumFloat16x8(agg, element);
|
||||
return MlasMaximum(agg, element);
|
||||
}
|
||||
|
||||
template<>
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X4
|
||||
PoolAggregate16x4<MaxPoolAggregation>(MLAS_FLOAT16X4 agg, MLAS_FLOAT16X4 element)
|
||||
{
|
||||
return MlasMaximumFloat16x4(agg, element);
|
||||
return MlasMaximum(agg, element);
|
||||
}
|
||||
|
||||
template<>
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X8
|
||||
PoolSummary16x8<MaxPoolAggregation>(MLAS_FLOAT16X8 agg, size_t size)
|
||||
{
|
||||
|
|
@ -105,7 +105,7 @@ PoolSummary16x8<MaxPoolAggregation>(MLAS_FLOAT16X8 agg, size_t size)
|
|||
}
|
||||
|
||||
template<>
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_FLOAT16X4
|
||||
PoolSummary16x4<MaxPoolAggregation>(MLAS_FLOAT16X4 agg, size_t size)
|
||||
{
|
||||
|
|
@ -144,14 +144,14 @@ template <>
|
|||
MLAS_FORCEINLINE MLAS_FLOAT16X8
|
||||
PoolAggregate16x8<AveragePoolAggregation>(MLAS_FLOAT16X8 agg, MLAS_FLOAT16X8 element)
|
||||
{
|
||||
return MlasAddFloat16x8(agg, element);
|
||||
return MlasAdd(agg, element);
|
||||
}
|
||||
|
||||
template <>
|
||||
MLAS_FORCEINLINE MLAS_FLOAT16X4
|
||||
PoolAggregate16x4<AveragePoolAggregation>(MLAS_FLOAT16X4 agg, MLAS_FLOAT16X4 element)
|
||||
{
|
||||
return MlasAddFloat16x4(agg, element);
|
||||
return MlasAdd(agg, element);
|
||||
}
|
||||
|
||||
template <>
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ struct MLAS_SOFTMAX_DISPATCH {
|
|||
|
||||
/**
|
||||
* @brief Compute the expotential function for each element of the input array and returnt he sum. It has smaller
|
||||
* dynamic range for the input than Exp_Fp16_Fn.
|
||||
* dynamic range for the input than Exp_Fp16_Fn thus is faster.
|
||||
* @param Input Address of the input array
|
||||
* @param Output Address of the output array. Could be the same as the input array or nullptr.
|
||||
* @param N Number of elements in the input array
|
||||
|
|
|
|||
|
|
@ -22,7 +22,453 @@ Abstract:
|
|||
|
||||
namespace softmax_neon {
|
||||
|
||||
// tanh kernel for fp16. Output and input can be the same buffer.
|
||||
template <typename T>
|
||||
struct MlasExpConstants {
|
||||
T LowerRange;
|
||||
T UpperRange;
|
||||
T LowerRangeSumExp;
|
||||
T UpperRangeSumExp;
|
||||
T RoundingBias;
|
||||
T Log2Reciprocal;
|
||||
T Log2High;
|
||||
T Log2Mid;
|
||||
T Log2Low;
|
||||
T poly_0;
|
||||
T poly_1;
|
||||
T poly_2;
|
||||
T poly_3;
|
||||
T poly_4;
|
||||
T poly_56;
|
||||
T MinimumExponent;
|
||||
T MaximumExponent;
|
||||
};
|
||||
|
||||
const MlasExpConstants<_mlas_fp16_> ExpConstantsFp16 = {
|
||||
0xcc55, // -25 * ln2
|
||||
0x498c, // 16 * ln2
|
||||
0xc95f, // -15.5 * ln2
|
||||
0x495f, // 15.5 * ln2
|
||||
0x6600, // 1.5 * 2^10
|
||||
0x3dc5, // 1/ln2
|
||||
0xb98b, // -6.9287109375e-1f16
|
||||
0x8c85, // -2.758502960205078e-4f16
|
||||
0x8004, // -2.384185791015625e-7f16
|
||||
0x15b0, // 1/6!
|
||||
0x2044, // 1/5!
|
||||
0x2955, // 1/4!
|
||||
0x3155, // 1/3!
|
||||
0x3800, // 1/2!
|
||||
0x3c00, // 1/1!
|
||||
0xC800, // -14
|
||||
0x3C00, // 15
|
||||
};
|
||||
|
||||
const MlasExpConstants<float16x8_t> ExpConstantsFp16x8 = {
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.LowerRange),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.UpperRange),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.LowerRangeSumExp),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.UpperRangeSumExp),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.RoundingBias),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.Log2Reciprocal),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.Log2High),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.Log2Mid),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.Log2Low),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.poly_0),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.poly_1),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.poly_2),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.poly_3),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.poly_4),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.poly_56),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.MinimumExponent),
|
||||
MlasBroadcastFloat16x8(ExpConstantsFp16.MaximumExponent),
|
||||
};
|
||||
|
||||
const MlasExpConstants<float16x4_t> ExpConstantsFp16x4 = {
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.LowerRange),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.UpperRange),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.LowerRangeSumExp),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.UpperRangeSumExp),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.RoundingBias),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.Log2Reciprocal),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.Log2High),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.Log2Mid),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.Log2Low),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.poly_0),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.poly_1),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.poly_2),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.poly_3),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.poly_4),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.poly_56),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.MinimumExponent),
|
||||
MlasBroadcastFloat16x4(ExpConstantsFp16.MaximumExponent),
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
MLAS_FORCEINLINE
|
||||
MlasExpConstants<T> Get_Exp_Constants();
|
||||
|
||||
template <>
|
||||
MLAS_FORCEINLINE
|
||||
MlasExpConstants<float16x8_t> Get_Exp_Constants<float16x8_t>() {
|
||||
return ExpConstantsFp16x8;
|
||||
}
|
||||
|
||||
template <>
|
||||
MLAS_FORCEINLINE
|
||||
MlasExpConstants<float16x4_t> Get_Exp_Constants<float16x4_t>() {
|
||||
return ExpConstantsFp16x4;
|
||||
}
|
||||
|
||||
// Range reduction + polynomial approximation. Refer algorithm details to MlasComputeExpVector.
|
||||
template<typename T>
|
||||
MLAS_FORCEINLINE
|
||||
T Exp_Vector_Fp16(T x) {
|
||||
const auto constants = Get_Exp_Constants<T>();
|
||||
auto clamped_x = MlasClamp(x, constants.LowerRange, constants.UpperRange);
|
||||
|
||||
// integral
|
||||
auto biased = MlasMultiplyAdd(clamped_x, constants.Log2Reciprocal, constants.RoundingBias);
|
||||
auto m = MlasSubtract(biased, constants.RoundingBias);
|
||||
|
||||
// residual
|
||||
auto r = MlasMultiplyAdd(m, constants.Log2High, clamped_x);
|
||||
r = MlasMultiplyAdd(m, constants.Log2Mid, r);
|
||||
r = MlasMultiplyAdd(m, constants.Log2Low, r);
|
||||
|
||||
// handle overflow
|
||||
auto overflow = MlasShiftLeft<10>(MlasReinterpretAsInt16(biased));
|
||||
auto normal = overflow;
|
||||
|
||||
auto minimum_exponent = MlasReinterpretAsInt16(constants.MinimumExponent);
|
||||
auto maximum_exponent = MlasReinterpretAsInt16(constants.MaximumExponent);
|
||||
normal = MlasClamp(normal, minimum_exponent, maximum_exponent);
|
||||
|
||||
overflow = MlasSubtract(overflow, normal);
|
||||
overflow = MlasAdd(overflow, maximum_exponent);
|
||||
normal = MlasAdd(normal, maximum_exponent);
|
||||
|
||||
// polynomial approximation
|
||||
auto p = MlasMultiplyAdd(constants.poly_0, r, constants.poly_1);
|
||||
p = MlasMultiplyAdd(p, r, constants.poly_2);
|
||||
p = MlasMultiplyAdd(p, r, constants.poly_3);
|
||||
p = MlasMultiplyAdd(p, r, constants.poly_4);
|
||||
p = MlasMultiplyAdd(p, r, constants.poly_56);
|
||||
|
||||
auto overflow_f = MlasReinterpretAsFloat16(overflow);
|
||||
r = MlasMultiply(r, overflow_f);
|
||||
p = MlasMultiplyAdd(p, r, overflow_f);
|
||||
p = MlasMultiply(p, MlasReinterpretAsFloat16(normal));
|
||||
|
||||
return p;
|
||||
}
|
||||
|
||||
void Exp_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N) {
|
||||
const auto* input = reinterpret_cast<const _mlas_fp16_*>(Input);
|
||||
auto* output = reinterpret_cast<_mlas_fp16_*>(Output);
|
||||
|
||||
while (N >= 32) {
|
||||
auto v0 = MlasLoadFloat16x8(input);
|
||||
auto v1 = MlasLoadFloat16x8(input + 8);
|
||||
auto v2 = MlasLoadFloat16x8(input + 16);
|
||||
auto v3 = MlasLoadFloat16x8(input + 24);
|
||||
|
||||
auto r0 = Exp_Vector_Fp16(v0);
|
||||
auto r1 = Exp_Vector_Fp16(v1);
|
||||
auto r2 = Exp_Vector_Fp16(v2);
|
||||
auto r3 = Exp_Vector_Fp16(v3);
|
||||
|
||||
MlasStoreFloat16x8(output, r0);
|
||||
MlasStoreFloat16x8(output + 8, r1);
|
||||
MlasStoreFloat16x8(output + 16, r2);
|
||||
MlasStoreFloat16x8(output + 24, r3);
|
||||
|
||||
input += 32;
|
||||
output += 32;
|
||||
N -= 32;
|
||||
}
|
||||
|
||||
if (N & 16) {
|
||||
auto v0 = MlasLoadFloat16x8(input);
|
||||
auto v1 = MlasLoadFloat16x8(input + 8);
|
||||
|
||||
auto r0 = Exp_Vector_Fp16(v0);
|
||||
auto r1 = Exp_Vector_Fp16(v1);
|
||||
|
||||
MlasStoreFloat16x8(output, r0);
|
||||
MlasStoreFloat16x8(output + 8, r1);
|
||||
|
||||
input += 16;
|
||||
output += 16;
|
||||
N -= 16;
|
||||
}
|
||||
|
||||
if (N & 8) {
|
||||
auto v0 = MlasLoadFloat16x8(input);
|
||||
auto r0 = Exp_Vector_Fp16(v0);
|
||||
MlasStoreFloat16x8(output, r0);
|
||||
|
||||
input += 8;
|
||||
output += 8;
|
||||
N -= 8;
|
||||
}
|
||||
|
||||
if (N & 4) {
|
||||
auto v0 = MlasLoadFloat16x4(input);
|
||||
auto r0 = Exp_Vector_Fp16(v0);
|
||||
MlasStoreFloat16x4(output, r0);
|
||||
|
||||
input += 4;
|
||||
output += 4;
|
||||
N -= 4;
|
||||
}
|
||||
|
||||
if (N == 3) {
|
||||
auto v0 = MlasLoadPartialFloat16x4(input, 3);
|
||||
auto r0 = Exp_Vector_Fp16(v0);
|
||||
MlasStorePartialFloat16x4(output, r0, 3);
|
||||
} else if (N == 2) {
|
||||
auto v0 = MlasLoadPartialFloat16x4(input, 2);
|
||||
auto r0 = Exp_Vector_Fp16(v0);
|
||||
MlasStorePartialFloat16x4(output, r0, 2);
|
||||
} else if (N == 1) {
|
||||
auto v0 = MlasLoadPartialFloat16x4(input, 1);
|
||||
auto r0 = Exp_Vector_Fp16(v0);
|
||||
MlasStorePartialFloat16x4(output, r0, 1);
|
||||
}
|
||||
}
|
||||
|
||||
// assume no overflow
|
||||
template<typename T>
|
||||
MLAS_FORCEINLINE
|
||||
T SumExp_Vector_Fp16(T x, T negative_maximum) {
|
||||
const auto constants = Get_Exp_Constants<T>();
|
||||
auto clamped_x = MlasMaximum(MlasAdd(x, negative_maximum), constants.LowerRangeSumExp);
|
||||
|
||||
// integral
|
||||
auto biased = MlasMultiplyAdd(clamped_x, constants.Log2Reciprocal, constants.RoundingBias);
|
||||
auto m = MlasSubtract(biased, constants.RoundingBias);
|
||||
|
||||
// residual
|
||||
auto r = MlasMultiplyAdd(m, constants.Log2High, clamped_x);
|
||||
r = MlasMultiplyAdd(m, constants.Log2Mid, r);
|
||||
r = MlasMultiplyAdd(m, constants.Log2Low, r);
|
||||
|
||||
// 2^m
|
||||
auto normal = MlasShiftLeft<10>(MlasReinterpretAsInt16(biased));
|
||||
normal = MlasAdd(normal, MlasReinterpretAsInt16(constants.MaximumExponent));
|
||||
|
||||
// polynomial approximation
|
||||
auto p = MlasMultiplyAdd(constants.poly_0, r, constants.poly_1);
|
||||
p = MlasMultiplyAdd(p, r, constants.poly_2);
|
||||
p = MlasMultiplyAdd(p, r, constants.poly_3);
|
||||
p = MlasMultiplyAdd(p, r, constants.poly_4);
|
||||
p = MlasMultiplyAdd(p, r, constants.poly_56);
|
||||
p = MlasMultiplyAdd(p, r, constants.poly_56);
|
||||
|
||||
p = MlasMultiply(p, MlasReinterpretAsFloat16(normal));
|
||||
|
||||
return p;
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
float16x8_t AddUp(float16x8_t v0, float16x8_t v1, float16x8_t v2, float16x8_t v3, float16x8_t v4) {
|
||||
v0 = MlasAdd(v0, v1);
|
||||
v2 = MlasAdd(v2, v3);
|
||||
return MlasAdd(MlasAdd(v0, v2), v4);
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
float16x8_t AddUp(float16x8_t v0, float16x8_t v1, float16x8_t v2) {
|
||||
return MlasAdd(MlasAdd(v0, v1), v2);
|
||||
}
|
||||
|
||||
MLAS_FP16 SumExp_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N, const MLAS_FP16 NegativeMaximum) {
|
||||
const auto* input = reinterpret_cast<const _mlas_fp16_*>(Input);
|
||||
auto* output = reinterpret_cast<_mlas_fp16_*>(Output);
|
||||
float16x8_t negative_maximum8 = MlasBroadcastFloat16x8(NegativeMaximum.val);
|
||||
float16x4_t negative_maximum4 = MlasBroadcastFloat16x4(NegativeMaximum.val);
|
||||
const bool store_output = Output != nullptr;
|
||||
float16x8_t accumulator8 = MlasZeroFloat16x8();
|
||||
float16x4_t accumulator4 = MlasZeroFloat16x4();
|
||||
_mlas_fp16_ buffer[4] = {0};
|
||||
|
||||
while (N >= 32) {
|
||||
auto v0 = MlasLoadFloat16x8(input);
|
||||
auto v1 = MlasLoadFloat16x8(input + 8);
|
||||
auto v2 = MlasLoadFloat16x8(input + 16);
|
||||
auto v3 = MlasLoadFloat16x8(input + 24);
|
||||
|
||||
auto r0 = SumExp_Vector_Fp16(v0, negative_maximum8);
|
||||
auto r1 = SumExp_Vector_Fp16(v1, negative_maximum8);
|
||||
auto r2 = SumExp_Vector_Fp16(v2, negative_maximum8);
|
||||
auto r3 = SumExp_Vector_Fp16(v3, negative_maximum8);
|
||||
|
||||
accumulator8 = AddUp(r0, r1, r2, r3, accumulator8);
|
||||
|
||||
if (store_output) {
|
||||
MlasStoreFloat16x8(output, r0);
|
||||
MlasStoreFloat16x8(output + 8, r1);
|
||||
MlasStoreFloat16x8(output + 16, r2);
|
||||
MlasStoreFloat16x8(output + 24, r3);
|
||||
output += 32;
|
||||
}
|
||||
|
||||
input += 32;
|
||||
N -= 32;
|
||||
}
|
||||
|
||||
if (N & 16) {
|
||||
auto v0 = MlasLoadFloat16x8(input);
|
||||
auto v1 = MlasLoadFloat16x8(input + 8);
|
||||
|
||||
auto r0 = SumExp_Vector_Fp16(v0, negative_maximum8);
|
||||
auto r1 = SumExp_Vector_Fp16(v1, negative_maximum8);
|
||||
|
||||
accumulator8 = AddUp(r0, r1, accumulator8);
|
||||
|
||||
if (store_output) {
|
||||
MlasStoreFloat16x8(output, r0);
|
||||
MlasStoreFloat16x8(output + 8, r1);
|
||||
output += 16;
|
||||
}
|
||||
|
||||
input += 16;
|
||||
N -= 16;
|
||||
}
|
||||
|
||||
if (N & 8) {
|
||||
auto v0 = MlasLoadFloat16x8(input);
|
||||
auto r0 = SumExp_Vector_Fp16(v0, negative_maximum8);
|
||||
accumulator8 = MlasAdd(r0, accumulator8);
|
||||
|
||||
if (store_output) {
|
||||
MlasStoreFloat16x8(output, r0);
|
||||
output += 8;
|
||||
}
|
||||
|
||||
input += 8;
|
||||
N -= 8;
|
||||
}
|
||||
|
||||
if (N & 4) {
|
||||
auto v0 = MlasLoadFloat16x4(input);
|
||||
auto r0 = SumExp_Vector_Fp16(v0, negative_maximum4);
|
||||
accumulator4 = MlasAdd(r0, accumulator4);
|
||||
|
||||
if (store_output) {
|
||||
MlasStoreFloat16x4(output, r0);
|
||||
output += 4;
|
||||
}
|
||||
|
||||
input += 4;
|
||||
N -= 4;
|
||||
}
|
||||
|
||||
if (N == 3) {
|
||||
auto v0 = MlasLoadPartialFloat16x4(input, 3);
|
||||
auto r0 = SumExp_Vector_Fp16(v0, negative_maximum4);
|
||||
|
||||
if (store_output) {
|
||||
MlasStorePartialFloat16x4(output, r0, 3);
|
||||
}
|
||||
|
||||
r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 3));
|
||||
accumulator4 = MlasAdd(r0, accumulator4);
|
||||
} else if (N == 2) {
|
||||
auto v0 = MlasLoadPartialFloat16x4(input, 2);
|
||||
auto r0 = SumExp_Vector_Fp16(v0, negative_maximum4);
|
||||
|
||||
if (store_output) {
|
||||
MlasStorePartialFloat16x4(output, r0, 2);
|
||||
}
|
||||
|
||||
r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 3));
|
||||
r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 2));
|
||||
accumulator4 = MlasAdd(r0, accumulator4);
|
||||
} else if (N == 1) {
|
||||
auto v0 = MlasLoadPartialFloat16x4(input, 1);
|
||||
auto r0 = SumExp_Vector_Fp16(v0, negative_maximum4);
|
||||
|
||||
if (store_output) {
|
||||
MlasStorePartialFloat16x4(output, r0, 1);
|
||||
}
|
||||
|
||||
r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 3));
|
||||
r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 2));
|
||||
r0 = MlasReinterpretAsFloat16(vset_lane_s16(0, MlasReinterpretAsInt16(r0), 1));
|
||||
accumulator4 = MlasAdd(r0, accumulator4);
|
||||
}
|
||||
|
||||
auto t = MlasAdd(vget_low_f16(accumulator8), vget_high_f16(accumulator8));
|
||||
t = MlasAdd(t, accumulator4);
|
||||
_mlas_fp16_ result = MlasReduceAdd(t);
|
||||
return MLAS_FP16::FromBits(result);
|
||||
}
|
||||
|
||||
const struct {
|
||||
_mlas_fp16_ LowerRange;
|
||||
_mlas_fp16_ UpperRange;
|
||||
_mlas_fp16_ alpha_9;
|
||||
_mlas_fp16_ alpha_7;
|
||||
_mlas_fp16_ alpha_5;
|
||||
_mlas_fp16_ alpha_3;
|
||||
_mlas_fp16_ alpha_1;
|
||||
_mlas_fp16_ beta_10;
|
||||
_mlas_fp16_ beta_8;
|
||||
_mlas_fp16_ beta_6;
|
||||
_mlas_fp16_ beta_4;
|
||||
_mlas_fp16_ beta_2;
|
||||
_mlas_fp16_ beta_0;
|
||||
} MlasTanh16Constants = {
|
||||
0xc500, // -5.0f16
|
||||
0x4500, // 5.0f16
|
||||
0x002e, // 1/9!
|
||||
0x0a80, // 1/7!
|
||||
0x2044, // 1/5!
|
||||
0x3155, // 1/3!
|
||||
0x3c00, // 1
|
||||
0x0005, // 1/10!
|
||||
0x01a0, // 1/8!
|
||||
0x15b0, // 1/6!
|
||||
0x2955, // 1/4!
|
||||
0x3800, // 1/2!
|
||||
0x3c00, // 1
|
||||
};
|
||||
|
||||
// _Float16 my_tanh(_Float16 Value) {
|
||||
// _Float16 v_tmp;
|
||||
// v_tmp = (Value < MlasTanh16Constants.LowerRange) ? MlasTanh16Constants.LowerRange : Value;
|
||||
// Value = (v_tmp > MlasTanh16Constants.UpperRange) ? MlasTanh16Constants.UpperRange : v_tmp;
|
||||
|
||||
// _Float16 ValueSquared = Value * Value;
|
||||
|
||||
// _Float16 p = MlasTanh16Constants.alpha_9;
|
||||
// p = p * ValueSquared + MlasTanh16Constants.alpha_7;
|
||||
// p = p * ValueSquared + MlasTanh16Constants.alpha_5;
|
||||
// p = p * ValueSquared + MlasTanh16Constants.alpha_3;
|
||||
// p = p * ValueSquared + MlasTanh16Constants.alpha_1;
|
||||
// p = p * Value;
|
||||
|
||||
// _Float16 q = MlasTanh16Constants.beta_10;
|
||||
// q = q * ValueSquared + MlasTanh16Constants.beta_8;
|
||||
// q = q * ValueSquared + MlasTanh16Constants.beta_6;
|
||||
// q = q * ValueSquared + MlasTanh16Constants.beta_4;
|
||||
// q = q * ValueSquared + MlasTanh16Constants.beta_2;
|
||||
// q = q * ValueSquared + MlasTanh16Constants.beta_0;
|
||||
|
||||
// return (p / q);
|
||||
// }
|
||||
|
||||
|
||||
// _Float16 my_tanh_no_overflow(_Float16 Value) {
|
||||
// if (Value > 0.5f16) {
|
||||
// _Float16 exp = my_exp(Value);
|
||||
// return (exp - 1.0f16/exp) / (exp + 1.0f16/exp);
|
||||
// } else {
|
||||
// return my_tanh(Value);
|
||||
// }
|
||||
// }
|
||||
|
||||
void Tanh_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N) {
|
||||
|
||||
}
|
||||
|
|
@ -31,20 +477,10 @@ void Softcap_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N, co
|
|||
|
||||
}
|
||||
|
||||
// exp kernel for fp16. Output and input can be the same buffer.
|
||||
void Exp_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N) {
|
||||
|
||||
}
|
||||
|
||||
// reduce max kernel for fp16
|
||||
MLAS_FP16 ReduceMax_Kernel_Fp16(const MLAS_FP16* Input, size_t N) {
|
||||
return MLAS_FP16::FromBits(0);
|
||||
}
|
||||
|
||||
MLAS_FP16 SumExp_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N, const MLAS_FP16 NegativeMaximum) {
|
||||
return MLAS_FP16::FromBits(0);
|
||||
}
|
||||
|
||||
void Softmax_Kernel_Fp16(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N, const MLAS_FP16 scale) {
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -200,9 +200,10 @@ MlasComputeTanh<MLAS_FP16>(
|
|||
dispatch->Tanh_Fp16(Input, Output, N);
|
||||
}
|
||||
|
||||
template <>
|
||||
void
|
||||
MLASCALL
|
||||
MlasComputeSoftcap(
|
||||
MlasComputeSoftcap<MLAS_FP16>(
|
||||
const MLAS_FP16* Input,
|
||||
MLAS_FP16* Output,
|
||||
size_t N,
|
||||
|
|
|
|||
|
|
@ -111,12 +111,12 @@ const struct {
|
|||
int16_t MinimumExponent;
|
||||
int16_t MaximumExponent;
|
||||
} MlasExp16Constants = {
|
||||
-17.328679513f16, // -25 * ln2
|
||||
11.090354888f16, // 16 * ln2
|
||||
-10.743781298f16,
|
||||
10.743781298f16, // 15.5 * ln2
|
||||
1536.f16, // 1.5 * 2^10
|
||||
1.4423828125f16,
|
||||
-17.328679513f16, // -25 * ln2 cc55
|
||||
11.090354888f16, // 16 * ln2 498c
|
||||
-10.743781298f16, // -15.5 * ln2 c95f
|
||||
10.743781298f16, // 15.5 * ln2 495f
|
||||
1536.f16, // 1.5 * 2^10 6600
|
||||
1.4423828125f16, // 1/ln2, 3dc5
|
||||
-6.9287109375e-1f16, // 0xb98b
|
||||
-2.758502960205078e-4f16, // 0x8c85
|
||||
-2.384185791015625e-7f16, // 0x8004
|
||||
|
|
@ -125,7 +125,7 @@ const struct {
|
|||
4.1666666666666664e-2f16, // 1/4! 0x2955
|
||||
1.6666666e-1f16, // 1/3! 0x3155
|
||||
0.5f16, // 1/2! 0x3800
|
||||
1.0f16,
|
||||
1.0f16, // 1/1! 0x3c00
|
||||
int16_t(0xC800), // -14
|
||||
int16_t(0x3C00), // 15
|
||||
};
|
||||
|
|
@ -338,8 +338,8 @@ const struct {
|
|||
_Float16 beta_2;
|
||||
_Float16 beta_0;
|
||||
} MlasTanh16Constants = {
|
||||
-5.0f16,
|
||||
5.0f16,
|
||||
-5.0f16, // c500
|
||||
5.0f16, // 4500
|
||||
2.755731922398589e-06f16, // 0x002e
|
||||
0.00019841269841269839f16, // 0xa80
|
||||
0.008333333333333333f16, // 0x2044
|
||||
|
|
@ -409,9 +409,9 @@ const struct {
|
|||
}
|
||||
|
||||
_Float16 my_tanh_no_overflow(_Float16 Value) {
|
||||
if (Value > 0.45f16) {
|
||||
_Float16 exp = my_exp(2.f16 * Value);
|
||||
return 1 - 2.f16 / (1 + exp);
|
||||
if (Value > 0.5f16) {
|
||||
_Float16 exp = my_exp(Value);
|
||||
return (exp - 1.0f16/exp) / (exp + 1.0f16/exp);
|
||||
} else {
|
||||
return my_tanh(Value);
|
||||
}
|
||||
|
|
@ -433,7 +433,7 @@ const struct {
|
|||
|
||||
void test_tanh_no_overflow(_Float16 x) {
|
||||
float ref = std::tanh((float)x);
|
||||
float out = fast_tanh(x);
|
||||
float out = my_tanh_no_overflow(x);
|
||||
float diff = std::abs(out - ref);
|
||||
std::cout << "x " << (float)x << ", result: " << out << ", expecting: " << ref << " diff " << diff / ref << std::endl;
|
||||
}
|
||||
|
|
@ -445,6 +445,12 @@ const struct {
|
|||
}
|
||||
|
||||
void ExecuteShort(void) override {
|
||||
// print_hex("lower range ", MlasExp16Constants.LowerRange);
|
||||
// print_hex("upper range ", MlasExp16Constants.UpperRange);
|
||||
// print_hex("lower range sum exp ", MlasExp16Constants.LowerRangeSumExp);
|
||||
// print_hex("upper range sum exp ", MlasExp16Constants.UpperRangeSumExp);
|
||||
// print_hex("rounding bias ", MlasExp16Constants.RoundingBias);
|
||||
// print_hex("log2 reciprocal ", MlasExp16Constants.Log2Reciprocal);
|
||||
// print_hex("h ", (_Float16)MlasExp16Constants.Log2High);
|
||||
// print_hex("l ", (_Float16)MlasExp16Constants.Log2Low);
|
||||
// print_hex("ll ", MlasExp16Constants.Log2Lowest);
|
||||
|
|
@ -455,20 +461,20 @@ const struct {
|
|||
// print_hex("poly3 ", (_Float16)MlasExp16Constants.poly_3);
|
||||
// print_hex("poly4 ", (_Float16)MlasExp16Constants.poly_4);
|
||||
// Test(.01f16);
|
||||
// print_hex("alpha_13 ", MlasTanh16Constants.alpha_13);
|
||||
// print_hex("alpha_11 ", MlasTanh16Constants.alpha_11);
|
||||
// print_hex("alpha_9 ", MlasTanh16Constants.alpha_9);
|
||||
// print_hex("alpha_7 ", MlasTanh16Constants.alpha_7);
|
||||
// print_hex("alpha_5 ", MlasTanh16Constants.alpha_5);
|
||||
// print_hex("alpha_3 ", MlasTanh16Constants.alpha_3);
|
||||
// print_hex("alpha_1 ", MlasTanh16Constants.alpha_1);
|
||||
// print_hex("beta_10 ", MlasTanh16Constants.beta_10);
|
||||
// print_hex("beta_8 ", MlasTanh16Constants.beta_8);
|
||||
// print_hex("beta_6 ", MlasTanh16Constants.beta_6);
|
||||
// print_hex("beta_4 ", MlasTanh16Constants.beta_4);
|
||||
// print_hex("beta_2 ", MlasTanh16Constants.beta_2);
|
||||
// print_hex("beta_0 ", MlasTanh16Constants.beta_0);
|
||||
for (_Float16 x = 0.f16; x <= 5.f16; x += 0.005f16) {
|
||||
print_hex("lower range ", MlasTanh16Constants.LowerRange);
|
||||
print_hex("upper range ", MlasTanh16Constants.UpperRange);
|
||||
print_hex("alpha_9 ", MlasTanh16Constants.alpha_9);
|
||||
print_hex("alpha_7 ", MlasTanh16Constants.alpha_7);
|
||||
print_hex("alpha_5 ", MlasTanh16Constants.alpha_5);
|
||||
print_hex("alpha_3 ", MlasTanh16Constants.alpha_3);
|
||||
print_hex("alpha_1 ", MlasTanh16Constants.alpha_1);
|
||||
print_hex("beta_10 ", MlasTanh16Constants.beta_10);
|
||||
print_hex("beta_8 ", MlasTanh16Constants.beta_8);
|
||||
print_hex("beta_6 ", MlasTanh16Constants.beta_6);
|
||||
print_hex("beta_4 ", MlasTanh16Constants.beta_4);
|
||||
print_hex("beta_2 ", MlasTanh16Constants.beta_2);
|
||||
print_hex("beta_0 ", MlasTanh16Constants.beta_0);
|
||||
for (_Float16 x = 0.f16; x <= 9.f16; x += 0.005f16) {
|
||||
test_tanh_no_overflow(x);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue