[js/webgpu] Fix Tanh explosion (#19201)

### Description
```math
\tanh(x)=\frac{e^x-e^{-x}}{e^x+e^{-x}}=
\left\{
\begin{array}{cc}
-\frac{1-e^{-2\cdot(-x)}}{1+e^{-2\cdot(-x)}}, & x<0 \\
0, & x=0 \\
\frac{1-e^{-2x}}{1+e^{-2x}}, & x>0
\end{array}
\right.
```

### Motivation and Context
On some platforms,
$$\tanh(1000)=\frac{e^{1000}-e^{-1000}}{e^{1000}+e^{-1000}}$$ would
produce NaN instead of 0.999... or 1 (imagine $e^{1000}=\infty$ and
$\frac{\infty}{\infty}$ explodes).
This commit is contained in:
Jiajie Hu 2024-01-26 00:25:35 +08:00 committed by GitHub
parent 1c92e56dc0
commit 5b06505073
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 30 additions and 1 deletions

View file

@ -279,7 +279,9 @@ export const tan = (context: ComputeContext): void => {
};
export const tanh = (context: ComputeContext): void => {
context.compute(createElementwiseProgramInfo(context.inputs[0], 'Tanh', 'tanh'));
// TODO: revisit after https://github.com/gpuweb/gpuweb/issues/4458 is resolved
context.compute(createElementwiseProgramInfo(
context.inputs[0], 'Tanh', a => `sign(${a}) * (1 - exp(-2 * abs(${a}))) / (1 + exp(-2 * abs(${a})))`));
};
export const thresholdedRelu = (context: ComputeContext, attributes: AlphaAttributes): number => {

View file

@ -0,0 +1,26 @@
[
{
"name": "tanh with no attributes",
"operator": "Tanh",
"attributes": [],
"cases": [
{
"name": "T[2,4]",
"inputs": [
{
"data": [-1000, -1, 0, 0.1, 0.2, 0.3, 0.4, 1000],
"dims": [2, 4],
"type": "float32"
}
],
"outputs": [
{
"data": [-1, -0.761594, 0, 0.099668, 0.197375, 0.291313, 0.379949, 1],
"dims": [2, 4],
"type": "float32"
}
]
}
]
}
]

View file

@ -1389,6 +1389,7 @@
"sub.jsonc",
"sub_int32.jsonc",
"tan.jsonc",
"tanh.jsonc",
"tile.jsonc",
"transpose.jsonc",
"transpose_int32_uint32.jsonc",