Adding Peter's Swish Op ULP analysis. (#42573)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/42573

* Generate the ULP png files for different ranges.

Test Plan: test_op_ulp_error.py

Reviewed By: hyuen

Differential Revision: D22938572

fbshipit-source-id: 6374bef6d44c38e1141030d44029dee99112cd18

caffe2/contrib/fakelowp/test/test_op_nnpi_fp16.py

@@ -3,22 +3,17 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import ctypes
 import numpy as np
-import os
 
 import caffe2.python.fakelowp.init_shared_libs  # noqa
-
-from hypothesis import given, settings
+from hypothesis import given
 from hypothesis import strategies as st
-
-
 from caffe2.proto import caffe2_pb2
-from caffe2.python import dyndep
 from caffe2.python import core
 from caffe2.python import workspace
 from caffe2.python.onnx.onnxifi import onnxifi_caffe2_net
 from caffe2.python.fakelowp.test_utils import print_test_debug_info
+from caffe2.python.oss.fakelowp.test_utils import compute_ulp_error
 import caffe2.python.serialized_test.serialized_test_util as serial
 
 core.GlobalInit(["caffe2", "--caffe2_log_level=-3", "--glow_global_fp16=1"])
@@ -119,12 +114,9 @@ class ArithmeticOpsTest(serial.SerializedTestCase):
 
 
 class UnaryOpTest(serial.SerializedTestCase):
-    def _test_unary_op(self, opname, value, rtol=1e-5, atol=1e-8):
+    def _test_unary_op(self, opname, X, rtol=1e-5, atol=1e-8):
         workspace.ResetWorkspace()
-        n = 1
-        m = 10001
 
-        X = np.linspace(-value, value, num=m, dtype=np.float32)
         pred_net = caffe2_pb2.NetDef()
         pred_net.name = "pred"
         pred_net.external_input.append("X")
@@ -147,7 +139,7 @@ class UnaryOpTest(serial.SerializedTestCase):
         )
         print("REF NET = {}".format(ref_net))
 
-        shape_hints = {"X": (n, m)}
+        shape_hints = {"X": X.shape}
         pred_net_onnxified = onnxifi_caffe2_net(pred_net,
                                                 shape_hints,
                                                 debug=True,
@@ -167,8 +159,6 @@ class UnaryOpTest(serial.SerializedTestCase):
         workspace.RunNet(ref_net.name)
         Y_c2 = workspace.FetchBlob('Y')
 
-
-
         if not np.allclose(Y_c2, Y_glow, rtol=atol, atol=atol):
             diff = np.abs(Y_c2 - Y_glow)
             np.save('/tmp/' + opname + 'diff', diff)
@@ -181,19 +171,42 @@ class UnaryOpTest(serial.SerializedTestCase):
             })
             assert(0)
 
+        return Y_glow
+
+    def _test_op_w_ulp_error(self, opname, regions, atol=0, err_threshold=2):
+        ulp_err = 0
+        for x0, x1 in regions:
+            X = np.linspace(x0, x1, num=1025, dtype=np.float16).astype(np.float32)
+            Y_glow = self._test_unary_op(opname, X, atol=atol)
+            region_err = compute_ulp_error(opname, X, Y_glow)
+            ulp_err = max(np.max(np.abs(region_err)), ulp_err)
+        if (ulp_err > err_threshold):
+            print(r'{} Op detected ulp_err={}'.format(opname, ulp_err))
+            assert(0)
+
     # These tests doesn't need to run multiple times given that it is a
     # linear sweep and it is deterministic.
     # Once hypothesis.testing version is updated, we can re-enable
     # testing with different hypothesis examples.
     def test_sigmoid(self):
-        self._test_unary_op("Sigmoid", value=20)
+        opname = "Sigmoid"
+        regions = [[-8., -4.], [-4., -2.], [-2., -1.], [-1., -.5], [-.5, -.25],
+                   [-.25, .25], [.25, .5], [.5, 1.], [1., 2.], [2., 4.],
+                   [4., 8.]]
+        self._test_op_w_ulp_error(opname, regions, atol=0, err_threshold=2.5)
 
     # These tests doesn't need to run multiple times given that it is a
     # linear sweep and it is deterministic.
     # Once hypothesis.testing version is updated, we can re-enable
     # testing with different hypothesis examples.
     def test_tanh(self):
-        self._test_unary_op("Tanh", value=20)
+        opname = "Tanh"
+        regions = [[2.**(-9), 2.**(-8)], [2.**(-8), 2.**(-7)],
+                   [2.**(-7), 2.**(-6)], [2.**(-6), 2.**(-5)],
+                   [2.**(-5), 2.**(-4)], [2.**(-4), 2.**(-3)],
+                   [2.**(-3), 2.**(-2)], [2.**(-2), 2.**(-1)],
+                   [2.**(-1), 1.], [1., 2.], [2., 4.], [4., 8.]]
+        self._test_op_w_ulp_error(opname, regions, atol=0, err_threshold=2)
 
     # These tests doesn't need to run multiple times given that it is a
     # linear sweep and it is deterministic.
@@ -201,7 +214,10 @@ class UnaryOpTest(serial.SerializedTestCase):
     # testing with different hypothesis examples.
     # TODO: move atol to 1e-8 once we get a non-lowered swish implementation
     def test_swish(self):
-        self._test_unary_op("Swish", value=20, atol=0.008)
+        opname = "Swish"
+        regions = [[-20.5, -11.], [-11., -8.], [-8., -1.], [-1., -0.1],
+                   [-1. / 8., 1. / 8.], [1. / 8, 5.], [5., 8.]]
+        self._test_op_w_ulp_error(opname, regions, atol=0.008, err_threshold=384)
 
     # These tests doesn't need to run multiple times given that it is a
     # linear sweep and it is deterministic.
@@ -328,6 +344,6 @@ class ReluTest(serial.SerializedTestCase):
         if not np.allclose(Y_c2, Y_glow):
             diff = np.abs((Y_glow - Y_c2) / (Y_c2 + kEpsilon))
             print_test_debug_info("Relu", {
-                "seed":seed, "X": X,
+                "seed": seed, "X": X,
                 "Y_glow": Y_glow, "Y_c2": Y_c2, "diff": diff})
             assert(0)

caffe2/python/fakelowp/test_utils.py

@@ -6,7 +6,6 @@ from __future__ import unicode_literals
 import sys
 import numpy as np
 
-
 def print_test_debug_info(testname, items_dict):
     filename = "debug_operator_onnxifi_" + testname + ".txt"
     np.set_printoptions(threshold=sys.maxsize)
@@ -16,7 +15,6 @@ def print_test_debug_info(testname, items_dict):
             f.write("{}\n".format(key))
             f.write("{}\n".format(value))
 
-
 def print_net(net):
     for i in net.external_input:
         print("Input: {}".format(i))
@@ -28,3 +26,40 @@ def print_net(net):
             print("  input: {}".format(x))
         for y in op.output:
             print("  output: {}".format(y))
+
+def _sigmoid(x):
+    return 1. / (1. + np.exp(np.float64(-x)))
+
+def _tanh(x):
+    return np.tanh(np.float64(x))
+
+def _swish(x):
+    return np.float64(x) * _sigmoid(x)
+
+def _gelu_by_sigmoid(x):
+    return np.float64(x) / (1. + np.exp(np.float64(x) * 1.702))
+
+
+def _acc_func(opname, x):
+    if opname == "Swish":
+        return _swish(x)
+    elif opname == "Sigmoid":
+        return _sigmoid(x)
+    elif opname == "Tanh":
+        return _tanh(x)
+    elif opname == "Gelu":
+        return _gelu_by_sigmoid(x)
+    else:
+        return x
+
+def _get_ulp16(x):
+    abs_x = np.abs(x)
+    mask = (abs_x > 2.**(-14))
+    abs_x = mask * abs_x + (1 - mask) * 2.**(-14)
+    k = np.floor(np.log2(abs_x))
+    return 2.**(k - 10)
+
+def compute_ulp_error(opname, xvec, y_nnpi):
+    y_acc = _acc_func(opname, np.float64(xvec))
+    scale = 1. / _get_ulp16(y_acc)
+    return (y_nnpi - y_acc) * scale