From e95fbaaba38445dff170d06668b6845903722b2a Mon Sep 17 00:00:00 2001 From: Venkata Chintapalli Date: Fri, 7 Aug 2020 19:11:26 -0700 Subject: [PATCH] Adding Peter's Swish Op ULP analysis. (#42573) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/42573 * Generate the ULP png files for different ranges. Test Plan: test_op_ulp_error.py Reviewed By: hyuen Differential Revision: D22938572 fbshipit-source-id: 6374bef6d44c38e1141030d44029dee99112cd18 --- .../fakelowp/test/test_op_nnpi_fp16.py | 52 ++++++++++++------- caffe2/python/fakelowp/test_utils.py | 39 +++++++++++++- 2 files changed, 71 insertions(+), 20 deletions(-) diff --git a/caffe2/contrib/fakelowp/test/test_op_nnpi_fp16.py b/caffe2/contrib/fakelowp/test/test_op_nnpi_fp16.py index 36577f785d8..f9615c5ace8 100644 --- a/caffe2/contrib/fakelowp/test/test_op_nnpi_fp16.py +++ b/caffe2/contrib/fakelowp/test/test_op_nnpi_fp16.py @@ -3,22 +3,17 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -import ctypes import numpy as np -import os import caffe2.python.fakelowp.init_shared_libs # noqa - -from hypothesis import given, settings +from hypothesis import given from hypothesis import strategies as st - - from caffe2.proto import caffe2_pb2 -from caffe2.python import dyndep from caffe2.python import core from caffe2.python import workspace from caffe2.python.onnx.onnxifi import onnxifi_caffe2_net from caffe2.python.fakelowp.test_utils import print_test_debug_info +from caffe2.python.oss.fakelowp.test_utils import compute_ulp_error import caffe2.python.serialized_test.serialized_test_util as serial core.GlobalInit(["caffe2", "--caffe2_log_level=-3", "--glow_global_fp16=1"]) @@ -119,12 +114,9 @@ class ArithmeticOpsTest(serial.SerializedTestCase): class UnaryOpTest(serial.SerializedTestCase): - def _test_unary_op(self, opname, value, rtol=1e-5, atol=1e-8): + def _test_unary_op(self, opname, X, rtol=1e-5, atol=1e-8): workspace.ResetWorkspace() - n = 1 - m = 10001 - X = np.linspace(-value, value, num=m, dtype=np.float32) pred_net = caffe2_pb2.NetDef() pred_net.name = "pred" pred_net.external_input.append("X") @@ -147,7 +139,7 @@ class UnaryOpTest(serial.SerializedTestCase): ) print("REF NET = {}".format(ref_net)) - shape_hints = {"X": (n, m)} + shape_hints = {"X": X.shape} pred_net_onnxified = onnxifi_caffe2_net(pred_net, shape_hints, debug=True, @@ -167,8 +159,6 @@ class UnaryOpTest(serial.SerializedTestCase): workspace.RunNet(ref_net.name) Y_c2 = workspace.FetchBlob('Y') - - if not np.allclose(Y_c2, Y_glow, rtol=atol, atol=atol): diff = np.abs(Y_c2 - Y_glow) np.save('/tmp/' + opname + 'diff', diff) @@ -181,19 +171,42 @@ class UnaryOpTest(serial.SerializedTestCase): }) assert(0) + return Y_glow + + def _test_op_w_ulp_error(self, opname, regions, atol=0, err_threshold=2): + ulp_err = 0 + for x0, x1 in regions: + X = np.linspace(x0, x1, num=1025, dtype=np.float16).astype(np.float32) + Y_glow = self._test_unary_op(opname, X, atol=atol) + region_err = compute_ulp_error(opname, X, Y_glow) + ulp_err = max(np.max(np.abs(region_err)), ulp_err) + if (ulp_err > err_threshold): + print(r'{} Op detected ulp_err={}'.format(opname, ulp_err)) + assert(0) + # These tests doesn't need to run multiple times given that it is a # linear sweep and it is deterministic. # Once hypothesis.testing version is updated, we can re-enable # testing with different hypothesis examples. def test_sigmoid(self): - self._test_unary_op("Sigmoid", value=20) + opname = "Sigmoid" + regions = [[-8., -4.], [-4., -2.], [-2., -1.], [-1., -.5], [-.5, -.25], + [-.25, .25], [.25, .5], [.5, 1.], [1., 2.], [2., 4.], + [4., 8.]] + self._test_op_w_ulp_error(opname, regions, atol=0, err_threshold=2.5) # These tests doesn't need to run multiple times given that it is a # linear sweep and it is deterministic. # Once hypothesis.testing version is updated, we can re-enable # testing with different hypothesis examples. def test_tanh(self): - self._test_unary_op("Tanh", value=20) + opname = "Tanh" + regions = [[2.**(-9), 2.**(-8)], [2.**(-8), 2.**(-7)], + [2.**(-7), 2.**(-6)], [2.**(-6), 2.**(-5)], + [2.**(-5), 2.**(-4)], [2.**(-4), 2.**(-3)], + [2.**(-3), 2.**(-2)], [2.**(-2), 2.**(-1)], + [2.**(-1), 1.], [1., 2.], [2., 4.], [4., 8.]] + self._test_op_w_ulp_error(opname, regions, atol=0, err_threshold=2) # These tests doesn't need to run multiple times given that it is a # linear sweep and it is deterministic. @@ -201,7 +214,10 @@ class UnaryOpTest(serial.SerializedTestCase): # testing with different hypothesis examples. # TODO: move atol to 1e-8 once we get a non-lowered swish implementation def test_swish(self): - self._test_unary_op("Swish", value=20, atol=0.008) + opname = "Swish" + regions = [[-20.5, -11.], [-11., -8.], [-8., -1.], [-1., -0.1], + [-1. / 8., 1. / 8.], [1. / 8, 5.], [5., 8.]] + self._test_op_w_ulp_error(opname, regions, atol=0.008, err_threshold=384) # These tests doesn't need to run multiple times given that it is a # linear sweep and it is deterministic. @@ -328,6 +344,6 @@ class ReluTest(serial.SerializedTestCase): if not np.allclose(Y_c2, Y_glow): diff = np.abs((Y_glow - Y_c2) / (Y_c2 + kEpsilon)) print_test_debug_info("Relu", { - "seed":seed, "X": X, + "seed": seed, "X": X, "Y_glow": Y_glow, "Y_c2": Y_c2, "diff": diff}) assert(0) diff --git a/caffe2/python/fakelowp/test_utils.py b/caffe2/python/fakelowp/test_utils.py index 275289bf08e..75e4422f3cc 100644 --- a/caffe2/python/fakelowp/test_utils.py +++ b/caffe2/python/fakelowp/test_utils.py @@ -6,7 +6,6 @@ from __future__ import unicode_literals import sys import numpy as np - def print_test_debug_info(testname, items_dict): filename = "debug_operator_onnxifi_" + testname + ".txt" np.set_printoptions(threshold=sys.maxsize) @@ -16,7 +15,6 @@ def print_test_debug_info(testname, items_dict): f.write("{}\n".format(key)) f.write("{}\n".format(value)) - def print_net(net): for i in net.external_input: print("Input: {}".format(i)) @@ -28,3 +26,40 @@ def print_net(net): print(" input: {}".format(x)) for y in op.output: print(" output: {}".format(y)) + +def _sigmoid(x): + return 1. / (1. + np.exp(np.float64(-x))) + +def _tanh(x): + return np.tanh(np.float64(x)) + +def _swish(x): + return np.float64(x) * _sigmoid(x) + +def _gelu_by_sigmoid(x): + return np.float64(x) / (1. + np.exp(np.float64(x) * 1.702)) + + +def _acc_func(opname, x): + if opname == "Swish": + return _swish(x) + elif opname == "Sigmoid": + return _sigmoid(x) + elif opname == "Tanh": + return _tanh(x) + elif opname == "Gelu": + return _gelu_by_sigmoid(x) + else: + return x + +def _get_ulp16(x): + abs_x = np.abs(x) + mask = (abs_x > 2.**(-14)) + abs_x = mask * abs_x + (1 - mask) * 2.**(-14) + k = np.floor(np.log2(abs_x)) + return 2.**(k - 10) + +def compute_ulp_error(opname, xvec, y_nnpi): + y_acc = _acc_func(opname, np.float64(xvec)) + scale = 1. / _get_ulp16(y_acc) + return (y_nnpi - y_acc) * scale