From 8289981f0e40f8cfaad661c35a2c094972e7ab2d Mon Sep 17 00:00:00 2001 From: Zhang Lei Date: Fri, 4 Sep 2020 09:37:17 -0700 Subject: [PATCH] Implement QLinearSigmoid. (#5015) Refactor QLinearLeakyRelu using QLinearLookupBase. Paralleling the lookup phase. --- .../contrib_ops/cpu/cpu_contrib_kernels.cc | 4 + .../contrib_ops/cpu/qlinear_lookup_table.cc | 117 ++++++++++++++---- .../contrib_ops/cpu/qlinear_lookup_table.h | 40 +++++- .../core/graph/contrib_ops/contrib_defs.cc | 29 +++++ .../contrib_ops/qlinear_lookup_table_test.cc | 43 ++++++- 5 files changed, 201 insertions(+), 32 deletions(-) diff --git a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc index 0b9e24790d..49ba8f163f 100644 --- a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc @@ -41,6 +41,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QuantizeLinear); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearLeakyRelu); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearLeakyRelu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearSigmoid); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearSigmoid); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearAdd); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearAdd); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QAttention); @@ -115,6 +117,8 @@ Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.cc b/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.cc index ee52faae74..d27ee51b28 100644 --- a/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.cc +++ b/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.cc @@ -4,11 +4,12 @@ #include "qlinear_lookup_table.h" #include "core/providers/common.h" #include "core/mlas/inc/mlas.h" +#include "core/platform/threadpool.h" namespace onnxruntime { namespace contrib { -static void QLinearLookupTableTransform(const uint8_t* x, const uint8_t table[256], uint8_t* y, size_t n) { +static void QLinearLookupTableTransform(const uint8_t* x, const uint8_t* table, uint8_t* y, size_t n) { for (; n >= 4; n -= 4) { const size_t x_value0 = x[0]; const size_t x_value1 = x[1]; @@ -34,12 +35,12 @@ static void QLinearLookupTableTransform(const uint8_t* x, const uint8_t table[25 } template -static void BuildQLinearLeakyReluLookupTable(uint8_t table[256], - const Tensor* tensor_x_scale, - const Tensor* tensor_x_zero_point, - const Tensor* tensor_y_scale, - const Tensor* tensor_y_zero_point, - float alpha) { +static void QlinearBuildLookupTable(uint8_t* table, + const Tensor* tensor_x_scale, + const Tensor* tensor_x_zero_point, + const Tensor* tensor_y_scale, + const Tensor* tensor_y_zero_point, + const LookupTableArrayTransformer& array_values_transformer) { ORT_ENFORCE(IsScalarOr1ElementVector(tensor_x_scale), "QLinearLeakyRelu : input X_scale must be a scalar or 1D tensor of size 1"); ORT_ENFORCE(tensor_x_zero_point == nullptr || IsScalarOr1ElementVector(tensor_x_zero_point), @@ -54,18 +55,36 @@ static void BuildQLinearLeakyReluLookupTable(uint8_t table[256], const float Y_scale = *(tensor_y_scale->Data()); const T Y_zero_point = (tensor_y_zero_point == nullptr) ? static_cast(0) : *(tensor_y_zero_point->template Data()); - float dequantized_vector[256]; + float dequantized_input[256]; + float dequantized_output[256]; for (int i = 0; i < 256; ++i) { T x = static_cast(i); - float x_dequantized = X_scale * (static_cast(x) - static_cast(X_zero_point)); - dequantized_vector[i] = x_dequantized >= 0.0f ? x_dequantized : alpha * x_dequantized; + dequantized_input[i] = X_scale * (static_cast(x) - static_cast(X_zero_point)); } - MlasQuantizeLinear(dequantized_vector, (T*)table, 256, Y_scale, Y_zero_point); + array_values_transformer(dequantized_input, dequantized_output, 256); + MlasQuantizeLinear(dequantized_output, (T*)table, 256, Y_scale, Y_zero_point); } template -QLinearLeakyRelu::QLinearLeakyRelu(const OpKernelInfo& info) - : OpKernel(info), alpha_(info.GetAttrOrDefault("alpha", 0.01f)) { +static void QlinearBuildLookupTable(uint8_t* table, + const Tensor* tensor_x_scale, + const Tensor* tensor_x_zero_point, + const Tensor* tensor_y_scale, + const Tensor* tensor_y_zero_point, + const LookupTableScalarTransformer& value_transformer) { + LookupTableArrayTransformer array_values_transformer = + [&value_transformer](const float* input, float* output, size_t length) { + for (size_t i = 0; i < length; ++i) { + *output++ = value_transformer(*input++); + } + }; + return QlinearBuildLookupTable(table, tensor_x_scale, tensor_x_zero_point, + tensor_y_scale, tensor_y_zero_point, array_values_transformer); +} + +template +template +void QLinearLookupBase::BuildLookupTableIfFixed(const OpKernelInfo& info, Transformer fn) { const Tensor* tensor_x_scale = nullptr; const Tensor* tensor_x_zero_point = nullptr; const Tensor* tensor_y_scale = nullptr; @@ -75,38 +94,80 @@ QLinearLeakyRelu::QLinearLeakyRelu(const OpKernelInfo& info) bool get_x_zero_point = !info.node().InputDefs()[2]->Exists() || info.TryGetConstantInput(2, &tensor_x_zero_point); bool get_y_scale = info.TryGetConstantInput(3, &tensor_y_scale); bool get_y_zero_point = !info.node().InputDefs()[4]->Exists() || info.TryGetConstantInput(4, &tensor_y_zero_point); - is_fixed_parameters_ = get_x_scale && get_x_zero_point && get_y_scale && get_y_zero_point; + bool is_fixed_parameters = get_x_scale && get_x_zero_point && get_y_scale && get_y_zero_point; - if (is_fixed_parameters_) { - BuildQLinearLeakyReluLookupTable( - fixed_lookup_table_, tensor_x_scale, tensor_x_zero_point, - tensor_y_scale, tensor_y_zero_point, alpha_); + if (is_fixed_parameters) { + fixed_lookup_table_.resize(256); + QlinearBuildLookupTable( + fixed_lookup_table_.data(), tensor_x_scale, tensor_x_zero_point, + tensor_y_scale, tensor_y_zero_point, fn); } } template -Status QLinearLeakyRelu::Compute(OpKernelContext* context) const { +template +Status QLinearLookupBase::ComputeBase(OpKernelContext* context, Transformer fn) const { const auto& X = *context->Input(0); const auto& input_shape = X.Shape(); const auto N = input_shape.Size(); auto& Y = *context->Output(0, input_shape); uint8_t table[256]; - if (!is_fixed_parameters_) { - BuildQLinearLeakyReluLookupTable( + if (fixed_lookup_table_.size() == 0) { + QlinearBuildLookupTable( table, context->Input(1), context->Input(2), - context->Input(3), context->Input(4), alpha_); + context->Input(3), context->Input(4), fn); } - QLinearLookupTableTransform( - reinterpret_cast(X.template Data()), - is_fixed_parameters_ ? fixed_lookup_table_ : table, - reinterpret_cast(Y.template MutableData()), - static_cast(N)); + using onnxruntime::concurrency::ThreadPool; + using onnxruntime::TensorOpCost; + ThreadPool* tp = context->GetOperatorThreadPool(); + const uint8_t* x_data = reinterpret_cast(X.template Data()); + uint8_t* y_data = reinterpret_cast(Y.template MutableData()); + ThreadPool::TryParallelFor( + tp, N, TensorOpCost{1.0, 1.0, 1.0}, + [this, x_data, y_data, &table](std::ptrdiff_t first, std::ptrdiff_t last) { + QLinearLookupTableTransform( + x_data + first, + fixed_lookup_table_.size() ? fixed_lookup_table_.data() : table, + y_data + first, + last - first); + }); return Status::OK(); } +// Derived classes from QLinearLookupBase +template +QLinearLeakyRelu::QLinearLeakyRelu(const OpKernelInfo& info) + : QLinearLookupBase(info), alpha_(info.GetAttrOrDefault("alpha", 0.01f)) { + this->BuildLookupTableIfFixed(info, [this](float v) -> float { + return v >= 0.0f ? v : alpha_ * v; + }); +} + +template +Status QLinearLeakyRelu::Compute(OpKernelContext* context) const { + return this->ComputeBase(context, [this](float v) -> float { + return v >= 0.0f ? v : alpha_ * v; + }); +} + +template +QLinearSigmoid::QLinearSigmoid(const OpKernelInfo& info) + : QLinearLookupBase(info) { + this->BuildLookupTableIfFixed(info, [](const float* input, float* output, size_t length) { + MlasComputeLogistic(input, output, length); + }); +} + +template +Status QLinearSigmoid::Compute(OpKernelContext* context) const { + return this->ComputeBase(context, [](const float* input, float* output, size_t length) { + MlasComputeLogistic(input, output, length); + }); +} + #define REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(op_name, version, data_type, KERNEL_CLASS) \ ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( \ op_name, version, data_type, \ @@ -116,6 +177,8 @@ Status QLinearLeakyRelu::Compute(OpKernelContext* context) const { REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearLeakyRelu, 1, int8_t, QLinearLeakyRelu); REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearLeakyRelu, 1, uint8_t, QLinearLeakyRelu); +REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearSigmoid, 1, int8_t, QLinearSigmoid); +REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearSigmoid, 1, uint8_t, QLinearSigmoid); } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.h b/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.h index 214f56f6f4..279c747807 100644 --- a/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.h +++ b/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.h @@ -5,12 +5,40 @@ #include "core/common/common.h" #include "core/framework/op_kernel.h" +#include namespace onnxruntime { namespace contrib { +// function that transform array of input value to array of output value of length +typedef std::function LookupTableArrayTransformer; + +// function that transform single value +typedef std::function LookupTableScalarTransformer; + + template -class QLinearLeakyRelu final : public OpKernel { +class QLinearLookupBase : public OpKernel { + public: + QLinearLookupBase(const OpKernelInfo& info) + : OpKernel(info), fixed_lookup_table_() { + } + +// protected: + template + Status ComputeBase(OpKernelContext* context, Transformer fn) const; + + // Should be called in derived class's constructor + template + void BuildLookupTableIfFixed(const OpKernelInfo& info, Transformer fn); + + // when input quantizaton parameters are const, pre-compute table value. + // After construction, non-zero size means pre-computed. Save space when not pre-computed. + std::vector fixed_lookup_table_; +}; + +template +class QLinearLeakyRelu final : public QLinearLookupBase { public: QLinearLeakyRelu(const OpKernelInfo& info); @@ -18,8 +46,14 @@ class QLinearLeakyRelu final : public OpKernel { private: const float alpha_; - bool is_fixed_parameters_; // Fixed Scale and Zero Point for both x and y - uint8_t fixed_lookup_table_[256]; // when is const paramter, table value is here. +}; + +template +class QLinearSigmoid final : public QLinearLookupBase { + public: + QLinearSigmoid(const OpKernelInfo& info); + + Status Compute(OpKernelContext* context) const override; }; } // namespace contrib diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index 36963944ab..48d1287092 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -2272,6 +2272,35 @@ and produces one output data (Tensor) where the function `f(x) = quantize(alp "Constrain input and output types to 8 bit tensors.") .TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput); + const char* QLinearSigmoidDoc_ver1 = R"DOC( +QLinearSigmoid takes quantized input data (Tensor), and quantize parameter for output, and produces one output data +(Tensor) where the function `f(x) = quantize(Sigmoid(dequantize(x)))`, is applied to the data tensor elementwise. +Wwhere the function `Sigmoid(x) = 1 / (1 + exp(-x))` )DOC"; + + ONNX_CONTRIB_OPERATOR_SCHEMA(QLinearSigmoid) + .SetDomain(kMSDomain) + .SinceVersion(1) + .SetDoc(QLinearSigmoidDoc_ver1) + .Input(0, "X", "Input tensor", "T") + .Input(1, "X_scale", + "Input X's scale. It's a scalar, which means a per-tensor/layer quantization.", + "tensor(float)") + .Input(2, "X_zero_point", + "Input X's zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", OpSchema::Optional) + .Input(3, "Y_scale", + "Output Y's scale. It's a scalar, which means a per-tensor/layer quantization.", + "tensor(float)") + .Input(4, "Y_zero_point", + "Output Y's zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.", + "T", OpSchema::Optional) + .Output(0, "Y", "Output tensor", "T") + .TypeConstraint( + "T", + {"tensor(uint8)", "tensor(int8)"}, + "Constrain input and output types to 8 bit tensors.") + .TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput); + ONNX_CONTRIB_OPERATOR_SCHEMA(MurmurHash3) .SetDomain(kMSDomain) .SinceVersion(1) diff --git a/onnxruntime/test/contrib_ops/qlinear_lookup_table_test.cc b/onnxruntime/test/contrib_ops/qlinear_lookup_table_test.cc index 2c7a642bcd..fcf2b5b6c6 100644 --- a/onnxruntime/test/contrib_ops/qlinear_lookup_table_test.cc +++ b/onnxruntime/test/contrib_ops/qlinear_lookup_table_test.cc @@ -18,7 +18,7 @@ TEST(QLinearLookupTableBasedOperatorTests, QLinearLeakyRelu_Int8) { std::vector dims = {16}; test.AddInput("X", dims, {0, 16, 17, 18, 19, 90, 91, 127, -128, -110, -108, -100, -16, -17, -18, -1}); test.AddInput("X_scale", {}, {X_scale}); - test.AddMissingOptionalInput(); // optional "X_zero_point" using default value here + test.AddMissingOptionalInput(); // optional "X_zero_point" using default value here test.AddInput("Y_scale", {}, {Y_scale}); test.AddInput("Y_zero_point", {}, {Y_zero_point}); test.AddOutput("Y", dims, {-100, -60, -58, -55, -52, 125, 127, 127, -128, -128, -127, -125, -104, -104, -104, -100}); @@ -28,7 +28,6 @@ TEST(QLinearLookupTableBasedOperatorTests, QLinearLeakyRelu_Int8) { std::fesetround(origin_round_mode); } - TEST(QLinearLookupTableBasedOperatorTests, QLinearLeakyRelu_UInt8) { OpTester test("QLinearLeakyRelu", 1, onnxruntime::kMSDomain); test.AddAttribute("alpha", 0.1f); @@ -50,5 +49,45 @@ TEST(QLinearLookupTableBasedOperatorTests, QLinearLeakyRelu_UInt8) { std::fesetround(origin_round_mode); } +TEST(QLinearLookupTableBasedOperatorTests, QLinearSigmoid_Int8) { + OpTester test("QLinearSigmoid", 1, onnxruntime::kMSDomain); + float X_scale = 0.025f; + //int8_t X_zero_point = 0; + float Y_scale = 1.0f / 256.0f; + int8_t Y_zero_point = -120; + + std::vector dims = {16}; + test.AddInput("X", dims, {0, 16, 17, 18, 19, 90, 91, 127, -128, -110, -108, -100, -16, -17, -18, -1}); + test.AddInput("X_scale", {}, {X_scale}); + test.AddMissingOptionalInput(); // optional "X_zero_point" using default value here + test.AddInput("Y_scale", {}, {Y_scale}); + test.AddInput("Y_zero_point", {}, {Y_zero_point}); + test.AddOutput("Y", dims, {8, 33, 35, 36, 38, 112, 112, 126, -110, -105, -104, -101, -17, -19, -20, 6}); + auto origin_round_mode = std::fegetround(); + std::fesetround(FE_TONEAREST); + test.Run(); + std::fesetround(origin_round_mode); } + +TEST(QLinearLookupTableBasedOperatorTests, QLinearSigmoid_UInt8) { + OpTester test("QLinearSigmoid", 1, onnxruntime::kMSDomain); + float X_scale = 0.025f; + uint8_t X_zero_point = 128; + float Y_scale = 1.0f / 256.0f; + uint8_t Y_zero_point = 8; + + std::vector dims = {16}; + test.AddInput("X", dims, {0, 16, 17, 18, 19, 90, 91, 127, 128, 136, 137, 138, 216, 217, 218, 255}); + test.AddInput("X_scale", {}, {X_scale}); + test.AddInput("X_zero_point", {}, {X_zero_point}); + test.AddInput("Y_scale", {}, {Y_scale}); + test.AddInput("Y_zero_point", {}, {Y_zero_point}); + test.AddOutput("Y", dims, {18, 23, 23, 23, 24, 79, 81, 134, 136, 149, 150, 152, 238, 239, 240, 254}); + auto origin_round_mode = std::fegetround(); + std::fesetround(FE_TONEAREST); + test.Run(); + std::fesetround(origin_round_mode); } + +} // namespace test +} // namespace onnxruntime