From 62d1458ea8effcf1d82590c0ea738c1ed488bb04 Mon Sep 17 00:00:00 2001 From: Nick Kreeger Date: Tue, 6 Jul 2021 18:31:05 -0500 Subject: [PATCH] Move kernel implementations outside of lookup table utility functions. (#8306) --- .../contrib_ops/cpu/qlinear_activations.cc | 113 ++++++++++++++++++ .../contrib_ops/cpu/qlinear_activations.h | 54 +++++++++ .../contrib_ops/cpu/qlinear_lookup_table.cc | 108 +---------------- .../contrib_ops/cpu/qlinear_lookup_table.h | 44 +------ 4 files changed, 177 insertions(+), 142 deletions(-) create mode 100644 onnxruntime/contrib_ops/cpu/qlinear_activations.cc create mode 100644 onnxruntime/contrib_ops/cpu/qlinear_activations.h diff --git a/onnxruntime/contrib_ops/cpu/qlinear_activations.cc b/onnxruntime/contrib_ops/cpu/qlinear_activations.cc new file mode 100644 index 0000000000..b1e9778373 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/qlinear_activations.cc @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "qlinear_activations.h" + +#include "qlinear_lookup_table.h" +#include "core/mlas/inc/mlas.h" +#include "core/platform/threadpool.h" + +namespace onnxruntime { +namespace contrib { + +template +template +void QLinearLookupBase::BuildLookupTableIfFixed(const OpKernelInfo& info, Transformer fn) { + const Tensor* tensor_x_scale = nullptr; + const Tensor* tensor_x_zero_point = nullptr; + const Tensor* tensor_y_scale = nullptr; + const Tensor* tensor_y_zero_point = nullptr; + + bool get_x_scale = info.TryGetConstantInput(1, &tensor_x_scale); + bool get_x_zero_point = !info.node().InputDefs()[2]->Exists() || info.TryGetConstantInput(2, &tensor_x_zero_point); + bool get_y_scale = info.TryGetConstantInput(3, &tensor_y_scale); + bool get_y_zero_point = !info.node().InputDefs()[4]->Exists() || info.TryGetConstantInput(4, &tensor_y_zero_point); + bool is_fixed_parameters = get_x_scale && get_x_zero_point && get_y_scale && get_y_zero_point; + + if (is_fixed_parameters) { + fixed_lookup_table_.resize(256); + QlinearBuildLookupTable( + fixed_lookup_table_.data(), tensor_x_scale, tensor_x_zero_point, + tensor_y_scale, tensor_y_zero_point, fn); + } +} + +template +template +Status QLinearLookupBase::ComputeBase(OpKernelContext* context, Transformer fn) const { + const auto& X = *context->Input(0); + const auto& input_shape = X.Shape(); + const auto N = input_shape.Size(); + auto& Y = *context->Output(0, input_shape); + + uint8_t table[256]; + if (fixed_lookup_table_.size() == 0) { + QlinearBuildLookupTable( + table, context->Input(1), context->Input(2), + context->Input(3), context->Input(4), fn); + } + + using onnxruntime::TensorOpCost; + using onnxruntime::concurrency::ThreadPool; + ThreadPool* tp = context->GetOperatorThreadPool(); + const uint8_t* x_data = reinterpret_cast(X.template Data()); + uint8_t* y_data = reinterpret_cast(Y.template MutableData()); + ThreadPool::TryParallelFor( + tp, N, TensorOpCost{1.0, 1.0, 1.0}, + [this, x_data, y_data, &table](std::ptrdiff_t first, std::ptrdiff_t last) { + QLinearLookupTableTransform( + x_data + first, + fixed_lookup_table_.size() ? fixed_lookup_table_.data() : table, + y_data + first, + last - first); + }); + + return Status::OK(); +} + +// Derived classes from QLinearLookupBase +template +QLinearLeakyRelu::QLinearLeakyRelu(const OpKernelInfo& info) + : QLinearLookupBase(info), alpha_(info.GetAttrOrDefault("alpha", 0.01f)) { + this->BuildLookupTableIfFixed(info, [this](float v) -> float { + return v >= 0.0f ? v : alpha_ * v; + }); +} + +template +Status QLinearLeakyRelu::Compute(OpKernelContext* context) const { + return this->ComputeBase(context, [this](float v) -> float { + return v >= 0.0f ? v : alpha_ * v; + }); +} + +template +QLinearSigmoid::QLinearSigmoid(const OpKernelInfo& info) + : QLinearLookupBase(info) { + this->BuildLookupTableIfFixed(info, [](const float* input, float* output, size_t length) { + MlasComputeLogistic(input, output, length); + }); +} + +template +Status QLinearSigmoid::Compute(OpKernelContext* context) const { + return this->ComputeBase(context, [](const float* input, float* output, size_t length) { + MlasComputeLogistic(input, output, length); + }); +} + +#define REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(op_name, version, data_type, KERNEL_CLASS) \ + ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( \ + op_name, version, data_type, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + KERNEL_CLASS); + +REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearLeakyRelu, 1, int8_t, QLinearLeakyRelu); +REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearLeakyRelu, 1, uint8_t, QLinearLeakyRelu); +REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearSigmoid, 1, int8_t, QLinearSigmoid); +REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearSigmoid, 1, uint8_t, QLinearSigmoid); + +} // namespace contrib +} // namespace onnxruntime + diff --git a/onnxruntime/contrib_ops/cpu/qlinear_activations.h b/onnxruntime/contrib_ops/cpu/qlinear_activations.h new file mode 100644 index 0000000000..3ac8eb55d1 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/qlinear_activations.h @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace contrib { + +template +class QLinearLookupBase : public OpKernel { + public: + QLinearLookupBase(const OpKernelInfo& info) + : OpKernel(info), fixed_lookup_table_() { + } + + // protected: + template + Status ComputeBase(OpKernelContext* context, Transformer fn) const; + + // Should be called in derived class's constructor + template + void BuildLookupTableIfFixed(const OpKernelInfo& info, Transformer fn); + + // when input quantizaton parameters are const, pre-compute table value. + // After construction, non-zero size means pre-computed. Save space when not pre-computed. + std::vector fixed_lookup_table_; +}; + +template +class QLinearLeakyRelu final : public QLinearLookupBase { + public: + QLinearLeakyRelu(const OpKernelInfo& info); + + Status Compute(OpKernelContext* context) const override; + + private: + const float alpha_; +}; + +template +class QLinearSigmoid final : public QLinearLookupBase { + public: + QLinearSigmoid(const OpKernelInfo& info); + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace contrib +} // namespace onnxruntime + diff --git a/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.cc b/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.cc index 6ff5c5661d..6de919dc54 100644 --- a/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.cc +++ b/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.cc @@ -2,9 +2,9 @@ // Licensed under the MIT License. #include "qlinear_lookup_table.h" -#include "core/providers/common.h" + #include "core/mlas/inc/mlas.h" -#include "core/platform/threadpool.h" +#include "core/providers/common.h" namespace onnxruntime { namespace contrib { @@ -51,9 +51,11 @@ void QlinearBuildLookupTable(uint8_t* table, "QlinearBuildLookupTable : input Y_zero_point must be a scalar or 1D tensor of size 1"); const float X_scale = *(tensor_x_scale->Data()); - const T X_zero_point = (tensor_x_zero_point == nullptr) ? static_cast(0) : *(tensor_x_zero_point->template Data()); + const T X_zero_point = + (tensor_x_zero_point == nullptr) ? static_cast(0) : *(tensor_x_zero_point->template Data()); const float Y_scale = *(tensor_y_scale->Data()); - const T Y_zero_point = (tensor_y_zero_point == nullptr) ? static_cast(0) : *(tensor_y_zero_point->template Data()); + const T Y_zero_point = + (tensor_y_zero_point == nullptr) ? static_cast(0) : *(tensor_y_zero_point->template Data()); float dequantized_input[256]; float dequantized_output[256]; @@ -96,103 +98,5 @@ template void QlinearBuildLookupTable(uint8_t* table, const Tensor* tensor_y_zero_point, const LookupTableScalarTransformer& value_transformer); -template -template -void QLinearLookupBase::BuildLookupTableIfFixed(const OpKernelInfo& info, Transformer fn) { - const Tensor* tensor_x_scale = nullptr; - const Tensor* tensor_x_zero_point = nullptr; - const Tensor* tensor_y_scale = nullptr; - const Tensor* tensor_y_zero_point = nullptr; - - bool get_x_scale = info.TryGetConstantInput(1, &tensor_x_scale); - bool get_x_zero_point = !info.node().InputDefs()[2]->Exists() || info.TryGetConstantInput(2, &tensor_x_zero_point); - bool get_y_scale = info.TryGetConstantInput(3, &tensor_y_scale); - bool get_y_zero_point = !info.node().InputDefs()[4]->Exists() || info.TryGetConstantInput(4, &tensor_y_zero_point); - bool is_fixed_parameters = get_x_scale && get_x_zero_point && get_y_scale && get_y_zero_point; - - if (is_fixed_parameters) { - fixed_lookup_table_.resize(256); - QlinearBuildLookupTable( - fixed_lookup_table_.data(), tensor_x_scale, tensor_x_zero_point, - tensor_y_scale, tensor_y_zero_point, fn); - } -} - -template -template -Status QLinearLookupBase::ComputeBase(OpKernelContext* context, Transformer fn) const { - const auto& X = *context->Input(0); - const auto& input_shape = X.Shape(); - const auto N = input_shape.Size(); - auto& Y = *context->Output(0, input_shape); - - uint8_t table[256]; - if (fixed_lookup_table_.size() == 0) { - QlinearBuildLookupTable( - table, context->Input(1), context->Input(2), - context->Input(3), context->Input(4), fn); - } - - using onnxruntime::TensorOpCost; - using onnxruntime::concurrency::ThreadPool; - ThreadPool* tp = context->GetOperatorThreadPool(); - const uint8_t* x_data = reinterpret_cast(X.template Data()); - uint8_t* y_data = reinterpret_cast(Y.template MutableData()); - ThreadPool::TryParallelFor( - tp, N, TensorOpCost{1.0, 1.0, 1.0}, - [this, x_data, y_data, &table](std::ptrdiff_t first, std::ptrdiff_t last) { - QLinearLookupTableTransform( - x_data + first, - fixed_lookup_table_.size() ? fixed_lookup_table_.data() : table, - y_data + first, - last - first); - }); - - return Status::OK(); -} - -// Derived classes from QLinearLookupBase -template -QLinearLeakyRelu::QLinearLeakyRelu(const OpKernelInfo& info) - : QLinearLookupBase(info), alpha_(info.GetAttrOrDefault("alpha", 0.01f)) { - this->BuildLookupTableIfFixed(info, [this](float v) -> float { - return v >= 0.0f ? v : alpha_ * v; - }); -} - -template -Status QLinearLeakyRelu::Compute(OpKernelContext* context) const { - return this->ComputeBase(context, [this](float v) -> float { - return v >= 0.0f ? v : alpha_ * v; - }); -} - -template -QLinearSigmoid::QLinearSigmoid(const OpKernelInfo& info) - : QLinearLookupBase(info) { - this->BuildLookupTableIfFixed(info, [](const float* input, float* output, size_t length) { - MlasComputeLogistic(input, output, length); - }); -} - -template -Status QLinearSigmoid::Compute(OpKernelContext* context) const { - return this->ComputeBase(context, [](const float* input, float* output, size_t length) { - MlasComputeLogistic(input, output, length); - }); -} - -#define REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(op_name, version, data_type, KERNEL_CLASS) \ - ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( \ - op_name, version, data_type, \ - KernelDefBuilder() \ - .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - KERNEL_CLASS); - -REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearLeakyRelu, 1, int8_t, QLinearLeakyRelu); -REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearLeakyRelu, 1, uint8_t, QLinearLeakyRelu); -REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearSigmoid, 1, int8_t, QLinearSigmoid); -REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearSigmoid, 1, uint8_t, QLinearSigmoid); - } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.h b/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.h index 746b67bb38..2df4d47226 100644 --- a/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.h +++ b/onnxruntime/contrib_ops/cpu/qlinear_lookup_table.h @@ -3,10 +3,12 @@ #pragma once -#include "core/common/common.h" -#include "core/framework/op_kernel.h" +#include "core/framework/tensor.h" +#include +#include #include +// TODO(kreeger): Move this folder to a quantization utils/toolkit folder. namespace onnxruntime { namespace contrib { @@ -34,44 +36,6 @@ void QlinearBuildLookupTable(uint8_t* table, void QLinearLookupTableTransform(const uint8_t* x, const uint8_t* table, uint8_t* y, size_t n); -template -class QLinearLookupBase : public OpKernel { - public: - QLinearLookupBase(const OpKernelInfo& info) - : OpKernel(info), fixed_lookup_table_() { - } - - // protected: - template - Status ComputeBase(OpKernelContext* context, Transformer fn) const; - - // Should be called in derived class's constructor - template - void BuildLookupTableIfFixed(const OpKernelInfo& info, Transformer fn); - - // when input quantizaton parameters are const, pre-compute table value. - // After construction, non-zero size means pre-computed. Save space when not pre-computed. - std::vector fixed_lookup_table_; -}; - -template -class QLinearLeakyRelu final : public QLinearLookupBase { - public: - QLinearLeakyRelu(const OpKernelInfo& info); - - Status Compute(OpKernelContext* context) const override; - - private: - const float alpha_; -}; - -template -class QLinearSigmoid final : public QLinearLookupBase { - public: - QLinearSigmoid(const OpKernelInfo& info); - - Status Compute(OpKernelContext* context) const override; -}; } // namespace contrib } // namespace onnxruntime