saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement QLinearSigmoid. (#5015)

Browse source 
Refactor QLinearLeakyRelu using QLinearLookupBase.
Paralleling the lookup phase.
This commit is contained in:
Zhang Lei 2020-09-04 09:37:17 -07:00 committed by GitHub
parent 0fc9c504fe
commit 8289981f0e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 201 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc
View file

@ -41,6 +41,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1,
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QuantizeLinear);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearLeakyRelu);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearLeakyRelu);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearSigmoid);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearSigmoid);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearAdd);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearAdd);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QAttention);
@ -115,6 +117,8 @@ Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QuantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearLeakyRelu)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearLeakyRelu)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearSigmoid)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearSigmoid)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearAdd)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearAdd)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QAttention)>,

117
onnxruntime/contrib_ops/cpu/qlinear_lookup_table.cc
View file

@ -4,11 +4,12 @@
#include "qlinear_lookup_table.h"
#include "core/providers/common.h"
#include "core/mlas/inc/mlas.h"
#include "core/platform/threadpool.h"
namespace onnxruntime {
namespace contrib {
static void QLinearLookupTableTransform(const uint8_t* x, const uint8_t table[256], uint8_t* y, size_t n) {
static void QLinearLookupTableTransform(const uint8_t* x, const uint8_t* table, uint8_t* y, size_t n) {
for (; n >= 4; n -= 4) {
const size_t x_value0 = x[0];
const size_t x_value1 = x[1];
@ -34,12 +35,12 @@ static void QLinearLookupTableTransform(const uint8_t* x, const uint8_t table[25
}
template <typename T>
static void BuildQLinearLeakyReluLookupTable(uint8_t table[256],
const Tensor* tensor_x_scale,
const Tensor* tensor_x_zero_point,
const Tensor* tensor_y_scale,
const Tensor* tensor_y_zero_point,
float alpha) {
static void QlinearBuildLookupTable(uint8_t* table,
const Tensor* tensor_x_scale,
const Tensor* tensor_x_zero_point,
const Tensor* tensor_y_scale,
const Tensor* tensor_y_zero_point,
const LookupTableArrayTransformer& array_values_transformer) {
ORT_ENFORCE(IsScalarOr1ElementVector(tensor_x_scale),
"QLinearLeakyRelu : input X_scale must be a scalar or 1D tensor of size 1");
ORT_ENFORCE(tensor_x_zero_point == nullptr || IsScalarOr1ElementVector(tensor_x_zero_point),
@ -54,18 +55,36 @@ static void BuildQLinearLeakyReluLookupTable(uint8_t table[256],
const float Y_scale = *(tensor_y_scale->Data<float>());
const T Y_zero_point = (tensor_y_zero_point == nullptr) ? static_cast<T>(0) : *(tensor_y_zero_point->template Data<T>());
float dequantized_vector[256];
float dequantized_input[256];
float dequantized_output[256];
for (int i = 0; i < 256; ++i) {
T x = static_cast<T>(i);
float x_dequantized = X_scale * (static_cast<int>(x) - static_cast<int>(X_zero_point));
dequantized_vector[i] = x_dequantized >= 0.0f ? x_dequantized : alpha * x_dequantized;
dequantized_input[i] = X_scale * (static_cast<int>(x) - static_cast<int>(X_zero_point));
}
MlasQuantizeLinear(dequantized_vector, (T*)table, 256, Y_scale, Y_zero_point);
array_values_transformer(dequantized_input, dequantized_output, 256);
MlasQuantizeLinear(dequantized_output, (T*)table, 256, Y_scale, Y_zero_point);
}
template <typename T>
QLinearLeakyRelu<T>::QLinearLeakyRelu(const OpKernelInfo& info)
: OpKernel(info), alpha_(info.GetAttrOrDefault("alpha", 0.01f)) {
static void QlinearBuildLookupTable(uint8_t* table,
const Tensor* tensor_x_scale,
const Tensor* tensor_x_zero_point,
const Tensor* tensor_y_scale,
const Tensor* tensor_y_zero_point,
const LookupTableScalarTransformer& value_transformer) {
LookupTableArrayTransformer array_values_transformer =
[&value_transformer](const float* input, float* output, size_t length) {
for (size_t i = 0; i < length; ++i) {
*output++ = value_transformer(*input++);
}
};
return QlinearBuildLookupTable<T>(table, tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, array_values_transformer);
}
template <typename T>
template <typename Transformer>
void QLinearLookupBase<T>::BuildLookupTableIfFixed(const OpKernelInfo& info, Transformer fn) {
const Tensor* tensor_x_scale = nullptr;
const Tensor* tensor_x_zero_point = nullptr;
const Tensor* tensor_y_scale = nullptr;
@ -75,38 +94,80 @@ QLinearLeakyRelu<T>::QLinearLeakyRelu(const OpKernelInfo& info)
bool get_x_zero_point = !info.node().InputDefs()[2]->Exists() || info.TryGetConstantInput(2, &tensor_x_zero_point);
bool get_y_scale = info.TryGetConstantInput(3, &tensor_y_scale);
bool get_y_zero_point = !info.node().InputDefs()[4]->Exists() || info.TryGetConstantInput(4, &tensor_y_zero_point);
is_fixed_parameters_ = get_x_scale && get_x_zero_point && get_y_scale && get_y_zero_point;
bool is_fixed_parameters = get_x_scale && get_x_zero_point && get_y_scale && get_y_zero_point;
if (is_fixed_parameters_) {
BuildQLinearLeakyReluLookupTable<T>(
fixed_lookup_table_, tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, alpha_);
if (is_fixed_parameters) {
fixed_lookup_table_.resize(256);
QlinearBuildLookupTable<T>(
fixed_lookup_table_.data(), tensor_x_scale, tensor_x_zero_point,
tensor_y_scale, tensor_y_zero_point, fn);
}
}
template <typename T>
Status QLinearLeakyRelu<T>::Compute(OpKernelContext* context) const {
template <typename Transformer>
Status QLinearLookupBase<T>::ComputeBase(OpKernelContext* context, Transformer fn) const {
const auto& X = *context->Input<Tensor>(0);
const auto& input_shape = X.Shape();
const auto N = input_shape.Size();
auto& Y = *context->Output(0, input_shape);
uint8_t table[256];
if (!is_fixed_parameters_) {
BuildQLinearLeakyReluLookupTable<T>(
if (fixed_lookup_table_.size() == 0) {
QlinearBuildLookupTable<T>(
table, context->Input<Tensor>(1), context->Input<Tensor>(2),
context->Input<Tensor>(3), context->Input<Tensor>(4), alpha_);
context->Input<Tensor>(3), context->Input<Tensor>(4), fn);
}
QLinearLookupTableTransform(
reinterpret_cast<const uint8_t*>(X.template Data<T>()),
is_fixed_parameters_ ? fixed_lookup_table_ : table,
reinterpret_cast<uint8_t*>(Y.template MutableData<T>()),
static_cast<size_t>(N));
using onnxruntime::concurrency::ThreadPool;
using onnxruntime::TensorOpCost;
ThreadPool* tp = context->GetOperatorThreadPool();
const uint8_t* x_data = reinterpret_cast<const uint8_t*>(X.template Data<T>());
uint8_t* y_data = reinterpret_cast<uint8_t*>(Y.template MutableData<T>());
ThreadPool::TryParallelFor(
tp, N, TensorOpCost{1.0, 1.0, 1.0},
[this, x_data, y_data, &table](std::ptrdiff_t first, std::ptrdiff_t last) {
QLinearLookupTableTransform(
x_data + first,
fixed_lookup_table_.size() ? fixed_lookup_table_.data() : table,
y_data + first,
last - first);
});
return Status::OK();
}
// Derived classes from QLinearLookupBase
template <typename T>
QLinearLeakyRelu<T>::QLinearLeakyRelu(const OpKernelInfo& info)
: QLinearLookupBase<T>(info), alpha_(info.GetAttrOrDefault("alpha", 0.01f)) {
this->BuildLookupTableIfFixed(info, [this](float v) -> float {
return v >= 0.0f ? v : alpha_ * v;
});
}
template <typename T>
Status QLinearLeakyRelu<T>::Compute(OpKernelContext* context) const {
return this->ComputeBase(context, [this](float v) -> float {
return v >= 0.0f ? v : alpha_ * v;
});
}
template <typename T>
QLinearSigmoid<T>::QLinearSigmoid(const OpKernelInfo& info)
: QLinearLookupBase<T>(info) {
this->BuildLookupTableIfFixed(info, [](const float* input, float* output, size_t length) {
MlasComputeLogistic(input, output, length);
});
}
template <typename T>
Status QLinearSigmoid<T>::Compute(OpKernelContext* context) const {
return this->ComputeBase(context, [](const float* input, float* output, size_t length) {
MlasComputeLogistic(input, output, length);
});
}
#define REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(op_name, version, data_type, KERNEL_CLASS) \
ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( \
op_name, version, data_type, \
@ -116,6 +177,8 @@ Status QLinearLeakyRelu<T>::Compute(OpKernelContext* context) const {
REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearLeakyRelu, 1, int8_t, QLinearLeakyRelu);
REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearLeakyRelu, 1, uint8_t, QLinearLeakyRelu);
REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearSigmoid, 1, int8_t, QLinearSigmoid);
REGISTER_QLINEAR_LOOKUPTABLE_TYPED_KERNEL(QLinearSigmoid, 1, uint8_t, QLinearSigmoid);
} // namespace contrib
} // namespace onnxruntime

40
onnxruntime/contrib_ops/cpu/qlinear_lookup_table.h
View file

@ -5,12 +5,40 @@
#include "core/common/common.h"
#include "core/framework/op_kernel.h"
#include <vector>
namespace onnxruntime {
namespace contrib {
// function that transform array of input value to array of output value of length
typedef std::function<void(const float* input, float* output, size_t length)> LookupTableArrayTransformer;
// function that transform single value
typedef std::function<float(float)> LookupTableScalarTransformer;
template <typename T>
class QLinearLeakyRelu final : public OpKernel {
class QLinearLookupBase : public OpKernel {
public:
QLinearLookupBase(const OpKernelInfo& info)
: OpKernel(info), fixed_lookup_table_() {
}
// protected:
template <typename Transformer>
Status ComputeBase(OpKernelContext* context, Transformer fn) const;
// Should be called in derived class's constructor
template <typename Transformer>
void BuildLookupTableIfFixed(const OpKernelInfo& info, Transformer fn);
// when input quantizaton parameters are const, pre-compute table value.
// After construction, non-zero size means pre-computed. Save space when not pre-computed.
std::vector<uint8_t> fixed_lookup_table_;
};
template <typename T>
class QLinearLeakyRelu final : public QLinearLookupBase<T> {
public:
QLinearLeakyRelu(const OpKernelInfo& info);
@ -18,8 +46,14 @@ class QLinearLeakyRelu final : public OpKernel {
private:
const float alpha_;
bool is_fixed_parameters_; // Fixed Scale and Zero Point for both x and y
uint8_t fixed_lookup_table_[256]; // when is const paramter, table value is here.
};
template <typename T>
class QLinearSigmoid final : public QLinearLookupBase<T> {
public:
QLinearSigmoid(const OpKernelInfo& info);
Status Compute(OpKernelContext* context) const override;
};
} // namespace contrib

29
onnxruntime/core/graph/contrib_ops/contrib_defs.cc
View file

@ -2272,6 +2272,35 @@ and produces one output data (Tensor<T>) where the function `f(x) = quantize(alp
"Constrain input and output types to 8 bit tensors.")
.TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput);
const char* QLinearSigmoidDoc_ver1 = R"DOC(
QLinearSigmoid takes quantized input data (Tensor), and quantize parameter for output, and produces one output data
(Tensor<T>) where the function `f(x) = quantize(Sigmoid(dequantize(x)))`, is applied to the data tensor elementwise.
Wwhere the function `Sigmoid(x) = 1 / (1 + exp(-x))` )DOC";
ONNX_CONTRIB_OPERATOR_SCHEMA(QLinearSigmoid)
.SetDomain(kMSDomain)
.SinceVersion(1)
.SetDoc(QLinearSigmoidDoc_ver1)
.Input(0, "X", "Input tensor", "T")
.Input(1, "X_scale",
"Input X's scale. It's a scalar, which means a per-tensor/layer quantization.",
"tensor(float)")
.Input(2, "X_zero_point",
"Input X's zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.",
"T", OpSchema::Optional)
.Input(3, "Y_scale",
"Output Y's scale. It's a scalar, which means a per-tensor/layer quantization.",
"tensor(float)")
.Input(4, "Y_zero_point",
"Output Y's zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.",
"T", OpSchema::Optional)
.Output(0, "Y", "Output tensor", "T")
.TypeConstraint(
"T",
{"tensor(uint8)", "tensor(int8)"},
"Constrain input and output types to 8 bit tensors.")
.TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput);
ONNX_CONTRIB_OPERATOR_SCHEMA(MurmurHash3)
.SetDomain(kMSDomain)
.SinceVersion(1)

43
onnxruntime/test/contrib_ops/qlinear_lookup_table_test.cc
View file

@ -18,7 +18,7 @@ TEST(QLinearLookupTableBasedOperatorTests, QLinearLeakyRelu_Int8) {
std::vector<int64_t> dims = {16};
test.AddInput<int8_t>("X", dims, {0, 16, 17, 18, 19, 90, 91, 127, -128, -110, -108, -100, -16, -17, -18, -1});
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddMissingOptionalInput<int8_t>(); // optional "X_zero_point" using default value here
test.AddMissingOptionalInput<int8_t>(); // optional "X_zero_point" using default value here
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<int8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<int8_t>("Y", dims, {-100, -60, -58, -55, -52, 125, 127, 127, -128, -128, -127, -125, -104, -104, -104, -100});
@ -28,7 +28,6 @@ TEST(QLinearLookupTableBasedOperatorTests, QLinearLeakyRelu_Int8) {
std::fesetround(origin_round_mode);
}
TEST(QLinearLookupTableBasedOperatorTests, QLinearLeakyRelu_UInt8) {
OpTester test("QLinearLeakyRelu", 1, onnxruntime::kMSDomain);
test.AddAttribute<float>("alpha", 0.1f);
@ -50,5 +49,45 @@ TEST(QLinearLookupTableBasedOperatorTests, QLinearLeakyRelu_UInt8) {
std::fesetround(origin_round_mode);
}
TEST(QLinearLookupTableBasedOperatorTests, QLinearSigmoid_Int8) {
OpTester test("QLinearSigmoid", 1, onnxruntime::kMSDomain);
float X_scale = 0.025f;
//int8_t X_zero_point = 0;
float Y_scale = 1.0f / 256.0f;
int8_t Y_zero_point = -120;
std::vector<int64_t> dims = {16};
test.AddInput<int8_t>("X", dims, {0, 16, 17, 18, 19, 90, 91, 127, -128, -110, -108, -100, -16, -17, -18, -1});
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddMissingOptionalInput<int8_t>(); // optional "X_zero_point" using default value here
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<int8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<int8_t>("Y", dims, {8, 33, 35, 36, 38, 112, 112, 126, -110, -105, -104, -101, -17, -19, -20, 6});
auto origin_round_mode = std::fegetround();
std::fesetround(FE_TONEAREST);
test.Run();
std::fesetround(origin_round_mode);
}
TEST(QLinearLookupTableBasedOperatorTests, QLinearSigmoid_UInt8) {
OpTester test("QLinearSigmoid", 1, onnxruntime::kMSDomain);
float X_scale = 0.025f;
uint8_t X_zero_point = 128;
float Y_scale = 1.0f / 256.0f;
uint8_t Y_zero_point = 8;
std::vector<int64_t> dims = {16};
test.AddInput<uint8_t>("X", dims, {0, 16, 17, 18, 19, 90, 91, 127, 128, 136, 137, 138, 216, 217, 218, 255});
test.AddInput<float>("X_scale", {}, {X_scale});
test.AddInput<uint8_t>("X_zero_point", {}, {X_zero_point});
test.AddInput<float>("Y_scale", {}, {Y_scale});
test.AddInput<uint8_t>("Y_zero_point", {}, {Y_zero_point});
test.AddOutput<uint8_t>("Y", dims, {18, 23, 23, 23, 24, 79, 81, 134, 136, 149, 150, 152, 238, 239, 240, 254});
auto origin_round_mode = std::fegetround();
std::fesetround(FE_TONEAREST);
test.Run();
std::fesetround(origin_round_mode);
}
} // namespace test
} // namespace onnxruntime