saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

implement per-channel for quantizelinear and dequantizelinear (#4759)

Browse source 
* update onnx to latest master

* implement per-channel for quantizelinear and dequantizelinear

* refine the unit test

* exclude sequence_insert tests

* refine onnx cmake

* add failure tests to broken_tests

* move qdq common code to a seperate function

* refine code
This commit is contained in:
Yufeng Li 2020-08-21 12:08:50 -07:00 committed by GitHub
parent 5427a7e9af
commit fb43aa0de0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 618 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cgmanifests/submodules/cgmanifest.json
View file

@ -335,7 +335,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "a82c6a7010e2e332d8f74ad5b0c726fd47c85376",
"commitHash": "c443abd2acad2411103593600319ff81a676afbc",
"repositoryUrl": "https://github.com/onnx/onnx"
},
"comments": "git submodule at cmake/external/onnx"

2
cmake/external/onnx vendored

@ -1 +1 @@
Subproject commit a82c6a7010e2e332d8f74ad5b0c726fd47c85376
Subproject commit c443abd2acad2411103593600319ff81a676afbc

5
cmake/onnx/CMakeLists.txt
View file

@ -3,7 +3,10 @@
#TODO: if protobuf is a shared lib and onnxruntime_USE_FULL_PROTOBUF is ON, then onnx_proto should be built as a shared lib instead of a static lib. Otherwise any code outside onnxruntime.dll can't use onnx protobuf definitions if they share the protobuf.dll with onnxruntime. For example, if protobuf is a shared lib and onnx_proto is a static lib then onnxruntime_perf_test won't work.
add_library(onnx_proto ${ONNXRUNTIME_ROOT}/core/protobuf/onnx-ml.proto ${ONNXRUNTIME_ROOT}/core/protobuf/onnx-operators-ml.proto)
add_library(onnx_proto
${ONNXRUNTIME_ROOT}/core/protobuf/onnx-ml.proto
${ONNXRUNTIME_ROOT}/core/protobuf/onnx-operators-ml.proto
${ONNXRUNTIME_ROOT}/core/protobuf/onnx-data.proto)
target_include_directories(onnx_proto PUBLIC $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_INCLUDE_DIRECTORIES> "${CMAKE_CURRENT_BINARY_DIR}/..")
target_compile_definitions(onnx_proto PUBLIC $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_COMPILE_DEFINITIONS>)
onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS ${ONNXRUNTIME_ROOT}/core/protobuf TARGET onnx_proto)

101
onnxruntime/core/protobuf/onnx-data.in.proto Normal file
View file

@ -0,0 +1,101 @@
// Copyright (c) ONNX Project Contributors.
// Licensed under the MIT license.
syntax = "proto2";
package {PACKAGE_NAME};
// #if ONNX-ML
import "onnx-ml.proto";
// #else
import "onnx.proto";
// #endif
// This file contains the proto definitions for MapProto and
// SequenceProto. These protos are used to represent the data structures
// of maps and sequence for use in test data or ModelProto.
// Sequences
//
// Defines a dense, ordered, collection of elements that are of homogeneous types.
// Sequences can be made out of tensors, maps, or sequences.
//
// If a sequence is made out of tensors, the tensors must have the same element
// type (i.e. int32). In some cases, the tensors in a sequence can have different
// shapes. Whether the tensors can have different shapes or not depends on the
// type/shape associated with the corresponding "ValueInfo". For example,
// "Sequence<Tensor<float, [M,N]>" means that all tensors have same shape. However,
// "Sequence<Tensor<float, [omitted,omitted]>" means they can have different
// shapes (all of rank 2), where "omitted" means the corresponding dimension has
// no symbolic/constant value. Finally, "Sequence<Tensor<float, omitted>>" means
// that the different tensors can have different ranks, when the "shape" itself
// is omitted from the tensor-type. For a more complete description, refer to
// https://github.com/onnx/onnx/blob/master/docs/IR.md#static-tensor-shapes.
//
message SequenceProto {
optional string name = 1;
enum DataType {
UNDEFINED = 0;
TENSOR = 1;
SPARSE_TENSOR = 2;
SEQUENCE = 3;
MAP = 4;
}
// The data type of the element.
// This field MUST have a valid SequenceProto.DataType value
optional int32 elem_type = 2;
// For TensorProto values.
// When this field is present, the elem_type field MUST be TENSOR.
repeated TensorProto tensor_values = 3;
// For SparseTensorProto values.
// When this field is present, the elem_type field MUST be SPARSE_TENSOR.
repeated SparseTensorProto sparse_tensor_values = 4;
// For SequenceProto values, allowing sequences to be of themselves.
// When this field is present, the elem_type field MUST be SEQUENCE.
repeated SequenceProto sequence_values = 5;
// For MapProto values.
// When this field is present, the elem_type field MUST be MAP.
repeated MapProto map_values = 6;
}
// Maps
//
// Specifies an associative table, defined by keys and values.
// MapProto is formed with a repeated field of keys (of type INT8, INT16, INT32,
// INT64, UINT8, UINT16, UINT32, UINT64, or STRING) and values (of type TENSOR,
// SPARSE_TENSOR, SEQUENCE, or MAP). Key types and value types have to remain
// the same throughout the instantiation of the MapProto.
//
message MapProto {
optional string name = 1;
// All MapProto data types must have the same length of keys and values.
// The data type of the key.
// This field MUST have a valid TensorProto.DataType value of
// INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING
optional int32 key_type = 2;
// Every element of keys has to be one of the following data types
// INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING.
// The integer cases are represented by the repeated int64 field keys below.
repeated int64 keys = 3;
// If keys are strings, they are represented by the repeated bytes field
// string_keys below.
repeated bytes string_keys = 4;
// MapProto values are represented in a SequenceProto of the same length as the
// repeated keys field and have to be one of the following data types
// TENSOR, SPARSE_TENSOR, MAP, SEQUENCE.
optional SequenceProto values = 5;
}

106
onnxruntime/core/protobuf/onnx-data.proto Normal file
View file

@ -0,0 +1,106 @@
//
// WARNING: This file is automatically generated! Please edit onnx.in.proto.
//
// Copyright (c) ONNX Project Contributors.
// Licensed under the MIT license.
syntax = "proto2";
package onnx;
import "onnx-ml.proto";
// This file contains the proto definitions for MapProto and
// SequenceProto. These protos are used to represent the data structures
// of maps and sequence for use in test data or ModelProto.
// Sequences
//
// Defines a dense, ordered, collection of elements that are of homogeneous types.
// Sequences can be made out of tensors, maps, or sequences.
//
// If a sequence is made out of tensors, the tensors must have the same element
// type (i.e. int32). In some cases, the tensors in a sequence can have different
// shapes. Whether the tensors can have different shapes or not depends on the
// type/shape associated with the corresponding "ValueInfo". For example,
// "Sequence<Tensor<float, [M,N]>" means that all tensors have same shape. However,
// "Sequence<Tensor<float, [omitted,omitted]>" means they can have different
// shapes (all of rank 2), where "omitted" means the corresponding dimension has
// no symbolic/constant value. Finally, "Sequence<Tensor<float, omitted>>" means
// that the different tensors can have different ranks, when the "shape" itself
// is omitted from the tensor-type. For a more complete description, refer to
// https://github.com/onnx/onnx/blob/master/docs/IR.md#static-tensor-shapes.
//
message SequenceProto {
optional string name = 1;
enum DataType {
UNDEFINED = 0;
TENSOR = 1;
SPARSE_TENSOR = 2;
SEQUENCE = 3;
MAP = 4;
}
// The data type of the element.
// This field MUST have a valid SequenceProto.DataType value
optional int32 elem_type = 2;
// For TensorProto values.
// When this field is present, the elem_type field MUST be TENSOR.
repeated TensorProto tensor_values = 3;
// For SparseTensorProto values.
// When this field is present, the elem_type field MUST be SPARSE_TENSOR.
repeated SparseTensorProto sparse_tensor_values = 4;
// For SequenceProto values, allowing sequences to be of themselves.
// When this field is present, the elem_type field MUST be SEQUENCE.
repeated SequenceProto sequence_values = 5;
// For MapProto values.
// When this field is present, the elem_type field MUST be MAP.
repeated MapProto map_values = 6;
}
// Maps
//
// Specifies an associative table, defined by keys and values.
// MapProto is formed with a repeated field of keys (of type INT8, INT16, INT32,
// INT64, UINT8, UINT16, UINT32, UINT64, or STRING) and values (of type TENSOR,
// SPARSE_TENSOR, SEQUENCE, or MAP). Key types and value types have to remain
// the same throughout the instantiation of the MapProto.
//
message MapProto {
optional string name = 1;
// All MapProto data types must have the same length of keys and values.
// The data type of the key.
// This field MUST have a valid TensorProto.DataType value of
// INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING
optional int32 key_type = 2;
// Every element of keys has to be one of the following data types
// INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING.
// The integer cases are represented by the repeated int64 field keys below.
repeated int64 keys = 3;
// If keys are strings, they are represented by the repeated bytes field
// string_keys below.
repeated bytes string_keys = 4;
// MapProto values are represented in a SequenceProto of the same length as the
// repeated keys field and have to be one of the following data types
// TENSOR, SPARSE_TENSOR, MAP, SEQUENCE.
optional SequenceProto values = 5;
}
// For using protobuf-lite
option optimize_for = LITE_RUNTIME;

106
onnxruntime/core/protobuf/onnx-data.proto3 Normal file
View file

@ -0,0 +1,106 @@
//
// WARNING: This file is automatically generated! Please edit onnx.in.proto.
//
// Copyright (c) ONNX Project Contributors.
// Licensed under the MIT license.
syntax = "proto3";
package onnx;
import "onnx-ml.proto3";
// This file contains the proto definitions for MapProto and
// SequenceProto. These protos are used to represent the data structures
// of maps and sequence for use in test data or ModelProto.
// Sequences
//
// Defines a dense, ordered, collection of elements that are of homogeneous types.
// Sequences can be made out of tensors, maps, or sequences.
//
// If a sequence is made out of tensors, the tensors must have the same element
// type (i.e. int32). In some cases, the tensors in a sequence can have different
// shapes. Whether the tensors can have different shapes or not depends on the
// type/shape associated with the corresponding "ValueInfo". For example,
// "Sequence<Tensor<float, [M,N]>" means that all tensors have same shape. However,
// "Sequence<Tensor<float, [omitted,omitted]>" means they can have different
// shapes (all of rank 2), where "omitted" means the corresponding dimension has
// no symbolic/constant value. Finally, "Sequence<Tensor<float, omitted>>" means
// that the different tensors can have different ranks, when the "shape" itself
// is omitted from the tensor-type. For a more complete description, refer to
// https://github.com/onnx/onnx/blob/master/docs/IR.md#static-tensor-shapes.
//
message SequenceProto {
string name = 1;
enum DataType {
UNDEFINED = 0;
TENSOR = 1;
SPARSE_TENSOR = 2;
SEQUENCE = 3;
MAP = 4;
}
// The data type of the element.
// This field MUST have a valid SequenceProto.DataType value
int32 elem_type = 2;
// For TensorProto values.
// When this field is present, the elem_type field MUST be TENSOR.
repeated TensorProto tensor_values = 3;
// For SparseTensorProto values.
// When this field is present, the elem_type field MUST be SPARSE_TENSOR.
repeated SparseTensorProto sparse_tensor_values = 4;
// For SequenceProto values, allowing sequences to be of themselves.
// When this field is present, the elem_type field MUST be SEQUENCE.
repeated SequenceProto sequence_values = 5;
// For MapProto values.
// When this field is present, the elem_type field MUST be MAP.
repeated MapProto map_values = 6;
}
// Maps
//
// Specifies an associative table, defined by keys and values.
// MapProto is formed with a repeated field of keys (of type INT8, INT16, INT32,
// INT64, UINT8, UINT16, UINT32, UINT64, or STRING) and values (of type TENSOR,
// SPARSE_TENSOR, SEQUENCE, or MAP). Key types and value types have to remain
// the same throughout the instantiation of the MapProto.
//
message MapProto {
string name = 1;
// All MapProto data types must have the same length of keys and values.
// The data type of the key.
// This field MUST have a valid TensorProto.DataType value of
// INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING
int32 key_type = 2;
// Every element of keys has to be one of the following data types
// INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING.
// The integer cases are represented by the repeated int64 field keys below.
repeated int64 keys = 3;
// If keys are strings, they are represented by the repeated bytes field
// string_keys below.
repeated bytes string_keys = 4;
// MapProto values are represented in a SequenceProto of the same length as the
// repeated keys field and have to be one of the following data types
// TENSOR, SPARSE_TENSOR, MAP, SEQUENCE.
SequenceProto values = 5;
}
// For using protobuf-lite
option optimize_for = LITE_RUNTIME;

2
onnxruntime/core/providers/common.h
View file

@ -26,7 +26,7 @@ Returns true if given tensor is a scalar or 1D tensor of size 1
**/
inline bool IsScalarOr1ElementVector(const Tensor* input) {
if (input->Shape().NumDimensions() == 0 ||
(input->Shape().NumDimensions() == 1 && input->Shape().GetDims().size() == 1)) {
(input->Shape().NumDimensions() == 1 && input->Shape().Size() == 1)) {
return true;
} else {
return false;

36
onnxruntime/core/providers/cpu/cpu_execution_provider.cc
View file

@ -278,10 +278,10 @@ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOn
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, int32_t, Resize);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, uint8_t, Resize);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, ThresholdedRelu);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t, DequantizeLinear);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, int8_t, DequantizeLinear);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t, QuantizeLinear);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, int8_t, QuantizeLinear);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, uint8_t, DequantizeLinear);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, int8_t, DequantizeLinear);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, uint8_t, QuantizeLinear);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, int8_t, QuantizeLinear);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, QLinearMatMul);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t, MatMulInteger);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, ConvInteger);
@ -440,6 +440,10 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain,
// opset 13
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Cast);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Sign);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, uint8_t, DequantizeLinear);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, int8_t, DequantizeLinear);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, uint8_t, QuantizeLinear);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, int8_t, QuantizeLinear);
Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
static const BuildKernelCreateInfoFn function_table[] = {
@ -841,14 +845,14 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10,
uint8_t, Resize)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, ThresholdedRelu)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t,
DequantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, int8_t,
DequantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t,
QuantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, int8_t,
QuantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, uint8_t,
DequantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, int8_t,
DequantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, uint8_t,
QuantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, int8_t,
QuantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, QLinearMatMul)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t,
MatMulInteger)>,
@ -1100,6 +1104,14 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
// opset 13
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Cast)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Sign)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, uint8_t,
DequantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, int8_t,
DequantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, uint8_t,
QuantizeLinear)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, int8_t,
QuantizeLinear)>,
};
for (auto& function_table_entry : function_table) {

156
onnxruntime/core/providers/cpu/tensor/quantize_linear.cc
View file

@ -7,21 +7,60 @@
namespace onnxruntime {
ONNX_CPU_OPERATOR_TYPED_KERNEL(
DequantizeLinear,
10,
uint8_t,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<uint8_t>()),
DequantizeLinear<uint8_t>);
static void PrepareForQDQ(const TensorShape& input_shape,
const Tensor& scale,
const Tensor* zero_point_ptr,
int64_t axis,
int64_t& block_count,
int64_t& broadcast_dim,
int64_t& block_size) {
if (IsScalarOr1ElementVector(&scale)) { // per-tensor QuantizeLinear/DequantizeLinear
block_count = 1;
broadcast_dim = 1;
block_size = static_cast<size_t>(input_shape.Size());
ONNX_CPU_OPERATOR_TYPED_KERNEL(
DequantizeLinear,
10,
int8_t,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<int8_t>()),
DequantizeLinear<int8_t>);
// enforce that zero point are scalars
ORT_ENFORCE(zero_point_ptr == nullptr || IsScalarOr1ElementVector(zero_point_ptr),
"x_zero_point must be null or a scalar or 1D tensor or size 1.");
} else { // per-channel QuantizeLinear/DequantizeLinear
const int64_t axis_no_neg = HandleNegativeAxis(axis, input_shape.NumDimensions());
block_count = input_shape.SizeToDimension(axis_no_neg);
broadcast_dim = input_shape[axis_no_neg];
block_size = input_shape.SizeFromDimension(axis_no_neg + 1);
// if an axis was specified, ensure the scale and zero point are compatible
ORT_ENFORCE(scale.Shape().NumDimensions() == 1 && scale.Shape()[0] == broadcast_dim,
"scale must be 1D tensor with size ",
broadcast_dim);
ORT_ENFORCE(zero_point_ptr == nullptr || (zero_point_ptr->Shape().NumDimensions() == 1 && zero_point_ptr->Shape()[0] == broadcast_dim),
"x_zero_point must be null or 1D tensor with size ",
broadcast_dim);
}
}
#define REGISTER_DEQUANTIZELINEAR(T) \
ONNX_CPU_OPERATOR_TYPED_KERNEL( \
DequantizeLinear, \
13, \
T, \
KernelDefBuilder() \
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
DequantizeLinear<T>);
#define REGISTER_DEQUANTIZELINEAR_VERSIONED(T) \
ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( \
DequantizeLinear, \
10, \
12, \
T, \
KernelDefBuilder() \
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
DequantizeLinear<T>);
REGISTER_DEQUANTIZELINEAR(int8_t)
REGISTER_DEQUANTIZELINEAR(uint8_t)
REGISTER_DEQUANTIZELINEAR_VERSIONED(int8_t)
REGISTER_DEQUANTIZELINEAR_VERSIONED(uint8_t)
template <typename T>
// formula is Y = (X - ZeroPoint) * Scale
@ -37,28 +76,7 @@ Status DequantizeLinear<T>::Compute(OpKernelContext* ctx) const {
int64_t broadcast_dim;
int64_t block_size;
if (has_axis_) { // custom DequantizeLinear only
const int64_t axis = HandleNegativeAxis(axis_, x_shape.NumDimensions());
N = x_shape.SizeToDimension(axis);
broadcast_dim = x_shape[axis];
block_size = x_shape.SizeFromDimension(axis + 1);
// if an axis was specified, ensure the scale and zero point are compatible
ORT_ENFORCE(x_scale.Shape().NumDimensions() == 1 && x_scale.Shape().Size() == broadcast_dim,
"x_scale must be 1D tensor with size ",
broadcast_dim);
ORT_ENFORCE(x_zero_point != nullptr && x_zero_point->Shape().NumDimensions() == 1 && x_zero_point->Shape().Size() == broadcast_dim,
"x_zero_point must be 1D tensor with size ",
broadcast_dim);
} else {
N = 1;
broadcast_dim = 1;
block_size = static_cast<size_t>(x_shape.Size());
// if no axis, enforce that scale and zero point are scalars
ORT_ENFORCE(IsScalarOr1ElementVector(&x_scale), "x_scale must be a scalar or 1D tensor or size 1.");
ORT_ENFORCE(x_zero_point == nullptr || IsScalarOr1ElementVector(x_zero_point), "x_zero_point must be a scalar or 1D tensor or size 1.");
}
PrepareForQDQ(x.Shape(), x_scale, x_zero_point, axis_, N, broadcast_dim, block_size);
const T* zero_point = x_zero_point ? x_zero_point->template Data<T>() : nullptr;
const float* scale = x_scale.template Data<float>();
@ -79,23 +97,31 @@ Status DequantizeLinear<T>::Compute(OpKernelContext* ctx) const {
return Status::OK();
}
ONNX_CPU_OPERATOR_TYPED_KERNEL(
QuantizeLinear,
10,
uint8_t,
KernelDefBuilder()
.TypeConstraint("T1", DataTypeImpl::GetTensorType<float>())
.TypeConstraint("T2", DataTypeImpl::GetTensorType<uint8_t>()),
QuantizeLinear<uint8_t>);
#define REGISTER_QUANTIZELINEAR(T) \
ONNX_CPU_OPERATOR_TYPED_KERNEL( \
QuantizeLinear, \
13, \
T, \
KernelDefBuilder() \
.TypeConstraint("T1", DataTypeImpl::GetTensorType<float>()) \
.TypeConstraint("T2", DataTypeImpl::GetTensorType<T>()), \
QuantizeLinear<T>);
ONNX_CPU_OPERATOR_TYPED_KERNEL(
QuantizeLinear,
10,
int8_t,
KernelDefBuilder()
.TypeConstraint("T1", DataTypeImpl::GetTensorType<float>())
.TypeConstraint("T2", DataTypeImpl::GetTensorType<int8_t>()),
QuantizeLinear<int8_t>);
#define REGISTER_QUANTIZELINEAR_VERSIONED(T) \
ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( \
QuantizeLinear, \
10, \
12, \
T, \
KernelDefBuilder() \
.TypeConstraint("T1", DataTypeImpl::GetTensorType<float>()) \
.TypeConstraint("T2", DataTypeImpl::GetTensorType<T>()), \
QuantizeLinear<T>);
REGISTER_QUANTIZELINEAR(int8_t)
REGISTER_QUANTIZELINEAR(uint8_t)
REGISTER_QUANTIZELINEAR_VERSIONED(int8_t)
REGISTER_QUANTIZELINEAR_VERSIONED(uint8_t)
template <typename T>
// formula is Y = X / Scale + ZeroPoint
@ -109,31 +135,7 @@ Status QuantizeLinear<T>::Compute(OpKernelContext* ctx) const {
int64_t N;
int64_t broadcast_dim;
int64_t block_size;
if (has_axis_) { // custom QuantizeLinear only
const int64_t axis = HandleNegativeAxis(axis_, x_shape.NumDimensions());
N = x_shape.SizeToDimension(axis);
broadcast_dim = x_shape[axis];
block_size = x_shape.SizeFromDimension(axis + 1);
// if an axis was specified, ensure the scale and zero point are compatible
ORT_ENFORCE(y_scale.Shape().NumDimensions() == 1 && y_scale.Shape().Size() == broadcast_dim,
"y_scale must be 1D tensor with size ",
broadcast_dim);
ORT_ENFORCE(y_zero_point != nullptr && y_zero_point->Shape().NumDimensions() == 1 && y_zero_point->Shape().Size() == broadcast_dim,
"y_zero_point must be 1D tensor with size ",
broadcast_dim);
} else {
N = 1;
broadcast_dim = 1;
block_size = x_shape.Size();
// if no axis, enforce that scale and zero point are scalars
ORT_ENFORCE(IsScalarOr1ElementVector(&y_scale),
"y_scale must be a scalar or 1D tensor or size 1.");
ORT_ENFORCE(y_zero_point == nullptr || IsScalarOr1ElementVector(y_zero_point),
"y_zero_point must be a scalar or 1D tensor or size 1.");
}
PrepareForQDQ(x.Shape(), y_scale, y_zero_point, axis_, N, broadcast_dim, block_size);
const T* zero_point = y_zero_point != nullptr ? y_zero_point->template Data<T>() : nullptr;
const float* scale = y_scale.template Data<float>();

18
onnxruntime/core/providers/cpu/tensor/quantize_linear.h
View file

@ -13,27 +13,29 @@ template <typename T>
class DequantizeLinear final : public OpKernel {
public:
DequantizeLinear(const OpKernelInfo& info) : OpKernel(info) {
has_axis_ = info.GetAttr<int64_t>("axis", &axis_).IsOK();
if (!info.GetAttr<int64_t>("axis", &axis_).IsOK()) {
axis_ = 1;
}
}
Status Compute(OpKernelContext* context) const override;
private:
int64_t axis_ = 0;
bool has_axis_;
private:
int64_t axis_;
};
template <typename T>
class QuantizeLinear final : public OpKernel {
public:
QuantizeLinear(const OpKernelInfo& info) : OpKernel(info) {
has_axis_ = info.GetAttr<int64_t>("axis", &axis_).IsOK();
if (!info.GetAttr<int64_t>("axis", &axis_).IsOK()) {
axis_ = 1;
}
}
Status Compute(OpKernelContext* context) const override;
private:
int64_t axis_ = 0;
bool has_axis_;
int64_t axis_;
};
} // namespace onnxruntime

10
onnxruntime/test/onnx/main.cc
View file

@ -547,6 +547,8 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
{"nesterov_momentum", "not a registered function/op", {}}, // Op not registered.
{"cast_FLOAT_to_BFLOAT16", "onnx generate bfloat tensor as uint16 type", {}},
{"cast_BFLOAT16_to_FLOAT", "onnx generate bfloat tensor as uint16 type", {}},
{"sequence_insert_at_back", "onnx currently not supporting loading segment", {}},
{"sequence_insert_at_front", "onnx currently not supporting loading segment", {}},
};
#ifdef DISABLE_ML_OPS
@ -756,6 +758,14 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
broken_tests.insert({"nllloss_NCd1d2_with_weight_reduction_mean_expanded", "wait for investigation"});
broken_tests.insert({"nllloss_NCd1d2d3d4d5_mean_weight", "wait for investigation"});
broken_tests.insert({"nllloss_NCd1d2d3d4d5_mean_weight_expanded", "wait for investigation"});
broken_tests.insert({"nllloss_NCd1_ii", "wait for investigation"});
broken_tests.insert({"nllloss_NCd1_ii_expanded", "wait for investigation"});
broken_tests.insert({"nllloss_NCd1_mean_weight_negative_ii", "wait for investigation"});
broken_tests.insert({"nllloss_NCd1_mean_weight_negative_ii_expanded", "wait for investigation"});
broken_tests.insert({"nllloss_NCd1_weight_ii", "wait for investigation"});
broken_tests.insert({"nllloss_NCd1_weight_ii_expanded", "wait for investigation"});
broken_tests.insert({"nllloss_NCd1d2_no_weight_reduction_mean_ii", "wait for investigation"});
broken_tests.insert({"nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", "wait for investigation"});
}
if (enable_tensorrt) {

184
onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc
View file

@ -7,7 +7,7 @@
namespace onnxruntime {
namespace test {
// scalar zero & scale with uint8
TEST(DequantizeLinearOpTest, DequantizeLinear_0) {
TEST(DequantizeLinearOpTest, Uint8) {
OpTester test("DequantizeLinear", 10);
std::vector<int64_t> dims{4};
test.AddInput<uint8_t>("x", dims, {0, 3, 128, 255});
@ -18,7 +18,7 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_0) {
}
// scalar zero & scale with int8
TEST(DequantizeLinearOpTest, DequantizeLinear_1) {
TEST(DequantizeLinearOpTest, Int8) {
OpTester test("DequantizeLinear", 10);
std::vector<int64_t> dims{4};
test.AddInput<int8_t>("x", dims, {-30, -3, 100, 127});
@ -29,7 +29,7 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_1) {
}
// 2d inputs
TEST(DequantizeLinearOpTest, DequantizeLinear_2D) {
TEST(DequantizeLinearOpTest, 2D) {
OpTester test("DequantizeLinear", 10);
std::vector<int64_t> dims{3, 4};
test.AddInput<uint8_t>("X", dims,
@ -46,7 +46,7 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_2D) {
}
// dequantize with scalar data
TEST(DequantizeLinearOpTest, DequantizeLinear_Scalar) {
TEST(DequantizeLinearOpTest, Scalar) {
OpTester test("DequantizeLinear", 10);
test.AddInput<int8_t>("x", {}, {100});
test.AddInput<float>("x_scale", {}, {2.0f});
@ -56,7 +56,7 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_Scalar) {
}
// dequantize without zero point
TEST(DequantizeLinearOpTest, DequantizeLinear_Without_Zero_Point) {
TEST(DequantizeLinearOpTest, Without_Zero_Point) {
OpTester test("DequantizeLinear", 10);
test.AddInput<int8_t>("x", {}, {100});
test.AddInput<float>("x_scale", {}, {2.0f});
@ -64,8 +64,119 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_Without_Zero_Point) {
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider});
}
// 1d zero & scale with default axis
TEST(DequantizeLinearOpTest, Per_Channel_Axis_Default) {
OpTester test("DequantizeLinear", 13);
std::vector<int64_t> dims{2, 3, 2, 4};
test.AddInput<int8_t>("X", dims,
{7, 9, 10, 10,
5, 8, 9, 1,
8, 6, 7, 9,
10, 0, 7, 10,
8, 2, 6, 0,
5, 9, 8, 1,
2, 7, 5, 3,
2, 4, 1, 3,
8, 7, 4, 8,
10, 1, 5, 5,
7, 7, 0, 2,
4, 4, 0, 5});
test.AddInput<float>("scale", {3}, {1, 10, 7});
test.AddInput<int8_t>("zero_point", {3}, {10, 2, 1});
test.AddOutput<float>("Y", dims,
{-3, -1, 0, 0,
-5, -2, -1, -9,
60, 40, 50, 70,
80, -20, 50, 80,
49, 7, 35, -7,
28, 56, 49, 0,
-8, -3, -5, -7,
-8, -6, -9, -7,
60, 50, 20, 60,
80, -10, 30, 30,
42, 42, -7, 7,
21, 21, -7, 28});
test.Run();
}
// 1d zero & scale with uint8 broadcast axis 0
TEST(DequantizeLinearOpTest, Per_Channel_Axis_0) {
OpTester test("DequantizeLinear", 13);
std::vector<int64_t> dims{3, 4};
test.AddInput<uint8_t>("X", dims,
{0, 1, 2, 3,
0, 1, 2, 3,
0, 10, 20, 30});
test.AddAttribute<int64_t>("axis", 0);
test.AddInput<float>("scale", {3},
{1.0f,
2.0f,
4.0f});
test.AddInput<uint8_t>("zero_point", {3},
{0,
0,
0});
test.AddOutput<float>("Y", dims,
{0, 1, 2, 3,
0, 2, 4, 6,
0, 40, 80, 120});
test.Run();
}
// 1d zero & scale with int8 broadcast axis 1
TEST(DequantizeLinearOpTest, Per_Channel_Axis_1) {
OpTester test("DequantizeLinear", 13);
std::vector<int64_t> dims{3, 4};
test.AddInput<int8_t>("X", dims,
{0, 1, 2, 3,
0, 2, 4, 6,
0, 10, 20, 30});
test.AddAttribute<int64_t>("axis", 1);
test.AddInput<float>("scale", {4}, {1, 2, 4, 8});
test.AddInput<int8_t>("zero_point", {4}, {0, -10, -20, -30});
test.AddOutput<float>("Y", dims,
{0, 22, 88, 264,
0, 24, 96, 288,
0, 40, 160, 480});
test.Run();
}
// 1d zero & scale with uint8 broadcast axis -2 (-2 resolves to axis 0)
TEST(DequantizeLinearOpTest, Per_Channel_Neg_2) {
OpTester test("DequantizeLinear", 13);
std::vector<int64_t> dims{3, 4};
test.AddInput<uint8_t>("X", dims,
{0, 1, 2, 3,
0, 1, 2, 3,
0, 10, 20, 30});
test.AddAttribute<int64_t>("axis", -2);
test.AddInput<float>("scale", {3},
{1.0f,
2.0f,
4.0f});
test.AddInput<uint8_t>("zero_point", {3},
{0,
0,
0});
test.AddOutput<float>("Y", dims,
{0, 1, 2, 3,
0, 2, 4, 6,
0, 40, 80, 120});
test.Run();
}
// quantize with scalar zero point and scale
TEST(QuantizeLinearOpTest, QuantizeLinear_uint8) {
TEST(QuantizeLinearOpTest, Uint8) {
OpTester test("QuantizeLinear", 10);
std::vector<int64_t> dims{6};
test.AddInput<float>("x", dims, {0, 2, 3, 1000, -254, -1000});
@ -76,7 +187,7 @@ TEST(QuantizeLinearOpTest, QuantizeLinear_uint8) {
}
// quantize with scalar zero point and scale
TEST(QuantizeLinearOpTest, QuantizeLinear_int8) {
TEST(QuantizeLinearOpTest, Int8) {
OpTester test("QuantizeLinear", 10);
std::vector<int64_t> dims{6};
test.AddInput<float>("x", dims, {0, 2, 3, 5, -2, -5});
@ -87,7 +198,7 @@ TEST(QuantizeLinearOpTest, QuantizeLinear_int8) {
}
// quantize with scalar zero point and scale
TEST(QuantizeLinearOpTest, QuantizeLinear_int8_NegativeZeroPoint) {
TEST(QuantizeLinearOpTest, Int8_NegativeZeroPoint) {
OpTester test("QuantizeLinear", 10);
std::vector<int64_t> dims{8};
test.AddInput<float>("x", dims, {0, 2, 3, 5, 6, -2, -5, -6});
@ -98,7 +209,7 @@ TEST(QuantizeLinearOpTest, QuantizeLinear_int8_NegativeZeroPoint) {
}
// quantize with scalar zero point and scale
TEST(QuantizeLinearOpTest, QuantizeLinear_int8_PositiveZeroPoint) {
TEST(QuantizeLinearOpTest, Int8_PositiveZeroPoint) {
OpTester test("QuantizeLinear", 10);
std::vector<int64_t> dims{8};
test.AddInput<float>("x", dims, {0, 2, 3, 5, 6, -2, -5, -6});
@ -109,7 +220,7 @@ TEST(QuantizeLinearOpTest, QuantizeLinear_int8_PositiveZeroPoint) {
}
// quantize with 2D data
TEST(QuantizeLinearOpTest, QuantizeLinear_2D) {
TEST(QuantizeLinearOpTest, 2D) {
OpTester test("QuantizeLinear", 10);
std::vector<int64_t> dims{3, 4};
test.AddInput<float>("X", dims,
@ -126,7 +237,7 @@ TEST(QuantizeLinearOpTest, QuantizeLinear_2D) {
}
// quantize with scalar data
TEST(QuantizeLinearOpTest, QuantizeLinear_Scalar) {
TEST(QuantizeLinearOpTest, Scalar) {
OpTester test("QuantizeLinear", 10);
test.AddInput<float>("x", {}, {3});
test.AddInput<float>("y_scale", {}, {2.0f});
@ -144,5 +255,56 @@ TEST(QuantizeLinearOpTest, DISABLED_QuantizeLinear_Without_Zero_Point) {
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider});
}
TEST(QuantizeLinearOpTest, Per_Channel_Axis_Default) {
OpTester test("QuantizeLinear", 13);
std::vector<int64_t> dims{3, 4};
test.AddInput<float>("X", dims,
{0, 2, 1, 1001,
1, 1, 2, 1100,
2, 4.2f, 3, 1200});
test.AddInput<float>("scale", {4}, {1, 2, 3, 20});
test.AddInput<uint8_t>("zero_point", {4}, {64, 100, 127, 127});
test.AddOutput<uint8_t>("Y", dims,
{64, 101, 127, 177,
65, 100, 128, 182,
66, 102, 128, 187});
test.Run();
}
TEST(QuantizeLinearOpTest, Per_Channel_Axis_0) {
OpTester test("QuantizeLinear", 13);
std::vector<int64_t> dims{3, 4};
test.AddInput<float>("X", dims,
{0, 2, 3, 1000,
0, 2, 3, 1000,
0, 2, 3, 1000});
test.AddAttribute<int64_t>("axis", 0);
test.AddInput<float>("scale", {3}, {1, 2, 4});
test.AddInput<uint8_t>("zero_point", {3}, {0, 0, 0});
test.AddOutput<uint8_t>("Y", dims,
{0, 2, 3, 255,
0, 1, 2, 255,
0, 0, 1, 250});
test.Run();
}
// quantize with per-channel and negative axis (-2 resolves to axis 0)
TEST(QuantizeLinearOpTest, Per_Channel_Axis_neg) {
OpTester test("QuantizeLinear", 13);
std::vector<int64_t> dims{3, 4};
test.AddInput<float>("X", dims,
{0, 2, 3, 1000,
0, 2, 3, 1000,
0, 2, 3, 1000});
test.AddAttribute<int64_t>("axis", -2);
test.AddInput<float>("scale", {3}, {1, 2, 4});
test.AddInput<uint8_t>("zero_point", {3}, {0, 0, 0});
test.AddOutput<uint8_t>("Y", dims,
{0, 2, 3, 255,
0, 1, 2, 255,
0, 0, 1, 250});
test.Run();
}
} // namespace test
} // namespace onnxruntime

2
onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc vendored
View file

@ -21,6 +21,8 @@
"^test_nesterov_momentum",
"^test_pow_types_float32_uint32",
"^test_pow_types_float32_uint64",
"^test_sequence_insert_at_back_cpu", // numpy_helper currently not supporting loading segments.
"^test_sequence_insert_at_front_cpu", // numpy_helper currently not supporting loading segments.
"^test_gradient_of_add_and_mul",
"^test_gradient_of_add",
"^test_batchnorm_example_training_mode",