From fb43aa0de07604860f7b4c7586fc9bd506a188b6 Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Fri, 21 Aug 2020 12:08:50 -0700 Subject: [PATCH] implement per-channel for quantizelinear and dequantizelinear (#4759) * update onnx to latest master * implement per-channel for quantizelinear and dequantizelinear * refine the unit test * exclude sequence_insert tests * refine onnx cmake * add failure tests to broken_tests * move qdq common code to a seperate function * refine code --- cgmanifests/submodules/cgmanifest.json | 2 +- cmake/external/onnx | 2 +- cmake/onnx/CMakeLists.txt | 5 +- onnxruntime/core/protobuf/onnx-data.in.proto | 101 ++++++++++ onnxruntime/core/protobuf/onnx-data.proto | 106 ++++++++++ onnxruntime/core/protobuf/onnx-data.proto3 | 106 ++++++++++ onnxruntime/core/providers/common.h | 2 +- .../providers/cpu/cpu_execution_provider.cc | 36 ++-- .../providers/cpu/tensor/quantize_linear.cc | 156 +++++++-------- .../providers/cpu/tensor/quantize_linear.h | 18 +- onnxruntime/test/onnx/main.cc | 10 + .../cpu/tensor/quantize_linear_test.cc | 184 ++++++++++++++++-- .../onnx_backend_test_series_filters.jsonc | 2 + 13 files changed, 618 insertions(+), 112 deletions(-) create mode 100644 onnxruntime/core/protobuf/onnx-data.in.proto create mode 100644 onnxruntime/core/protobuf/onnx-data.proto create mode 100644 onnxruntime/core/protobuf/onnx-data.proto3 diff --git a/cgmanifests/submodules/cgmanifest.json b/cgmanifests/submodules/cgmanifest.json index de7d36acc8..dc04424dfd 100644 --- a/cgmanifests/submodules/cgmanifest.json +++ b/cgmanifests/submodules/cgmanifest.json @@ -335,7 +335,7 @@ "component": { "type": "git", "git": { - "commitHash": "a82c6a7010e2e332d8f74ad5b0c726fd47c85376", + "commitHash": "c443abd2acad2411103593600319ff81a676afbc", "repositoryUrl": "https://github.com/onnx/onnx" }, "comments": "git submodule at cmake/external/onnx" diff --git a/cmake/external/onnx b/cmake/external/onnx index a82c6a7010..c443abd2ac 160000 --- a/cmake/external/onnx +++ b/cmake/external/onnx @@ -1 +1 @@ -Subproject commit a82c6a7010e2e332d8f74ad5b0c726fd47c85376 +Subproject commit c443abd2acad2411103593600319ff81a676afbc diff --git a/cmake/onnx/CMakeLists.txt b/cmake/onnx/CMakeLists.txt index 79177911da..dfd161b305 100644 --- a/cmake/onnx/CMakeLists.txt +++ b/cmake/onnx/CMakeLists.txt @@ -3,7 +3,10 @@ #TODO: if protobuf is a shared lib and onnxruntime_USE_FULL_PROTOBUF is ON, then onnx_proto should be built as a shared lib instead of a static lib. Otherwise any code outside onnxruntime.dll can't use onnx protobuf definitions if they share the protobuf.dll with onnxruntime. For example, if protobuf is a shared lib and onnx_proto is a static lib then onnxruntime_perf_test won't work. -add_library(onnx_proto ${ONNXRUNTIME_ROOT}/core/protobuf/onnx-ml.proto ${ONNXRUNTIME_ROOT}/core/protobuf/onnx-operators-ml.proto) +add_library(onnx_proto + ${ONNXRUNTIME_ROOT}/core/protobuf/onnx-ml.proto + ${ONNXRUNTIME_ROOT}/core/protobuf/onnx-operators-ml.proto + ${ONNXRUNTIME_ROOT}/core/protobuf/onnx-data.proto) target_include_directories(onnx_proto PUBLIC $ "${CMAKE_CURRENT_BINARY_DIR}/..") target_compile_definitions(onnx_proto PUBLIC $) onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS ${ONNXRUNTIME_ROOT}/core/protobuf TARGET onnx_proto) diff --git a/onnxruntime/core/protobuf/onnx-data.in.proto b/onnxruntime/core/protobuf/onnx-data.in.proto new file mode 100644 index 0000000000..5e583dd2ca --- /dev/null +++ b/onnxruntime/core/protobuf/onnx-data.in.proto @@ -0,0 +1,101 @@ +// Copyright (c) ONNX Project Contributors. +// Licensed under the MIT license. + +syntax = "proto2"; + +package {PACKAGE_NAME}; +// #if ONNX-ML +import "onnx-ml.proto"; +// #else +import "onnx.proto"; +// #endif + +// This file contains the proto definitions for MapProto and +// SequenceProto. These protos are used to represent the data structures +// of maps and sequence for use in test data or ModelProto. + +// Sequences +// +// Defines a dense, ordered, collection of elements that are of homogeneous types. +// Sequences can be made out of tensors, maps, or sequences. +// +// If a sequence is made out of tensors, the tensors must have the same element +// type (i.e. int32). In some cases, the tensors in a sequence can have different +// shapes. Whether the tensors can have different shapes or not depends on the +// type/shape associated with the corresponding "ValueInfo". For example, +// "Sequence" means that all tensors have same shape. However, +// "Sequence" means they can have different +// shapes (all of rank 2), where "omitted" means the corresponding dimension has +// no symbolic/constant value. Finally, "Sequence>" means +// that the different tensors can have different ranks, when the "shape" itself +// is omitted from the tensor-type. For a more complete description, refer to +// https://github.com/onnx/onnx/blob/master/docs/IR.md#static-tensor-shapes. +// +message SequenceProto { + + optional string name = 1; + + enum DataType { + UNDEFINED = 0; + TENSOR = 1; + SPARSE_TENSOR = 2; + SEQUENCE = 3; + MAP = 4; + } + + // The data type of the element. + // This field MUST have a valid SequenceProto.DataType value + optional int32 elem_type = 2; + + // For TensorProto values. + // When this field is present, the elem_type field MUST be TENSOR. + repeated TensorProto tensor_values = 3; + + // For SparseTensorProto values. + // When this field is present, the elem_type field MUST be SPARSE_TENSOR. + repeated SparseTensorProto sparse_tensor_values = 4; + + // For SequenceProto values, allowing sequences to be of themselves. + // When this field is present, the elem_type field MUST be SEQUENCE. + repeated SequenceProto sequence_values = 5; + + // For MapProto values. + // When this field is present, the elem_type field MUST be MAP. + repeated MapProto map_values = 6; + +} + + +// Maps +// +// Specifies an associative table, defined by keys and values. +// MapProto is formed with a repeated field of keys (of type INT8, INT16, INT32, +// INT64, UINT8, UINT16, UINT32, UINT64, or STRING) and values (of type TENSOR, +// SPARSE_TENSOR, SEQUENCE, or MAP). Key types and value types have to remain +// the same throughout the instantiation of the MapProto. +// +message MapProto { + + optional string name = 1; + + // All MapProto data types must have the same length of keys and values. + + // The data type of the key. + // This field MUST have a valid TensorProto.DataType value of + // INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING + optional int32 key_type = 2; + + // Every element of keys has to be one of the following data types + // INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING. + // The integer cases are represented by the repeated int64 field keys below. + repeated int64 keys = 3; + + // If keys are strings, they are represented by the repeated bytes field + // string_keys below. + repeated bytes string_keys = 4; + + // MapProto values are represented in a SequenceProto of the same length as the + // repeated keys field and have to be one of the following data types + // TENSOR, SPARSE_TENSOR, MAP, SEQUENCE. + optional SequenceProto values = 5; +} diff --git a/onnxruntime/core/protobuf/onnx-data.proto b/onnxruntime/core/protobuf/onnx-data.proto new file mode 100644 index 0000000000..aa2290ae3b --- /dev/null +++ b/onnxruntime/core/protobuf/onnx-data.proto @@ -0,0 +1,106 @@ +// +// WARNING: This file is automatically generated! Please edit onnx.in.proto. +// + + +// Copyright (c) ONNX Project Contributors. +// Licensed under the MIT license. + +syntax = "proto2"; + +package onnx; +import "onnx-ml.proto"; + +// This file contains the proto definitions for MapProto and +// SequenceProto. These protos are used to represent the data structures +// of maps and sequence for use in test data or ModelProto. + +// Sequences +// +// Defines a dense, ordered, collection of elements that are of homogeneous types. +// Sequences can be made out of tensors, maps, or sequences. +// +// If a sequence is made out of tensors, the tensors must have the same element +// type (i.e. int32). In some cases, the tensors in a sequence can have different +// shapes. Whether the tensors can have different shapes or not depends on the +// type/shape associated with the corresponding "ValueInfo". For example, +// "Sequence" means that all tensors have same shape. However, +// "Sequence" means they can have different +// shapes (all of rank 2), where "omitted" means the corresponding dimension has +// no symbolic/constant value. Finally, "Sequence>" means +// that the different tensors can have different ranks, when the "shape" itself +// is omitted from the tensor-type. For a more complete description, refer to +// https://github.com/onnx/onnx/blob/master/docs/IR.md#static-tensor-shapes. +// +message SequenceProto { + + optional string name = 1; + + enum DataType { + UNDEFINED = 0; + TENSOR = 1; + SPARSE_TENSOR = 2; + SEQUENCE = 3; + MAP = 4; + } + + // The data type of the element. + // This field MUST have a valid SequenceProto.DataType value + optional int32 elem_type = 2; + + // For TensorProto values. + // When this field is present, the elem_type field MUST be TENSOR. + repeated TensorProto tensor_values = 3; + + // For SparseTensorProto values. + // When this field is present, the elem_type field MUST be SPARSE_TENSOR. + repeated SparseTensorProto sparse_tensor_values = 4; + + // For SequenceProto values, allowing sequences to be of themselves. + // When this field is present, the elem_type field MUST be SEQUENCE. + repeated SequenceProto sequence_values = 5; + + // For MapProto values. + // When this field is present, the elem_type field MUST be MAP. + repeated MapProto map_values = 6; + +} + + +// Maps +// +// Specifies an associative table, defined by keys and values. +// MapProto is formed with a repeated field of keys (of type INT8, INT16, INT32, +// INT64, UINT8, UINT16, UINT32, UINT64, or STRING) and values (of type TENSOR, +// SPARSE_TENSOR, SEQUENCE, or MAP). Key types and value types have to remain +// the same throughout the instantiation of the MapProto. +// +message MapProto { + + optional string name = 1; + + // All MapProto data types must have the same length of keys and values. + + // The data type of the key. + // This field MUST have a valid TensorProto.DataType value of + // INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING + optional int32 key_type = 2; + + // Every element of keys has to be one of the following data types + // INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING. + // The integer cases are represented by the repeated int64 field keys below. + repeated int64 keys = 3; + + // If keys are strings, they are represented by the repeated bytes field + // string_keys below. + repeated bytes string_keys = 4; + + // MapProto values are represented in a SequenceProto of the same length as the + // repeated keys field and have to be one of the following data types + // TENSOR, SPARSE_TENSOR, MAP, SEQUENCE. + optional SequenceProto values = 5; +} + +// For using protobuf-lite +option optimize_for = LITE_RUNTIME; + diff --git a/onnxruntime/core/protobuf/onnx-data.proto3 b/onnxruntime/core/protobuf/onnx-data.proto3 new file mode 100644 index 0000000000..00e4969094 --- /dev/null +++ b/onnxruntime/core/protobuf/onnx-data.proto3 @@ -0,0 +1,106 @@ +// +// WARNING: This file is automatically generated! Please edit onnx.in.proto. +// + + +// Copyright (c) ONNX Project Contributors. +// Licensed under the MIT license. + +syntax = "proto3"; + +package onnx; +import "onnx-ml.proto3"; + +// This file contains the proto definitions for MapProto and +// SequenceProto. These protos are used to represent the data structures +// of maps and sequence for use in test data or ModelProto. + +// Sequences +// +// Defines a dense, ordered, collection of elements that are of homogeneous types. +// Sequences can be made out of tensors, maps, or sequences. +// +// If a sequence is made out of tensors, the tensors must have the same element +// type (i.e. int32). In some cases, the tensors in a sequence can have different +// shapes. Whether the tensors can have different shapes or not depends on the +// type/shape associated with the corresponding "ValueInfo". For example, +// "Sequence" means that all tensors have same shape. However, +// "Sequence" means they can have different +// shapes (all of rank 2), where "omitted" means the corresponding dimension has +// no symbolic/constant value. Finally, "Sequence>" means +// that the different tensors can have different ranks, when the "shape" itself +// is omitted from the tensor-type. For a more complete description, refer to +// https://github.com/onnx/onnx/blob/master/docs/IR.md#static-tensor-shapes. +// +message SequenceProto { + + string name = 1; + + enum DataType { + UNDEFINED = 0; + TENSOR = 1; + SPARSE_TENSOR = 2; + SEQUENCE = 3; + MAP = 4; + } + + // The data type of the element. + // This field MUST have a valid SequenceProto.DataType value + int32 elem_type = 2; + + // For TensorProto values. + // When this field is present, the elem_type field MUST be TENSOR. + repeated TensorProto tensor_values = 3; + + // For SparseTensorProto values. + // When this field is present, the elem_type field MUST be SPARSE_TENSOR. + repeated SparseTensorProto sparse_tensor_values = 4; + + // For SequenceProto values, allowing sequences to be of themselves. + // When this field is present, the elem_type field MUST be SEQUENCE. + repeated SequenceProto sequence_values = 5; + + // For MapProto values. + // When this field is present, the elem_type field MUST be MAP. + repeated MapProto map_values = 6; + +} + + +// Maps +// +// Specifies an associative table, defined by keys and values. +// MapProto is formed with a repeated field of keys (of type INT8, INT16, INT32, +// INT64, UINT8, UINT16, UINT32, UINT64, or STRING) and values (of type TENSOR, +// SPARSE_TENSOR, SEQUENCE, or MAP). Key types and value types have to remain +// the same throughout the instantiation of the MapProto. +// +message MapProto { + + string name = 1; + + // All MapProto data types must have the same length of keys and values. + + // The data type of the key. + // This field MUST have a valid TensorProto.DataType value of + // INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING + int32 key_type = 2; + + // Every element of keys has to be one of the following data types + // INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, or STRING. + // The integer cases are represented by the repeated int64 field keys below. + repeated int64 keys = 3; + + // If keys are strings, they are represented by the repeated bytes field + // string_keys below. + repeated bytes string_keys = 4; + + // MapProto values are represented in a SequenceProto of the same length as the + // repeated keys field and have to be one of the following data types + // TENSOR, SPARSE_TENSOR, MAP, SEQUENCE. + SequenceProto values = 5; +} + +// For using protobuf-lite +option optimize_for = LITE_RUNTIME; + diff --git a/onnxruntime/core/providers/common.h b/onnxruntime/core/providers/common.h index 51903ca01f..628c29654c 100644 --- a/onnxruntime/core/providers/common.h +++ b/onnxruntime/core/providers/common.h @@ -26,7 +26,7 @@ Returns true if given tensor is a scalar or 1D tensor of size 1 **/ inline bool IsScalarOr1ElementVector(const Tensor* input) { if (input->Shape().NumDimensions() == 0 || - (input->Shape().NumDimensions() == 1 && input->Shape().GetDims().size() == 1)) { + (input->Shape().NumDimensions() == 1 && input->Shape().Size() == 1)) { return true; } else { return false; diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index d046fbb4f0..e9508374f9 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -278,10 +278,10 @@ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOn class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, int32_t, Resize); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 10, uint8_t, Resize); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, ThresholdedRelu); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t, DequantizeLinear); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, int8_t, DequantizeLinear); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t, QuantizeLinear); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, int8_t, QuantizeLinear); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, uint8_t, DequantizeLinear); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, int8_t, DequantizeLinear); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, uint8_t, QuantizeLinear); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, 12, int8_t, QuantizeLinear); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, QLinearMatMul); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, uint8_t, MatMulInteger); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, ConvInteger); @@ -440,6 +440,10 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, // opset 13 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Cast); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Sign); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, uint8_t, DequantizeLinear); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, int8_t, DequantizeLinear); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, uint8_t, QuantizeLinear); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, int8_t, QuantizeLinear); Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { static const BuildKernelCreateInfoFn function_table[] = { @@ -841,14 +845,14 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, @@ -1100,6 +1104,14 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { // opset 13 BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, }; for (auto& function_table_entry : function_table) { diff --git a/onnxruntime/core/providers/cpu/tensor/quantize_linear.cc b/onnxruntime/core/providers/cpu/tensor/quantize_linear.cc index d5426029a6..5cfa4d1887 100644 --- a/onnxruntime/core/providers/cpu/tensor/quantize_linear.cc +++ b/onnxruntime/core/providers/cpu/tensor/quantize_linear.cc @@ -7,21 +7,60 @@ namespace onnxruntime { -ONNX_CPU_OPERATOR_TYPED_KERNEL( - DequantizeLinear, - 10, - uint8_t, - KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()), - DequantizeLinear); +static void PrepareForQDQ(const TensorShape& input_shape, + const Tensor& scale, + const Tensor* zero_point_ptr, + int64_t axis, + int64_t& block_count, + int64_t& broadcast_dim, + int64_t& block_size) { + if (IsScalarOr1ElementVector(&scale)) { // per-tensor QuantizeLinear/DequantizeLinear + block_count = 1; + broadcast_dim = 1; + block_size = static_cast(input_shape.Size()); -ONNX_CPU_OPERATOR_TYPED_KERNEL( - DequantizeLinear, - 10, - int8_t, - KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()), - DequantizeLinear); + // enforce that zero point are scalars + ORT_ENFORCE(zero_point_ptr == nullptr || IsScalarOr1ElementVector(zero_point_ptr), + "x_zero_point must be null or a scalar or 1D tensor or size 1."); + } else { // per-channel QuantizeLinear/DequantizeLinear + const int64_t axis_no_neg = HandleNegativeAxis(axis, input_shape.NumDimensions()); + block_count = input_shape.SizeToDimension(axis_no_neg); + broadcast_dim = input_shape[axis_no_neg]; + block_size = input_shape.SizeFromDimension(axis_no_neg + 1); + + // if an axis was specified, ensure the scale and zero point are compatible + ORT_ENFORCE(scale.Shape().NumDimensions() == 1 && scale.Shape()[0] == broadcast_dim, + "scale must be 1D tensor with size ", + broadcast_dim); + ORT_ENFORCE(zero_point_ptr == nullptr || (zero_point_ptr->Shape().NumDimensions() == 1 && zero_point_ptr->Shape()[0] == broadcast_dim), + "x_zero_point must be null or 1D tensor with size ", + broadcast_dim); + } +} + +#define REGISTER_DEQUANTIZELINEAR(T) \ + ONNX_CPU_OPERATOR_TYPED_KERNEL( \ + DequantizeLinear, \ + 13, \ + T, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + DequantizeLinear); + +#define REGISTER_DEQUANTIZELINEAR_VERSIONED(T) \ + ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( \ + DequantizeLinear, \ + 10, \ + 12, \ + T, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + DequantizeLinear); + +REGISTER_DEQUANTIZELINEAR(int8_t) +REGISTER_DEQUANTIZELINEAR(uint8_t) +REGISTER_DEQUANTIZELINEAR_VERSIONED(int8_t) +REGISTER_DEQUANTIZELINEAR_VERSIONED(uint8_t) template // formula is Y = (X - ZeroPoint) * Scale @@ -37,28 +76,7 @@ Status DequantizeLinear::Compute(OpKernelContext* ctx) const { int64_t broadcast_dim; int64_t block_size; - if (has_axis_) { // custom DequantizeLinear only - const int64_t axis = HandleNegativeAxis(axis_, x_shape.NumDimensions()); - N = x_shape.SizeToDimension(axis); - broadcast_dim = x_shape[axis]; - block_size = x_shape.SizeFromDimension(axis + 1); - - // if an axis was specified, ensure the scale and zero point are compatible - ORT_ENFORCE(x_scale.Shape().NumDimensions() == 1 && x_scale.Shape().Size() == broadcast_dim, - "x_scale must be 1D tensor with size ", - broadcast_dim); - ORT_ENFORCE(x_zero_point != nullptr && x_zero_point->Shape().NumDimensions() == 1 && x_zero_point->Shape().Size() == broadcast_dim, - "x_zero_point must be 1D tensor with size ", - broadcast_dim); - } else { - N = 1; - broadcast_dim = 1; - block_size = static_cast(x_shape.Size()); - - // if no axis, enforce that scale and zero point are scalars - ORT_ENFORCE(IsScalarOr1ElementVector(&x_scale), "x_scale must be a scalar or 1D tensor or size 1."); - ORT_ENFORCE(x_zero_point == nullptr || IsScalarOr1ElementVector(x_zero_point), "x_zero_point must be a scalar or 1D tensor or size 1."); - } + PrepareForQDQ(x.Shape(), x_scale, x_zero_point, axis_, N, broadcast_dim, block_size); const T* zero_point = x_zero_point ? x_zero_point->template Data() : nullptr; const float* scale = x_scale.template Data(); @@ -79,23 +97,31 @@ Status DequantizeLinear::Compute(OpKernelContext* ctx) const { return Status::OK(); } -ONNX_CPU_OPERATOR_TYPED_KERNEL( - QuantizeLinear, - 10, - uint8_t, - KernelDefBuilder() - .TypeConstraint("T1", DataTypeImpl::GetTensorType()) - .TypeConstraint("T2", DataTypeImpl::GetTensorType()), - QuantizeLinear); +#define REGISTER_QUANTIZELINEAR(T) \ + ONNX_CPU_OPERATOR_TYPED_KERNEL( \ + QuantizeLinear, \ + 13, \ + T, \ + KernelDefBuilder() \ + .TypeConstraint("T1", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("T2", DataTypeImpl::GetTensorType()), \ + QuantizeLinear); -ONNX_CPU_OPERATOR_TYPED_KERNEL( - QuantizeLinear, - 10, - int8_t, - KernelDefBuilder() - .TypeConstraint("T1", DataTypeImpl::GetTensorType()) - .TypeConstraint("T2", DataTypeImpl::GetTensorType()), - QuantizeLinear); +#define REGISTER_QUANTIZELINEAR_VERSIONED(T) \ + ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( \ + QuantizeLinear, \ + 10, \ + 12, \ + T, \ + KernelDefBuilder() \ + .TypeConstraint("T1", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("T2", DataTypeImpl::GetTensorType()), \ + QuantizeLinear); + +REGISTER_QUANTIZELINEAR(int8_t) +REGISTER_QUANTIZELINEAR(uint8_t) +REGISTER_QUANTIZELINEAR_VERSIONED(int8_t) +REGISTER_QUANTIZELINEAR_VERSIONED(uint8_t) template // formula is Y = X / Scale + ZeroPoint @@ -109,31 +135,7 @@ Status QuantizeLinear::Compute(OpKernelContext* ctx) const { int64_t N; int64_t broadcast_dim; int64_t block_size; - - if (has_axis_) { // custom QuantizeLinear only - const int64_t axis = HandleNegativeAxis(axis_, x_shape.NumDimensions()); - N = x_shape.SizeToDimension(axis); - broadcast_dim = x_shape[axis]; - block_size = x_shape.SizeFromDimension(axis + 1); - - // if an axis was specified, ensure the scale and zero point are compatible - ORT_ENFORCE(y_scale.Shape().NumDimensions() == 1 && y_scale.Shape().Size() == broadcast_dim, - "y_scale must be 1D tensor with size ", - broadcast_dim); - ORT_ENFORCE(y_zero_point != nullptr && y_zero_point->Shape().NumDimensions() == 1 && y_zero_point->Shape().Size() == broadcast_dim, - "y_zero_point must be 1D tensor with size ", - broadcast_dim); - } else { - N = 1; - broadcast_dim = 1; - block_size = x_shape.Size(); - - // if no axis, enforce that scale and zero point are scalars - ORT_ENFORCE(IsScalarOr1ElementVector(&y_scale), - "y_scale must be a scalar or 1D tensor or size 1."); - ORT_ENFORCE(y_zero_point == nullptr || IsScalarOr1ElementVector(y_zero_point), - "y_zero_point must be a scalar or 1D tensor or size 1."); - } + PrepareForQDQ(x.Shape(), y_scale, y_zero_point, axis_, N, broadcast_dim, block_size); const T* zero_point = y_zero_point != nullptr ? y_zero_point->template Data() : nullptr; const float* scale = y_scale.template Data(); diff --git a/onnxruntime/core/providers/cpu/tensor/quantize_linear.h b/onnxruntime/core/providers/cpu/tensor/quantize_linear.h index 073edcf297..9f443f1439 100644 --- a/onnxruntime/core/providers/cpu/tensor/quantize_linear.h +++ b/onnxruntime/core/providers/cpu/tensor/quantize_linear.h @@ -13,27 +13,29 @@ template class DequantizeLinear final : public OpKernel { public: DequantizeLinear(const OpKernelInfo& info) : OpKernel(info) { - has_axis_ = info.GetAttr("axis", &axis_).IsOK(); + if (!info.GetAttr("axis", &axis_).IsOK()) { + axis_ = 1; + } } Status Compute(OpKernelContext* context) const override; - private: - int64_t axis_ = 0; - bool has_axis_; + private: + int64_t axis_; }; template class QuantizeLinear final : public OpKernel { public: QuantizeLinear(const OpKernelInfo& info) : OpKernel(info) { - has_axis_ = info.GetAttr("axis", &axis_).IsOK(); + if (!info.GetAttr("axis", &axis_).IsOK()) { + axis_ = 1; + } } - + Status Compute(OpKernelContext* context) const override; private: - int64_t axis_ = 0; - bool has_axis_; + int64_t axis_; }; } // namespace onnxruntime diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 8509374aed..8af5200f06 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -547,6 +547,8 @@ int real_main(int argc, char* argv[], Ort::Env& env) { {"nesterov_momentum", "not a registered function/op", {}}, // Op not registered. {"cast_FLOAT_to_BFLOAT16", "onnx generate bfloat tensor as uint16 type", {}}, {"cast_BFLOAT16_to_FLOAT", "onnx generate bfloat tensor as uint16 type", {}}, + {"sequence_insert_at_back", "onnx currently not supporting loading segment", {}}, + {"sequence_insert_at_front", "onnx currently not supporting loading segment", {}}, }; #ifdef DISABLE_ML_OPS @@ -756,6 +758,14 @@ int real_main(int argc, char* argv[], Ort::Env& env) { broken_tests.insert({"nllloss_NCd1d2_with_weight_reduction_mean_expanded", "wait for investigation"}); broken_tests.insert({"nllloss_NCd1d2d3d4d5_mean_weight", "wait for investigation"}); broken_tests.insert({"nllloss_NCd1d2d3d4d5_mean_weight_expanded", "wait for investigation"}); + broken_tests.insert({"nllloss_NCd1_ii", "wait for investigation"}); + broken_tests.insert({"nllloss_NCd1_ii_expanded", "wait for investigation"}); + broken_tests.insert({"nllloss_NCd1_mean_weight_negative_ii", "wait for investigation"}); + broken_tests.insert({"nllloss_NCd1_mean_weight_negative_ii_expanded", "wait for investigation"}); + broken_tests.insert({"nllloss_NCd1_weight_ii", "wait for investigation"}); + broken_tests.insert({"nllloss_NCd1_weight_ii_expanded", "wait for investigation"}); + broken_tests.insert({"nllloss_NCd1d2_no_weight_reduction_mean_ii", "wait for investigation"}); + broken_tests.insert({"nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", "wait for investigation"}); } if (enable_tensorrt) { diff --git a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc index ce94c173a8..5ae5f3d76e 100644 --- a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc @@ -7,7 +7,7 @@ namespace onnxruntime { namespace test { // scalar zero & scale with uint8 -TEST(DequantizeLinearOpTest, DequantizeLinear_0) { +TEST(DequantizeLinearOpTest, Uint8) { OpTester test("DequantizeLinear", 10); std::vector dims{4}; test.AddInput("x", dims, {0, 3, 128, 255}); @@ -18,7 +18,7 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_0) { } // scalar zero & scale with int8 -TEST(DequantizeLinearOpTest, DequantizeLinear_1) { +TEST(DequantizeLinearOpTest, Int8) { OpTester test("DequantizeLinear", 10); std::vector dims{4}; test.AddInput("x", dims, {-30, -3, 100, 127}); @@ -29,7 +29,7 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_1) { } // 2d inputs -TEST(DequantizeLinearOpTest, DequantizeLinear_2D) { +TEST(DequantizeLinearOpTest, 2D) { OpTester test("DequantizeLinear", 10); std::vector dims{3, 4}; test.AddInput("X", dims, @@ -46,7 +46,7 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_2D) { } // dequantize with scalar data -TEST(DequantizeLinearOpTest, DequantizeLinear_Scalar) { +TEST(DequantizeLinearOpTest, Scalar) { OpTester test("DequantizeLinear", 10); test.AddInput("x", {}, {100}); test.AddInput("x_scale", {}, {2.0f}); @@ -56,7 +56,7 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_Scalar) { } // dequantize without zero point -TEST(DequantizeLinearOpTest, DequantizeLinear_Without_Zero_Point) { +TEST(DequantizeLinearOpTest, Without_Zero_Point) { OpTester test("DequantizeLinear", 10); test.AddInput("x", {}, {100}); test.AddInput("x_scale", {}, {2.0f}); @@ -64,8 +64,119 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_Without_Zero_Point) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider}); } +// 1d zero & scale with default axis +TEST(DequantizeLinearOpTest, Per_Channel_Axis_Default) { + OpTester test("DequantizeLinear", 13); + std::vector dims{2, 3, 2, 4}; + test.AddInput("X", dims, + {7, 9, 10, 10, + 5, 8, 9, 1, + + 8, 6, 7, 9, + 10, 0, 7, 10, + + 8, 2, 6, 0, + 5, 9, 8, 1, + + 2, 7, 5, 3, + 2, 4, 1, 3, + + 8, 7, 4, 8, + 10, 1, 5, 5, + + 7, 7, 0, 2, + 4, 4, 0, 5}); + test.AddInput("scale", {3}, {1, 10, 7}); + test.AddInput("zero_point", {3}, {10, 2, 1}); + test.AddOutput("Y", dims, + {-3, -1, 0, 0, + -5, -2, -1, -9, + + 60, 40, 50, 70, + 80, -20, 50, 80, + + 49, 7, 35, -7, + 28, 56, 49, 0, + + -8, -3, -5, -7, + -8, -6, -9, -7, + + 60, 50, 20, 60, + 80, -10, 30, 30, + + 42, 42, -7, 7, + 21, 21, -7, 28}); + test.Run(); +} + +// 1d zero & scale with uint8 broadcast axis 0 +TEST(DequantizeLinearOpTest, Per_Channel_Axis_0) { + OpTester test("DequantizeLinear", 13); + std::vector dims{3, 4}; + test.AddInput("X", dims, + {0, 1, 2, 3, + 0, 1, 2, 3, + 0, 10, 20, 30}); + test.AddAttribute("axis", 0); + test.AddInput("scale", {3}, + {1.0f, + 2.0f, + 4.0f}); + test.AddInput("zero_point", {3}, + {0, + 0, + 0}); + test.AddOutput("Y", dims, + {0, 1, 2, 3, + 0, 2, 4, 6, + 0, 40, 80, 120}); + test.Run(); +} + +// 1d zero & scale with int8 broadcast axis 1 +TEST(DequantizeLinearOpTest, Per_Channel_Axis_1) { + OpTester test("DequantizeLinear", 13); + std::vector dims{3, 4}; + test.AddInput("X", dims, + {0, 1, 2, 3, + 0, 2, 4, 6, + 0, 10, 20, 30}); + test.AddAttribute("axis", 1); + test.AddInput("scale", {4}, {1, 2, 4, 8}); + test.AddInput("zero_point", {4}, {0, -10, -20, -30}); + test.AddOutput("Y", dims, + {0, 22, 88, 264, + 0, 24, 96, 288, + 0, 40, 160, 480}); + test.Run(); +} + +// 1d zero & scale with uint8 broadcast axis -2 (-2 resolves to axis 0) +TEST(DequantizeLinearOpTest, Per_Channel_Neg_2) { + OpTester test("DequantizeLinear", 13); + std::vector dims{3, 4}; + test.AddInput("X", dims, + {0, 1, 2, 3, + 0, 1, 2, 3, + 0, 10, 20, 30}); + test.AddAttribute("axis", -2); + test.AddInput("scale", {3}, + {1.0f, + 2.0f, + 4.0f}); + test.AddInput("zero_point", {3}, + {0, + 0, + 0}); + test.AddOutput("Y", dims, + {0, 1, 2, 3, + 0, 2, 4, 6, + 0, 40, 80, 120}); + test.Run(); +} + // quantize with scalar zero point and scale -TEST(QuantizeLinearOpTest, QuantizeLinear_uint8) { +TEST(QuantizeLinearOpTest, Uint8) { OpTester test("QuantizeLinear", 10); std::vector dims{6}; test.AddInput("x", dims, {0, 2, 3, 1000, -254, -1000}); @@ -76,7 +187,7 @@ TEST(QuantizeLinearOpTest, QuantizeLinear_uint8) { } // quantize with scalar zero point and scale -TEST(QuantizeLinearOpTest, QuantizeLinear_int8) { +TEST(QuantizeLinearOpTest, Int8) { OpTester test("QuantizeLinear", 10); std::vector dims{6}; test.AddInput("x", dims, {0, 2, 3, 5, -2, -5}); @@ -87,7 +198,7 @@ TEST(QuantizeLinearOpTest, QuantizeLinear_int8) { } // quantize with scalar zero point and scale -TEST(QuantizeLinearOpTest, QuantizeLinear_int8_NegativeZeroPoint) { +TEST(QuantizeLinearOpTest, Int8_NegativeZeroPoint) { OpTester test("QuantizeLinear", 10); std::vector dims{8}; test.AddInput("x", dims, {0, 2, 3, 5, 6, -2, -5, -6}); @@ -98,7 +209,7 @@ TEST(QuantizeLinearOpTest, QuantizeLinear_int8_NegativeZeroPoint) { } // quantize with scalar zero point and scale -TEST(QuantizeLinearOpTest, QuantizeLinear_int8_PositiveZeroPoint) { +TEST(QuantizeLinearOpTest, Int8_PositiveZeroPoint) { OpTester test("QuantizeLinear", 10); std::vector dims{8}; test.AddInput("x", dims, {0, 2, 3, 5, 6, -2, -5, -6}); @@ -109,7 +220,7 @@ TEST(QuantizeLinearOpTest, QuantizeLinear_int8_PositiveZeroPoint) { } // quantize with 2D data -TEST(QuantizeLinearOpTest, QuantizeLinear_2D) { +TEST(QuantizeLinearOpTest, 2D) { OpTester test("QuantizeLinear", 10); std::vector dims{3, 4}; test.AddInput("X", dims, @@ -126,7 +237,7 @@ TEST(QuantizeLinearOpTest, QuantizeLinear_2D) { } // quantize with scalar data -TEST(QuantizeLinearOpTest, QuantizeLinear_Scalar) { +TEST(QuantizeLinearOpTest, Scalar) { OpTester test("QuantizeLinear", 10); test.AddInput("x", {}, {3}); test.AddInput("y_scale", {}, {2.0f}); @@ -144,5 +255,56 @@ TEST(QuantizeLinearOpTest, DISABLED_QuantizeLinear_Without_Zero_Point) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider}); } +TEST(QuantizeLinearOpTest, Per_Channel_Axis_Default) { + OpTester test("QuantizeLinear", 13); + std::vector dims{3, 4}; + test.AddInput("X", dims, + {0, 2, 1, 1001, + 1, 1, 2, 1100, + 2, 4.2f, 3, 1200}); + test.AddInput("scale", {4}, {1, 2, 3, 20}); + test.AddInput("zero_point", {4}, {64, 100, 127, 127}); + test.AddOutput("Y", dims, + {64, 101, 127, 177, + 65, 100, 128, 182, + 66, 102, 128, 187}); + test.Run(); +} + +TEST(QuantizeLinearOpTest, Per_Channel_Axis_0) { + OpTester test("QuantizeLinear", 13); + std::vector dims{3, 4}; + test.AddInput("X", dims, + {0, 2, 3, 1000, + 0, 2, 3, 1000, + 0, 2, 3, 1000}); + test.AddAttribute("axis", 0); + test.AddInput("scale", {3}, {1, 2, 4}); + test.AddInput("zero_point", {3}, {0, 0, 0}); + test.AddOutput("Y", dims, + {0, 2, 3, 255, + 0, 1, 2, 255, + 0, 0, 1, 250}); + test.Run(); +} + +// quantize with per-channel and negative axis (-2 resolves to axis 0) +TEST(QuantizeLinearOpTest, Per_Channel_Axis_neg) { + OpTester test("QuantizeLinear", 13); + std::vector dims{3, 4}; + test.AddInput("X", dims, + {0, 2, 3, 1000, + 0, 2, 3, 1000, + 0, 2, 3, 1000}); + test.AddAttribute("axis", -2); + test.AddInput("scale", {3}, {1, 2, 4}); + test.AddInput("zero_point", {3}, {0, 0, 0}); + test.AddOutput("Y", dims, + {0, 2, 3, 255, + 0, 1, 2, 255, + 0, 0, 1, 250}); + test.Run(); +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc index efcdba7551..fb421d5765 100644 --- a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc +++ b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc @@ -21,6 +21,8 @@ "^test_nesterov_momentum", "^test_pow_types_float32_uint32", "^test_pow_types_float32_uint64", + "^test_sequence_insert_at_back_cpu", // numpy_helper currently not supporting loading segments. + "^test_sequence_insert_at_front_cpu", // numpy_helper currently not supporting loading segments. "^test_gradient_of_add_and_mul", "^test_gradient_of_add", "^test_batchnorm_example_training_mode",