saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

NNAPI EP, add quantization support (#4530)

Browse source 
* nnapi quantization work, 1. add SymmPerChannelQuantParams to operand types

* add ways for operand_type to change dimension

* remove per-channel quantization

* Squashed commit of the following:

commit 4857c3a732298c2f96efb61800b7621251d61c1b
Author: gwang <wanggy@outlook.com>
Date:   Tue Jul 14 23:02:28 2020 -0700

    remove per-channel quantization

commit 775e4b2960f531496b8d11eef27d64e5b85c3c66
Author: gwang <wanggy@outlook.com>
Date:   Mon Jul 13 11:54:02 2020 -0700

    add ways for operand_type to change dimension

commit e56a494de67c66f8122d908270fbc2bb17e38423
Author: gwang <wanggy@outlook.com>
Date:   Wed Jul 8 15:18:55 2020 -0700

    nnapi quantization work, 1. add SymmPerChannelQuantParams to operand types

* add support for QuantizeLinear

* add dequantizelinear support

* minor style update

* minor bug fix

* add quantization support for qlinearmatmul, minor issue fix

* add quantized input support, minor bug fix

* fix issues in the qlinearmatmul

* add verify scale and zeropoint for qlinearmatmul

* add test for [de]qunatizelinear ops

* add qlinearconv support

* fixed small issue causing test failure

* fix test exception

* fix for centos test failure

* fix centos test failure

* fix issue causing win-tensorRT ci failure

* addressed comments
This commit is contained in:
gwang-msft 2020-07-20 16:43:31 -07:00 committed by GitHub
parent 7f9d9557b1
commit ca0dd8246c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 1034 additions and 233 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc
View file

@ -186,6 +186,33 @@ void ModelBuilder::PreprocessInitializers() {
}
}
// Help to get all quantized operators' input and the node(s) using the input
std::unordered_map<std::string, vector<const Node*>> GetAllQuantizedOpInputs(const GraphViewer& graph_view) {
std::unordered_map<std::string, vector<const Node*>> all_quantized_op_inputs;
const auto& node_indices = graph_view.GetNodesInTopologicalOrder();
for (const auto& node_idx : node_indices) {
const auto* node(graph_view.GetNode(node_idx));
const auto& op_type = node->OpType();
if (op_type == "DequantizeLinear" || op_type == "QLinearMatMul" || op_type == "QLinearConv") {
const auto& input_name = node->InputDefs()[0]->Name();
if (Contains(all_quantized_op_inputs, input_name))
all_quantized_op_inputs.at(input_name).push_back(node);
else
all_quantized_op_inputs.emplace(input_name, vector<const Node*>{node});
}
if (op_type == "QLinearMatMul" || op_type == "QLinearConv") {
const auto& input_name = node->InputDefs()[3]->Name();
if (Contains(all_quantized_op_inputs, input_name))
all_quantized_op_inputs.at(input_name).push_back(node);
else
all_quantized_op_inputs.emplace(input_name, vector<const Node*>{node});
}
}
return all_quantized_op_inputs;
}
void ModelBuilder::RegisterInitializers() {
// First pass to get all the stats of the initializers
auto initializer_size = initializers_.size();
@ -256,6 +283,7 @@ void ModelBuilder::RegisterInitializers() {
}
void ModelBuilder::RegisterModelInputs() {
const auto all_quantized_op_inputs = GetAllQuantizedOpInputs(graph_view_);
for (const auto* node_arg : graph_view_.GetInputs()) {
const auto& input_name = node_arg->Name();
@ -277,6 +305,8 @@ void ModelBuilder::RegisterModelInputs() {
}
Type type = Type::TENSOR_FLOAT32;
float scale = 0.0f;
int32_t zero_point = 0;
const auto* type_proto = node_arg->TypeAsProto();
if (!type_proto || !type_proto->tensor_type().has_elem_type()) {
ORT_THROW("The input of graph doesn't have elem_type: " + input_name);
@ -285,6 +315,23 @@ void ModelBuilder::RegisterModelInputs() {
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
type = Type::TENSOR_FLOAT32;
break;
case ONNX_NAMESPACE::TensorProto_DataType_UINT8: {
// For ONNX the quantized input does not carry scale and zero point info
// So we will need to search the operator using this input
// And dig out the scale and zero point as the input initializers to the operator
type = Type::TENSOR_QUANT8_ASYMM;
if (!Contains(all_quantized_op_inputs, input_name)) {
// We current do not support uint8 input if it is not a quantized input
ORT_THROW("The input of graph doesn't have valid type, name: " +
input_name + " type: " +
std::to_string(type_proto->tensor_type().elem_type()));
}
// TODO, verify the scale and zero point match if there are multiple op using same input
std::tie(scale, zero_point) =
GetQuantizedInputScaleAndZeroPoint(*this, *all_quantized_op_inputs.at(input_name)[0], input_name);
break;
}
default:
// TODO: support other type
ORT_THROW("The input of graph doesn't have valid type, name: " +
@ -293,7 +340,7 @@ void ModelBuilder::RegisterModelInputs() {
}
}
OperandType operand_type(type, shape);
OperandType operand_type(type, shape, scale, zero_point);
shaper_.AddShape(input_name, operand_type.dimensions);
auto index = AddNewOperand(input_name, operand_type, false /* is_nhwc */);

863
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc
View file

File diff suppressed because it is too large Load diff

6
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h
View file

@ -27,8 +27,12 @@ class IOpBuilder {
// for different onnx operators
std::unordered_map<std::string, std::shared_ptr<IOpBuilder>> CreateOpBuilders();
// Transpose the NHWCinput to NCHW output
// Transpose the NHWC input to NCHW output
void TransposeNHWCToNCHW(ModelBuilder& model_builder, const std::string& input, const std::string& output);
// Get the quantized input's scale and zero point for the given input
std::pair<float, int32_t> GetQuantizedInputScaleAndZeroPoint(const ModelBuilder& model_builder,
const Node& node, const std::string& input_name);
} // namespace nnapi
} // namespace onnxruntime

3
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc
View file

@ -296,6 +296,9 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<onnxruntime::No
case Type::TENSOR_INT32:
output_buffer = ort.GetTensorMutableData<int32_t>(output_tensor);
break;
case Type::TENSOR_QUANT8_ASYMM:
output_buffer = ort.GetTensorMutableData<uint8_t>(output_tensor);
break;
default:
return Status(common::ONNXRUNTIME, common::FAIL,
"Unsupported output type: " + TypeToStr(model_output_type.type));

43
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc
View file

@ -21,14 +21,14 @@ namespace android {
namespace nn {
namespace wrapper {
bool isScalarType(const Type& type) {
bool IsScalarType(const Type& type) {
return type == Type::FLOAT16 || type == Type::FLOAT32 || type == Type::INT32 || type == Type::BOOL || type == Type::UINT32;
}
OperandType::OperandType(Type type, const std::vector<uint32_t>& d, float scale, int32_t zeroPoint)
: type(type), dimensions(std::move(d)) {
: type(type), dimensions(d) {
if (dimensions.empty()) {
if (!isScalarType(type)) {
if (!IsScalarType(type)) {
dimensions = {1};
}
}
@ -45,38 +45,30 @@ OperandType::OperandType(Type type, const std::vector<uint32_t>& d, float scale,
OperandType::OperandType(const OperandType& other) {
type = other.type;
dimensions = other.dimensions;
if (dimensions.empty()) {
if (!isScalarType(type)) {
if (!IsScalarType(type)) {
dimensions = {1};
}
}
operandType = {
.type = static_cast<int32_t>(type),
.dimensionCount = static_cast<uint32_t>(dimensions.size()),
.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr,
.scale = other.operandType.scale,
.zeroPoint = other.operandType.zeroPoint,
};
} // namespace wrapper
operandType = other.operandType;
operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr;
}
OperandType& OperandType::operator=(const OperandType& other) {
if (this != &other) {
type = other.type;
dimensions = other.dimensions;
if (dimensions.empty()) {
if (!isScalarType(type)) {
if (!IsScalarType(type)) {
dimensions = {1};
}
}
operandType = {
.type = static_cast<int32_t>(type),
.dimensionCount = static_cast<uint32_t>(dimensions.size()),
.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr,
.scale = other.operandType.scale,
.zeroPoint = other.operandType.zeroPoint,
};
operandType = other.operandType;
operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr;
}
return *this;
@ -121,6 +113,17 @@ size_t OperandType::GetOperandBlobByteSize() const {
return Product(dimensions) * GetElementByteSize();
}
void OperandType::SetDimensions(const std::vector<uint32_t>& d) {
dimensions = d;
if (dimensions.empty()) {
if (!IsScalarType(type)) {
dimensions = {1};
}
}
operandType.dimensionCount = dimensions.size();
operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr;
}
} // namespace wrapper
} // namespace nn
} // namespace android

2
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h
View file

@ -113,6 +113,8 @@ struct OperandType {
// Get the whole blob size in bytes
size_t GetOperandBlobByteSize() const;
void SetDimensions(const std::vector<uint32_t>& d);
operator ANeuralNetworksOperandType() const { return operandType; }
};
} // namespace wrapper

108
onnxruntime/test/contrib_ops/quantize_ops_test.cc
View file

@ -9,16 +9,25 @@ namespace onnxruntime {
namespace test {
// scalar zero & scale with uint8
TEST(DequantizeLinearOpTest, DequantizeLinear_per_tensor_float_uint8) {
void TestDequantizeLinearPerTensorFloatUint8(bool use_initializer_except_x) {
OpTester test("DequantizeLinear", 1, onnxruntime::kMSDomain);
std::vector<int64_t> dims{4};
test.AddInput<uint8_t>("x", dims, {0, 3, 128, 255});
test.AddInput<float>("x_scale", {}, {2.0f});
test.AddInput<uint8_t>("x_zero_point", {}, {128});
test.AddInput<float>("x_scale", {}, {2.0f}, use_initializer_except_x);
test.AddInput<uint8_t>("x_zero_point", {}, {128}, use_initializer_except_x);
test.AddOutput<float>("y", dims, {-256.0f, -250.0f, 0.0f, 254.0f});
test.Run();
}
TEST(DequantizeLinearOpTest, DequantizeLinear_per_tensor_float_uint8) {
TestDequantizeLinearPerTensorFloatUint8(false);
}
// NNAPI EP requires weight to be an initializer
TEST(DequantizeLinearOpTest, DequantizeLinear_per_tensor_float_uint8_use_initializer_except_x) {
TestDequantizeLinearPerTensorFloatUint8(true);
}
// scalar zero & scale with int8
TEST(DequantizeLinearOpTest, DequantizeLinear_per_tensor_float_int8) {
OpTester test("DequantizeLinear", 1, onnxruntime::kMSDomain);
@ -166,11 +175,11 @@ TEST(DequantizeLinearContribOpTest, DequantizeLinear_3) {
}
// quantize with scalar zero point and scale
TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint8) {
void TestQuantizeLinearPerTensorFloatUint8(bool use_initializer_except_x) {
OpTester test("QuantizeLinear", 1, onnxruntime::kMSDomain);
std::vector<int64_t> dims{16};
test.AddInput<float>("x", dims, {
0.f, 2.f,
0.f, 2.f, //
3.f, -3.f, // rounding half to even
2.9f, -2.9f, // low case
3.1f, -3.1f, // up case
@ -179,24 +188,36 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint8) {
256.f, -258.f, // critical point
1000.f, -1000.f // saturate case
});
test.AddInput<float>("y_scale", {}, {2.0f});
test.AddInput<uint8_t>("y_zero_point", {}, {128});
test.AddOutput<uint8_t>("y", dims, {128, 129,
130, 126,
129, 127,
130, 126,
255, 0,
255, 0,
255, 0,
255, 0});
test.AddInput<float>("y_scale", {}, {2.0f}, use_initializer_except_x);
test.AddInput<uint8_t>("y_zero_point", {}, {128}, use_initializer_except_x);
test.AddOutput<uint8_t>("y", dims,
{128, 129,
130, 126,
129, 127,
130, 126,
255, 0,
255, 0,
255, 0,
255, 0});
test.Run();
}
TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint8) {
TestQuantizeLinearPerTensorFloatUint8(false);
}
// Only NNAPI EP requires weight to be an initializer
#ifdef USE_NNAPI
TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint8_use_initializer_except_x) {
TestQuantizeLinearPerTensorFloatUint8(true);
}
#endif
TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_int8) {
OpTester test("QuantizeLinear", 1, onnxruntime::kMSDomain);
std::vector<int64_t> dims{16};
test.AddInput<float>("x", dims, {
0.f, 2.f,
0.f, 2.f, //
3.f, -3.f, // rounding half to even
2.9f, -2.9f, // low case
3.1f, -3.1f, // up case
@ -207,14 +228,15 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_int8) {
});
test.AddInput<float>("y_scale", {}, {2.0f});
test.AddInput<int8_t>("y_zero_point", {}, {1});
test.AddOutput<int8_t>("y", dims, {1, 2,
3, -1,
2, 0,
3, -1,
127, -127,
127, -127,
127, -128,
127, -128});
test.AddOutput<int8_t>("y", dims,
{1, 2,
3, -1,
2, 0,
3, -1,
127, -127,
127, -127,
127, -128,
127, -128});
test.Run();
}
@ -223,7 +245,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_half_uint8) {
OpTester test("QuantizeLinear", 1, onnxruntime::kMSDomain);
std::vector<int64_t> dims{16};
test.AddInput<MLFloat16>("x", dims, ToFloat16({
0.f, 2.f,
0.f, 2.f, //
3.f, -3.f, // rounding half to even
2.9f, -2.9f, // low case
3.1f, -3.1f, // up case
@ -234,14 +256,15 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_half_uint8) {
}));
test.AddInput<MLFloat16>("y_scale", {}, ToFloat16({2.0f}));
test.AddInput<uint8_t>("y_zero_point", {}, {128});
test.AddOutput<uint8_t>("y", dims, {128, 129,
130, 126,
129, 127,
130, 126,
255, 0,
255, 0,
255, 0,
255, 0});
test.AddOutput<uint8_t>("y", dims,
{128, 129,
130, 126,
129, 127,
130, 126,
255, 0,
255, 0,
255, 0,
255, 0});
test.Run();
}
@ -249,7 +272,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_half_int8) {
OpTester test("QuantizeLinear", 1, onnxruntime::kMSDomain);
std::vector<int64_t> dims{16};
test.AddInput<MLFloat16>("x", dims, ToFloat16({
0.f, 2.f,
0.f, 2.f, //
3.f, -3.f, // rounding half to even
2.9f, -2.9f, // low case
3.1f, -3.1f, // up case
@ -260,14 +283,15 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_half_int8) {
}));
test.AddInput<MLFloat16>("y_scale", {}, ToFloat16({2.0f}));
test.AddInput<int8_t>("y_zero_point", {}, {1});
test.AddOutput<int8_t>("y", dims, {1, 2,
3, -1,
2, 0,
3, -1,
127, -127,
127, -127,
127, -128,
127, -128});
test.AddOutput<int8_t>("y", dims,
{1, 2,
3, -1,
2, 0,
3, -1,
127, -127,
127, -127,
127, -128,
127, -128});
test.Run();
}
#endif

75
onnxruntime/test/providers/cpu/math/gemm_test.cc
View file

@ -8,7 +8,7 @@
namespace onnxruntime {
namespace test {
TEST(GemmOpTest, GemmNoTrans) {
void TestGemmNoTrans(bool b_is_initializer) {
OpTester test("Gemm");
test.AddAttribute("transA", (int64_t)0);
@ -19,7 +19,7 @@ TEST(GemmOpTest, GemmNoTrans) {
test.AddInput<float>("A", {2, 4},
{1.0f, 2.0f, 3.0f, 4.0f,
-1.0f, -2.0f, -3.0f, -4.0f});
test.AddInput<float>("B", {4, 3}, std::vector<float>(12, 1.0f));
test.AddInput<float>("B", {4, 3}, std::vector<float>(12, 1.0f), b_is_initializer);
test.AddInput<float>("C", {2, 3}, std::vector<float>(6, 1.0f));
test.AddOutput<float>("Y", {2, 3},
{11.0f, 11.0f, 11.0f,
@ -27,6 +27,15 @@ TEST(GemmOpTest, GemmNoTrans) {
test.Run();
}
TEST(GemmOpTest, GemmNoTrans) {
TestGemmNoTrans(false);
}
// NNAPI EP requires weight to be an initializer
TEST(GemmOpTest, GemmNoTransBIsInitializer) {
TestGemmNoTrans(true);
}
// Only CUDA kernel has float 16 support
#ifdef USE_CUDA
TEST(GemmOpTest, GemmNoTrans_f16) {
@ -43,7 +52,7 @@ TEST(GemmOpTest, GemmNoTrans_f16) {
test.AddAttribute("beta", 1.0f);
std::vector<float> A{1.0f, 2.0f, 3.0f, 4.0f,
-1.0f, -2.0f, -3.0f, -4.0f};
-1.0f, -2.0f, -3.0f, -4.0f};
std::vector<float> B(12, 1.0f);
std::vector<float> C(6, 1.0f);
std::vector<float> Y{11.0f, 11.0f, 11.0f,
@ -62,7 +71,7 @@ TEST(GemmOpTest, GemmNoTrans_f16) {
test.AddInput<MLFloat16>("B", {4, 3}, f_B);
test.AddInput<MLFloat16>("C", {2, 3}, f_C);
test.AddOutput<MLFloat16>("Y", {2, 3}, f_Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: fp16 is not supported
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: fp16 is not supported
}
#endif
@ -82,11 +91,11 @@ TEST(GemmOpTest, GemmBroadcast) {
test.AddOutput<float>("Y", {2, 3},
{11.0f, 12.0f, 13.0f,
-9.0f, -8.0f, -7.0f});
#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32)
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); // OpenVINO : Temporarily disabled due to accuracy issues
#else
test.Run();
#endif
#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32)
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); // OpenVINO : Temporarily disabled due to accuracy issues
#else
test.Run();
#endif
}
TEST(GemmOpTest, GemmTrans) {
@ -107,11 +116,35 @@ TEST(GemmOpTest, GemmTrans) {
test.AddOutput<float>("Y", {2, 3},
{11.0f, 11.0f, 11.0f,
-9.0f, -9.0f, -9.0f});
#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32)
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues
#else
test.Run();
#endif
#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32)
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues
#else
test.Run();
#endif
}
// NNAPI EP's GEMM only works as A*B', add case only B is transposed
TEST(GemmOpTest, GemmTransB) {
OpTester test("Gemm");
test.AddAttribute("transA", (int64_t)0);
test.AddAttribute("transB", (int64_t)1);
test.AddAttribute("alpha", 1.0f);
test.AddAttribute("beta", 1.0f);
test.AddInput<float>("A", {2, 4},
{1.0f, 2.0f, 3.0f, 4.0f,
-1.0f, -2.0f, -3.0f, -4.0f});
test.AddInput<float>("B", {3, 4}, std::vector<float>(12, 1.0f));
test.AddInput<float>("C", {3}, std::vector<float>(3, 1.0f));
test.AddOutput<float>("Y", {2, 3},
{11.0f, 11.0f, 11.0f,
-9.0f, -9.0f, -9.0f});
#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32)
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues
#else
test.Run();
#endif
}
TEST(GemmOpTest, GemmAlphaBeta) {
@ -130,11 +163,11 @@ TEST(GemmOpTest, GemmAlphaBeta) {
test.AddOutput<float>("Y", {2, 3},
{7.0f, 7.0f, 7.0f,
-3.0f, -3.0f, -3.0f});
#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32)
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider,kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues
#else
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Seg fault in parser
#endif
#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32)
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues
#else
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Seg fault in parser
#endif
}
TEST(GemmOpTest, GemmNaN) {
@ -153,7 +186,7 @@ TEST(GemmOpTest, GemmNaN) {
test.AddOutput<float>("Y", {2, 3},
{10.0f, 10.0f, 10.0f,
-10.0f, -10.0f, -10.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Seg fault in parser
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Seg fault in parser
}
TEST(GemmOpTest, GemmScalarBroadcast) {
@ -247,7 +280,7 @@ TEST(GemmOpTest, GemmEmptyTensor) {
test.AddInput<float>("C", {3}, std::vector<float>(3, 1.0f));
test.AddOutput<float>("Y", {0, 3},
{});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kDnnlExecutionProvider}); //TensorRT: doesn't support dynamic shape yet
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kDnnlExecutionProvider}); //TensorRT: doesn't support dynamic shape yet
}
TEST(GemmOpTest, GemmNoBiasOpset11) {

43
onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc
View file

@ -12,7 +12,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D) {
test.AddInput<uint8_t>("T1", {2, 2, 4},
{208, 236, 0, 238,
3, 214, 255, 29,
208, 236, 0, 238,
3, 214, 255, 29});
@ -21,42 +21,51 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D) {
test.AddInput<uint8_t>("T2", {2, 4, 3},
{152, 51, 244,
60, 26, 255,
0, 127, 246,
60, 26, 255,
0, 127, 246,
127, 254, 247,
152, 51, 244,
60, 26, 255,
0, 127, 246,
127, 254, 247});
test.AddInput<float>("b_scale", {}, {0.00705f});
test.AddInput<uint8_t>("b_zero_point", {}, {114});
test.AddInput<float>("y_scale", {}, {0.0107f});
test.AddInput<uint8_t>("y_zero_point", {}, {118});
test.AddOutput<uint8_t>("T3", {2, 2, 3},
test.AddOutput<uint8_t>("T3", {2, 2, 3},
{168, 115, 255,
1, 66, 151,
168, 115, 255,
1, 66, 151});
test.Run();
}
static void QLinearMatMul2DTest(bool only_t1_not_initializer) {
OpTester test("QLinearMatMul", 10);
test.AddInput<uint8_t>("T1", {2, 4}, {208, 236, 0, 238, 3, 214, 255, 29});
test.AddInput<float>("a_scale", {1}, {0.0066f}, only_t1_not_initializer);
test.AddInput<uint8_t>("a_zero_point", {1}, {113}, only_t1_not_initializer);
test.AddInput<uint8_t>("T2", {4, 3}, {152, 51, 244, 60, 26, 255, 0, 127, 246, 127, 254, 247}, only_t1_not_initializer);
test.AddInput<float>("b_scale", {1}, {0.00705f}, only_t1_not_initializer);
test.AddInput<uint8_t>("b_zero_point", {1}, {114}, only_t1_not_initializer);
test.AddInput<float>("y_scale", {1}, {0.0107f}, only_t1_not_initializer);
test.AddInput<uint8_t>("y_zero_point", {1}, {118}, only_t1_not_initializer);
test.AddOutput<uint8_t>("T3", {2, 3}, {168, 115, 255, 1, 66, 151});
test.Run();
}
TEST(QuantizeLinearMatmulOpTest, QLinearMatMul) {
OpTester test("QLinearMatMul", 10);
test.AddInput<uint8_t>("T1", {2, 4}, {208, 236, 0, 238, 3, 214, 255, 29});
test.AddInput<float>("a_scale", {}, {0.0066f});
test.AddInput<uint8_t>("a_zero_point", {}, {113});
test.AddInput<uint8_t>("T2", {4, 3}, {152, 51, 244, 60, 26, 255, 0, 127, 246, 127, 254, 247});
test.AddInput<float>("b_scale", {}, {0.00705f});
test.AddInput<uint8_t>("b_zero_point", {}, {114});
test.AddInput<float>("y_scale", {}, {0.0107f});
test.AddInput<uint8_t>("y_zero_point", {}, {118});
test.AddOutput<uint8_t>("T3", {2, 3}, {168, 115, 255, 1, 66, 151});
test.Run();
QLinearMatMul2DTest(false);
}
// NNAPI EP requires weight to be an initializer
TEST(QuantizeLinearMatmulOpTest, QLinearMatMulAllInputExceptT1AreInitializers) {
QLinearMatMul2DTest(true);
}
} // namespace test
} // namespace onnxruntime

32
onnxruntime/test/providers/cpu/nn/conv_op_test.cc
View file

@ -24,6 +24,7 @@ void TestConvOp(const ConvOpAndTestAttributes& attributes,
const vector<vector<int64_t>>& input_shapes,
const std::initializer_list<float>& expected_output,
const vector<int64_t>& expected_output_shape,
bool weight_is_initializer = false,
OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess,
const std::string& err_str = "",
int opset = 7) {
@ -46,9 +47,11 @@ void TestConvOp(const ConvOpAndTestAttributes& attributes,
ORT_ENFORCE(inputs.size() <= 3, "Our name array is only setup to handle 3 inputs");
const char* szNames[] = {"X", "W", "B"};
for (size_t i = 0; i < inputs.size(); i++) {
test.AddInput<float>(szNames[i], input_shapes[i], inputs[i]);
}
test.AddInput<float>(szNames[0], input_shapes[0], inputs[0]);
test.AddInput<float>(szNames[1], input_shapes[1], inputs[1], weight_is_initializer);
if (inputs.size() == 3)
test.AddInput<float>(szNames[2], input_shapes[2], inputs[2]);
test.AddOutput<float>("Y", expected_output_shape, expected_output);
std::unordered_set<std::string> excluded_providers(attributes.excluded_providers);
@ -199,6 +202,9 @@ TEST(ConvTest, Conv2D_1) {
attrs.excluded_providers.insert(kCudaExecutionProvider); // asymmetric padding is not supported by cudnn
TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape);
// NNAPI EP requires weight to be an initializer
TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true);
}
TEST(ConvTest, Conv1D_Invalid_Input_Shape) {
@ -216,7 +222,7 @@ TEST(ConvTest, Conv1D_Invalid_Input_Shape) {
vector<int64_t> X_shape = {1, 1, 1};
vector<int64_t> dummy_shape = {1, 1, 2};
auto dummy_vals = {0.0f, 0.0f};
TestConvOp(attrs, {X, dummy_vals}, {X_shape, dummy_shape}, dummy_vals, dummy_shape,
TestConvOp(attrs, {X, dummy_vals}, {X_shape, dummy_shape}, dummy_vals, dummy_shape, false,
OpTester::ExpectResult::kExpectFailure,
"Node:node1 Output:Y [ShapeInferenceError] Can't merge shape info. "
"Both source and target dimension have values but they differ. Source=0 Target=2 Dimension=2",
@ -239,7 +245,7 @@ TEST(ConvTest, Conv2D_Invalid_Input_Shape) {
vector<int64_t> dummy_shape = {2, 2, 1, 2};
auto dummy_vals = {-0.0f, 0.0f, -0.0f, -0.0f,
-0.0f, 0.0f, -0.0f, -0.0f};
TestConvOp(attrs, {X, dummy_vals}, {X_shape, dummy_shape}, dummy_vals, dummy_shape,
TestConvOp(attrs, {X, dummy_vals}, {X_shape, dummy_shape}, dummy_vals, dummy_shape, false,
OpTester::ExpectResult::kExpectFailure,
"Node:node1 Output:Y [ShapeInferenceError] Can't merge shape info. "
"Both source and target dimension have values but they differ. Source=1 Target=2 Dimension=0",
@ -289,6 +295,9 @@ TEST(ConvTest, Conv2D_2) {
0.06516310572624207f, -0.015176207758486271f, 0.14682966470718384f, -0.02665453404188156f,
-0.18779225647449493f};
TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape);
// NNAPI EP requires weight to be an initializer
TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true);
}
TEST(ConvTest, Conv2D_Bias_1) {
@ -312,6 +321,9 @@ TEST(ConvTest, Conv2D_Bias_1) {
auto expected_vals = {13.0f, 17.0f, 25.0f, 29.0f, 11.0f, 15.0f, 23.0f, 27.0f};
TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape);
// NNAPI EP requires weight to be an initializer
TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape, true);
}
// Conv48
@ -363,6 +375,8 @@ TEST(ConvTest, Conv2D_Bias_2) {
attrs.excluded_providers.insert(kCudaExecutionProvider); // asymmetric padding is not supported by cudnn
TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape);
TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape, true);
}
TEST(ConvTest, Conv2D_AutoPad1) {
@ -390,6 +404,9 @@ TEST(ConvTest, Conv2D_AutoPad1) {
27.0f, 36.0f, 36.0f, 36.0f, 21.0f,
12.0f, 15.0f, 15.0f, 15.0f, 8.0f};
TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape);
// NNAPI EP requires weight to be an initializer
TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true);
}
TEST(ConvTest, Conv2D_AutoPad2) {
@ -421,6 +438,9 @@ TEST(ConvTest, Conv2D_AutoPad2) {
12.0f, 24.0f, 12.0f, 24.0f, 12.0f,
5.0f, 10.0f, 5.0f, 10.0f, 5.0f};
TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape);
// NNAPI EP requires weight to be an initializer
TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true);
}
// Conv10
@ -629,7 +649,7 @@ TEST(ConvTest, ConvDimWithZero) {
attrs.excluded_providers.insert(kNGraphExecutionProvider);
attrs.excluded_providers.insert(kAclExecutionProvider);
TestConvOp(attrs, {X, W}, {X_shape, W_shape}, {}, out_shape, OpTester::ExpectResult::kExpectSuccess, "", 10);
TestConvOp(attrs, {X, W}, {X_shape, W_shape}, {}, out_shape, false, OpTester::ExpectResult::kExpectSuccess, "", 10);
}
} // namespace test

43
onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc
View file

@ -73,22 +73,22 @@ void TestQLinearConvOp(OpTester& test,
const QuantizedBiasTensor* B,
const QuantizedTensor& Y,
const std::vector<int64_t>& Y_shape,
bool all_input_initializer_except_x = false,
const std::unordered_set<std::string>& excluded_provider_types = {}) {
test.AddInput<uint8_t>("x", X_shape, X.quantized_);
test.AddInput<float>("x_scale", {}, {X.scale_});
test.AddInput<uint8_t>("x_zero_point", {}, {X.zero_point_});
test.AddInput<float>("x_scale", {}, {X.scale_}, all_input_initializer_except_x);
test.AddInput<uint8_t>("x_zero_point", {}, {X.zero_point_}, all_input_initializer_except_x);
test.AddInput<uint8_t>("w", W_shape, W.quantized_);
test.AddInput<float>("w_scale", {}, {W.scale_});
test.AddInput<uint8_t>("w_zero_point", {}, {W.zero_point_});
test.AddInput<uint8_t>("w", W_shape, W.quantized_, all_input_initializer_except_x);
test.AddInput<float>("w_scale", {}, {W.scale_}, all_input_initializer_except_x);
test.AddInput<uint8_t>("w_zero_point", {}, {W.zero_point_}, all_input_initializer_except_x);
test.AddInput<float>("y_scale", {}, {Y.scale_});
test.AddInput<uint8_t>("y_zero_point", {}, {Y.zero_point_});
test.AddInput<float>("y_scale", {}, {Y.scale_}, all_input_initializer_except_x);
test.AddInput<uint8_t>("y_zero_point", {}, {Y.zero_point_}, all_input_initializer_except_x);
if (B != nullptr) {
const std::vector<int64_t> B_shape{static_cast<int64_t>(B->quantized_.size())};
test.AddInput<int32_t>("b", B_shape, B->quantized_);
test.AddInput<int32_t>("b", B_shape, B->quantized_, all_input_initializer_except_x);
}
test.AddOutput<uint8_t>("y", Y_shape, Y.quantized_);
@ -96,7 +96,7 @@ void TestQLinearConvOp(OpTester& test,
test.Run(OpTester::ExpectResult::kExpectSuccess, "", excluded_provider_types);
}
TEST(QLinearConvTest, Conv2DTest) {
void RunConv2DTest(bool all_input_initializer_except_x) {
QuantizedTensor X({0.45246148109436035f, 0.15498268604278564f, 0.11199361085891724f, -0.39421093463897705f,
0.2626858949661255f, 0.13414543867111206f, -0.27184486389160156f, -0.43028733134269714f,
-0.26825493574142456f, 0.3893144130706787f, -0.13631996512413025f, -0.009590476751327515f,
@ -131,7 +131,16 @@ TEST(QLinearConvTest, Conv2DTest) {
X, {1, 1, 7, 7},
W, {1, 1, 1, 1},
nullptr,
Y, {1, 1, 7, 7});
Y, {1, 1, 7, 7},
all_input_initializer_except_x);
}
TEST(QLinearConvTest, Conv2DTest) {
RunConv2DTest(false);
}
TEST(QLinearConvTest, Conv2DTestAllInputInitializerExceptX) {
RunConv2DTest(true);
}
TEST(QLinearConvTest, Conv3DTest) {
@ -172,7 +181,7 @@ TEST(QLinearConvTest, Conv3DTest) {
Y, {1, 1, 4, 4, 4});
}
TEST(QLinearConvTest, WithBias_2D) {
void RunConv2DWithBiasTest(bool all_input_initializer_except_x) {
QuantizedTensor X({6, 81, 214, 151, 234, 42, 50, 89, 30, 91, 125, 141, 52, 31, 58, 224, 84, 251, 67, 137,
223, 119, 79, 220, 249, 75, 131, 246, 113, 56, 54, 197, 110, 142, 126, 171, 53, 228,
240, 83, 229, 218, 185, 9, 80, 116, 176, 193, 175, 253},
@ -203,9 +212,18 @@ TEST(QLinearConvTest, WithBias_2D) {
W, {4, 2, 3, 3},
&B,
Y, {1, 4, 5, 5},
all_input_initializer_except_x,
{kNGraphExecutionProvider});
}
TEST(QLinearConvTest, WithBias_2D) {
RunConv2DWithBiasTest(false);
}
TEST(QLinearConvTest, WithBias_2D_AllInputInitializerExceptX) {
RunConv2DWithBiasTest(true);
}
TEST(QLinearConvTest, WithGroup_2D) {
QuantizedTensor X({98, 166, 219, 195, 46, 97, 27, 211, 239, 1, 28, 208, 143, 144, 215, 252, 79, 5, 154,
56, 122, 191, 94, 25, 221, 48, 37, 182, 68, 245, 210, 206, 183, 22, 163, 104, 242,
@ -236,6 +254,7 @@ TEST(QLinearConvTest, WithGroup_2D) {
W, {6, 2, 2, 2},
&B,
Y, {1, 6, 2, 3},
false,
{kNGraphExecutionProvider});
}