saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[Qnn EP]Enable Gelu op support (#15631)

Browse source 
### Description
Enable Gelu contrib op support

### Motivation and Context
unblock models with contrib op Gelu
This commit is contained in:
Hector Li 2023-04-21 16:54:34 -07:00 committed by GitHub
parent 0080bb0331
commit fab3e33105
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 99 additions and 172 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc
View file

@ -49,6 +49,7 @@ static const OpVersionsAndSelector::OpVersionsMap GetUnaryOpVersionsMap() {
{"ReduceProd", {}},
{"ReduceSum", {}},
{"Relu", {}},
{"Gelu", {}},
{"Sigmoid", {}},
{"Slice", {}},
{"Softmax", {}},
@ -190,7 +191,8 @@ std::vector<NodeGroup> SelectorManager::GetQDQSelections(const GraphViewer& grap
const auto* node = graph_viewer.GetNode(index);
// post layout transformation all the layout sensitive nodes are converted to domain
// kMSInternalNHWCDomain. Therefore need to allow this domain as well.
if (node->Domain() != kOnnxDomain && node->Domain() != kMSInternalNHWCDomain) {
// Allow kMSDomain for contrib op like Gelu
if (node->Domain() != kOnnxDomain && node->Domain() != kMSInternalNHWCDomain && node->Domain() != kMSDomain) {
continue;
}

9
onnxruntime/core/providers/qnn/builder/op_builder_factory.cc
View file

@ -38,6 +38,7 @@ OpBuilderRegistrations::OpBuilderRegistrations() {
CreateSimpleOpBuilder("Pow", *this);
CreateSimpleOpBuilder("PRelu", *this);
CreateSimpleOpBuilder("Relu", *this);
CreateSimpleOpBuilder("Gelu", *this);
CreateSimpleOpBuilder("Round", *this);
CreateSimpleOpBuilder("Where", *this);
CreateSimpleOpBuilder("Sigmoid", *this);
@ -51,6 +52,9 @@ OpBuilderRegistrations::OpBuilderRegistrations() {
CreateSimpleOpBuilder("LogSoftmax", *this);
CreateSimpleOpBuilder("MatMul", *this);
CreateSimpleOpBuilder("Concat", *this);
CreateSimpleOpBuilder("QuantizeLinear", *this);
CreateSimpleOpBuilder("DequantizeLinear", *this);
}
{
@ -75,11 +79,6 @@ OpBuilderRegistrations::OpBuilderRegistrations() {
CreatePoolOpBuilder("MaxPool", *this);
}
{
CreateQdqOpBuilder("QuantizeLinear", *this);
CreateQdqOpBuilder("DequantizeLinear", *this);
}
{
CreateReshapeOpBuilder("Reshape", *this);
CreateReshapeOpBuilder("Flatten", *this);

4
onnxruntime/core/providers/qnn/builder/opbuilder/base_op_builder.h
View file

@ -135,9 +135,13 @@ class BaseOpBuilder : public IOpBuilder {
{"Tanh", "Tanh"},
{"Transpose", "Transpose"},
{"DequantizeLinear", "Dequantize"},
{"QuantizeLinear", "Quantize"},
{"MatMul", "MatMul"},
{"Relu", "Relu"},
{"Gelu", "Gelu"},
{"Sigmoid", "Sigmoid"},
{"Conv", "Conv2d"},

165
onnxruntime/core/providers/qnn/builder/opbuilder/qdq_op_builder.cc
View file

@ -1,165 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/common.h"
#include "core/providers/shared/utils/utils.h"
#include "core/providers/qnn/builder/qnn_model_wrapper.h"
#include "core/providers/qnn/builder/op_builder_factory.h"
#include "base_op_builder.h"
namespace onnxruntime {
namespace qnn {
class QdqOpBuilder : public BaseOpBuilder {
public:
QdqOpBuilder() : BaseOpBuilder("QdqOpBuilder") {}
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(QdqOpBuilder);
protected:
Status ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
const logging::Logger& logger,
bool is_quantized_model,
std::vector<std::string>& input_names,
bool do_op_validation) const override ORT_MUST_USE_RESULT;
private:
Status AddQuantizeNodeOnModelInput(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
const logging::Logger& logger) const;
Status AddDequantizeNodeOnModelOutput(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
const logging::Logger& logger) const;
};
Status QdqOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
const logging::Logger& logger,
bool is_quantized_model,
std::vector<std::string>& input_names,
bool do_op_validation) const {
ORT_UNUSED_PARAMETER(input_names);
ORT_UNUSED_PARAMETER(is_quantized_model);
ORT_UNUSED_PARAMETER(do_op_validation);
if (node_unit.OpType() == "QuantizeLinear") {
return AddQuantizeNodeOnModelInput(qnn_model_wrapper, node_unit, logger);
} else if (node_unit.OpType() == "DequantizeLinear") {
return AddDequantizeNodeOnModelOutput(qnn_model_wrapper, node_unit, logger);
}
return Status::OK();
}
Status QdqOpBuilder::AddQuantizeNodeOnModelInput(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
const logging::Logger& logger) const {
auto& input_name = node_unit.Inputs()[0].node_arg.Name();
auto& output_name = node_unit.Outputs()[0].node_arg.Name();
LOGS(logger, VERBOSE) << "AddQuantizeNodeOnModelInput: " << input_name << "->" << output_name;
const auto* type_proto = node_unit.GetNode().InputDefs()[2]->TypeAsProto();
Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32;
ORT_RETURN_IF_ERROR(GetQnnDataType(true, type_proto, qnn_data_type));
std::vector<uint32_t> input_shape;
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(node_unit.Inputs()[0].node_arg, input_shape), "Cannot get shape");
std::vector<uint32_t> output_shape = input_shape;
float scale_value = 0.0f;
int32_t offset_value = 0;
const auto& scale_name = node_unit.GetNode().InputDefs()[1]->Name();
ORT_RETURN_IF_NOT(qnn_model_wrapper.ProcessScale(scale_name, scale_value), "ProcessScale failed");
const auto& zero_point_name = node_unit.GetNode().InputDefs()[2]->Name();
ORT_RETURN_IF_NOT(qnn_model_wrapper.ProcessOffset(zero_point_name, offset_value), "ProcessOffset failed");
Qnn_TensorType_t input_tensor_type = QNN_TENSOR_TYPE_APP_WRITE;
Qnn_QuantizeParams_t quantize_params = QNN_QUANTIZE_PARAMS_INIT;
QnnTensorWrapper input_tensorwrapper(input_name,
input_tensor_type,
QNN_DATATYPE_FLOAT_32,
quantize_params,
std::move(input_shape));
ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(input_tensorwrapper)), "Failed to add tensor.");
Qnn_TensorType_t output_tensor_type = QNN_TENSOR_TYPE_NATIVE;
quantize_params.encodingDefinition = QNN_DEFINITION_DEFINED;
quantize_params.quantizationEncoding = QNN_QUANTIZATION_ENCODING_SCALE_OFFSET;
quantize_params.scaleOffsetEncoding.scale = scale_value;
quantize_params.scaleOffsetEncoding.offset = offset_value;
QnnTensorWrapper output_tensorwrapper(output_name,
output_tensor_type,
qnn_data_type,
quantize_params,
std::move(output_shape));
ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(output_tensorwrapper)), "Failed to add tensor.");
std::vector<QnnTensorWrapper> output_tensors;
output_tensors.emplace_back(std::move(output_tensorwrapper));
const static std::string qnn_op_type = "Quantize";
ORT_RETURN_IF_NOT(qnn_model_wrapper.CreateQnnNode(output_name,
qnn_def::package_name,
qnn_op_type,
{input_name},
{output_name},
{}),
"Failed to add node.");
return Status::OK();
}
Status QdqOpBuilder::AddDequantizeNodeOnModelOutput(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
const logging::Logger& logger) const {
auto& input_name = node_unit.Inputs()[0].node_arg.Name();
auto& output_name = node_unit.Outputs()[0].node_arg.Name();
LOGS(logger, VERBOSE) << "AddDequantizeNodeOnModelOutput: " << input_name << "->" << output_name;
// get type from zero_point
const auto* type_proto = node_unit.GetNode().InputDefs()[2]->TypeAsProto();
Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32;
ORT_RETURN_IF_ERROR(GetQnnDataType(true, type_proto, qnn_data_type));
std::vector<uint32_t> output_shape;
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(node_unit.Outputs()[0].node_arg, output_shape),
"Cannot get shape");
float scale_value = 0.0f;
int32_t offset_value = 0;
const auto& scale_name = node_unit.GetNode().InputDefs()[1]->Name();
ORT_RETURN_IF_NOT(qnn_model_wrapper.ProcessScale(scale_name, scale_value), "ProcessScale failed");
const auto& zero_point_name = node_unit.GetNode().InputDefs()[2]->Name();
ORT_RETURN_IF_NOT(qnn_model_wrapper.ProcessOffset(zero_point_name, offset_value), "ProcessOffset failed");
Qnn_TensorType_t input_tensor_type = QNN_TENSOR_TYPE_NATIVE;
Qnn_QuantizeParams_t quantize_params = QNN_QUANTIZE_PARAMS_INIT;
quantize_params.encodingDefinition = QNN_DEFINITION_DEFINED;
quantize_params.quantizationEncoding = QNN_QUANTIZATION_ENCODING_SCALE_OFFSET;
quantize_params.scaleOffsetEncoding.scale = scale_value;
quantize_params.scaleOffsetEncoding.offset = offset_value;
std::vector<uint32_t> input_shape = output_shape;
QnnTensorWrapper input_tensorwrapper(input_name, input_tensor_type, qnn_data_type, quantize_params,
std::move(input_shape));
ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(input_tensorwrapper)), "Failed to add tensor.");
bool is_graph_output = qnn_model_wrapper.IsGraphOutput(output_name);
Qnn_TensorType_t output_tensor_type = is_graph_output ? QNN_TENSOR_TYPE_APP_READ : QNN_TENSOR_TYPE_NATIVE;
QnnTensorWrapper output_tensorwrapper(output_name, output_tensor_type, QNN_DATATYPE_FLOAT_32,
QNN_QUANTIZE_PARAMS_INIT, std::move(output_shape));
ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(output_tensorwrapper)), "Failed to add tensor.");
const static std::string qnn_op_type = "Dequantize";
ORT_RETURN_IF_NOT(qnn_model_wrapper.CreateQnnNode(output_name,
qnn_def::package_name,
qnn_op_type,
{input_name},
{output_name},
{}),
"Failed to add node.");
return Status::OK();
}
void CreateQdqOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations) {
op_registrations.AddOpBuilder(op_type, std::make_unique<QdqOpBuilder>());
}
} // namespace qnn
} // namespace onnxruntime

3
onnxruntime/test/providers/qnn/qnn_test_utils.cc
View file

@ -25,7 +25,8 @@ void RunQnnModelTest(const GetTestModelFn& build_test_case, const ProviderOption
verification_params.ep_node_assignment = expected_ep_assignment;
verification_params.graph_verifier = &graph_verify;
verification_params.fp32_abs_err = fp32_abs_err;
const std::unordered_map<std::string, int> domain_to_version = {{"", opset_version}};
// Add kMSDomain to cover contrib op like Gelu
const std::unordered_map<std::string, int> domain_to_version = {{"", opset_version}, {kMSDomain, 1}};
onnxruntime::Model model(test_description, false, ModelMetaData(), PathString(),
IOnnxRuntimeOpSchemaRegistryList(), domain_to_version, {},

86
onnxruntime/test/providers/qnn/simple_op_htp_test.cc Normal file
View file

@ -0,0 +1,86 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#if !defined(ORT_MINIMAL_BUILD)
#include <string>
#include "core/graph/graph.h"
#include "test/optimizer/qdq_test_utils.h"
#include "test/providers/qnn/qnn_test_utils.h"
#include "gtest/gtest.h"
namespace onnxruntime {
namespace test {
#if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__)
// Creates the graph:
// _______________________
// | |
// input_u8 -> DQ -> | SimpleOp | -> Q -> output_u8
// |_______________________|
//
// Currently used to test QNN EP.
template <typename InputQType>
GetQDQTestCaseFn BuildQDQSingleInputOpTestCase(const std::vector<int64_t>& input_shape, const std::string& op_type) {
return [input_shape, op_type](ModelTestBuilder& builder) {
const InputQType quant_zero_point = 0;
const float quant_scale = 1.0f;
auto* input = builder.MakeInput<InputQType>(input_shape, std::numeric_limits<InputQType>::min(),
std::numeric_limits<InputQType>::max());
auto* dq_input = builder.MakeIntermediate();
builder.AddDequantizeLinearNode<InputQType>(input, quant_scale, quant_zero_point, dq_input);
auto* op_output = builder.MakeIntermediate();
builder.AddNode(op_type, {dq_input}, {op_output}, kMSDomain);
auto* q_output = builder.MakeIntermediate();
builder.AddQuantizeLinearNode<InputQType>(op_output, quant_scale, quant_zero_point, q_output);
auto* final_output = builder.MakeOutput();
builder.AddDequantizeLinearNode<InputQType>(q_output, quant_scale, quant_zero_point, final_output);
};
}
/**
* Runs an BatchNormalization model on the QNN HTP backend. Checks the graph node assignment, and that inference
* outputs for QNN and CPU match.
*
* \param input_shape The input's shape.
* \param test_description Description of the test for error reporting.
* \param expected_ep_assignment How many nodes are expected to be assigned to QNN (All, Some, or None).
* \param num_modes_in_graph The number of expected nodes in the graph.
*/
static void RunQDQSingleInputOpTest(const std::vector<int64_t>& input_shape, const std::string& op_type,
const char* test_description,
ExpectedEPNodeAssignment expected_ep_assignment, int num_nodes_in_graph) {
ProviderOptions provider_options;
#if defined(_WIN32)
provider_options["backend_path"] = "QnnHtp.dll";
#else
provider_options["backend_path"] = "libQnnHtp.so";
#endif
// Runs model with DQ-> InstanceNorm -> Q and compares the outputs of the CPU and QNN EPs.
RunQnnModelTest(BuildQDQSingleInputOpTestCase<uint8_t>(input_shape, op_type),
provider_options,
11,
expected_ep_assignment,
num_nodes_in_graph,
test_description);
}
// Check that QNN compiles DQ -> BatchNormalization -> Q as a single unit.
// Use an input of rank 3.
TEST_F(QnnHTPBackendTests, TestQDQGeluTest) {
RunQDQSingleInputOpTest({1, 2, 3}, "Gelu", "TestQDQGeluTest", ExpectedEPNodeAssignment::All, 1);
}
#endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__)
} // namespace test
} // namespace onnxruntime
#endif