diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc index 0e343bbc60..132db0ae49 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc @@ -49,6 +49,7 @@ static const OpVersionsAndSelector::OpVersionsMap GetUnaryOpVersionsMap() { {"ReduceProd", {}}, {"ReduceSum", {}}, {"Relu", {}}, + {"Gelu", {}}, {"Sigmoid", {}}, {"Slice", {}}, {"Softmax", {}}, @@ -190,7 +191,8 @@ std::vector SelectorManager::GetQDQSelections(const GraphViewer& grap const auto* node = graph_viewer.GetNode(index); // post layout transformation all the layout sensitive nodes are converted to domain // kMSInternalNHWCDomain. Therefore need to allow this domain as well. - if (node->Domain() != kOnnxDomain && node->Domain() != kMSInternalNHWCDomain) { + // Allow kMSDomain for contrib op like Gelu + if (node->Domain() != kOnnxDomain && node->Domain() != kMSInternalNHWCDomain && node->Domain() != kMSDomain) { continue; } diff --git a/onnxruntime/core/providers/qnn/builder/op_builder_factory.cc b/onnxruntime/core/providers/qnn/builder/op_builder_factory.cc index f2a6695b3a..fa2d5c9733 100644 --- a/onnxruntime/core/providers/qnn/builder/op_builder_factory.cc +++ b/onnxruntime/core/providers/qnn/builder/op_builder_factory.cc @@ -38,6 +38,7 @@ OpBuilderRegistrations::OpBuilderRegistrations() { CreateSimpleOpBuilder("Pow", *this); CreateSimpleOpBuilder("PRelu", *this); CreateSimpleOpBuilder("Relu", *this); + CreateSimpleOpBuilder("Gelu", *this); CreateSimpleOpBuilder("Round", *this); CreateSimpleOpBuilder("Where", *this); CreateSimpleOpBuilder("Sigmoid", *this); @@ -51,6 +52,9 @@ OpBuilderRegistrations::OpBuilderRegistrations() { CreateSimpleOpBuilder("LogSoftmax", *this); CreateSimpleOpBuilder("MatMul", *this); CreateSimpleOpBuilder("Concat", *this); + + CreateSimpleOpBuilder("QuantizeLinear", *this); + CreateSimpleOpBuilder("DequantizeLinear", *this); } { @@ -75,11 +79,6 @@ OpBuilderRegistrations::OpBuilderRegistrations() { CreatePoolOpBuilder("MaxPool", *this); } - { - CreateQdqOpBuilder("QuantizeLinear", *this); - CreateQdqOpBuilder("DequantizeLinear", *this); - } - { CreateReshapeOpBuilder("Reshape", *this); CreateReshapeOpBuilder("Flatten", *this); diff --git a/onnxruntime/core/providers/qnn/builder/opbuilder/base_op_builder.h b/onnxruntime/core/providers/qnn/builder/opbuilder/base_op_builder.h index a86790fdf8..7e1a6024bb 100644 --- a/onnxruntime/core/providers/qnn/builder/opbuilder/base_op_builder.h +++ b/onnxruntime/core/providers/qnn/builder/opbuilder/base_op_builder.h @@ -135,9 +135,13 @@ class BaseOpBuilder : public IOpBuilder { {"Tanh", "Tanh"}, {"Transpose", "Transpose"}, + {"DequantizeLinear", "Dequantize"}, + {"QuantizeLinear", "Quantize"}, + {"MatMul", "MatMul"}, {"Relu", "Relu"}, + {"Gelu", "Gelu"}, {"Sigmoid", "Sigmoid"}, {"Conv", "Conv2d"}, diff --git a/onnxruntime/core/providers/qnn/builder/opbuilder/qdq_op_builder.cc b/onnxruntime/core/providers/qnn/builder/opbuilder/qdq_op_builder.cc deleted file mode 100644 index e961294b9a..0000000000 --- a/onnxruntime/core/providers/qnn/builder/opbuilder/qdq_op_builder.cc +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/common.h" -#include "core/providers/shared/utils/utils.h" -#include "core/providers/qnn/builder/qnn_model_wrapper.h" -#include "core/providers/qnn/builder/op_builder_factory.h" - -#include "base_op_builder.h" - -namespace onnxruntime { -namespace qnn { - -class QdqOpBuilder : public BaseOpBuilder { - public: - QdqOpBuilder() : BaseOpBuilder("QdqOpBuilder") {} - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(QdqOpBuilder); - - protected: - Status ProcessInputs(QnnModelWrapper& qnn_model_wrapper, - const NodeUnit& node_unit, - const logging::Logger& logger, - bool is_quantized_model, - std::vector& input_names, - bool do_op_validation) const override ORT_MUST_USE_RESULT; - - private: - Status AddQuantizeNodeOnModelInput(QnnModelWrapper& qnn_model_wrapper, - const NodeUnit& node_unit, - const logging::Logger& logger) const; - Status AddDequantizeNodeOnModelOutput(QnnModelWrapper& qnn_model_wrapper, - const NodeUnit& node_unit, - const logging::Logger& logger) const; -}; - -Status QdqOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper, - const NodeUnit& node_unit, - const logging::Logger& logger, - bool is_quantized_model, - std::vector& input_names, - bool do_op_validation) const { - ORT_UNUSED_PARAMETER(input_names); - ORT_UNUSED_PARAMETER(is_quantized_model); - ORT_UNUSED_PARAMETER(do_op_validation); - if (node_unit.OpType() == "QuantizeLinear") { - return AddQuantizeNodeOnModelInput(qnn_model_wrapper, node_unit, logger); - } else if (node_unit.OpType() == "DequantizeLinear") { - return AddDequantizeNodeOnModelOutput(qnn_model_wrapper, node_unit, logger); - } - - return Status::OK(); -} - -Status QdqOpBuilder::AddQuantizeNodeOnModelInput(QnnModelWrapper& qnn_model_wrapper, - const NodeUnit& node_unit, - const logging::Logger& logger) const { - auto& input_name = node_unit.Inputs()[0].node_arg.Name(); - auto& output_name = node_unit.Outputs()[0].node_arg.Name(); - LOGS(logger, VERBOSE) << "AddQuantizeNodeOnModelInput: " << input_name << "->" << output_name; - - const auto* type_proto = node_unit.GetNode().InputDefs()[2]->TypeAsProto(); - Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; - ORT_RETURN_IF_ERROR(GetQnnDataType(true, type_proto, qnn_data_type)); - - std::vector input_shape; - ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(node_unit.Inputs()[0].node_arg, input_shape), "Cannot get shape"); - std::vector output_shape = input_shape; - - float scale_value = 0.0f; - int32_t offset_value = 0; - const auto& scale_name = node_unit.GetNode().InputDefs()[1]->Name(); - ORT_RETURN_IF_NOT(qnn_model_wrapper.ProcessScale(scale_name, scale_value), "ProcessScale failed"); - const auto& zero_point_name = node_unit.GetNode().InputDefs()[2]->Name(); - ORT_RETURN_IF_NOT(qnn_model_wrapper.ProcessOffset(zero_point_name, offset_value), "ProcessOffset failed"); - - Qnn_TensorType_t input_tensor_type = QNN_TENSOR_TYPE_APP_WRITE; - Qnn_QuantizeParams_t quantize_params = QNN_QUANTIZE_PARAMS_INIT; - QnnTensorWrapper input_tensorwrapper(input_name, - input_tensor_type, - QNN_DATATYPE_FLOAT_32, - quantize_params, - std::move(input_shape)); - ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(input_tensorwrapper)), "Failed to add tensor."); - Qnn_TensorType_t output_tensor_type = QNN_TENSOR_TYPE_NATIVE; - quantize_params.encodingDefinition = QNN_DEFINITION_DEFINED; - quantize_params.quantizationEncoding = QNN_QUANTIZATION_ENCODING_SCALE_OFFSET; - quantize_params.scaleOffsetEncoding.scale = scale_value; - quantize_params.scaleOffsetEncoding.offset = offset_value; - QnnTensorWrapper output_tensorwrapper(output_name, - output_tensor_type, - qnn_data_type, - quantize_params, - std::move(output_shape)); - ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(output_tensorwrapper)), "Failed to add tensor."); - - std::vector output_tensors; - output_tensors.emplace_back(std::move(output_tensorwrapper)); - const static std::string qnn_op_type = "Quantize"; - ORT_RETURN_IF_NOT(qnn_model_wrapper.CreateQnnNode(output_name, - qnn_def::package_name, - qnn_op_type, - {input_name}, - {output_name}, - {}), - "Failed to add node."); - - return Status::OK(); -} - -Status QdqOpBuilder::AddDequantizeNodeOnModelOutput(QnnModelWrapper& qnn_model_wrapper, - const NodeUnit& node_unit, - const logging::Logger& logger) const { - auto& input_name = node_unit.Inputs()[0].node_arg.Name(); - auto& output_name = node_unit.Outputs()[0].node_arg.Name(); - LOGS(logger, VERBOSE) << "AddDequantizeNodeOnModelOutput: " << input_name << "->" << output_name; - - // get type from zero_point - const auto* type_proto = node_unit.GetNode().InputDefs()[2]->TypeAsProto(); - Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32; - ORT_RETURN_IF_ERROR(GetQnnDataType(true, type_proto, qnn_data_type)); - - std::vector output_shape; - ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(node_unit.Outputs()[0].node_arg, output_shape), - "Cannot get shape"); - float scale_value = 0.0f; - int32_t offset_value = 0; - const auto& scale_name = node_unit.GetNode().InputDefs()[1]->Name(); - ORT_RETURN_IF_NOT(qnn_model_wrapper.ProcessScale(scale_name, scale_value), "ProcessScale failed"); - const auto& zero_point_name = node_unit.GetNode().InputDefs()[2]->Name(); - ORT_RETURN_IF_NOT(qnn_model_wrapper.ProcessOffset(zero_point_name, offset_value), "ProcessOffset failed"); - Qnn_TensorType_t input_tensor_type = QNN_TENSOR_TYPE_NATIVE; - Qnn_QuantizeParams_t quantize_params = QNN_QUANTIZE_PARAMS_INIT; - quantize_params.encodingDefinition = QNN_DEFINITION_DEFINED; - quantize_params.quantizationEncoding = QNN_QUANTIZATION_ENCODING_SCALE_OFFSET; - quantize_params.scaleOffsetEncoding.scale = scale_value; - quantize_params.scaleOffsetEncoding.offset = offset_value; - - std::vector input_shape = output_shape; - QnnTensorWrapper input_tensorwrapper(input_name, input_tensor_type, qnn_data_type, quantize_params, - std::move(input_shape)); - ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(input_tensorwrapper)), "Failed to add tensor."); - bool is_graph_output = qnn_model_wrapper.IsGraphOutput(output_name); - Qnn_TensorType_t output_tensor_type = is_graph_output ? QNN_TENSOR_TYPE_APP_READ : QNN_TENSOR_TYPE_NATIVE; - QnnTensorWrapper output_tensorwrapper(output_name, output_tensor_type, QNN_DATATYPE_FLOAT_32, - QNN_QUANTIZE_PARAMS_INIT, std::move(output_shape)); - ORT_RETURN_IF_NOT(qnn_model_wrapper.AddTensorWrapper(std::move(output_tensorwrapper)), "Failed to add tensor."); - const static std::string qnn_op_type = "Dequantize"; - - ORT_RETURN_IF_NOT(qnn_model_wrapper.CreateQnnNode(output_name, - qnn_def::package_name, - qnn_op_type, - {input_name}, - {output_name}, - {}), - "Failed to add node."); - - return Status::OK(); -} - -void CreateQdqOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations) { - op_registrations.AddOpBuilder(op_type, std::make_unique()); -} - -} // namespace qnn -} // namespace onnxruntime diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.cc b/onnxruntime/test/providers/qnn/qnn_test_utils.cc index cc945a4971..e6d50c87fb 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.cc +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.cc @@ -25,7 +25,8 @@ void RunQnnModelTest(const GetTestModelFn& build_test_case, const ProviderOption verification_params.ep_node_assignment = expected_ep_assignment; verification_params.graph_verifier = &graph_verify; verification_params.fp32_abs_err = fp32_abs_err; - const std::unordered_map domain_to_version = {{"", opset_version}}; + // Add kMSDomain to cover contrib op like Gelu + const std::unordered_map domain_to_version = {{"", opset_version}, {kMSDomain, 1}}; onnxruntime::Model model(test_description, false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), domain_to_version, {}, diff --git a/onnxruntime/test/providers/qnn/simple_op_htp_test.cc b/onnxruntime/test/providers/qnn/simple_op_htp_test.cc new file mode 100644 index 0000000000..ea7f718b79 --- /dev/null +++ b/onnxruntime/test/providers/qnn/simple_op_htp_test.cc @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#if !defined(ORT_MINIMAL_BUILD) + +#include +#include "core/graph/graph.h" + +#include "test/optimizer/qdq_test_utils.h" +#include "test/providers/qnn/qnn_test_utils.h" + +#include "gtest/gtest.h" + +namespace onnxruntime { +namespace test { +#if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) + +// Creates the graph: +// _______________________ +// | | +// input_u8 -> DQ -> | SimpleOp | -> Q -> output_u8 +// |_______________________| +// +// Currently used to test QNN EP. +template +GetQDQTestCaseFn BuildQDQSingleInputOpTestCase(const std::vector& input_shape, const std::string& op_type) { + return [input_shape, op_type](ModelTestBuilder& builder) { + const InputQType quant_zero_point = 0; + const float quant_scale = 1.0f; + + auto* input = builder.MakeInput(input_shape, std::numeric_limits::min(), + std::numeric_limits::max()); + auto* dq_input = builder.MakeIntermediate(); + builder.AddDequantizeLinearNode(input, quant_scale, quant_zero_point, dq_input); + + auto* op_output = builder.MakeIntermediate(); + builder.AddNode(op_type, {dq_input}, {op_output}, kMSDomain); + + auto* q_output = builder.MakeIntermediate(); + builder.AddQuantizeLinearNode(op_output, quant_scale, quant_zero_point, q_output); + + auto* final_output = builder.MakeOutput(); + builder.AddDequantizeLinearNode(q_output, quant_scale, quant_zero_point, final_output); + }; +} + +/** + * Runs an BatchNormalization model on the QNN HTP backend. Checks the graph node assignment, and that inference + * outputs for QNN and CPU match. + * + * \param input_shape The input's shape. + * \param test_description Description of the test for error reporting. + * \param expected_ep_assignment How many nodes are expected to be assigned to QNN (All, Some, or None). + * \param num_modes_in_graph The number of expected nodes in the graph. + */ +static void RunQDQSingleInputOpTest(const std::vector& input_shape, const std::string& op_type, + const char* test_description, + ExpectedEPNodeAssignment expected_ep_assignment, int num_nodes_in_graph) { + ProviderOptions provider_options; +#if defined(_WIN32) + provider_options["backend_path"] = "QnnHtp.dll"; +#else + provider_options["backend_path"] = "libQnnHtp.so"; +#endif + + // Runs model with DQ-> InstanceNorm -> Q and compares the outputs of the CPU and QNN EPs. + RunQnnModelTest(BuildQDQSingleInputOpTestCase(input_shape, op_type), + provider_options, + 11, + expected_ep_assignment, + num_nodes_in_graph, + test_description); +} + +// Check that QNN compiles DQ -> BatchNormalization -> Q as a single unit. +// Use an input of rank 3. +TEST_F(QnnHTPBackendTests, TestQDQGeluTest) { + RunQDQSingleInputOpTest({1, 2, 3}, "Gelu", "TestQDQGeluTest", ExpectedEPNodeAssignment::All, 1); +} + +#endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) + +} // namespace test +} // namespace onnxruntime + +#endif \ No newline at end of file