diff --git a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc index d0f0f48350..28759ea668 100644 --- a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc @@ -20,7 +20,6 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Range); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, WordConvEmbedding); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, GatherND); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MatMulInteger16); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, TransposeMatMul); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MurmurHash3); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, MaxpoolWithMask); @@ -28,19 +27,25 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Pad); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Unique); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, ConvTransposeWithDynamicPads); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, CropAndResize); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, CDist); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, double, CDist); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Gelu); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, BiasGelu); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, FastGelu); + +// ******** Start: Quantization ******************* // +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MatMulInteger16); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, DequantizeLinear); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, DequantizeLinear); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QuantizeLinear); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QuantizeLinear); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearLeakyRelu); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearLeakyRelu); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, CDist); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, double, CDist); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Gelu); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, BiasGelu); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, FastGelu); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float_uint8_t_uint8_t, QAttention); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float_uint8_t_int8_t, QAttention); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, DynamicQuantizeMatMul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, DynamicQuantizeMatMul); +// ******** End: Quantization ******************* // // This section includes all op kernel declarations for former experimental ops which have now been removed from onnx. // To maintain backward compatibility these are added as contrib ops. @@ -86,6 +91,27 @@ Status RegisterNchwcKernels(KernelRegistry& kernel_registry) { return Status::OK(); } +Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) { + static const BuildKernelCreateInfoFn function_table[] = { + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + }; + + for (auto& function_table_entry : function_table) { + ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry())); + } + return Status::OK(); +} + Status RegisterCpuContribKernels(KernelRegistry& kernel_registry) { static const BuildKernelCreateInfoFn function_table[] = { BuildKernelCreateInfo, @@ -102,26 +128,18 @@ Status RegisterCpuContribKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, + // These ops were experimental ops in onnx domain which have been removed now. We add them here as // contrib ops to main backward compatibility BuildKernelCreateInfo, @@ -148,6 +166,9 @@ Status RegisterCpuContribKernels(KernelRegistry& kernel_registry) { if (MlasNchwcGetBlockSize() > 1) { ORT_RETURN_IF_ERROR(RegisterNchwcKernels(kernel_registry)); } + + RegisterQuantizationKernels(kernel_registry); + return Status::OK(); } diff --git a/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc new file mode 100644 index 0000000000..2e7566e27b --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.cc @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "dynamic_quantize_matmul.h" + +#include "core/common/safeint.h" +#include "core/mlas/inc/mlas.h" +#include "core/providers/common.h" +#include "core/providers/cpu/math/matmul_helper.h" +#include "core/util/math_cpuonly.h" +#include "core/util/qmath.h" + +#include + +namespace onnxruntime { +namespace contrib { + +#define REGISTER_DYNAMIC_QUANTIZE_MATMUL(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + DynamicQuantizeMatMul, \ + kMSDomain, \ + 1, \ + T, \ + kCpuExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T1", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("T2", DataTypeImpl::GetTensorType()), \ + DynamicQuantizeMatMul); + +REGISTER_DYNAMIC_QUANTIZE_MATMUL(int8_t) +REGISTER_DYNAMIC_QUANTIZE_MATMUL(uint8_t) + +static void GetQuantizationParameter(const float* data, int64_t num_of_elements, float& scale, uint8_t& zp) { + // find input range min and max + float min = ConstEigenVectorMap(data, num_of_elements).minCoeff(); + float max = ConstEigenVectorMap(data, num_of_elements).maxCoeff(); + + // ensure the input range includes zero + min = std::min(min, 0.0f); + max = std::max(max, 0.0f); + + // find scale and zero point + uint8_t qmin = 0; + uint8_t qmax = 255; + scale = max == min ? 1.0f : (max - min) / (qmax - qmin); + + float initial_zero_point = qmin - min / scale; + zp = static_cast(RoundHalfToEven(std::max(float(qmin), std::min(float(qmax), initial_zero_point)))); +} + +template +Status DynamicQuantizeMatMul::Compute(OpKernelContext* ctx) const { + auto* a = ctx->Input(0); + auto* b = ctx->Input(1); + ORT_ENFORCE(a != nullptr && b != nullptr); + + auto* b_scale_tensor = ctx->Input(2); + ORT_ENFORCE(IsScalarOr1ElementVector(b_scale_tensor), + "MatmulInteger : input B scale must be a scalar or 1D tensor of size 1. Per-Channel is not supported yet."); + + float b_scale = *b_scale_tensor->template Data(); + + auto* b_zp_tensor = ctx->Input(3); + T b_zp = 0; + if (b_zp_tensor != nullptr) { + ORT_ENFORCE(IsScalarOr1ElementVector(b_zp_tensor), + "MatmulInteger : input B zero point must be a scalar or 1D tensor of size 1. Per-Channel is not supported yet."); + b_zp = *b_zp_tensor->template Data(); + } + + // calculate quantization parameter of a + const float* a_data = a->template Data(); + int64_t num_of_elements = a->Shape().Size(); + + float a_scale; + uint8_t a_zp; + GetQuantizationParameter(a_data, num_of_elements, a_scale, a_zp); + + AllocatorPtr allocator; + ORT_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&allocator)); + uint8_t* a_data_quant = static_cast(allocator->Alloc(SafeInt(num_of_elements) * sizeof(uint8_t))); + BufferUniquePtr a_buffer_quant_holder(a_data_quant, BufferDeleter(allocator)); + // quantize the data + MlasQuantizeLinear(a_data, a_data_quant, num_of_elements, a_scale, a_zp); + + MatMulComputeHelper helper; + ORT_RETURN_IF_ERROR(helper.Compute(a->Shape(), b->Shape())); + + int32_t* matmul_output = static_cast(allocator->Alloc(SafeInt(helper.OutputShape().Size()) * sizeof(int32_t))); + BufferUniquePtr matmul_output_holder(matmul_output, BufferDeleter(allocator)); + + concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool(); + for (size_t i = 0; i < helper.OutputOffsets().size(); i++) { + QGemm(static_cast(helper.M()), + static_cast(helper.N()), + static_cast(helper.K()), + a_data_quant + helper.LeftOffsets()[i], + static_cast(helper.K()), + a_zp, + b->template Data() + helper.RightOffsets()[i], + static_cast(helper.N()), + b_zp, + matmul_output + helper.OutputOffsets()[i], + static_cast(helper.N()), + thread_pool); + } + + Tensor* y = ctx->Output(0, helper.OutputShape()); + float* y_data = y->template MutableData(); + + float multiplier = a_scale * b_scale; + for (int64_t i = 0; i < helper.OutputShape().Size(); i++) { + y_data[i] = matmul_output[i] * multiplier; + } + return Status::OK(); +} + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.h b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.h new file mode 100644 index 0000000000..28cd21bc00 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_matmul.h @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace contrib { + +template +class DynamicQuantizeMatMul final : public OpKernel { + public: + DynamicQuantizeMatMul(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const override; +}; +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index af85126923..7b65660efd 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -1656,7 +1656,41 @@ Matrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy- // Right now we only support int32 y_type->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto::INT32); - matmulShapeInference(ctx, 0, 1); + ONNX_NAMESPACE::matmulShapeInference(ctx, 0, 1); + }); + + ONNX_CONTRIB_OPERATOR_SCHEMA(DynamicQuantizeMatMul) + .SetDomain(kMSDomain) + .SinceVersion(1) + .Input(0, "A", "N-dimensional matrix A", "T1") + .Input(1, "B", "N-dimensional matrix B", "T2") + .Input( + 2, + "b_scale", + "Scale of quantized input 'B'. It could be a scalar or a 1-D tensor, " + "which means a per-tensor or per-column quantization. If it's a 1-D tensor, its number " + "of elements should be equal to the number of columns of input 'B'.", + "T1") + .Input( + 3, + "b_zero_point", + "Zero point tensor for input 'B'. It's optional and default value is 0. It could be a scalar or a 1-D tensor, " + "which means a per-tensor or per-column quantization. If it's a 1-D tensor, its number " + "of elements should be equal to the number of columns of input 'B'.", + "T2", + OpSchema::Optional) + .Output(0, "Y", "Matrix multiply results from A * B", "T1") + .TypeConstraint( + "T1", + {"tensor(float)"}, + "Constrain input A, b_scale and output Y data type as float tensor.") + .TypeConstraint( + "T2", + {"tensor(int8)", "tensor(uint8)"}, + "Constrain input B data type to 8-bit integer tensor.") + .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromInputToOutput(ctx, 0, 0); + ONNX_NAMESPACE::matmulShapeInference(ctx, 0, 1); }); static const char* TransposeMatMul_doc = R"DOC( diff --git a/onnxruntime/core/graph/graph_utils.cc b/onnxruntime/core/graph/graph_utils.cc index fd6d1fbfcb..c2a6a7b0dc 100644 --- a/onnxruntime/core/graph/graph_utils.cc +++ b/onnxruntime/core/graph/graph_utils.cc @@ -265,11 +265,10 @@ static void MoveAllNodeOutputs(Graph& graph, Node& src_node, Node& target_node) int GetNodeInputIndexFromInputName(const Node& node, const std::string& input_name) { auto itr = std::find_if(node.InputDefs().begin(), node.InputDefs().end(), [&input_name](const NodeArg* input) { return input->Name() == input_name; }); - ORT_ENFORCE(itr != node.InputDefs().end(), - "Attempting to get index for an input which does not exist."); + ORT_ENFORCE(itr != node.InputDefs().end(), + "Attempting to get index for an input which does not exist."); auto index = std::distance(node.InputDefs().begin(), itr); return static_cast(index); - } const std::string& GetNodeInputName(const Node& node, int index) { @@ -716,7 +715,7 @@ bool FindPath(const Node& node, bool is_input_edge, const std::vectorInputEdgesBegin() : current_node->OutputEdgesBegin(); auto edges_end = is_input_edge ? current_node->InputEdgesEnd() : current_node->OutputEdgesEnd(); @@ -756,21 +755,37 @@ bool FindPath(const Node& node, bool is_input_edge, const std::vector& edges_to_match, std::vector>& result, const logging::Logger& logger) { + result.clear(); + + std::vector edge_ends; + if (!FindPath(node, is_input_edge, edges_to_match, edge_ends, logger)) { + return false; + } + + result.reserve(edges_to_match.size()); + std::transform(edge_ends.begin(), edge_ends.end(), std::back_inserter(result), [&graph](const Node::EdgeEnd* edge_end) -> Node& { + return *graph.GetNode(edge_end->GetNode().Index()); + }); + + return true; +} + bool RemoveNodesWithOneOutputBottomUp(Graph& graph, const Node& start_node) { std::queue q; std::vector nodes_to_remove; q.push(&start_node); - // From the current node, remove nodes bottom-up util it reaches a node with multiple outputs/graph output. + // From the current node, remove nodes bottom-up util it reaches a node with multiple outputs/graph output. while (q.size() != 0) { const Node& cur_node = *(q.front()); q.pop(); - // Each eligible node in the subgraph must have less than one output edge and no output should be + // Each eligible node in the subgraph must have less than one output edge and no output should be // the graph output if (cur_node.GetOutputEdgesCount() > 1 || !graph.GetNodeOutputsInGraphOutputs(cur_node).empty()) { continue; } nodes_to_remove.push_back(cur_node.Index()); - // push the parents of current node to the queue. + // push the parents of current node to the queue. for (unsigned int i = 0; i < cur_node.InputDefs().size(); ++i) { const std::string& input_name = GetNodeInputName(cur_node, i); if (IsInitializer(graph, input_name, true) || IsGraphInput(graph, cur_node.InputDefs()[i])) { @@ -787,7 +802,7 @@ bool RemoveNodesWithOneOutputBottomUp(Graph& graph, const Node& start_node) { // Remove nodes that are not used anymore. for (const auto& node_index : nodes_to_remove) { Node* node = graph.GetNode(node_index); - RemoveNodeOutputEdges(graph, *node); + RemoveNodeOutputEdges(graph, *node); graph.RemoveNode(node->Index()); } return true; diff --git a/onnxruntime/core/graph/graph_utils.h b/onnxruntime/core/graph/graph_utils.h index b5d927eff0..546619f9f1 100644 --- a/onnxruntime/core/graph/graph_utils.h +++ b/onnxruntime/core/graph/graph_utils.h @@ -92,7 +92,7 @@ bool GetRepeatedNodeAttributeValues(const Node& node, } /** Find the first child of the specified op type. */ -const Node* FirstChildByType(Node& node, const std::string& child_type); +const Node* FirstChildByType(Node& node, const std::string& child_type); /** Find the first parent of the specified op type. */ const Node* FirstParentByType(Node& node, const std::string& parent_type); @@ -202,7 +202,6 @@ const Node::EdgeEnd* GetInputEdge(const Node& node, int arg_index); */ const Node* GetInputNode(const Node& node, int arg_index); - /** Expected edge end information for matching input or output edge. For input edge, the node in the edge end refers to the source node, otherwise the destination node. */ @@ -247,6 +246,10 @@ struct EdgeEndToMatch { */ bool FindPath(const Node& node, bool is_input_edge, const std::vector& edges_to_match, std::vector& result, const logging::Logger& logger); +/** Same as FindPath above, but return the references of matched Node +*/ +bool FindPath(Graph& graph, const Node& node, bool is_input_edge, const std::vector& edges_to_match, std::vector>& result, const logging::Logger& logger); + /** * Remove nodes with only one output edge using bottom-up bfs traversal. * @param node: The node to start with. diff --git a/onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.cc b/onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.cc new file mode 100644 index 0000000000..dce6dd57d7 --- /dev/null +++ b/onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.cc @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "dynamic_quantize_matmul_fusion.h" + +#include "core/graph/graph_utils.h" +#include "core/optimizer/initializer.h" +#include "core/optimizer/utils.h" +#include + +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::common; +namespace onnxruntime { + +/** +DynamicQuantizeMatMulFusion will fuse subgraph like below into DynamicQuantizeMatMul: + (input) + | + v +DynamicQuantizeLinear --------+ + | | + v v +MatMulInteger (B const) Mul (B const) + | | + v v + Cast ------------------>Mul + | + v + (output) +*/ +Status DynamicQuantizeMatMulFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const { + GraphViewer graph_viewer(graph); + const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder(); + std::vector> nodes_to_remove; + + for (auto node_index : node_topology_list) { + auto* node_ptr = graph.GetNode(node_index); + if (nullptr == node_ptr) + continue; // node was removed + + auto& mul_node = *node_ptr; + + ORT_RETURN_IF_ERROR(Recurse(mul_node, modified, graph_level, logger)); + + if (!graph_utils::IsSupportedOptypeVersionAndDomain(mul_node, "Mul", {7}) || + !graph_utils::IsSupportedProvider(mul_node, GetCompatibleExecutionProviders())) { + continue; + } + + // Left Parents path + std::vector left_parent_path{ + {0, 0, "Cast", {6, 9}, kOnnxDomain}, + {0, 0, "MatMulInteger", {10}, kOnnxDomain}, + {0, 0, "DynamicQuantizeLinear", {11}, kOnnxDomain}}; + + std::vector right_parent_path{ + {0, 1, "Mul", {7}, kOnnxDomain}, + {1, 0, "DynamicQuantizeLinear", {11}, kOnnxDomain}}; + + std::vector> left_nodes; + std::vector> right_nodes; + if (!graph_utils::FindPath(graph, mul_node, true, left_parent_path, left_nodes, logger) || + !graph_utils::FindPath(graph, mul_node, true, right_parent_path, right_nodes, logger)) { + continue; + } + + Node& cast_node = left_nodes[0]; + Node& matmulinteger_node = left_nodes[1]; + Node& dql_node_left = left_nodes[2]; + + Node& mul_node_right = right_nodes[0]; + Node& dql_node_right = right_nodes[1]; + + // Check if left DynamicQuantizeLinear is same as right DynamicQuantizeLinear + if (dql_node_left.Index() != dql_node_right.Index()) { + continue; + } + + // Check Nodes' Edges count and Nodes' outputs are not in Graph output + if (!optimizer_utils::CheckOutputEdges(graph, cast_node, 1) || + !optimizer_utils::CheckOutputEdges(graph, matmulinteger_node, 1) || + !optimizer_utils::CheckOutputEdges(graph, mul_node_right, 1) || + !optimizer_utils::CheckOutputEdges(graph, dql_node_left, 3)) { + continue; + } + + const NodeArg& matmulinteger_B = *(matmulinteger_node.InputDefs()[1]); + if (!graph_utils::IsConstantInitializer(graph, matmulinteger_B.Name(), true)) { + continue; + } + + const NodeArg& mul_right_B = *(mul_node_right.InputDefs()[1]); + if (!graph_utils::IsConstantInitializer(graph, mul_right_B.Name(), true)) { + continue; + } + + std::vector input_defs{dql_node_left.MutableInputDefs()[0], + matmulinteger_node.MutableInputDefs()[1], + mul_node_right.MutableInputDefs()[1]}; + + if (matmulinteger_node.InputDefs().size() == 4) { + const NodeArg& matmulinteger_B_zp = *(matmulinteger_node.InputDefs()[3]); + if (!graph_utils::IsConstantInitializer(graph, matmulinteger_B_zp.Name(), true)) { + continue; + } + input_defs.push_back(matmulinteger_node.MutableInputDefs()[3]); + } + + Node& fused_node = graph.AddNode(graph.GenerateNodeName("DynamicQuantizeMatMul"), + "DynamicQuantizeMatMul", + "fused DynamicQuantizeMatMul", + input_defs, + mul_node.MutableOutputDefs(), + nullptr, + kMSDomain); + + // Assign provider to this new node. Provider should be same as the provider for old node. + fused_node.SetExecutionProviderType(mul_node.GetExecutionProviderType()); + + nodes_to_remove.push_back(dql_node_left); + nodes_to_remove.push_back(matmulinteger_node); + nodes_to_remove.push_back(cast_node); + nodes_to_remove.push_back(mul_node_right); + nodes_to_remove.push_back(mul_node); + } + + for (const auto& node : nodes_to_remove) { + graph_utils::RemoveNodeOutputEdges(graph, node); + graph.RemoveNode(node.get().Index()); + } + + modified = true; + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.h b/onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.h new file mode 100644 index 0000000000..1272529baf --- /dev/null +++ b/onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/optimizer/graph_transformer.h" + +namespace onnxruntime { + +/** +@Class DynamicQuantizeMatMul +Fuse DynamicQuantizeLinear + MatMulInteger and following cast and mul to DynamicQuantizeMatMul +*/ +class DynamicQuantizeMatMulFusion : public GraphTransformer { + public: + DynamicQuantizeMatMulFusion(const std::unordered_set& compatible_execution_providers = {}) noexcept + : GraphTransformer("DynamicQuantizeMatMulFusion", compatible_execution_providers) { + } + + Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 19d01c9762..a84d4594fd 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -2,34 +2,36 @@ // Licensed under the MIT License. #include "core/optimizer/graph_transformer_utils.h" -#include "core/optimizer/identity_elimination.h" -#include "core/optimizer/slice_elimination.h" -#include "core/optimizer/conv_mul_fusion.h" -#include "core/optimizer/conv_bn_fusion.h" -#include "core/optimizer/conv_add_fusion.h" -#include "core/optimizer/constant_folding.h" -#include "core/optimizer/unsqueeze_elimination.h" -#include "core/optimizer/rule_based_graph_transformer.h" -#include "core/optimizer/conv_activation_fusion.h" -#include "core/optimizer/gemm_activation_fusion.h" -#include "core/optimizer/matmul_add_fusion.h" -#include "core/optimizer/dropout_elimination.h" -#include "core/optimizer/relu_clip_fusion.h" -#include "core/optimizer/shape_to_initializer.h" -#include "core/optimizer/nchwc_transformer.h" -#include "core/optimizer/free_dim_override_transformer.h" -#include "core/optimizer/bias_gelu_fusion.h" -#include "core/optimizer/gelu_fusion.h" -#include "core/optimizer/gelu_approximation.h" -#include "core/optimizer/fast_gelu_fusion.h" -#include "core/optimizer/layer_norm_fusion.h" -#include "core/optimizer/skip_layer_norm_fusion.h" -#include "core/optimizer/embed_layer_norm_fusion.h" -#include "core/optimizer/reshape_fusion.h" -#include "core/optimizer/attention_fusion.h" -#include "core/optimizer/expand_elimination.h" -#include "core/optimizer/cast_elimination.h" + #include "core/mlas/inc/mlas.h" +#include "core/optimizer/attention_fusion.h" +#include "core/optimizer/bias_gelu_fusion.h" +#include "core/optimizer/cast_elimination.h" +#include "core/optimizer/constant_folding.h" +#include "core/optimizer/conv_activation_fusion.h" +#include "core/optimizer/conv_add_fusion.h" +#include "core/optimizer/conv_bn_fusion.h" +#include "core/optimizer/conv_mul_fusion.h" +#include "core/optimizer/dropout_elimination.h" +#include "core/optimizer/dynamic_quantize_matmul_fusion.h" +#include "core/optimizer/embed_layer_norm_fusion.h" +#include "core/optimizer/expand_elimination.h" +#include "core/optimizer/fast_gelu_fusion.h" +#include "core/optimizer/free_dim_override_transformer.h" +#include "core/optimizer/gelu_approximation.h" +#include "core/optimizer/gelu_fusion.h" +#include "core/optimizer/gemm_activation_fusion.h" +#include "core/optimizer/identity_elimination.h" +#include "core/optimizer/layer_norm_fusion.h" +#include "core/optimizer/matmul_add_fusion.h" +#include "core/optimizer/nchwc_transformer.h" +#include "core/optimizer/relu_clip_fusion.h" +#include "core/optimizer/reshape_fusion.h" +#include "core/optimizer/rule_based_graph_transformer.h" +#include "core/optimizer/shape_to_initializer.h" +#include "core/optimizer/skip_layer_norm_fusion.h" +#include "core/optimizer/slice_elimination.h" +#include "core/optimizer/unsqueeze_elimination.h" namespace onnxruntime { @@ -126,6 +128,7 @@ std::vector> GenerateTransformers(TransformerL #ifndef DISABLE_CONTRIB_OPS transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); + transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); std::unordered_set cpu_acl_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider}; @@ -171,8 +174,8 @@ std::vector> GenerateTransformers(TransformerL // These transformers could only be enabled by custom transformer list. #ifndef DISABLE_CONTRIB_OPS if (level == TransformerLevel::Level2) { - std::unordered_set cuda_execution_providers = {onnxruntime::kCudaExecutionProvider}; - transformers.emplace_back(onnxruntime::make_unique(cuda_execution_providers)); + std::unordered_set cuda_execution_providers = {onnxruntime::kCudaExecutionProvider}; + transformers.emplace_back(onnxruntime::make_unique(cuda_execution_providers)); } #endif diff --git a/onnxruntime/core/providers/cpu/tensor/dynamicquantizelinear.cc b/onnxruntime/core/providers/cpu/tensor/dynamicquantizelinear.cc index 81b89903b0..74a3a1c530 100644 --- a/onnxruntime/core/providers/cpu/tensor/dynamicquantizelinear.cc +++ b/onnxruntime/core/providers/cpu/tensor/dynamicquantizelinear.cc @@ -2,11 +2,11 @@ // Licensed under the MIT License. #include "dynamicquantizelinear.h" + +#include "core/mlas/inc/mlas.h" #include "core/providers/common.h" #include "core/util/math_cpuonly.h" -#include "core/mlas/inc/mlas.h" -#include -#include +#include "core/util/qmath.h" namespace onnxruntime { @@ -18,12 +18,6 @@ ONNX_CPU_OPERATOR_TYPED_KERNEL( .TypeConstraint("T2", DataTypeImpl::GetTensorType()), DynamicQuantizeLinear); -static float RoundHalfToEven(float input) { - std::fesetround(FE_TONEAREST); - auto result = std::nearbyintf(input); - return result; -} - // formula is Y = X / Scale + ZeroPoint template Status DynamicQuantizeLinear::Compute(OpKernelContext* ctx) const { diff --git a/onnxruntime/core/util/qmath.h b/onnxruntime/core/util/qmath.h index 734fde6efe..b79775b3f8 100644 --- a/onnxruntime/core/util/qmath.h +++ b/onnxruntime/core/util/qmath.h @@ -5,6 +5,9 @@ #include "core/platform/threadpool.h" +#include +#include + #if defined(_M_AMD64) || defined(__x86_64__) || defined(_M_IX86) || defined(__i386__) #define MLAS_SUPPORTS_GEMM_U8X8 #endif @@ -25,4 +28,11 @@ void QGemm( OutputScalar* result_data, int ldc, concurrency::ThreadPool* thread_pool); + +inline float RoundHalfToEven(float input) { + std::fesetround(FE_TONEAREST); + auto result = std::nearbyintf(input); + return result; +} + } // namespace onnxruntime diff --git a/onnxruntime/test/common/tensor_op_test_utils.h b/onnxruntime/test/common/tensor_op_test_utils.h index e6b97e27ac..bb35f89082 100644 --- a/onnxruntime/test/common/tensor_op_test_utils.h +++ b/onnxruntime/test/common/tensor_op_test_utils.h @@ -131,5 +131,27 @@ inline std::vector ToFloat16(const std::vector& data) { return result; } +inline void CheckTensor(const Tensor& expected_tensor, const Tensor& output_tensor, double rtol, double atol) { + ORT_ENFORCE(expected_tensor.Shape() == output_tensor.Shape(), + "Expected output shape [" + expected_tensor.Shape().ToString() + + "] did not match run output shape [" + + output_tensor.Shape().ToString() + "]"); + + ASSERT_TRUE(expected_tensor.DataType() == DataTypeImpl::GetType()) << "Compare with non float number is not supported yet. "; + auto expected = expected_tensor.Data(); + auto output = output_tensor.Data(); + for (auto i = 0; i < expected_tensor.Shape().Size(); ++i) { + const auto expected_value = expected[i], actual_value = output[i]; + if (std::isnan(expected_value)) { + ASSERT_TRUE(std::isnan(actual_value)) << "value mismatch at index " << i << "; expected is NaN, actual is not NaN"; + } else if (std::isinf(expected_value)) { + ASSERT_EQ(expected_value, actual_value) << "value mismatch at index " << i; + } else { + double diff = fabs(expected_value - actual_value); + ASSERT_TRUE(diff <= (atol + rtol * fabs(expected_value))) << "value mismatch at index " << i << "; expected: " << expected_value << ", actual: " << actual_value; + } + } +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc b/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc new file mode 100644 index 0000000000..2b4be7cc8b --- /dev/null +++ b/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc @@ -0,0 +1,173 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/tensor.h" +#include "core/session/inference_session.h" +#include "test/common/tensor_op_test_utils.h" +#include "test/framework/test_utils.h" +#include "test/providers/provider_test_utils.h" +#include "test/util/include/default_providers.h" + +#include +#include + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +using namespace std; + +namespace onnxruntime { +namespace test { + +template +class DynamicQuantizeMatMulOpTester : public OpTester { + public: + DynamicQuantizeMatMulOpTester(const char* op, + const std::vector& A_dims, + const std::vector& B_dims, + const std::vector& Y_dims, + string&& model_file, + int opset_version = 1, + const char* domain = onnxruntime::kMSDomain) : OpTester(op, opset_version, domain), + A_dims_(A_dims), + B_dims_(B_dims), + Y_dims_(Y_dims), + model_file_(model_file) { + Init(); + } + void Init() { + // create rand inputs + RandomValueGenerator random{}; + A_data_ = random.Uniform(A_dims_, -1.0f, 1.0f); + + std::vector tmp_B_data = random.Uniform(B_dims_, std::numeric_limits::min(), std::numeric_limits::max()); + std::transform(tmp_B_data.begin(), tmp_B_data.end(), std::back_inserter(B_data_), [](int32_t v) -> T { + return static_cast(v); + }); + + B_zero_point_ = {static_cast(random.Uniform({1}, std::numeric_limits::min(), std::numeric_limits::max())[0])}; + B_scale_ = random.Uniform({1}, -0.1f, 0.1f); + + const int64_t Y_size = std::accumulate(Y_dims_.cbegin(), Y_dims_.cend(), static_cast(1), std::multiplies{}); + Y_data_.resize(Y_size); + + AddInput("A", A_dims_, A_data_); + AddInput("B", B_dims_, B_data_); + AddInput("b_scale", {1}, B_scale_); + AddInput("b_zero_point", {1}, B_zero_point_); + + AddOutput("Y", Y_dims_, Y_data_); + } + void Run() { +#ifndef NDEBUG + // run_called_ to true to avoid a complaining in the destructor of OpTester + run_called_ = true; +#endif + std::vector cpu_fetches; + std::vector subgraph_fetches; + Compute(cpu_fetches); + ComputeOriginalSubgraph(subgraph_fetches); + + // Compare CPU with original subgraph on the output(0) - Y + ASSERT_TRUE(cpu_fetches.size() >= subgraph_fetches.size()); + for (size_t i = 0; i < subgraph_fetches.size(); i++) { + if (cpu_fetches[i].IsTensor() && subgraph_fetches[i].IsTensor()) { + VLOGS_DEFAULT(1) << "Checking tensor " << i; + CheckTensor(subgraph_fetches[i].Get(), cpu_fetches[i].Get(), 1e-3, 1e-3); + } + } + } + + private: + void Compute(std::vector& cpu_fetches) { + auto p_model = BuildGraph(); + auto& graph = p_model->MainGraph(); + + Status status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()) << status; + + // Hookup the inputs and outputs + NameMLValMap feeds; + std::vector output_names; + FillFeedsAndOutputNames(feeds, output_names); + + SessionOptions so; + so.session_logid = op_; + so.session_log_verbosity_level = 1; + + RunOptions run_options; + run_options.run_tag = op_; + run_options.run_log_verbosity_level = 1; + + // run with DynamicQuantizeMatMul + InferenceSession session_object{so, GetEnvironment()}; + std::string s1; + p_model->ToProto().SerializeToString(&s1); + std::istringstream str(s1); + ASSERT_TRUE((status = session_object.Load(str)).IsOK()) << status; + ASSERT_TRUE((status = session_object.Initialize()).IsOK()) << status; + ASSERT_TRUE((status = session_object.Run(run_options, feeds, output_names, &cpu_fetches)).IsOK()); + } + + void ComputeOriginalSubgraph(std::vector& subgraph_fetches) { + NameMLValMap feeds; + OrtValue ml_value; + std::vector output_names; + FillFeedsAndOutputNames(feeds, output_names); + + SessionOptions so; + so.session_logid = op_; + so.session_log_verbosity_level = 1; + so.graph_optimization_level = TransformerLevel::Level1; + + RunOptions run_options; + run_options.run_tag = op_; + run_options.run_log_verbosity_level = 1; + + Status status; + InferenceSession subgraph_session_object{so, GetEnvironment()}; + ASSERT_TRUE((status = subgraph_session_object.Load(model_file_)).IsOK()) << status; + ASSERT_TRUE((status = subgraph_session_object.Initialize()).IsOK()) << status; + ASSERT_TRUE((status = subgraph_session_object.Run(run_options, feeds, output_names, &subgraph_fetches)).IsOK()) << status; + } + + private: + std::vector A_dims_; + std::vector B_dims_; + std::vector Y_dims_; + + std::vector A_data_; + std::vector B_data_; + std::vector B_scale_; + std::vector B_zero_point_; + std::vector Y_data_; + + std::string model_file_; +}; + +TEST(DynamicQuantizeMatMul, Int8_test) { + std::vector A_dims{4, 128}; + std::vector B_dims{128, 128}; + std::vector Y_dims{4, 128}; + DynamicQuantizeMatMulOpTester test("DynamicQuantizeMatMul", + A_dims, + B_dims, + Y_dims, + "testdata/dynamic_quantize_matmul_int8.onnx"); + test.Run(); +} + +TEST(DynamicQuantizeMatMul, UInt8_test) { + std::vector A_dims{4, 128}; + std::vector B_dims{128, 128}; + std::vector Y_dims{4, 128}; + DynamicQuantizeMatMulOpTester test("DynamicQuantizeMatMul", + A_dims, + B_dims, + Y_dims, + "testdata/dynamic_quantize_matmul_uint8.onnx"); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/layer_norm_op_test.cc b/onnxruntime/test/contrib_ops/layer_norm_op_test.cc index e9999eb2f5..63c4499e2a 100644 --- a/onnxruntime/test/contrib_ops/layer_norm_op_test.cc +++ b/onnxruntime/test/contrib_ops/layer_norm_op_test.cc @@ -18,28 +18,6 @@ using namespace std; namespace onnxruntime { namespace test { -static void CheckTensor(const Tensor& expected_tensor, const Tensor& output_tensor, double rtol, double atol) { - ORT_ENFORCE(expected_tensor.Shape() == output_tensor.Shape(), - "Expected output shape [" + expected_tensor.Shape().ToString() + - "] did not match run output shape [" + - output_tensor.Shape().ToString() + "]"); - - ASSERT_TRUE(expected_tensor.DataType() == DataTypeImpl::GetType()) << "Compare with non float number is not supported yet. "; - auto expected = expected_tensor.Data(); - auto output = output_tensor.Data(); - for (auto i = 0; i < expected_tensor.Shape().Size(); ++i) { - const auto expected_value = expected[i], actual_value = output[i]; - if (std::isnan(expected_value)) { - ASSERT_TRUE(std::isnan(actual_value)) << "value mismatch at index " << i << "; expected is NaN, actual is not NaN"; - } else if (std::isinf(expected_value)) { - ASSERT_EQ(expected_value, actual_value) << "value mismatch at index " << i; - } else { - double diff = fabs(expected_value - actual_value); - ASSERT_TRUE(diff <= (atol + rtol * fabs(expected_value))) << "value mismatch at index " << i << "; expected: " << expected_value << ", actual: " << actual_value; - } - } -} - class LayerNormOpTester : public OpTester { public: LayerNormOpTester(const char* op, diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index 722bdb1ece..d740a4e85d 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -7,53 +7,54 @@ #include "core/graph/onnx_protobuf.h" -#include "core/session/inference_session.h" +#include "asserts.h" #include "core/framework/data_types.h" #include "core/framework/ml_value.h" #include "core/graph/graph_utils.h" #include "core/graph/graph_viewer.h" #include "core/graph/model.h" +#include "core/optimizer/attention_fusion.h" +#include "core/optimizer/bias_gelu_fusion.h" +#include "core/optimizer/cast_elimination.h" #include "core/optimizer/constant_folding.h" +#include "core/optimizer/conv_activation_fusion.h" +#include "core/optimizer/conv_add_fusion.h" #include "core/optimizer/conv_bn_fusion.h" #include "core/optimizer/conv_mul_fusion.h" -#include "core/optimizer/conv_add_fusion.h" -#include "core/optimizer/conv_activation_fusion.h" #include "core/optimizer/dropout_elimination.h" -#include "core/optimizer/gemm_activation_fusion.h" -#include "core/optimizer/bias_gelu_fusion.h" -#include "core/optimizer/gelu_fusion.h" -#include "core/optimizer/gelu_approximation.h" -#include "core/optimizer/layer_norm_fusion.h" -#include "core/optimizer/skip_layer_norm_fusion.h" +#include "core/optimizer/dynamic_quantize_matmul_fusion.h" #include "core/optimizer/embed_layer_norm_fusion.h" +#include "core/optimizer/expand_elimination.h" +#include "core/optimizer/fast_gelu_fusion.h" +#include "core/optimizer/gelu_approximation.h" +#include "core/optimizer/gelu_fusion.h" +#include "core/optimizer/gemm_activation_fusion.h" #include "core/optimizer/graph_transformer.h" #include "core/optimizer/graph_transformer_mgr.h" #include "core/optimizer/identity_elimination.h" #include "core/optimizer/initializer.h" +#include "core/optimizer/layer_norm_fusion.h" #include "core/optimizer/matmul_add_fusion.h" #include "core/optimizer/matmul_transpose_fusion.h" #include "core/optimizer/relu_clip_fusion.h" +#include "core/optimizer/reshape_fusion.h" #include "core/optimizer/rule_based_graph_transformer.h" #include "core/optimizer/shape_to_initializer.h" +#include "core/optimizer/skip_layer_norm_fusion.h" #include "core/optimizer/slice_elimination.h" #include "core/optimizer/unsqueeze_elimination.h" -#include "core/optimizer/reshape_fusion.h" -#include "core/optimizer/attention_fusion.h" -#include "core/optimizer/fast_gelu_fusion.h" -#include "core/optimizer/expand_elimination.h" -#include "core/optimizer/cast_elimination.h" #include "core/optimizer/utils.h" #include "core/platform/env.h" +#include "core/session/inference_session.h" #include "core/util/math.h" +#include "gtest/gtest.h" #include "test/capturing_sink.h" +#include "test/common/tensor_op_test_utils.h" +#include "test/compare_ortvalue.h" #include "test/framework/test_utils.h" #include "test/optimizer/graph_transform_test_fixture.h" -#include "test/compare_ortvalue.h" -#include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" #include "test/test_environment.h" -#include "asserts.h" -#include "gtest/gtest.h" using namespace std; using namespace ONNX_NAMESPACE; @@ -1721,7 +1722,6 @@ TEST_F(GraphTransformationTests, BiasGeluTest) { ASSERT_TRUE(op_to_count["BiasGelu"] == 1); } - // BiasGelu allows input switching based on input dimensions. // This test validates the input edges are plugged correct in the optimized graph. TEST_F(GraphTransformationTests, BiasGeluSwitchedInputOrder) { @@ -2296,6 +2296,25 @@ TEST_F(GraphTransformationTests, EmbedLayerNormFusionFormat5) { } } +TEST_F(GraphTransformationTests, DynamicQuantizeMatMulTest) { + auto model_uri = MODEL_FOLDER "fusion/dynamic_quantize_matmul.onnx"; + std::shared_ptr p_model; + ASSERT_STATUS_OK(Model::Load(model_uri, p_model, nullptr, *logger_)); + Graph& graph = p_model->MainGraph(); + + onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); + auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_); + ASSERT_TRUE(ret.IsOK()); + + std::map op_to_count = CountOpsInGraph(graph); + EXPECT_EQ(op_to_count["DynamicQuantizeLinear"], 0); + EXPECT_EQ(op_to_count["MatMulInteger"], 0); + EXPECT_EQ(op_to_count["Cast"], 0); + EXPECT_EQ(op_to_count["Mul"], 0); + EXPECT_EQ(op_to_count["DynamicQuantizeMatMul"], 1); +} + #endif } // namespace test diff --git a/onnxruntime/test/testdata/dynamic_quantize_matmul_int8.onnx b/onnxruntime/test/testdata/dynamic_quantize_matmul_int8.onnx new file mode 100644 index 0000000000..9127e436a0 --- /dev/null +++ b/onnxruntime/test/testdata/dynamic_quantize_matmul_int8.onnx @@ -0,0 +1,43 @@ +:è +M +A a_quantizeda_scalea_zpDynamicQuantizeLinear"DynamicQuantizeLinear +W + a_quantized +B +a_zp + b_zero_pointmatmul_output_int32 MatMulInteger" MatMulInteger +. +a_scale +b_scale +multiplier mul_right"Mul +A +matmul_output_int32matmul_output_floatcast"Cast* +to  +5 +matmul_output_float + +multiplierY +mul_bottom"MulDynamicQuantizeLinear_fusionZ +A + + +M +KZ +B + + +K +NZ +b_scale + + +Z + b_zero_point + + +b +Y + + +M +NB \ No newline at end of file diff --git a/onnxruntime/test/testdata/dynamic_quantize_matmul_test.py b/onnxruntime/test/testdata/dynamic_quantize_matmul_test.py new file mode 100644 index 0000000000..635d968765 --- /dev/null +++ b/onnxruntime/test/testdata/dynamic_quantize_matmul_test.py @@ -0,0 +1,33 @@ +import onnx +from onnx import helper +from onnx import TensorProto +from enum import Enum + +def GenerateModel(model_name, sign): + nodes = [ # DynamicQuantizeMatMul subgraph + helper.make_node("DynamicQuantizeLinear", ["A"], ["a_quantized", "a_scale", "a_zp"], "DynamicQuantizeLinear"), + helper.make_node("MatMulInteger", ["a_quantized", "B", "a_zp", "b_zero_point"], ["matmul_output_int32"], "MatMulInteger"), + helper.make_node("Mul", ["a_scale", "b_scale"], ["multiplier"], "mul_right"), + helper.make_node("Cast", ["matmul_output_int32"], ["matmul_output_float"], "cast", to=1), + helper.make_node("Mul", ["matmul_output_float", "multiplier"], ["Y"], "mul_bottom"), + ] + + graph = helper.make_graph( + nodes, + "DynamicQuantizeMatMul_fusion", #name + [ # inputs + helper.make_tensor_value_info('A', TensorProto.FLOAT, ['M', 'K']), + helper.make_tensor_value_info('B', TensorProto.INT8 if sign else TensorProto.UINT8, ['K', 'N']), + helper.make_tensor_value_info('b_scale', TensorProto.FLOAT, [1]), + helper.make_tensor_value_info('b_zero_point', TensorProto.INT8 if sign else TensorProto.UINT8, [1]), + ], + [ # outputs + helper.make_tensor_value_info('Y', TensorProto.FLOAT, ['M', 'N']), + ]) + + model = helper.make_model(graph) + onnx.save(model, model_name) + +if __name__ == "__main__": + GenerateModel('dynamic_quantize_matmul_int8.onnx', True) + GenerateModel('dynamic_quantize_matmul_int8.onnx', False) \ No newline at end of file diff --git a/onnxruntime/test/testdata/dynamic_quantize_matmul_uint8.onnx b/onnxruntime/test/testdata/dynamic_quantize_matmul_uint8.onnx new file mode 100644 index 0000000000..a4305de244 --- /dev/null +++ b/onnxruntime/test/testdata/dynamic_quantize_matmul_uint8.onnx @@ -0,0 +1,43 @@ +:è +M +A a_quantizeda_scalea_zpDynamicQuantizeLinear"DynamicQuantizeLinear +W + a_quantized +B +a_zp + b_zero_pointmatmul_output_int32 MatMulInteger" MatMulInteger +. +a_scale +b_scale +multiplier mul_right"Mul +A +matmul_output_int32matmul_output_floatcast"Cast* +to  +5 +matmul_output_float + +multiplierY +mul_bottom"MulDynamicQuantizeLinear_fusionZ +A + + +M +KZ +B + + +K +NZ +b_scale + + +Z + b_zero_point + + +b +Y + + +M +NB \ No newline at end of file diff --git a/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.onnx b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.onnx new file mode 100644 index 0000000000..c2be298a4e --- /dev/null +++ b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.onnx @@ -0,0 +1,28 @@ +:ì +Q +input a_quantizeda_scalea_zpDynamicQuantizeLinear"DynamicQuantizeLinear +Y + a_quantized + b_quantized +a_zp +b_zpmatmul_output_int32 MatMulInteger" MatMulInteger +. +a_scale +b_scale +multiplier mul_right"Mul +A +matmul_output_int32matmul_output_floatcast"Cast* +to  +: +matmul_output_float + +multiplieroutput +mul_bottom"MulDynamicQuantizeLinear_fusion**B b_quantized* *€Bb_zp*"ffæ?Bb_scaleZ +input +  + +b +output +  + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.py b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.py new file mode 100644 index 0000000000..b22cdf8e1b --- /dev/null +++ b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.py @@ -0,0 +1,36 @@ +import onnx +from onnx import helper +from onnx import TensorProto +from enum import Enum + +def GenerateModel(model_name): + nodes = [ # LayerNorm subgraph + helper.make_node("DynamicQuantizeLinear", ["input"], ["a_quantized", "a_scale", "a_zp"], "DynamicQuantizeLinear"), + helper.make_node("MatMulInteger", ["a_quantized", "b_quantized", "a_zp", "b_zp"], ["matmul_output_int32"], "MatMulInteger"), + helper.make_node("Mul", ["a_scale", "b_scale"], ["multiplier"], "mul_right"), + helper.make_node("Cast", ["matmul_output_int32"], ["matmul_output_float"], "cast", to=1), + helper.make_node("Mul", ["matmul_output_float", "multiplier"], ["output"], "mul_bottom"), + ] + + initializers = [ # initializers + helper.make_tensor('b_quantized', TensorProto.UINT8, [2,3], [2, 4, 5, 6, 7, 8]), + helper.make_tensor('b_zp', TensorProto.UINT8, [], [128]), + helper.make_tensor('b_scale', TensorProto.FLOAT, [], [1.8]), + ] + + graph = helper.make_graph( + nodes, + "DynamicQuantizeLinear_fusion", #name + [ # inputs + helper.make_tensor_value_info('input', TensorProto.FLOAT, [3, 2]), + ], + [ # outputs + helper.make_tensor_value_info('output', TensorProto.FLOAT, [3, 3]), + ], + initializers) + + model = helper.make_model(graph) + onnx.save(model, model_name) + +if __name__ == "__main__": + GenerateModel('dynamic_quantize_matmul.onnx') \ No newline at end of file