mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
matmul integer fusion (#4195)
* Introduce DynamicQuantizeMatMul It fuses DynamicQuantizeLinear, MatMul and following cast, multiplier. It gets float in and float out for quantized matmul. We have a MLAS kernel in implementation for this op.
This commit is contained in:
Yufeng Li
GitHub
parent
2605faef88
commit
87d68d8531
20 changed files with 858 additions and 105 deletions
|
|
@ -20,7 +20,6 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1,
|
|||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Range);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, WordConvEmbedding);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, GatherND);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MatMulInteger16);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, TransposeMatMul);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MurmurHash3);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, MaxpoolWithMask);
|
||||
|
|
@ -28,19 +27,25 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Pad);
|
|||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Unique);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, ConvTransposeWithDynamicPads);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, CropAndResize);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, CDist);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, double, CDist);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Gelu);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, BiasGelu);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, FastGelu);
|
||||
|
||||
// ******** Start: Quantization ******************* //
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MatMulInteger16);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, DequantizeLinear);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, DequantizeLinear);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QuantizeLinear);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QuantizeLinear);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearLeakyRelu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearLeakyRelu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, CDist);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, double, CDist);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Gelu);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, BiasGelu);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, FastGelu);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float_uint8_t_uint8_t, QAttention);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float_uint8_t_int8_t, QAttention);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, DynamicQuantizeMatMul);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, DynamicQuantizeMatMul);
|
||||
// ******** End: Quantization ******************* //
|
||||
|
||||
// This section includes all op kernel declarations for former experimental ops which have now been removed from onnx.
|
||||
// To maintain backward compatibility these are added as contrib ops.
|
||||
|
|
@ -86,6 +91,27 @@ Status RegisterNchwcKernels(KernelRegistry& kernel_registry) {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) {
|
||||
static const BuildKernelCreateInfoFn function_table[] = {
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MatMulInteger16)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, DequantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, DequantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QuantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QuantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearLeakyRelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearLeakyRelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float_uint8_t_uint8_t, QAttention)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float_uint8_t_int8_t, QAttention)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, DynamicQuantizeMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, DynamicQuantizeMatMul)>,
|
||||
};
|
||||
|
||||
for (auto& function_table_entry : function_table) {
|
||||
ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry()));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status RegisterCpuContribKernels(KernelRegistry& kernel_registry) {
|
||||
static const BuildKernelCreateInfoFn function_table[] = {
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, SampleOp)>,
|
||||
|
|
@ -102,26 +128,18 @@ Status RegisterCpuContribKernels(KernelRegistry& kernel_registry) {
|
|||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, WordConvEmbedding)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, GatherND)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MurmurHash3)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MatMulInteger16)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, TransposeMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, MaxpoolWithMask)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Pad)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Unique)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, ConvTransposeWithDynamicPads)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, CropAndResize)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, DequantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, DequantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QuantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QuantizeLinear)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearLeakyRelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearLeakyRelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, CDist)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, double, CDist)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, BiasGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Gelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, FastGelu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float_uint8_t_uint8_t, QAttention)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float_uint8_t_int8_t, QAttention)>,
|
||||
|
||||
// These ops were experimental ops in onnx domain which have been removed now. We add them here as
|
||||
// contrib ops to main backward compatibility
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, Affine)>,
|
||||
|
|
@ -148,6 +166,9 @@ Status RegisterCpuContribKernels(KernelRegistry& kernel_registry) {
|
|||
if (MlasNchwcGetBlockSize() > 1) {
|
||||
ORT_RETURN_IF_ERROR(RegisterNchwcKernels(kernel_registry));
|
||||
}
|
||||
|
||||
RegisterQuantizationKernels(kernel_registry);
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,119 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "dynamic_quantize_matmul.h"
|
||||
|
||||
#include "core/common/safeint.h"
|
||||
#include "core/mlas/inc/mlas.h"
|
||||
#include "core/providers/common.h"
|
||||
#include "core/providers/cpu/math/matmul_helper.h"
|
||||
#include "core/util/math_cpuonly.h"
|
||||
#include "core/util/qmath.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace contrib {
|
||||
|
||||
#define REGISTER_DYNAMIC_QUANTIZE_MATMUL(T) \
|
||||
ONNX_OPERATOR_TYPED_KERNEL_EX( \
|
||||
DynamicQuantizeMatMul, \
|
||||
kMSDomain, \
|
||||
1, \
|
||||
T, \
|
||||
kCpuExecutionProvider, \
|
||||
KernelDefBuilder() \
|
||||
.TypeConstraint("T1", DataTypeImpl::GetTensorType<float>()) \
|
||||
.TypeConstraint("T2", DataTypeImpl::GetTensorType<T>()), \
|
||||
DynamicQuantizeMatMul<T>);
|
||||
|
||||
REGISTER_DYNAMIC_QUANTIZE_MATMUL(int8_t)
|
||||
REGISTER_DYNAMIC_QUANTIZE_MATMUL(uint8_t)
|
||||
|
||||
static void GetQuantizationParameter(const float* data, int64_t num_of_elements, float& scale, uint8_t& zp) {
|
||||
// find input range min and max
|
||||
float min = ConstEigenVectorMap<float>(data, num_of_elements).minCoeff();
|
||||
float max = ConstEigenVectorMap<float>(data, num_of_elements).maxCoeff();
|
||||
|
||||
// ensure the input range includes zero
|
||||
min = std::min(min, 0.0f);
|
||||
max = std::max(max, 0.0f);
|
||||
|
||||
// find scale and zero point
|
||||
uint8_t qmin = 0;
|
||||
uint8_t qmax = 255;
|
||||
scale = max == min ? 1.0f : (max - min) / (qmax - qmin);
|
||||
|
||||
float initial_zero_point = qmin - min / scale;
|
||||
zp = static_cast<uint8_t>(RoundHalfToEven(std::max(float(qmin), std::min(float(qmax), initial_zero_point))));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
Status DynamicQuantizeMatMul<T>::Compute(OpKernelContext* ctx) const {
|
||||
auto* a = ctx->Input<Tensor>(0);
|
||||
auto* b = ctx->Input<Tensor>(1);
|
||||
ORT_ENFORCE(a != nullptr && b != nullptr);
|
||||
|
||||
auto* b_scale_tensor = ctx->Input<Tensor>(2);
|
||||
ORT_ENFORCE(IsScalarOr1ElementVector(b_scale_tensor),
|
||||
"MatmulInteger : input B scale must be a scalar or 1D tensor of size 1. Per-Channel is not supported yet.");
|
||||
|
||||
float b_scale = *b_scale_tensor->template Data<float>();
|
||||
|
||||
auto* b_zp_tensor = ctx->Input<Tensor>(3);
|
||||
T b_zp = 0;
|
||||
if (b_zp_tensor != nullptr) {
|
||||
ORT_ENFORCE(IsScalarOr1ElementVector(b_zp_tensor),
|
||||
"MatmulInteger : input B zero point must be a scalar or 1D tensor of size 1. Per-Channel is not supported yet.");
|
||||
b_zp = *b_zp_tensor->template Data<T>();
|
||||
}
|
||||
|
||||
// calculate quantization parameter of a
|
||||
const float* a_data = a->template Data<float>();
|
||||
int64_t num_of_elements = a->Shape().Size();
|
||||
|
||||
float a_scale;
|
||||
uint8_t a_zp;
|
||||
GetQuantizationParameter(a_data, num_of_elements, a_scale, a_zp);
|
||||
|
||||
AllocatorPtr allocator;
|
||||
ORT_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&allocator));
|
||||
uint8_t* a_data_quant = static_cast<uint8_t*>(allocator->Alloc(SafeInt<size_t>(num_of_elements) * sizeof(uint8_t)));
|
||||
BufferUniquePtr a_buffer_quant_holder(a_data_quant, BufferDeleter(allocator));
|
||||
// quantize the data
|
||||
MlasQuantizeLinear(a_data, a_data_quant, num_of_elements, a_scale, a_zp);
|
||||
|
||||
MatMulComputeHelper helper;
|
||||
ORT_RETURN_IF_ERROR(helper.Compute(a->Shape(), b->Shape()));
|
||||
|
||||
int32_t* matmul_output = static_cast<int32_t*>(allocator->Alloc(SafeInt<size_t>(helper.OutputShape().Size()) * sizeof(int32_t)));
|
||||
BufferUniquePtr matmul_output_holder(matmul_output, BufferDeleter(allocator));
|
||||
|
||||
concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool();
|
||||
for (size_t i = 0; i < helper.OutputOffsets().size(); i++) {
|
||||
QGemm(static_cast<int>(helper.M()),
|
||||
static_cast<int>(helper.N()),
|
||||
static_cast<int>(helper.K()),
|
||||
a_data_quant + helper.LeftOffsets()[i],
|
||||
static_cast<int>(helper.K()),
|
||||
a_zp,
|
||||
b->template Data<T>() + helper.RightOffsets()[i],
|
||||
static_cast<int>(helper.N()),
|
||||
b_zp,
|
||||
matmul_output + helper.OutputOffsets()[i],
|
||||
static_cast<int>(helper.N()),
|
||||
thread_pool);
|
||||
}
|
||||
|
||||
Tensor* y = ctx->Output(0, helper.OutputShape());
|
||||
float* y_data = y->template MutableData<float>();
|
||||
|
||||
float multiplier = a_scale * b_scale;
|
||||
for (int64_t i = 0; i < helper.OutputShape().Size(); i++) {
|
||||
y_data[i] = matmul_output[i] * multiplier;
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace contrib
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "core/common/common.h"
|
||||
#include "core/framework/op_kernel.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace contrib {
|
||||
|
||||
template <typename T>
|
||||
class DynamicQuantizeMatMul final : public OpKernel {
|
||||
public:
|
||||
DynamicQuantizeMatMul(const OpKernelInfo& info) : OpKernel(info) {}
|
||||
|
||||
Status Compute(OpKernelContext* context) const override;
|
||||
};
|
||||
} // namespace contrib
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -1656,7 +1656,41 @@ Matrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-
|
|||
// Right now we only support int32
|
||||
y_type->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto::INT32);
|
||||
|
||||
matmulShapeInference(ctx, 0, 1);
|
||||
ONNX_NAMESPACE::matmulShapeInference(ctx, 0, 1);
|
||||
});
|
||||
|
||||
ONNX_CONTRIB_OPERATOR_SCHEMA(DynamicQuantizeMatMul)
|
||||
.SetDomain(kMSDomain)
|
||||
.SinceVersion(1)
|
||||
.Input(0, "A", "N-dimensional matrix A", "T1")
|
||||
.Input(1, "B", "N-dimensional matrix B", "T2")
|
||||
.Input(
|
||||
2,
|
||||
"b_scale",
|
||||
"Scale of quantized input 'B'. It could be a scalar or a 1-D tensor, "
|
||||
"which means a per-tensor or per-column quantization. If it's a 1-D tensor, its number "
|
||||
"of elements should be equal to the number of columns of input 'B'.",
|
||||
"T1")
|
||||
.Input(
|
||||
3,
|
||||
"b_zero_point",
|
||||
"Zero point tensor for input 'B'. It's optional and default value is 0. It could be a scalar or a 1-D tensor, "
|
||||
"which means a per-tensor or per-column quantization. If it's a 1-D tensor, its number "
|
||||
"of elements should be equal to the number of columns of input 'B'.",
|
||||
"T2",
|
||||
OpSchema::Optional)
|
||||
.Output(0, "Y", "Matrix multiply results from A * B", "T1")
|
||||
.TypeConstraint(
|
||||
"T1",
|
||||
{"tensor(float)"},
|
||||
"Constrain input A, b_scale and output Y data type as float tensor.")
|
||||
.TypeConstraint(
|
||||
"T2",
|
||||
{"tensor(int8)", "tensor(uint8)"},
|
||||
"Constrain input B data type to 8-bit integer tensor.")
|
||||
.TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) {
|
||||
propagateElemTypeFromInputToOutput(ctx, 0, 0);
|
||||
ONNX_NAMESPACE::matmulShapeInference(ctx, 0, 1);
|
||||
});
|
||||
|
||||
static const char* TransposeMatMul_doc = R"DOC(
|
||||
|
|
|
|||
|
|
@ -265,11 +265,10 @@ static void MoveAllNodeOutputs(Graph& graph, Node& src_node, Node& target_node)
|
|||
int GetNodeInputIndexFromInputName(const Node& node, const std::string& input_name) {
|
||||
auto itr = std::find_if(node.InputDefs().begin(), node.InputDefs().end(),
|
||||
[&input_name](const NodeArg* input) { return input->Name() == input_name; });
|
||||
ORT_ENFORCE(itr != node.InputDefs().end(),
|
||||
"Attempting to get index for an input which does not exist.");
|
||||
ORT_ENFORCE(itr != node.InputDefs().end(),
|
||||
"Attempting to get index for an input which does not exist.");
|
||||
auto index = std::distance(node.InputDefs().begin(), itr);
|
||||
return static_cast<int>(index);
|
||||
|
||||
}
|
||||
|
||||
const std::string& GetNodeInputName(const Node& node, int index) {
|
||||
|
|
@ -716,7 +715,7 @@ bool FindPath(const Node& node, bool is_input_edge, const std::vector<EdgeEndToM
|
|||
const Node::EdgeEnd* edge_found = nullptr;
|
||||
#ifndef NDEBUG
|
||||
LOGS(logger, VERBOSE) << (is_input_edge ? "I:" : "O:") << edge.src_arg_index << "," << edge.dst_arg_index
|
||||
<< "," << edge.op_type << "," << edge.domain << "," << ToString(edge.versions);
|
||||
<< "," << edge.op_type << "," << edge.domain << "," << ToString(edge.versions);
|
||||
#endif
|
||||
auto edges_begin = is_input_edge ? current_node->InputEdgesBegin() : current_node->OutputEdgesBegin();
|
||||
auto edges_end = is_input_edge ? current_node->InputEdgesEnd() : current_node->OutputEdgesEnd();
|
||||
|
|
@ -756,21 +755,37 @@ bool FindPath(const Node& node, bool is_input_edge, const std::vector<EdgeEndToM
|
|||
return true;
|
||||
}
|
||||
|
||||
bool FindPath(Graph& graph, const Node& node, bool is_input_edge, const std::vector<EdgeEndToMatch>& edges_to_match, std::vector<std::reference_wrapper<Node>>& result, const logging::Logger& logger) {
|
||||
result.clear();
|
||||
|
||||
std::vector<const Node::EdgeEnd*> edge_ends;
|
||||
if (!FindPath(node, is_input_edge, edges_to_match, edge_ends, logger)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
result.reserve(edges_to_match.size());
|
||||
std::transform(edge_ends.begin(), edge_ends.end(), std::back_inserter(result), [&graph](const Node::EdgeEnd* edge_end) -> Node& {
|
||||
return *graph.GetNode(edge_end->GetNode().Index());
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool RemoveNodesWithOneOutputBottomUp(Graph& graph, const Node& start_node) {
|
||||
std::queue<const Node*> q;
|
||||
std::vector<NodeIndex> nodes_to_remove;
|
||||
q.push(&start_node);
|
||||
// From the current node, remove nodes bottom-up util it reaches a node with multiple outputs/graph output.
|
||||
// From the current node, remove nodes bottom-up util it reaches a node with multiple outputs/graph output.
|
||||
while (q.size() != 0) {
|
||||
const Node& cur_node = *(q.front());
|
||||
q.pop();
|
||||
// Each eligible node in the subgraph must have less than one output edge and no output should be
|
||||
// Each eligible node in the subgraph must have less than one output edge and no output should be
|
||||
// the graph output
|
||||
if (cur_node.GetOutputEdgesCount() > 1 || !graph.GetNodeOutputsInGraphOutputs(cur_node).empty()) {
|
||||
continue;
|
||||
}
|
||||
nodes_to_remove.push_back(cur_node.Index());
|
||||
// push the parents of current node to the queue.
|
||||
// push the parents of current node to the queue.
|
||||
for (unsigned int i = 0; i < cur_node.InputDefs().size(); ++i) {
|
||||
const std::string& input_name = GetNodeInputName(cur_node, i);
|
||||
if (IsInitializer(graph, input_name, true) || IsGraphInput(graph, cur_node.InputDefs()[i])) {
|
||||
|
|
@ -787,7 +802,7 @@ bool RemoveNodesWithOneOutputBottomUp(Graph& graph, const Node& start_node) {
|
|||
// Remove nodes that are not used anymore.
|
||||
for (const auto& node_index : nodes_to_remove) {
|
||||
Node* node = graph.GetNode(node_index);
|
||||
RemoveNodeOutputEdges(graph, *node);
|
||||
RemoveNodeOutputEdges(graph, *node);
|
||||
graph.RemoveNode(node->Index());
|
||||
}
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ bool GetRepeatedNodeAttributeValues(const Node& node,
|
|||
}
|
||||
|
||||
/** Find the first child of the specified op type. */
|
||||
const Node* FirstChildByType(Node& node, const std::string& child_type);
|
||||
const Node* FirstChildByType(Node& node, const std::string& child_type);
|
||||
/** Find the first parent of the specified op type. */
|
||||
const Node* FirstParentByType(Node& node, const std::string& parent_type);
|
||||
|
||||
|
|
@ -202,7 +202,6 @@ const Node::EdgeEnd* GetInputEdge(const Node& node, int arg_index);
|
|||
*/
|
||||
const Node* GetInputNode(const Node& node, int arg_index);
|
||||
|
||||
|
||||
/** Expected edge end information for matching input or output edge.
|
||||
For input edge, the node in the edge end refers to the source node, otherwise the destination node.
|
||||
*/
|
||||
|
|
@ -247,6 +246,10 @@ struct EdgeEndToMatch {
|
|||
*/
|
||||
bool FindPath(const Node& node, bool is_input_edge, const std::vector<EdgeEndToMatch>& edges_to_match, std::vector<const Node::EdgeEnd*>& result, const logging::Logger& logger);
|
||||
|
||||
/** Same as FindPath above, but return the references of matched Node
|
||||
*/
|
||||
bool FindPath(Graph& graph, const Node& node, bool is_input_edge, const std::vector<EdgeEndToMatch>& edges_to_match, std::vector<std::reference_wrapper<Node>>& result, const logging::Logger& logger);
|
||||
|
||||
/**
|
||||
* Remove nodes with only one output edge using bottom-up bfs traversal.
|
||||
* @param node: The node to start with.
|
||||
|
|
|
|||
136
onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.cc
Normal file
136
onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.cc
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "dynamic_quantize_matmul_fusion.h"
|
||||
|
||||
#include "core/graph/graph_utils.h"
|
||||
#include "core/optimizer/initializer.h"
|
||||
#include "core/optimizer/utils.h"
|
||||
#include <deque>
|
||||
|
||||
using namespace ONNX_NAMESPACE;
|
||||
using namespace ::onnxruntime::common;
|
||||
namespace onnxruntime {
|
||||
|
||||
/**
|
||||
DynamicQuantizeMatMulFusion will fuse subgraph like below into DynamicQuantizeMatMul:
|
||||
(input)
|
||||
|
|
||||
v
|
||||
DynamicQuantizeLinear --------+
|
||||
| |
|
||||
v v
|
||||
MatMulInteger (B const) Mul (B const)
|
||||
| |
|
||||
v v
|
||||
Cast ------------------>Mul
|
||||
|
|
||||
v
|
||||
(output)
|
||||
*/
|
||||
Status DynamicQuantizeMatMulFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const {
|
||||
GraphViewer graph_viewer(graph);
|
||||
const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder();
|
||||
std::vector<std::reference_wrapper<Node>> nodes_to_remove;
|
||||
|
||||
for (auto node_index : node_topology_list) {
|
||||
auto* node_ptr = graph.GetNode(node_index);
|
||||
if (nullptr == node_ptr)
|
||||
continue; // node was removed
|
||||
|
||||
auto& mul_node = *node_ptr;
|
||||
|
||||
ORT_RETURN_IF_ERROR(Recurse(mul_node, modified, graph_level, logger));
|
||||
|
||||
if (!graph_utils::IsSupportedOptypeVersionAndDomain(mul_node, "Mul", {7}) ||
|
||||
!graph_utils::IsSupportedProvider(mul_node, GetCompatibleExecutionProviders())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Left Parents path
|
||||
std::vector<graph_utils::EdgeEndToMatch> left_parent_path{
|
||||
{0, 0, "Cast", {6, 9}, kOnnxDomain},
|
||||
{0, 0, "MatMulInteger", {10}, kOnnxDomain},
|
||||
{0, 0, "DynamicQuantizeLinear", {11}, kOnnxDomain}};
|
||||
|
||||
std::vector<graph_utils::EdgeEndToMatch> right_parent_path{
|
||||
{0, 1, "Mul", {7}, kOnnxDomain},
|
||||
{1, 0, "DynamicQuantizeLinear", {11}, kOnnxDomain}};
|
||||
|
||||
std::vector<std::reference_wrapper<Node>> left_nodes;
|
||||
std::vector<std::reference_wrapper<Node>> right_nodes;
|
||||
if (!graph_utils::FindPath(graph, mul_node, true, left_parent_path, left_nodes, logger) ||
|
||||
!graph_utils::FindPath(graph, mul_node, true, right_parent_path, right_nodes, logger)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Node& cast_node = left_nodes[0];
|
||||
Node& matmulinteger_node = left_nodes[1];
|
||||
Node& dql_node_left = left_nodes[2];
|
||||
|
||||
Node& mul_node_right = right_nodes[0];
|
||||
Node& dql_node_right = right_nodes[1];
|
||||
|
||||
// Check if left DynamicQuantizeLinear is same as right DynamicQuantizeLinear
|
||||
if (dql_node_left.Index() != dql_node_right.Index()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check Nodes' Edges count and Nodes' outputs are not in Graph output
|
||||
if (!optimizer_utils::CheckOutputEdges(graph, cast_node, 1) ||
|
||||
!optimizer_utils::CheckOutputEdges(graph, matmulinteger_node, 1) ||
|
||||
!optimizer_utils::CheckOutputEdges(graph, mul_node_right, 1) ||
|
||||
!optimizer_utils::CheckOutputEdges(graph, dql_node_left, 3)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const NodeArg& matmulinteger_B = *(matmulinteger_node.InputDefs()[1]);
|
||||
if (!graph_utils::IsConstantInitializer(graph, matmulinteger_B.Name(), true)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const NodeArg& mul_right_B = *(mul_node_right.InputDefs()[1]);
|
||||
if (!graph_utils::IsConstantInitializer(graph, mul_right_B.Name(), true)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<NodeArg*> input_defs{dql_node_left.MutableInputDefs()[0],
|
||||
matmulinteger_node.MutableInputDefs()[1],
|
||||
mul_node_right.MutableInputDefs()[1]};
|
||||
|
||||
if (matmulinteger_node.InputDefs().size() == 4) {
|
||||
const NodeArg& matmulinteger_B_zp = *(matmulinteger_node.InputDefs()[3]);
|
||||
if (!graph_utils::IsConstantInitializer(graph, matmulinteger_B_zp.Name(), true)) {
|
||||
continue;
|
||||
}
|
||||
input_defs.push_back(matmulinteger_node.MutableInputDefs()[3]);
|
||||
}
|
||||
|
||||
Node& fused_node = graph.AddNode(graph.GenerateNodeName("DynamicQuantizeMatMul"),
|
||||
"DynamicQuantizeMatMul",
|
||||
"fused DynamicQuantizeMatMul",
|
||||
input_defs,
|
||||
mul_node.MutableOutputDefs(),
|
||||
nullptr,
|
||||
kMSDomain);
|
||||
|
||||
// Assign provider to this new node. Provider should be same as the provider for old node.
|
||||
fused_node.SetExecutionProviderType(mul_node.GetExecutionProviderType());
|
||||
|
||||
nodes_to_remove.push_back(dql_node_left);
|
||||
nodes_to_remove.push_back(matmulinteger_node);
|
||||
nodes_to_remove.push_back(cast_node);
|
||||
nodes_to_remove.push_back(mul_node_right);
|
||||
nodes_to_remove.push_back(mul_node);
|
||||
}
|
||||
|
||||
for (const auto& node : nodes_to_remove) {
|
||||
graph_utils::RemoveNodeOutputEdges(graph, node);
|
||||
graph.RemoveNode(node.get().Index());
|
||||
}
|
||||
|
||||
modified = true;
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace onnxruntime
|
||||
23
onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.h
Normal file
23
onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.h
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "core/optimizer/graph_transformer.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
/**
|
||||
@Class DynamicQuantizeMatMul
|
||||
Fuse DynamicQuantizeLinear + MatMulInteger and following cast and mul to DynamicQuantizeMatMul
|
||||
*/
|
||||
class DynamicQuantizeMatMulFusion : public GraphTransformer {
|
||||
public:
|
||||
DynamicQuantizeMatMulFusion(const std::unordered_set<std::string>& compatible_execution_providers = {}) noexcept
|
||||
: GraphTransformer("DynamicQuantizeMatMulFusion", compatible_execution_providers) {
|
||||
}
|
||||
|
||||
Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override;
|
||||
};
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -2,34 +2,36 @@
|
|||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/optimizer/graph_transformer_utils.h"
|
||||
#include "core/optimizer/identity_elimination.h"
|
||||
#include "core/optimizer/slice_elimination.h"
|
||||
#include "core/optimizer/conv_mul_fusion.h"
|
||||
#include "core/optimizer/conv_bn_fusion.h"
|
||||
#include "core/optimizer/conv_add_fusion.h"
|
||||
#include "core/optimizer/constant_folding.h"
|
||||
#include "core/optimizer/unsqueeze_elimination.h"
|
||||
#include "core/optimizer/rule_based_graph_transformer.h"
|
||||
#include "core/optimizer/conv_activation_fusion.h"
|
||||
#include "core/optimizer/gemm_activation_fusion.h"
|
||||
#include "core/optimizer/matmul_add_fusion.h"
|
||||
#include "core/optimizer/dropout_elimination.h"
|
||||
#include "core/optimizer/relu_clip_fusion.h"
|
||||
#include "core/optimizer/shape_to_initializer.h"
|
||||
#include "core/optimizer/nchwc_transformer.h"
|
||||
#include "core/optimizer/free_dim_override_transformer.h"
|
||||
#include "core/optimizer/bias_gelu_fusion.h"
|
||||
#include "core/optimizer/gelu_fusion.h"
|
||||
#include "core/optimizer/gelu_approximation.h"
|
||||
#include "core/optimizer/fast_gelu_fusion.h"
|
||||
#include "core/optimizer/layer_norm_fusion.h"
|
||||
#include "core/optimizer/skip_layer_norm_fusion.h"
|
||||
#include "core/optimizer/embed_layer_norm_fusion.h"
|
||||
#include "core/optimizer/reshape_fusion.h"
|
||||
#include "core/optimizer/attention_fusion.h"
|
||||
#include "core/optimizer/expand_elimination.h"
|
||||
#include "core/optimizer/cast_elimination.h"
|
||||
|
||||
#include "core/mlas/inc/mlas.h"
|
||||
#include "core/optimizer/attention_fusion.h"
|
||||
#include "core/optimizer/bias_gelu_fusion.h"
|
||||
#include "core/optimizer/cast_elimination.h"
|
||||
#include "core/optimizer/constant_folding.h"
|
||||
#include "core/optimizer/conv_activation_fusion.h"
|
||||
#include "core/optimizer/conv_add_fusion.h"
|
||||
#include "core/optimizer/conv_bn_fusion.h"
|
||||
#include "core/optimizer/conv_mul_fusion.h"
|
||||
#include "core/optimizer/dropout_elimination.h"
|
||||
#include "core/optimizer/dynamic_quantize_matmul_fusion.h"
|
||||
#include "core/optimizer/embed_layer_norm_fusion.h"
|
||||
#include "core/optimizer/expand_elimination.h"
|
||||
#include "core/optimizer/fast_gelu_fusion.h"
|
||||
#include "core/optimizer/free_dim_override_transformer.h"
|
||||
#include "core/optimizer/gelu_approximation.h"
|
||||
#include "core/optimizer/gelu_fusion.h"
|
||||
#include "core/optimizer/gemm_activation_fusion.h"
|
||||
#include "core/optimizer/identity_elimination.h"
|
||||
#include "core/optimizer/layer_norm_fusion.h"
|
||||
#include "core/optimizer/matmul_add_fusion.h"
|
||||
#include "core/optimizer/nchwc_transformer.h"
|
||||
#include "core/optimizer/relu_clip_fusion.h"
|
||||
#include "core/optimizer/reshape_fusion.h"
|
||||
#include "core/optimizer/rule_based_graph_transformer.h"
|
||||
#include "core/optimizer/shape_to_initializer.h"
|
||||
#include "core/optimizer/skip_layer_norm_fusion.h"
|
||||
#include "core/optimizer/slice_elimination.h"
|
||||
#include "core/optimizer/unsqueeze_elimination.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
@ -126,6 +128,7 @@ std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(TransformerL
|
|||
|
||||
#ifndef DISABLE_CONTRIB_OPS
|
||||
transformers.emplace_back(onnxruntime::make_unique<GemmActivationFusion>(cpu_execution_providers));
|
||||
transformers.emplace_back(onnxruntime::make_unique<DynamicQuantizeMatMulFusion>(cpu_execution_providers));
|
||||
|
||||
std::unordered_set<std::string> cpu_acl_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider};
|
||||
|
||||
|
|
@ -171,8 +174,8 @@ std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(TransformerL
|
|||
// These transformers could only be enabled by custom transformer list.
|
||||
#ifndef DISABLE_CONTRIB_OPS
|
||||
if (level == TransformerLevel::Level2) {
|
||||
std::unordered_set<std::string> cuda_execution_providers = {onnxruntime::kCudaExecutionProvider};
|
||||
transformers.emplace_back(onnxruntime::make_unique<GeluApproximation>(cuda_execution_providers));
|
||||
std::unordered_set<std::string> cuda_execution_providers = {onnxruntime::kCudaExecutionProvider};
|
||||
transformers.emplace_back(onnxruntime::make_unique<GeluApproximation>(cuda_execution_providers));
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
// Licensed under the MIT License.
|
||||
|
||||
#include "dynamicquantizelinear.h"
|
||||
|
||||
#include "core/mlas/inc/mlas.h"
|
||||
#include "core/providers/common.h"
|
||||
#include "core/util/math_cpuonly.h"
|
||||
#include "core/mlas/inc/mlas.h"
|
||||
#include <cmath>
|
||||
#include <cfenv>
|
||||
#include "core/util/qmath.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
@ -18,12 +18,6 @@ ONNX_CPU_OPERATOR_TYPED_KERNEL(
|
|||
.TypeConstraint("T2", DataTypeImpl::GetTensorType<uint8_t>()),
|
||||
DynamicQuantizeLinear<uint8_t>);
|
||||
|
||||
static float RoundHalfToEven(float input) {
|
||||
std::fesetround(FE_TONEAREST);
|
||||
auto result = std::nearbyintf(input);
|
||||
return result;
|
||||
}
|
||||
|
||||
// formula is Y = X / Scale + ZeroPoint
|
||||
template <typename T>
|
||||
Status DynamicQuantizeLinear<T>::Compute(OpKernelContext* ctx) const {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@
|
|||
|
||||
#include "core/platform/threadpool.h"
|
||||
|
||||
#include <cfenv>
|
||||
#include <cmath>
|
||||
|
||||
#if defined(_M_AMD64) || defined(__x86_64__) || defined(_M_IX86) || defined(__i386__)
|
||||
#define MLAS_SUPPORTS_GEMM_U8X8
|
||||
#endif
|
||||
|
|
@ -25,4 +28,11 @@ void QGemm(
|
|||
OutputScalar* result_data,
|
||||
int ldc,
|
||||
concurrency::ThreadPool* thread_pool);
|
||||
|
||||
inline float RoundHalfToEven(float input) {
|
||||
std::fesetround(FE_TONEAREST);
|
||||
auto result = std::nearbyintf(input);
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -131,5 +131,27 @@ inline std::vector<MLFloat16> ToFloat16(const std::vector<float>& data) {
|
|||
return result;
|
||||
}
|
||||
|
||||
inline void CheckTensor(const Tensor& expected_tensor, const Tensor& output_tensor, double rtol, double atol) {
|
||||
ORT_ENFORCE(expected_tensor.Shape() == output_tensor.Shape(),
|
||||
"Expected output shape [" + expected_tensor.Shape().ToString() +
|
||||
"] did not match run output shape [" +
|
||||
output_tensor.Shape().ToString() + "]");
|
||||
|
||||
ASSERT_TRUE(expected_tensor.DataType() == DataTypeImpl::GetType<float>()) << "Compare with non float number is not supported yet. ";
|
||||
auto expected = expected_tensor.Data<float>();
|
||||
auto output = output_tensor.Data<float>();
|
||||
for (auto i = 0; i < expected_tensor.Shape().Size(); ++i) {
|
||||
const auto expected_value = expected[i], actual_value = output[i];
|
||||
if (std::isnan(expected_value)) {
|
||||
ASSERT_TRUE(std::isnan(actual_value)) << "value mismatch at index " << i << "; expected is NaN, actual is not NaN";
|
||||
} else if (std::isinf(expected_value)) {
|
||||
ASSERT_EQ(expected_value, actual_value) << "value mismatch at index " << i;
|
||||
} else {
|
||||
double diff = fabs(expected_value - actual_value);
|
||||
ASSERT_TRUE(diff <= (atol + rtol * fabs(expected_value))) << "value mismatch at index " << i << "; expected: " << expected_value << ", actual: " << actual_value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
173
onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc
Normal file
173
onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/framework/tensor.h"
|
||||
#include "core/session/inference_session.h"
|
||||
#include "test/common/tensor_op_test_utils.h"
|
||||
#include "test/framework/test_utils.h"
|
||||
#include "test/providers/provider_test_utils.h"
|
||||
#include "test/util/include/default_providers.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <random>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "gmock/gmock.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace test {
|
||||
|
||||
template <typename T>
|
||||
class DynamicQuantizeMatMulOpTester : public OpTester {
|
||||
public:
|
||||
DynamicQuantizeMatMulOpTester(const char* op,
|
||||
const std::vector<int64_t>& A_dims,
|
||||
const std::vector<int64_t>& B_dims,
|
||||
const std::vector<int64_t>& Y_dims,
|
||||
string&& model_file,
|
||||
int opset_version = 1,
|
||||
const char* domain = onnxruntime::kMSDomain) : OpTester(op, opset_version, domain),
|
||||
A_dims_(A_dims),
|
||||
B_dims_(B_dims),
|
||||
Y_dims_(Y_dims),
|
||||
model_file_(model_file) {
|
||||
Init();
|
||||
}
|
||||
void Init() {
|
||||
// create rand inputs
|
||||
RandomValueGenerator random{};
|
||||
A_data_ = random.Uniform<float>(A_dims_, -1.0f, 1.0f);
|
||||
|
||||
std::vector<int> tmp_B_data = random.Uniform<int32_t>(B_dims_, std::numeric_limits<T>::min(), std::numeric_limits<T>::max());
|
||||
std::transform(tmp_B_data.begin(), tmp_B_data.end(), std::back_inserter(B_data_), [](int32_t v) -> T {
|
||||
return static_cast<T>(v);
|
||||
});
|
||||
|
||||
B_zero_point_ = {static_cast<T>(random.Uniform<int32_t>({1}, std::numeric_limits<T>::min(), std::numeric_limits<T>::max())[0])};
|
||||
B_scale_ = random.Uniform<float>({1}, -0.1f, 0.1f);
|
||||
|
||||
const int64_t Y_size = std::accumulate(Y_dims_.cbegin(), Y_dims_.cend(), static_cast<int64_t>(1), std::multiplies<int64_t>{});
|
||||
Y_data_.resize(Y_size);
|
||||
|
||||
AddInput<float>("A", A_dims_, A_data_);
|
||||
AddInput<T>("B", B_dims_, B_data_);
|
||||
AddInput<float>("b_scale", {1}, B_scale_);
|
||||
AddInput<T>("b_zero_point", {1}, B_zero_point_);
|
||||
|
||||
AddOutput<float>("Y", Y_dims_, Y_data_);
|
||||
}
|
||||
void Run() {
|
||||
#ifndef NDEBUG
|
||||
// run_called_ to true to avoid a complaining in the destructor of OpTester
|
||||
run_called_ = true;
|
||||
#endif
|
||||
std::vector<MLValue> cpu_fetches;
|
||||
std::vector<MLValue> subgraph_fetches;
|
||||
Compute(cpu_fetches);
|
||||
ComputeOriginalSubgraph(subgraph_fetches);
|
||||
|
||||
// Compare CPU with original subgraph on the output(0) - Y
|
||||
ASSERT_TRUE(cpu_fetches.size() >= subgraph_fetches.size());
|
||||
for (size_t i = 0; i < subgraph_fetches.size(); i++) {
|
||||
if (cpu_fetches[i].IsTensor() && subgraph_fetches[i].IsTensor()) {
|
||||
VLOGS_DEFAULT(1) << "Checking tensor " << i;
|
||||
CheckTensor(subgraph_fetches[i].Get<Tensor>(), cpu_fetches[i].Get<Tensor>(), 1e-3, 1e-3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
void Compute(std::vector<MLValue>& cpu_fetches) {
|
||||
auto p_model = BuildGraph();
|
||||
auto& graph = p_model->MainGraph();
|
||||
|
||||
Status status = graph.Resolve();
|
||||
ASSERT_TRUE(status.IsOK()) << status;
|
||||
|
||||
// Hookup the inputs and outputs
|
||||
NameMLValMap feeds;
|
||||
std::vector<std::string> output_names;
|
||||
FillFeedsAndOutputNames(feeds, output_names);
|
||||
|
||||
SessionOptions so;
|
||||
so.session_logid = op_;
|
||||
so.session_log_verbosity_level = 1;
|
||||
|
||||
RunOptions run_options;
|
||||
run_options.run_tag = op_;
|
||||
run_options.run_log_verbosity_level = 1;
|
||||
|
||||
// run with DynamicQuantizeMatMul
|
||||
InferenceSession session_object{so, GetEnvironment()};
|
||||
std::string s1;
|
||||
p_model->ToProto().SerializeToString(&s1);
|
||||
std::istringstream str(s1);
|
||||
ASSERT_TRUE((status = session_object.Load(str)).IsOK()) << status;
|
||||
ASSERT_TRUE((status = session_object.Initialize()).IsOK()) << status;
|
||||
ASSERT_TRUE((status = session_object.Run(run_options, feeds, output_names, &cpu_fetches)).IsOK());
|
||||
}
|
||||
|
||||
void ComputeOriginalSubgraph(std::vector<MLValue>& subgraph_fetches) {
|
||||
NameMLValMap feeds;
|
||||
OrtValue ml_value;
|
||||
std::vector<std::string> output_names;
|
||||
FillFeedsAndOutputNames(feeds, output_names);
|
||||
|
||||
SessionOptions so;
|
||||
so.session_logid = op_;
|
||||
so.session_log_verbosity_level = 1;
|
||||
so.graph_optimization_level = TransformerLevel::Level1;
|
||||
|
||||
RunOptions run_options;
|
||||
run_options.run_tag = op_;
|
||||
run_options.run_log_verbosity_level = 1;
|
||||
|
||||
Status status;
|
||||
InferenceSession subgraph_session_object{so, GetEnvironment()};
|
||||
ASSERT_TRUE((status = subgraph_session_object.Load(model_file_)).IsOK()) << status;
|
||||
ASSERT_TRUE((status = subgraph_session_object.Initialize()).IsOK()) << status;
|
||||
ASSERT_TRUE((status = subgraph_session_object.Run(run_options, feeds, output_names, &subgraph_fetches)).IsOK()) << status;
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<int64_t> A_dims_;
|
||||
std::vector<int64_t> B_dims_;
|
||||
std::vector<int64_t> Y_dims_;
|
||||
|
||||
std::vector<float> A_data_;
|
||||
std::vector<T> B_data_;
|
||||
std::vector<float> B_scale_;
|
||||
std::vector<T> B_zero_point_;
|
||||
std::vector<float> Y_data_;
|
||||
|
||||
std::string model_file_;
|
||||
};
|
||||
|
||||
TEST(DynamicQuantizeMatMul, Int8_test) {
|
||||
std::vector<int64_t> A_dims{4, 128};
|
||||
std::vector<int64_t> B_dims{128, 128};
|
||||
std::vector<int64_t> Y_dims{4, 128};
|
||||
DynamicQuantizeMatMulOpTester<int8_t> test("DynamicQuantizeMatMul",
|
||||
A_dims,
|
||||
B_dims,
|
||||
Y_dims,
|
||||
"testdata/dynamic_quantize_matmul_int8.onnx");
|
||||
test.Run();
|
||||
}
|
||||
|
||||
TEST(DynamicQuantizeMatMul, UInt8_test) {
|
||||
std::vector<int64_t> A_dims{4, 128};
|
||||
std::vector<int64_t> B_dims{128, 128};
|
||||
std::vector<int64_t> Y_dims{4, 128};
|
||||
DynamicQuantizeMatMulOpTester<uint8_t> test("DynamicQuantizeMatMul",
|
||||
A_dims,
|
||||
B_dims,
|
||||
Y_dims,
|
||||
"testdata/dynamic_quantize_matmul_uint8.onnx");
|
||||
test.Run();
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -18,28 +18,6 @@ using namespace std;
|
|||
namespace onnxruntime {
|
||||
namespace test {
|
||||
|
||||
static void CheckTensor(const Tensor& expected_tensor, const Tensor& output_tensor, double rtol, double atol) {
|
||||
ORT_ENFORCE(expected_tensor.Shape() == output_tensor.Shape(),
|
||||
"Expected output shape [" + expected_tensor.Shape().ToString() +
|
||||
"] did not match run output shape [" +
|
||||
output_tensor.Shape().ToString() + "]");
|
||||
|
||||
ASSERT_TRUE(expected_tensor.DataType() == DataTypeImpl::GetType<float>()) << "Compare with non float number is not supported yet. ";
|
||||
auto expected = expected_tensor.Data<float>();
|
||||
auto output = output_tensor.Data<float>();
|
||||
for (auto i = 0; i < expected_tensor.Shape().Size(); ++i) {
|
||||
const auto expected_value = expected[i], actual_value = output[i];
|
||||
if (std::isnan(expected_value)) {
|
||||
ASSERT_TRUE(std::isnan(actual_value)) << "value mismatch at index " << i << "; expected is NaN, actual is not NaN";
|
||||
} else if (std::isinf(expected_value)) {
|
||||
ASSERT_EQ(expected_value, actual_value) << "value mismatch at index " << i;
|
||||
} else {
|
||||
double diff = fabs(expected_value - actual_value);
|
||||
ASSERT_TRUE(diff <= (atol + rtol * fabs(expected_value))) << "value mismatch at index " << i << "; expected: " << expected_value << ", actual: " << actual_value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class LayerNormOpTester : public OpTester {
|
||||
public:
|
||||
LayerNormOpTester(const char* op,
|
||||
|
|
|
|||
|
|
@ -7,53 +7,54 @@
|
|||
|
||||
#include "core/graph/onnx_protobuf.h"
|
||||
|
||||
#include "core/session/inference_session.h"
|
||||
#include "asserts.h"
|
||||
#include "core/framework/data_types.h"
|
||||
#include "core/framework/ml_value.h"
|
||||
#include "core/graph/graph_utils.h"
|
||||
#include "core/graph/graph_viewer.h"
|
||||
#include "core/graph/model.h"
|
||||
#include "core/optimizer/attention_fusion.h"
|
||||
#include "core/optimizer/bias_gelu_fusion.h"
|
||||
#include "core/optimizer/cast_elimination.h"
|
||||
#include "core/optimizer/constant_folding.h"
|
||||
#include "core/optimizer/conv_activation_fusion.h"
|
||||
#include "core/optimizer/conv_add_fusion.h"
|
||||
#include "core/optimizer/conv_bn_fusion.h"
|
||||
#include "core/optimizer/conv_mul_fusion.h"
|
||||
#include "core/optimizer/conv_add_fusion.h"
|
||||
#include "core/optimizer/conv_activation_fusion.h"
|
||||
#include "core/optimizer/dropout_elimination.h"
|
||||
#include "core/optimizer/gemm_activation_fusion.h"
|
||||
#include "core/optimizer/bias_gelu_fusion.h"
|
||||
#include "core/optimizer/gelu_fusion.h"
|
||||
#include "core/optimizer/gelu_approximation.h"
|
||||
#include "core/optimizer/layer_norm_fusion.h"
|
||||
#include "core/optimizer/skip_layer_norm_fusion.h"
|
||||
#include "core/optimizer/dynamic_quantize_matmul_fusion.h"
|
||||
#include "core/optimizer/embed_layer_norm_fusion.h"
|
||||
#include "core/optimizer/expand_elimination.h"
|
||||
#include "core/optimizer/fast_gelu_fusion.h"
|
||||
#include "core/optimizer/gelu_approximation.h"
|
||||
#include "core/optimizer/gelu_fusion.h"
|
||||
#include "core/optimizer/gemm_activation_fusion.h"
|
||||
#include "core/optimizer/graph_transformer.h"
|
||||
#include "core/optimizer/graph_transformer_mgr.h"
|
||||
#include "core/optimizer/identity_elimination.h"
|
||||
#include "core/optimizer/initializer.h"
|
||||
#include "core/optimizer/layer_norm_fusion.h"
|
||||
#include "core/optimizer/matmul_add_fusion.h"
|
||||
#include "core/optimizer/matmul_transpose_fusion.h"
|
||||
#include "core/optimizer/relu_clip_fusion.h"
|
||||
#include "core/optimizer/reshape_fusion.h"
|
||||
#include "core/optimizer/rule_based_graph_transformer.h"
|
||||
#include "core/optimizer/shape_to_initializer.h"
|
||||
#include "core/optimizer/skip_layer_norm_fusion.h"
|
||||
#include "core/optimizer/slice_elimination.h"
|
||||
#include "core/optimizer/unsqueeze_elimination.h"
|
||||
#include "core/optimizer/reshape_fusion.h"
|
||||
#include "core/optimizer/attention_fusion.h"
|
||||
#include "core/optimizer/fast_gelu_fusion.h"
|
||||
#include "core/optimizer/expand_elimination.h"
|
||||
#include "core/optimizer/cast_elimination.h"
|
||||
#include "core/optimizer/utils.h"
|
||||
#include "core/platform/env.h"
|
||||
#include "core/session/inference_session.h"
|
||||
#include "core/util/math.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "test/capturing_sink.h"
|
||||
#include "test/common/tensor_op_test_utils.h"
|
||||
#include "test/compare_ortvalue.h"
|
||||
#include "test/framework/test_utils.h"
|
||||
#include "test/optimizer/graph_transform_test_fixture.h"
|
||||
#include "test/compare_ortvalue.h"
|
||||
#include "test/common/tensor_op_test_utils.h"
|
||||
#include "test/providers/provider_test_utils.h"
|
||||
#include "test/test_environment.h"
|
||||
#include "asserts.h"
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace ONNX_NAMESPACE;
|
||||
|
|
@ -1721,7 +1722,6 @@ TEST_F(GraphTransformationTests, BiasGeluTest) {
|
|||
ASSERT_TRUE(op_to_count["BiasGelu"] == 1);
|
||||
}
|
||||
|
||||
|
||||
// BiasGelu allows input switching based on input dimensions.
|
||||
// This test validates the input edges are plugged correct in the optimized graph.
|
||||
TEST_F(GraphTransformationTests, BiasGeluSwitchedInputOrder) {
|
||||
|
|
@ -2296,6 +2296,25 @@ TEST_F(GraphTransformationTests, EmbedLayerNormFusionFormat5) {
|
|||
}
|
||||
}
|
||||
|
||||
TEST_F(GraphTransformationTests, DynamicQuantizeMatMulTest) {
|
||||
auto model_uri = MODEL_FOLDER "fusion/dynamic_quantize_matmul.onnx";
|
||||
std::shared_ptr<Model> p_model;
|
||||
ASSERT_STATUS_OK(Model::Load(model_uri, p_model, nullptr, *logger_));
|
||||
Graph& graph = p_model->MainGraph();
|
||||
|
||||
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
|
||||
graph_transformation_mgr.Register(onnxruntime::make_unique<DynamicQuantizeMatMulFusion>(), TransformerLevel::Level2);
|
||||
auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_);
|
||||
ASSERT_TRUE(ret.IsOK());
|
||||
|
||||
std::map<std::string, int> op_to_count = CountOpsInGraph(graph);
|
||||
EXPECT_EQ(op_to_count["DynamicQuantizeLinear"], 0);
|
||||
EXPECT_EQ(op_to_count["MatMulInteger"], 0);
|
||||
EXPECT_EQ(op_to_count["Cast"], 0);
|
||||
EXPECT_EQ(op_to_count["Mul"], 0);
|
||||
EXPECT_EQ(op_to_count["DynamicQuantizeMatMul"], 1);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace test
|
||||
|
|
|
|||
43
onnxruntime/test/testdata/dynamic_quantize_matmul_int8.onnx
vendored
Normal file
43
onnxruntime/test/testdata/dynamic_quantize_matmul_int8.onnx
vendored
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
:è
|
||||
M
|
||||
A a_quantizeda_scalea_zpDynamicQuantizeLinear"DynamicQuantizeLinear
|
||||
W
|
||||
a_quantized
|
||||
B
|
||||
a_zp
|
||||
b_zero_pointmatmul_output_int32
MatMulInteger"
MatMulInteger
|
||||
.
|
||||
a_scale
|
||||
b_scale
|
||||
multiplier mul_right"Mul
|
||||
A
|
||||
matmul_output_int32matmul_output_floatcast"Cast*
|
||||
to
|
||||
5
|
||||
matmul_output_float
|
||||
|
||||
multiplierY
|
||||
mul_bottom"Mul DynamicQuantizeLinear_fusionZ
|
||||
A
|
||||
|
||||
|
||||
M
|
||||
KZ
|
||||
B
|
||||
|
||||
|
||||
K
|
||||
NZ
|
||||
b_scale
|
||||
|
||||
|
||||
Z
|
||||
b_zero_point
|
||||
|
||||
|
||||
b
|
||||
Y
|
||||
|
||||
|
||||
M
|
||||
NB
|
||||
33
onnxruntime/test/testdata/dynamic_quantize_matmul_test.py
vendored
Normal file
33
onnxruntime/test/testdata/dynamic_quantize_matmul_test.py
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import onnx
|
||||
from onnx import helper
|
||||
from onnx import TensorProto
|
||||
from enum import Enum
|
||||
|
||||
def GenerateModel(model_name, sign):
|
||||
nodes = [ # DynamicQuantizeMatMul subgraph
|
||||
helper.make_node("DynamicQuantizeLinear", ["A"], ["a_quantized", "a_scale", "a_zp"], "DynamicQuantizeLinear"),
|
||||
helper.make_node("MatMulInteger", ["a_quantized", "B", "a_zp", "b_zero_point"], ["matmul_output_int32"], "MatMulInteger"),
|
||||
helper.make_node("Mul", ["a_scale", "b_scale"], ["multiplier"], "mul_right"),
|
||||
helper.make_node("Cast", ["matmul_output_int32"], ["matmul_output_float"], "cast", to=1),
|
||||
helper.make_node("Mul", ["matmul_output_float", "multiplier"], ["Y"], "mul_bottom"),
|
||||
]
|
||||
|
||||
graph = helper.make_graph(
|
||||
nodes,
|
||||
"DynamicQuantizeMatMul_fusion", #name
|
||||
[ # inputs
|
||||
helper.make_tensor_value_info('A', TensorProto.FLOAT, ['M', 'K']),
|
||||
helper.make_tensor_value_info('B', TensorProto.INT8 if sign else TensorProto.UINT8, ['K', 'N']),
|
||||
helper.make_tensor_value_info('b_scale', TensorProto.FLOAT, [1]),
|
||||
helper.make_tensor_value_info('b_zero_point', TensorProto.INT8 if sign else TensorProto.UINT8, [1]),
|
||||
],
|
||||
[ # outputs
|
||||
helper.make_tensor_value_info('Y', TensorProto.FLOAT, ['M', 'N']),
|
||||
])
|
||||
|
||||
model = helper.make_model(graph)
|
||||
onnx.save(model, model_name)
|
||||
|
||||
if __name__ == "__main__":
|
||||
GenerateModel('dynamic_quantize_matmul_int8.onnx', True)
|
||||
GenerateModel('dynamic_quantize_matmul_int8.onnx', False)
|
||||
43
onnxruntime/test/testdata/dynamic_quantize_matmul_uint8.onnx
vendored
Normal file
43
onnxruntime/test/testdata/dynamic_quantize_matmul_uint8.onnx
vendored
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
:è
|
||||
M
|
||||
A a_quantizeda_scalea_zpDynamicQuantizeLinear"DynamicQuantizeLinear
|
||||
W
|
||||
a_quantized
|
||||
B
|
||||
a_zp
|
||||
b_zero_pointmatmul_output_int32
MatMulInteger"
MatMulInteger
|
||||
.
|
||||
a_scale
|
||||
b_scale
|
||||
multiplier mul_right"Mul
|
||||
A
|
||||
matmul_output_int32matmul_output_floatcast"Cast*
|
||||
to
|
||||
5
|
||||
matmul_output_float
|
||||
|
||||
multiplierY
|
||||
mul_bottom"Mul DynamicQuantizeLinear_fusionZ
|
||||
A
|
||||
|
||||
|
||||
M
|
||||
KZ
|
||||
B
|
||||
|
||||
|
||||
K
|
||||
NZ
|
||||
b_scale
|
||||
|
||||
|
||||
Z
|
||||
b_zero_point
|
||||
|
||||
|
||||
b
|
||||
Y
|
||||
|
||||
|
||||
M
|
||||
NB
|
||||
28
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.onnx
vendored
Normal file
28
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.onnx
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
:ì
|
||||
Q
|
||||
input a_quantizeda_scalea_zpDynamicQuantizeLinear"DynamicQuantizeLinear
|
||||
Y
|
||||
a_quantized
|
||||
b_quantized
|
||||
a_zp
|
||||
b_zpmatmul_output_int32
MatMulInteger"
MatMulInteger
|
||||
.
|
||||
a_scale
|
||||
b_scale
|
||||
multiplier mul_right"Mul
|
||||
A
|
||||
matmul_output_int32matmul_output_floatcast"Cast*
|
||||
to
|
||||
:
|
||||
matmul_output_float
|
||||
|
||||
multiplieroutput
|
||||
mul_bottom"Mul DynamicQuantizeLinear_fusion**B b_quantized* *€Bb_zp*"ffæ?Bb_scaleZ
|
||||
input
|
||||
|
||||
|
||||
b
|
||||
output
|
||||
|
||||
|
||||
B
|
||||
36
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.py
vendored
Normal file
36
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.py
vendored
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import onnx
|
||||
from onnx import helper
|
||||
from onnx import TensorProto
|
||||
from enum import Enum
|
||||
|
||||
def GenerateModel(model_name):
|
||||
nodes = [ # LayerNorm subgraph
|
||||
helper.make_node("DynamicQuantizeLinear", ["input"], ["a_quantized", "a_scale", "a_zp"], "DynamicQuantizeLinear"),
|
||||
helper.make_node("MatMulInteger", ["a_quantized", "b_quantized", "a_zp", "b_zp"], ["matmul_output_int32"], "MatMulInteger"),
|
||||
helper.make_node("Mul", ["a_scale", "b_scale"], ["multiplier"], "mul_right"),
|
||||
helper.make_node("Cast", ["matmul_output_int32"], ["matmul_output_float"], "cast", to=1),
|
||||
helper.make_node("Mul", ["matmul_output_float", "multiplier"], ["output"], "mul_bottom"),
|
||||
]
|
||||
|
||||
initializers = [ # initializers
|
||||
helper.make_tensor('b_quantized', TensorProto.UINT8, [2,3], [2, 4, 5, 6, 7, 8]),
|
||||
helper.make_tensor('b_zp', TensorProto.UINT8, [], [128]),
|
||||
helper.make_tensor('b_scale', TensorProto.FLOAT, [], [1.8]),
|
||||
]
|
||||
|
||||
graph = helper.make_graph(
|
||||
nodes,
|
||||
"DynamicQuantizeLinear_fusion", #name
|
||||
[ # inputs
|
||||
helper.make_tensor_value_info('input', TensorProto.FLOAT, [3, 2]),
|
||||
],
|
||||
[ # outputs
|
||||
helper.make_tensor_value_info('output', TensorProto.FLOAT, [3, 3]),
|
||||
],
|
||||
initializers)
|
||||
|
||||
model = helper.make_model(graph)
|
||||
onnx.save(model, model_name)
|
||||
|
||||
if __name__ == "__main__":
|
||||
GenerateModel('dynamic_quantize_matmul.onnx')
|
||||
Loading…
Reference in a new issue