mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
Enable MatrixMultiplyIntegerToFloat on DML (#18275)
[Cherry Pick Reviewed] ### Description Commit all MatrixMultiplyIntegerToFloat PRs [MatrixMultiplyIntegerToFloat (](bf642a4d35)https://github.com/microsoft/onnxruntime/pull/16804[)] [MatMulIntToFloat Enable FP16 and update tensor ORT-DML indexing (](8237548d14)https://github.com/microsoft/onnxruntime/pull/16871[)] [Disable MatMulIntegerToFloat transformation for FP16 on CPU EP (](b16bf809de)https://github.com/microsoft/onnxruntime/pull/18239[)] ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. -->
This commit is contained in:
raoanag
GitHub
parent
823b65a60c
commit
deed125387
22 changed files with 359 additions and 92 deletions
|
|
@ -434,7 +434,7 @@ ONNX_MS_OPERATOR_SET_SCHEMA(
|
|||
.Output(0, "Y", "Matrix multiply results from A * B", "T3")
|
||||
.TypeConstraint("T1", {"tensor(int8)", "tensor(uint8)"}, "Constrain input A data type to 8-bit integer tensor.")
|
||||
.TypeConstraint("T2", {"tensor(int8)", "tensor(uint8)"}, "Constrain input B data type to 8-bit integer tensor.")
|
||||
.TypeConstraint("T3", {"tensor(float)"},
|
||||
.TypeConstraint("T3", {"tensor(float)", "tensor(float16)"},
|
||||
"Constrain input a_scale, b_scale and output Y data type as float tensor.")
|
||||
.TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) {
|
||||
propagateElemTypeFromInputToOutput(ctx, 2, 0);
|
||||
|
|
|
|||
|
|
@ -275,7 +275,8 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
|
|||
onnxruntime::kRocmExecutionProvider,
|
||||
onnxruntime::kAclExecutionProvider,
|
||||
onnxruntime::kArmNNExecutionProvider};
|
||||
|
||||
const InlinedHashSet<std::string_view> cpu_dml_eps = {onnxruntime::kCpuExecutionProvider,
|
||||
onnxruntime::kDmlExecutionProvider};
|
||||
#ifdef MLAS_TARGET_AMD64_IX86
|
||||
const bool avx2_precision_mode =
|
||||
session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsAvx2PrecisionMode, "0") == "1" && MlasPlatformU8S8Overflow();
|
||||
|
|
@ -293,7 +294,7 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
|
|||
}
|
||||
|
||||
transformers.emplace_back(std::make_unique<GemmActivationFusion>(cpu_ep));
|
||||
transformers.emplace_back(std::make_unique<MatMulIntegerToFloatFusion>(cpu_ep));
|
||||
transformers.emplace_back(std::make_unique<MatMulIntegerToFloatFusion>(cpu_dml_eps));
|
||||
transformers.emplace_back(std::make_unique<DynamicQuantizeMatMulFusion>(cpu_ep));
|
||||
|
||||
transformers.emplace_back(std::make_unique<ConvActivationFusion>(cpu_cuda_rocm_acl_armnn_eps));
|
||||
|
|
|
|||
|
|
@ -31,6 +31,24 @@ static bool CheckBiasShape(const TensorShapeProto* bias_shape) {
|
|||
return bias_last_dim > 1;
|
||||
}
|
||||
|
||||
bool HasElementDataType(const NodeArg& node_arg, int32_t data_type) {
|
||||
if (!node_arg.Exists()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto* type_proto = node_arg.TypeAsProto();
|
||||
if (!type_proto) {
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t actual_data_type;
|
||||
if (!utils::TryGetElementDataType(*type_proto, actual_data_type)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return data_type == actual_data_type;
|
||||
}
|
||||
|
||||
/**
|
||||
MatMulIntegerToFloatFusion will fuse subgraph like below into MatMulIntegerToFloat:
|
||||
|
||||
|
|
@ -63,9 +81,10 @@ Status MatMulIntegerToFloatFusion::ApplyImpl(Graph& graph, bool& modified, int g
|
|||
auto& mul_node = *node_ptr;
|
||||
|
||||
ORT_RETURN_IF_ERROR(Recurse(mul_node, modified, graph_level, logger));
|
||||
|
||||
const bool is_dml_ep = node_ptr->GetExecutionProviderType() == kDmlExecutionProvider;
|
||||
if (!graph_utils::IsSupportedOptypeVersionAndDomain(mul_node, "Mul", {7, 13, 14}) ||
|
||||
!graph_utils::IsSupportedProvider(mul_node, GetCompatibleExecutionProviders())) {
|
||||
!graph_utils::IsSupportedProvider(mul_node, GetCompatibleExecutionProviders()) ||
|
||||
(!is_dml_ep && HasElementDataType(*mul_node.InputDefs()[0], ONNX_NAMESPACE::TensorProto_DataType_FLOAT16))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -879,6 +879,12 @@ struct OperatorDescTraits<DML_QUANTIZED_LINEAR_MATRIX_MULTIPLY_OPERATOR_DESC>
|
|||
static constexpr DML_OPERATOR_TYPE Type = DML_OPERATOR_QUANTIZED_LINEAR_MATRIX_MULTIPLY;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OperatorDescTraits<DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_DESC>
|
||||
{
|
||||
static constexpr DML_OPERATOR_TYPE Type = (DML_OPERATOR_TYPE) DML_OPERATOR_MATRIX_MULTIPLY_INTEGER_TO_FLOAT;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OperatorDescTraits<DML_CONVOLUTION_INTEGER_OPERATOR_DESC>
|
||||
{
|
||||
|
|
@ -1430,12 +1436,6 @@ struct OperatorTypeTraits<(DML_OPERATOR_TYPE)DML_OPERATOR_ELEMENT_WISE_DEQUANTIZ
|
|||
using DescType = DML_ELEMENT_WISE_DEQUANTIZE_LINEAR_OPERATOR_DESC;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OperatorDescTraits<DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_DESC>
|
||||
{
|
||||
static constexpr DML_OPERATOR_TYPE Type = (DML_OPERATOR_TYPE) DML_OPERATOR_MATRIX_MULTIPLY_INTEGER_TO_FLOAT;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct OperatorTypeTraits<(DML_OPERATOR_TYPE)DML_OPERATOR_CONVOLUTION>
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1890,25 +1890,6 @@ constexpr DML_OPERATOR_SCHEMA DML_MATRIX_MULTIPLY_INTEGER_OPERATOR_SCHEMA {
|
|||
DML_MATRIX_MULTIPLY_INTEGER_OPERATOR_SCHEMA_FIELDS,
|
||||
};
|
||||
|
||||
constexpr DML_SCHEMA_FIELD DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA_FIELDS[8] {
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "ATensor", false },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "AScaleTensor", false },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "AZeroPointTensor", true },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BTensor", false },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BScaleTensor", false },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BZeroPointTensor", true },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BiasTensor", true },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_OUTPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "OutputTensor", false },
|
||||
};
|
||||
|
||||
constexpr DML_OPERATOR_SCHEMA DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA {
|
||||
"DML_OPERATOR_MATRIX_MULTIPLY_INTEGER_TO_FLOAT",
|
||||
static_cast<DML_OPERATOR_TYPE>(DML_OPERATOR_MATRIX_MULTIPLY_INTEGER_TO_FLOAT),
|
||||
DML_SCHEMA_OPERATOR_SUPPORT_FLAG_NONE,
|
||||
8,
|
||||
DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA_FIELDS,
|
||||
};
|
||||
|
||||
constexpr DML_SCHEMA_FIELD DML_QUANTIZED_LINEAR_MATRIX_MULTIPLY_OPERATOR_SCHEMA_FIELDS[9] {
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "ATensor", false },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "AScaleTensor", false },
|
||||
|
|
@ -1929,6 +1910,25 @@ constexpr DML_OPERATOR_SCHEMA DML_QUANTIZED_LINEAR_MATRIX_MULTIPLY_OPERATOR_SCHE
|
|||
DML_QUANTIZED_LINEAR_MATRIX_MULTIPLY_OPERATOR_SCHEMA_FIELDS,
|
||||
};
|
||||
|
||||
constexpr DML_SCHEMA_FIELD DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA_FIELDS[8] {
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "ATensor", false },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "AScaleTensor", false },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "AZeroPointTensor", true },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BTensor", false },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BScaleTensor", false },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BZeroPointTensor", true },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BiasTensor", true },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_OUTPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "OutputTensor", false },
|
||||
};
|
||||
|
||||
constexpr DML_OPERATOR_SCHEMA DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA {
|
||||
"DML_OPERATOR_MATRIX_MULTIPLY_INTEGER_TO_FLOAT",
|
||||
static_cast<DML_OPERATOR_TYPE>(DML_OPERATOR_MATRIX_MULTIPLY_INTEGER_TO_FLOAT),
|
||||
DML_SCHEMA_OPERATOR_SUPPORT_FLAG_NONE,
|
||||
8,
|
||||
DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA_FIELDS,
|
||||
};
|
||||
|
||||
constexpr DML_SCHEMA_FIELD DML_CONVOLUTION_INTEGER_OPERATOR_SCHEMA_FIELDS[11] {
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "InputTensor", false },
|
||||
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "InputZeroPointTensor", true },
|
||||
|
|
|
|||
|
|
@ -1158,6 +1158,19 @@ inline std::vector<OperatorField> GetFields(const DML_QUANTIZED_LINEAR_MATRIX_MU
|
|||
OperatorField(&DML_QUANTIZED_LINEAR_MATRIX_MULTIPLY_OPERATOR_SCHEMA.Fields[8], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.OutputTensor))),
|
||||
};
|
||||
}
|
||||
inline std::vector<OperatorField> GetFields(const DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_DESC& desc)
|
||||
{
|
||||
return {
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[0], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.ATensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[1], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.AScaleTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[2], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.AZeroPointTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[3], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[4], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BScaleTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[5], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BZeroPointTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[6], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BiasTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[7], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.OutputTensor))),
|
||||
};
|
||||
}
|
||||
inline std::vector<OperatorField> GetFields(const DML_CONVOLUTION_INTEGER_OPERATOR_DESC& desc)
|
||||
{
|
||||
return {
|
||||
|
|
@ -1673,19 +1686,6 @@ inline std::vector<OperatorField> GetFields(const DML_ACTIVATION_SHRINK_OPERATOR
|
|||
OperatorField(&DML_ACTIVATION_SHRINK_OPERATOR_SCHEMA.Fields[3], ToOperatorFieldType(static_cast<FLOAT>(desc.Threshold))),
|
||||
};
|
||||
}
|
||||
inline std::vector<OperatorField> GetFields(const DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_DESC& desc)
|
||||
{
|
||||
return {
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[0], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.ATensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[1], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.AScaleTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[2], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.AZeroPointTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[3], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[4], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BScaleTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[5], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BZeroPointTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[6], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BiasTensor))),
|
||||
OperatorField(&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA.Fields[7], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.OutputTensor))),
|
||||
};
|
||||
}
|
||||
inline std::vector<OperatorField> GetFields(const DML_ACTIVATION_GELU_OPERATOR_DESC& desc)
|
||||
{
|
||||
return {
|
||||
|
|
@ -1813,6 +1813,7 @@ inline const DML_OPERATOR_SCHEMA& GetSchema(DML_OPERATOR_TYPE operatorType)
|
|||
case DML_OPERATOR_RESAMPLE1: return DML_RESAMPLE1_OPERATOR_SCHEMA;
|
||||
case DML_OPERATOR_MATRIX_MULTIPLY_INTEGER: return DML_MATRIX_MULTIPLY_INTEGER_OPERATOR_SCHEMA;
|
||||
case DML_OPERATOR_QUANTIZED_LINEAR_MATRIX_MULTIPLY: return DML_QUANTIZED_LINEAR_MATRIX_MULTIPLY_OPERATOR_SCHEMA;
|
||||
case DML_OPERATOR_MATRIX_MULTIPLY_INTEGER_TO_FLOAT: return DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA;
|
||||
case DML_OPERATOR_CONVOLUTION_INTEGER: return DML_CONVOLUTION_INTEGER_OPERATOR_SCHEMA;
|
||||
case DML_OPERATOR_QUANTIZED_LINEAR_CONVOLUTION: return DML_QUANTIZED_LINEAR_CONVOLUTION_OPERATOR_SCHEMA;
|
||||
case DML_OPERATOR_ELEMENT_WISE_BIT_AND: return DML_ELEMENT_WISE_BIT_AND_OPERATOR_SCHEMA;
|
||||
|
|
@ -2340,6 +2341,10 @@ inline AbstractOperatorDesc ConvertOperatorDesc(const DML_OPERATOR_DESC& opDesc)
|
|||
return AbstractOperatorDesc(
|
||||
&DML_QUANTIZED_LINEAR_MATRIX_MULTIPLY_OPERATOR_SCHEMA,
|
||||
GetFields(*static_cast<const DML_QUANTIZED_LINEAR_MATRIX_MULTIPLY_OPERATOR_DESC*>(opDesc.Desc)));
|
||||
case DML_OPERATOR_MATRIX_MULTIPLY_INTEGER_TO_FLOAT:
|
||||
return AbstractOperatorDesc(
|
||||
&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA,
|
||||
GetFields(*static_cast<const DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_DESC*>(opDesc.Desc)));
|
||||
case DML_OPERATOR_CONVOLUTION_INTEGER:
|
||||
return AbstractOperatorDesc(
|
||||
&DML_CONVOLUTION_INTEGER_OPERATOR_SCHEMA,
|
||||
|
|
@ -2420,10 +2425,6 @@ inline AbstractOperatorDesc ConvertOperatorDesc(const DML_OPERATOR_DESC& opDesc)
|
|||
return AbstractOperatorDesc(
|
||||
&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD_OPERATOR_SCHEMA,
|
||||
GetFields(*static_cast<const DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD_OPERATOR_DESC*>(opDesc.Desc)));
|
||||
case DML_OPERATOR_MATRIX_MULTIPLY_INTEGER_TO_FLOAT:
|
||||
return AbstractOperatorDesc(
|
||||
&DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_SCHEMA,
|
||||
GetFields(*static_cast<const DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_DESC*>(opDesc.Desc)));
|
||||
case DML_OPERATOR_ROI_ALIGN_GRAD:
|
||||
return AbstractOperatorDesc(
|
||||
&DML_ROI_ALIGN_GRAD_OPERATOR_SCHEMA,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "precomp.h"
|
||||
|
||||
namespace Dml
|
||||
{
|
||||
|
||||
class DmlOperatorMatMulIntegerToFloat : public DmlOperator
|
||||
{
|
||||
enum OrtInputTensors : uint32_t
|
||||
{
|
||||
ortA,
|
||||
ortB,
|
||||
ortAScale,
|
||||
ortBScale,
|
||||
ortAZeroPoint,
|
||||
ortBZeroPoint,
|
||||
ortBias,
|
||||
ortInputCount
|
||||
};
|
||||
|
||||
enum DmlInputIndex : uint32_t
|
||||
{
|
||||
dmlA,
|
||||
dmlAScale,
|
||||
dmlAZeroPoint,
|
||||
dmlB,
|
||||
dmlBScale,
|
||||
dmlBZeroPoint,
|
||||
dmlBias,
|
||||
dmlInputCount,
|
||||
};
|
||||
|
||||
public:
|
||||
DmlOperatorMatMulIntegerToFloat(const MLOperatorKernelCreationContext& kernelInfo)
|
||||
: DmlOperator(kernelInfo)
|
||||
{
|
||||
std::vector<std::optional<uint32_t>> inputIndices = { OrtInputTensors::ortA, OrtInputTensors::ortAScale, OrtInputTensors::ortAZeroPoint, OrtInputTensors::ortB, OrtInputTensors::ortBScale, OrtInputTensors::ortBZeroPoint, OrtInputTensors::ortBias };
|
||||
DmlOperator::Initialize(kernelInfo, inputIndices);
|
||||
|
||||
std::vector<DimensionType> inputShape0 = kernelInfo.GetTensorShapeDescription().GetInputTensorShape(OrtInputTensors::ortA);
|
||||
std::vector<DimensionType> inputShape1 = kernelInfo.GetTensorShapeDescription().GetInputTensorShape(OrtInputTensors::ortB);
|
||||
std::vector<DimensionType> outputShape = kernelInfo.GetTensorShapeDescription().GetOutputTensorShape(0);
|
||||
|
||||
OperatorHelper::MatMulShapeMapping(inputShape0, inputShape1, outputShape);
|
||||
|
||||
// Initialize the input descriptions with broadcasting
|
||||
m_inputTensorDescs[DmlInputIndex::dmlA] = CreateTensorDescFromInput(kernelInfo, OrtInputTensors::ortA, TensorAxis::DoNotCoerce, TensorAxis::W, TensorAxis::RightAligned, inputShape0);
|
||||
m_inputTensorDescs[DmlInputIndex::dmlB] = CreateTensorDescFromInput(kernelInfo, OrtInputTensors::ortB, TensorAxis::DoNotCoerce, TensorAxis::W, TensorAxis::RightAligned, inputShape1);
|
||||
|
||||
// Broadcast Bias tensor to the shape of the output tensor.
|
||||
if(kernelInfo.IsInputValid(OrtInputTensors::ortBias)) {
|
||||
|
||||
m_inputTensorDescs[DmlInputIndex::dmlBias] = CreateTensorDescFromInput(kernelInfo, OrtInputTensors::ortBias, TensorAxis::DoNotCoerce,
|
||||
TensorAxis::W, TensorAxis::RightAligned, outputShape);
|
||||
}
|
||||
|
||||
uint32_t dmlDimSize = m_inputTensorDescs[DmlInputIndex::dmlA].GetDimensionCount();
|
||||
// Resize the A Scale to be the same dimension as the input tensor.
|
||||
// The 1D tensor needs to be moved to the H channel.
|
||||
m_inputTensorDescs[DmlInputIndex::dmlAScale] = CreateTensorDescFromInput(
|
||||
kernelInfo,
|
||||
OrtInputTensors::ortAScale,
|
||||
TensorAxis::DoNotCoerce,
|
||||
TensorAxis::H,
|
||||
TensorAxis::LeftAligned,
|
||||
std::nullopt,
|
||||
dmlDimSize
|
||||
);
|
||||
|
||||
// Resize the A ZeroPoint to be the same dimension as the input tensor.
|
||||
// The 1D tensor needs to be moved to the H channel.
|
||||
if (kernelInfo.IsInputValid(OrtInputTensors::ortAZeroPoint))
|
||||
{
|
||||
|
||||
m_inputTensorDescs[DmlInputIndex::dmlAZeroPoint] = CreateTensorDescFromInput(
|
||||
kernelInfo,
|
||||
OrtInputTensors::ortAZeroPoint,
|
||||
TensorAxis::DoNotCoerce,
|
||||
TensorAxis::H,
|
||||
TensorAxis::LeftAligned,
|
||||
std::nullopt,
|
||||
dmlDimSize
|
||||
);
|
||||
}
|
||||
|
||||
// B Zeropoint and BScale are already aligned in the W dimension so no need to align them
|
||||
|
||||
// Initialize the output description while overriding the shape
|
||||
m_outputTensorDescs[0] = CreateTensorDescFromOutput(kernelInfo, 0, TensorAxis::DoNotCoerce, TensorAxis::W, TensorAxis::RightAligned, outputShape);
|
||||
|
||||
std::vector<DML_TENSOR_DESC> inputDescs = GetDmlInputDescs();
|
||||
std::vector<DML_TENSOR_DESC> outputDescs = GetDmlOutputDescs();
|
||||
|
||||
DML_MATRIX_MULTIPLY_INTEGER_TO_FLOAT_OPERATOR_DESC matMulDesc = {};
|
||||
matMulDesc.ATensor = &inputDescs[DmlInputIndex::dmlA];
|
||||
matMulDesc.AScaleTensor = &inputDescs[DmlInputIndex::dmlAScale];
|
||||
matMulDesc.AZeroPointTensor = inputDescs[DmlInputIndex::dmlAZeroPoint].Desc != nullptr ? &inputDescs[DmlInputIndex::dmlAZeroPoint] : nullptr;
|
||||
matMulDesc.BTensor = &inputDescs[DmlInputIndex::dmlB];
|
||||
matMulDesc.BScaleTensor = &inputDescs[DmlInputIndex::dmlBScale];
|
||||
matMulDesc.BZeroPointTensor = inputDescs[DmlInputIndex::dmlBZeroPoint].Desc != nullptr ? &inputDescs[DmlInputIndex::dmlBZeroPoint] : nullptr;
|
||||
matMulDesc.BiasTensor = inputDescs[DmlInputIndex::dmlBias].Desc != nullptr ? &inputDescs[DmlInputIndex::dmlBias] : nullptr;
|
||||
matMulDesc.OutputTensor = &outputDescs[0];
|
||||
|
||||
DML_OPERATOR_DESC opDesc = { (DML_OPERATOR_TYPE) DML_OPERATOR_MATRIX_MULTIPLY_INTEGER_TO_FLOAT, &matMulDesc };
|
||||
SetDmlOperatorDesc(opDesc, kernelInfo);
|
||||
}
|
||||
};
|
||||
|
||||
DML_OP_DEFINE_CREATION_FUNCTION(MatMulIntegerToFloat, DmlOperatorMatMulIntegerToFloat);
|
||||
|
||||
} // namespace Dml
|
||||
|
|
@ -502,6 +502,7 @@ DML_OP_EXTERN_CREATION_FUNCTION(QLinearMatMul);
|
|||
DML_OP_EXTERN_CREATION_FUNCTION(QLinearConcat);
|
||||
DML_OP_EXTERN_CREATION_FUNCTION(DynamicQuantizeLinear);
|
||||
DML_OP_EXTERN_CREATION_FUNCTION(MatMulInteger);
|
||||
DML_OP_EXTERN_CREATION_FUNCTION(MatMulIntegerToFloat);
|
||||
DML_OP_EXTERN_CREATION_FUNCTION(ConvInteger);
|
||||
DML_OP_EXTERN_CREATION_FUNCTION(Trilu);
|
||||
DML_OP_EXTERN_CREATION_FUNCTION(Shape);
|
||||
|
|
@ -610,6 +611,13 @@ constexpr static std::array<SupportedTensorDataTypes, 3> supportedTypeListQLinea
|
|||
SupportedTensorDataTypes::Int8|SupportedTensorDataTypes::UInt8,
|
||||
SupportedTensorDataTypes::Int8|SupportedTensorDataTypes::UInt8
|
||||
};
|
||||
|
||||
constexpr static std::array<SupportedTensorDataTypes, 3> supportedTypeListMatMulIntegerToFloat = {
|
||||
SupportedTensorDataTypes::Ints8Bit,
|
||||
SupportedTensorDataTypes::Ints8Bit,
|
||||
SupportedTensorDataTypes::Float16to32
|
||||
};
|
||||
|
||||
constexpr static std::array<SupportedTensorDataTypes, 4> supportedTypeListQLinearConv = {
|
||||
SupportedTensorDataTypes::Int8|SupportedTensorDataTypes::UInt8,
|
||||
SupportedTensorDataTypes::Int8|SupportedTensorDataTypes::UInt8,
|
||||
|
|
@ -1050,6 +1058,7 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation
|
|||
{REG_INFO( 10, QLinearConv, typeNameListFour, supportedTypeListQLinearConv, DmlGraphSupport::Supported)},
|
||||
{REG_INFO( 10, QLinearMatMul, typeNameListThree, supportedTypeListQLinearMatMul, DmlGraphSupport::Supported)},
|
||||
{REG_INFO( 10, MatMulInteger, typeNameListThree, supportedTypeListInteger, DmlGraphSupport::Supported)},
|
||||
{REG_INFO_MS( 1, MatMulIntegerToFloat, typeNameListThree, supportedTypeListMatMulIntegerToFloat, DmlGraphSupport::Supported)},
|
||||
{REG_INFO( 10, ConvInteger, typeNameListThree, supportedTypeListInteger, DmlGraphSupport::Supported)},
|
||||
{REG_INFO( 11, DynamicQuantizeLinear, typeNameListTwo, supportedTypeListDynamicQuantizeLinear, DmlGraphSupport::Supported)},
|
||||
{REG_INFO( 7, LayerNormalization, typeNameListLayerNormContrib, supportedTypeListLayerNormalizationContrib, DmlGraphSupport::Supported, requiredConstantCpuInputs(), std::nullopt, QueryLayerNormalization)},
|
||||
|
|
|
|||
|
|
@ -826,6 +826,13 @@ public:
|
|||
QLinearMatMulHelper(const Info_t& info, const Shape_t& shape) : MatMulHelperBase(info, shape, 0, 3) {}
|
||||
};
|
||||
|
||||
class MatMulIntegerToFloatHelper : public MatMulHelperBase
|
||||
{
|
||||
public:
|
||||
template<typename Info_t, typename Shape_t>
|
||||
MatMulIntegerToFloatHelper(const Info_t& info, const Shape_t& shape) : MatMulHelperBase(info, shape, 0, 1) {}
|
||||
};
|
||||
|
||||
|
||||
class TopKHelper
|
||||
{
|
||||
|
|
@ -1698,6 +1705,7 @@ using ShapeInferenceHelper_Identity16 = GetOutputShapeAsInputShapeHelper;
|
|||
using ShapeInferenceHelper_MatMul = MatMulHelper;
|
||||
using ShapeInferenceHelper_MatMulInteger = MatMulHelper;
|
||||
using ShapeInferenceHelper_DynamicQuantizeMatMul = MatMulHelper;
|
||||
using ShapeInferenceHelper_MatMulIntegerToFloat = MatMulIntegerToFloatHelper;
|
||||
using ShapeInferenceHelper_QLinearMatMul = QLinearMatMulHelper;
|
||||
using ShapeInferenceHelper_QLinearAdd = GetBroadcastedOutputShapeHelper;
|
||||
using ShapeInferenceHelper_DynamicQuantizeLinear = GetOutputShapeAsInputShapeHelper;
|
||||
|
|
|
|||
|
|
@ -435,6 +435,7 @@ namespace OperatorHelper
|
|||
static const int sc_sinceVer_FusedMatMulActivation = 1;
|
||||
static const int sc_sinceVer_QLinearSigmoid = 1;
|
||||
static const int sc_sinceVer_Attention = 1;
|
||||
static const int sc_sinceVer_MatMulIntegerToFloat = 1;
|
||||
static const int sc_sinceVer_MultiHeadAttention = 1;
|
||||
static const int sc_sinceVer_SkipLayerNormalization = 1;
|
||||
static const int sc_sinceVer_EmbedLayerNormalization = 1;
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ using namespace std;
|
|||
namespace onnxruntime {
|
||||
namespace test {
|
||||
|
||||
template <typename IType, typename WType>
|
||||
template <typename IType, typename WType, typename OType>
|
||||
void TestMatMulIntegerToFloat(const std::vector<int64_t>& A_dims,
|
||||
std::vector<int64_t> B_dims,
|
||||
const std::string& reference_model,
|
||||
|
|
@ -50,11 +50,11 @@ void TestMatMulIntegerToFloat(const std::vector<int64_t>& A_dims,
|
|||
return static_cast<WType>(v);
|
||||
});
|
||||
|
||||
std::vector<float> A_scale = random.Uniform<float>(AsSpan<int64_t>({1}), -0.1f, 0.1f);
|
||||
std::vector<OType> A_scale = random.Uniform<OType>(AsSpan<int64_t>({1}), -0.1f, 0.1f);
|
||||
std::vector<IType> A_zero_point{(std::numeric_limits<IType>::lowest() + std::numeric_limits<IType>::max() + IType(2)) / 2};
|
||||
|
||||
int64_t b_scale_zp_size = per_column ? B_dims.back() : 1;
|
||||
std::vector<float> B_scale = random.Uniform<float>(AsSpan({b_scale_zp_size}), -0.1f, 0.1f);
|
||||
std::vector<OType> B_scale = random.Uniform<OType>(AsSpan({b_scale_zp_size}), -0.1f, 0.1f);
|
||||
|
||||
std::vector<WType> B_zero_point(b_scale_zp_size);
|
||||
std::for_each(B_zero_point.begin(),
|
||||
|
|
@ -65,13 +65,13 @@ void TestMatMulIntegerToFloat(const std::vector<int64_t>& A_dims,
|
|||
std::numeric_limits<WType>::max())[0]);
|
||||
});
|
||||
|
||||
std::vector<float> Bias = random.Uniform<float>(AsSpan({B_dims.back()}), -0.1f, 0.1f);
|
||||
std::vector<OType> Bias = random.Uniform<OType>(AsSpan({B_dims.back()}), -0.1f, 0.1f);
|
||||
|
||||
OpTester test("MatMulIntegerToFloat", 1, onnxruntime::kMSDomain);
|
||||
test.AddInput<IType>("A", A_dims, A_data);
|
||||
test.AddInput<WType>("B", B_dims, B_data, is_matrix_b_constant);
|
||||
test.AddInput<float>("a_scale", {1}, A_scale);
|
||||
test.AddInput<float>("b_scale", {b_scale_zp_size}, B_scale);
|
||||
test.AddInput<OType>("a_scale", {1}, A_scale);
|
||||
test.AddInput<OType>("b_scale", {b_scale_zp_size}, B_scale);
|
||||
|
||||
if (has_zp) {
|
||||
test.AddInput<IType>("a_zero_point", {1}, A_zero_point);
|
||||
|
|
@ -82,23 +82,38 @@ void TestMatMulIntegerToFloat(const std::vector<int64_t>& A_dims,
|
|||
}
|
||||
|
||||
if (has_bias) {
|
||||
test.AddInput<float>("bias", {B_dims.back()}, Bias);
|
||||
test.AddInput<OType>("bias", {B_dims.back()}, Bias);
|
||||
} else {
|
||||
test.AddOptionalInputEdge<float>();
|
||||
test.AddOptionalInputEdge<OType>();
|
||||
}
|
||||
|
||||
test.AddReferenceOutputs(reference_model);
|
||||
#if defined(USE_DML)
|
||||
if constexpr (std::is_same_v<OType, float>) {
|
||||
test.SetOutputRelErr("Y", 2e-2f);
|
||||
} else {
|
||||
test.SetOutputRelErr("Y", 2.0f);
|
||||
}
|
||||
#else
|
||||
test.SetOutputRelErr("Y", 1e-4f);
|
||||
test.Run();
|
||||
#endif
|
||||
|
||||
if constexpr (std::is_same_v<OType, float>) {
|
||||
test.Run();
|
||||
} else {
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCpuExecutionProvider});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
template <typename IType, typename WType, bool HasZeroPoint, bool HasBias>
|
||||
template <typename IType, typename WType, typename OType, bool HasZeroPoint, bool HasBias>
|
||||
void RunMatMulIntegerToFloatTest(const string& model_path) {
|
||||
std::vector<int64_t> A_dims{4, 128};
|
||||
std::vector<int64_t> B_dims{128, 128};
|
||||
std::vector<int64_t> Y_dims{4, 128};
|
||||
|
||||
TestMatMulIntegerToFloat<IType, WType>(A_dims,
|
||||
TestMatMulIntegerToFloat<IType, WType, OType>(
|
||||
A_dims,
|
||||
B_dims,
|
||||
model_path,
|
||||
false, /*is_matrix_b_constant*/
|
||||
|
|
@ -107,7 +122,8 @@ void RunMatMulIntegerToFloatTest(const string& model_path) {
|
|||
HasBias /*has_bias*/
|
||||
);
|
||||
|
||||
TestMatMulIntegerToFloat<IType, WType>(A_dims,
|
||||
TestMatMulIntegerToFloat<IType, WType, OType>(
|
||||
A_dims,
|
||||
B_dims,
|
||||
model_path,
|
||||
true, /*is_matrix_b_constant*/
|
||||
|
|
@ -116,7 +132,8 @@ void RunMatMulIntegerToFloatTest(const string& model_path) {
|
|||
HasBias /*has_bias*/
|
||||
);
|
||||
|
||||
TestMatMulIntegerToFloat<IType, WType>(A_dims,
|
||||
TestMatMulIntegerToFloat<IType, WType, OType>(
|
||||
A_dims,
|
||||
B_dims,
|
||||
model_path,
|
||||
false, /*is_matrix_b_constant*/
|
||||
|
|
@ -125,7 +142,8 @@ void RunMatMulIntegerToFloatTest(const string& model_path) {
|
|||
HasBias /*has_bias*/
|
||||
);
|
||||
|
||||
TestMatMulIntegerToFloat<IType, WType>(A_dims,
|
||||
TestMatMulIntegerToFloat<IType, WType, OType>(
|
||||
A_dims,
|
||||
B_dims,
|
||||
model_path,
|
||||
true, /*is_matrix_b_constant*/
|
||||
|
|
@ -135,22 +153,42 @@ void RunMatMulIntegerToFloatTest(const string& model_path) {
|
|||
);
|
||||
}
|
||||
|
||||
#if USE_DML
|
||||
TEST(MatMulIntegerToFloat, HasZeroPoint_NoBias_test_U8X8_FP16) {
|
||||
RunMatMulIntegerToFloatTest<uint8_t, int8_t, MLFloat16, true, false>("testdata/matmul_integer_to_float16_int8.onnx");
|
||||
RunMatMulIntegerToFloatTest<uint8_t, uint8_t, MLFloat16, true, false>("testdata/matmul_integer_to_float16_uint8.onnx");
|
||||
}
|
||||
|
||||
TEST(MatMulIntegerToFloat, NoZeroPoint_HasBias_test_U8X8_FP16) {
|
||||
RunMatMulIntegerToFloatTest<uint8_t, int8_t, MLFloat16, false, true>("testdata/matmul_integer_to_float16_int8_bias.onnx");
|
||||
RunMatMulIntegerToFloatTest<uint8_t, uint8_t, MLFloat16, false, true>("testdata/matmul_integer_to_float16_uint8_bias.onnx");
|
||||
}
|
||||
|
||||
TEST(MatMulIntegerToFloat, HasZeroPoint_NoBias_test_S8S8_FP16) {
|
||||
RunMatMulIntegerToFloatTest<int8_t, int8_t, MLFloat16, true, false>("testdata/matmul_integer_to_float16_int8_int8.onnx");
|
||||
}
|
||||
|
||||
TEST(MatMulIntegerToFloat, NoZeroPoint_HasBias_test_S8S8_FP16) {
|
||||
RunMatMulIntegerToFloatTest<int8_t, int8_t, MLFloat16, false, true>("testdata/matmul_integer_to_float16_int8_int8_bias.onnx");
|
||||
}
|
||||
#endif // USE_DML
|
||||
|
||||
TEST(MatMulIntegerToFloat, HasZeroPoint_NoBias_test_U8X8) {
|
||||
RunMatMulIntegerToFloatTest<uint8_t, int8_t, true, false>("testdata/matmul_integer_to_float_int8.onnx");
|
||||
RunMatMulIntegerToFloatTest<uint8_t, uint8_t, true, false>("testdata/matmul_integer_to_float_uint8.onnx");
|
||||
RunMatMulIntegerToFloatTest<uint8_t, int8_t, float, true, false>("testdata/matmul_integer_to_float_int8.onnx");
|
||||
RunMatMulIntegerToFloatTest<uint8_t, uint8_t, float, true, false>("testdata/matmul_integer_to_float_uint8.onnx");
|
||||
}
|
||||
|
||||
TEST(MatMulIntegerToFloat, NoZeroPoint_HasBias_test_U8X8) {
|
||||
RunMatMulIntegerToFloatTest<uint8_t, int8_t, false, true>("testdata/matmul_integer_to_float_int8_bias.onnx");
|
||||
RunMatMulIntegerToFloatTest<uint8_t, uint8_t, false, true>("testdata/matmul_integer_to_float_uint8_bias.onnx");
|
||||
RunMatMulIntegerToFloatTest<uint8_t, int8_t, float, false, true>("testdata/matmul_integer_to_float_int8_bias.onnx");
|
||||
RunMatMulIntegerToFloatTest<uint8_t, uint8_t, float, false, true>("testdata/matmul_integer_to_float_uint8_bias.onnx");
|
||||
}
|
||||
|
||||
TEST(MatMulIntegerToFloat, HasZeroPoint_NoBias_test_S8S8) {
|
||||
RunMatMulIntegerToFloatTest<int8_t, int8_t, true, false>("testdata/matmul_integer_to_float_int8_int8.onnx");
|
||||
RunMatMulIntegerToFloatTest<int8_t, int8_t, float, true, false>("testdata/matmul_integer_to_float_int8_int8.onnx");
|
||||
}
|
||||
|
||||
TEST(MatMulIntegerToFloat, NoZeroPoint_HasBias_test_S8S8) {
|
||||
RunMatMulIntegerToFloatTest<int8_t, int8_t, false, true>("testdata/matmul_integer_to_float_int8_int8_bias.onnx");
|
||||
RunMatMulIntegerToFloatTest<int8_t, int8_t, float, false, true>("testdata/matmul_integer_to_float_int8_int8_bias.onnx");
|
||||
}
|
||||
|
||||
TEST(MatMulIntegerToFloat, MatMulInteger_With_ZeroPoint) {
|
||||
|
|
|
|||
|
|
@ -5181,6 +5181,24 @@ TEST_F(GraphTransformationTests, MatMulIntegerToFloatTest) {
|
|||
EXPECT_EQ(op_to_count["Add"], 1);
|
||||
}
|
||||
|
||||
#ifdef USE_DML
|
||||
TEST_F(GraphTransformationTests, MatMulIntegerToFloat16Test) {
|
||||
constexpr const ORTCHAR_T* model_uri = MODEL_FOLDER "fusion/matmul_integer_to_float16_int8.onnx";
|
||||
std::shared_ptr<Model> p_model;
|
||||
ASSERT_STATUS_OK(Model::Load(model_uri, p_model, nullptr, *logger_));
|
||||
Graph& graph = p_model->MainGraph();
|
||||
|
||||
for (auto& node : graph.Nodes()) {
|
||||
node.SetExecutionProviderType(kDmlExecutionProvider);
|
||||
}
|
||||
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
|
||||
ASSERT_STATUS_OK(graph_transformation_mgr.Register(std::make_unique<MatMulIntegerToFloatFusion>(), TransformerLevel::Level2));
|
||||
ASSERT_STATUS_OK(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_));
|
||||
std::map<std::string, int> op_to_count = CountOpsInGraph(graph);
|
||||
EXPECT_EQ(op_to_count["com.microsoft.MatMulIntegerToFloat"], 1);
|
||||
}
|
||||
#endif // USE_DML
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef DISABLE_CONTRIB_OPS
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import onnx
|
|||
from onnx import TensorProto, helper
|
||||
|
||||
|
||||
def GenerateModel(model_name, sign_i, sign_w, has_zp=True, bias=False): # noqa: N802
|
||||
def GenerateModel(model_name, sign_i, sign_w, output_type_fp16, has_zp=True, bias=False): # noqa: N802
|
||||
nodes = [ # subgraph
|
||||
helper.make_node(
|
||||
"MatMulInteger",
|
||||
|
|
@ -13,7 +13,7 @@ def GenerateModel(model_name, sign_i, sign_w, has_zp=True, bias=False): # noqa:
|
|||
"MatMulInteger",
|
||||
),
|
||||
helper.make_node("Mul", ["a_scale", "b_scale"], ["multiplier"], "mul_right"),
|
||||
helper.make_node("Cast", ["matmul_output_int32"], ["matmul_output_float"], "cast", to=1),
|
||||
helper.make_node("Cast", ["matmul_output_int32"], ["matmul_output_float"], "cast", to=TensorProto.FLOAT16 if output_type_fp16 else TensorProto.FLOAT),
|
||||
helper.make_node(
|
||||
"Mul",
|
||||
["matmul_output_float", "multiplier"],
|
||||
|
|
@ -25,8 +25,8 @@ def GenerateModel(model_name, sign_i, sign_w, has_zp=True, bias=False): # noqa:
|
|||
inputs = [ # inputs
|
||||
helper.make_tensor_value_info("A", TensorProto.INT8 if sign_i else TensorProto.UINT8, ["M", "K"]),
|
||||
helper.make_tensor_value_info("B", TensorProto.INT8 if sign_w else TensorProto.UINT8, ["K", "N"]),
|
||||
helper.make_tensor_value_info("a_scale", TensorProto.FLOAT, [1]),
|
||||
helper.make_tensor_value_info("b_scale", TensorProto.FLOAT, ["C"]),
|
||||
helper.make_tensor_value_info("a_scale", TensorProto.FLOAT16 if output_type_fp16 else TensorProto.FLOAT, [1]),
|
||||
helper.make_tensor_value_info("b_scale", TensorProto.FLOAT16 if output_type_fp16 else TensorProto.FLOAT, ["C"]),
|
||||
]
|
||||
|
||||
if has_zp:
|
||||
|
|
@ -48,14 +48,14 @@ def GenerateModel(model_name, sign_i, sign_w, has_zp=True, bias=False): # noqa:
|
|||
if bias:
|
||||
nodes.extend([helper.make_node("Add", ["mul_bottom_output", "bias"], ["Y"], "add")])
|
||||
|
||||
inputs.extend([helper.make_tensor_value_info("bias", TensorProto.FLOAT, ["N"])])
|
||||
inputs.extend([helper.make_tensor_value_info("bias", TensorProto.FLOAT16 if output_type_fp16 else TensorProto.FLOAT, ["N"])])
|
||||
|
||||
graph = helper.make_graph(
|
||||
nodes,
|
||||
"DynamicQuantizeMatMul_fusion", # name
|
||||
inputs,
|
||||
[ # outputs
|
||||
helper.make_tensor_value_info("Y", TensorProto.FLOAT, ["M", "N"]),
|
||||
helper.make_tensor_value_info("Y", TensorProto.FLOAT16 if output_type_fp16 else TensorProto.FLOAT, ["M", "N"]),
|
||||
],
|
||||
)
|
||||
|
||||
|
|
@ -64,10 +64,18 @@ def GenerateModel(model_name, sign_i, sign_w, has_zp=True, bias=False): # noqa:
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
GenerateModel("matmul_integer_to_float_int8.onnx", False, True)
|
||||
GenerateModel("matmul_integer_to_float_uint8.onnx", False, False)
|
||||
GenerateModel("matmul_integer_to_float_int8_bias.onnx", False, True, False, True)
|
||||
GenerateModel("matmul_integer_to_float_uint8_bias.onnx", False, False, False, True)
|
||||
GenerateModel("matmul_integer_to_float16_int8.onnx", sign_i=False, sign_w=True, output_type_fp16=True)
|
||||
GenerateModel("matmul_integer_to_float16_uint8.onnx", sign_i=False, sign_w=False, output_type_fp16=True)
|
||||
GenerateModel("matmul_integer_to_float16_int8_bias.onnx", sign_i=False, sign_w=True, output_type_fp16=True, has_zp=False, bias=True)
|
||||
GenerateModel("matmul_integer_to_float16_uint8_bias.onnx", sign_i=False, sign_w=False, output_type_fp16=True, has_zp=False, bias=True)
|
||||
|
||||
GenerateModel("matmul_integer_to_float_int8_int8.onnx", True, True)
|
||||
GenerateModel("matmul_integer_to_float_int8_int8_bias.onnx", True, True, False, True)
|
||||
GenerateModel("matmul_integer_to_float16_int8_int8.onnx", sign_i=True, sign_w=True, output_type_fp16=True)
|
||||
GenerateModel("matmul_integer_to_float16_int8_int8_bias.onnx", sign_i=True, sign_w=True, output_type_fp16=True, has_zp=False, bias=True)
|
||||
|
||||
GenerateModel("matmul_integer_to_float_int8.onnx", sign_i=False, sign_w=True, output_type_fp16=False)
|
||||
GenerateModel("matmul_integer_to_float_uint8.onnx", sign_i=False, sign_w=False, output_type_fp16=False)
|
||||
GenerateModel("matmul_integer_to_float_int8_bias.onnx", sign_i=False, sign_w=True, output_type_fp16=False, has_zp=False, bias=True)
|
||||
GenerateModel("matmul_integer_to_float_uint8_bias.onnx", sign_i=False, sign_w=False, output_type_fp16=False, has_zp=False, bias=True)
|
||||
|
||||
GenerateModel("matmul_integer_to_float_int8_int8.onnx", sign_i=True, sign_w=True, output_type_fp16=False)
|
||||
GenerateModel("matmul_integer_to_float_int8_int8_bias.onnx", sign_i=True, sign_w=True, output_type_fp16=False, has_zp=False, bias=True)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
:ﾌ
|
||||
:ﾌ
|
||||
U
|
||||
A
|
||||
B
|
||||
|
|
@ -44,4 +44,4 @@ mul_bottom"Mul DynamicQuantizeMatMul_fusionZ
|
|||
|
||||
|
||||
M
|
||||
NB
|
||||
NB
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
:ﾄ
|
||||
:ﾄ
|
||||
9
|
||||
A
|
||||
Bmatmul_output_int32
MatMulInteger"
MatMulInteger
|
||||
|
|
@ -41,4 +41,4 @@ mul_bottom"Mul
|
|||
|
||||
|
||||
M
|
||||
NB
|
||||
NB
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
:ﾌ
|
||||
:ﾌ
|
||||
U
|
||||
A
|
||||
B
|
||||
|
|
@ -44,4 +44,4 @@ mul_bottom"Mul DynamicQuantizeMatMul_fusionZ
|
|||
|
||||
|
||||
M
|
||||
NB
|
||||
NB
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
:ﾄ
|
||||
:ﾄ
|
||||
9
|
||||
A
|
||||
Bmatmul_output_int32
MatMulInteger"
MatMulInteger
|
||||
|
|
@ -41,4 +41,4 @@ mul_bottom"Mul
|
|||
|
||||
|
||||
M
|
||||
NB
|
||||
NB
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
:ﾌ
|
||||
:ﾌ
|
||||
U
|
||||
A
|
||||
B
|
||||
|
|
@ -44,4 +44,4 @@ mul_bottom"Mul DynamicQuantizeMatMul_fusionZ
|
|||
|
||||
|
||||
M
|
||||
NB
|
||||
NB
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
:ﾄ
|
||||
:ﾄ
|
||||
9
|
||||
A
|
||||
Bmatmul_output_int32
MatMulInteger"
MatMulInteger
|
||||
|
|
@ -41,4 +41,4 @@ mul_bottom"Mul
|
|||
|
||||
|
||||
M
|
||||
NB
|
||||
NB
|
||||
Binary file not shown.
|
|
@ -104,4 +104,4 @@ def GenerateModel(model_name): # noqa: N802
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
GenerateModel("matmul_integer_to_float.onnx")
|
||||
GenerateModel("matmul_integer_to_float.onnx")
|
||||
51
onnxruntime/test/testdata/transform/fusion/matmul_integer_to_float16_int8.onnx
vendored
Normal file
51
onnxruntime/test/testdata/transform/fusion/matmul_integer_to_float16_int8.onnx
vendored
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
:Ě
|
||||
U
|
||||
A
|
||||
B
|
||||
a_zero_point
|
||||
b_zero_pointmatmul_output_int32
MatMulInteger"
MatMulInteger
|
||||
.
|
||||
a_scale
|
||||
b_scale
|
||||
multiplier mul_right"Mul
|
||||
A
|
||||
matmul_output_int32matmul_output_floatcast"Cast*
|
||||
to
|
||||
|
||||
5
|
||||
matmul_output_float
|
||||
|
||||
multiplierY
|
||||
mul_bottom"Mul DynamicQuantizeMatMul_fusionZ
|
||||
A
|
||||
|
||||
|
||||
M
|
||||
KZ
|
||||
B
|
||||
|
||||
|
||||
K
|
||||
NZ
|
||||
a_scale
|
||||
|
||||
|
||||
|
||||
Z
|
||||
b_scale
|
||||
|
||||
|
||||
CZ
|
||||
a_zero_point
|
||||
|
||||
|
||||
Z
|
||||
b_zero_point
|
||||
|
||||
Cb
|
||||
Y
|
||||
|
||||
|
||||
|
||||
M
|
||||
NB
|
||||
Loading…
Reference in a new issue