saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-16 18:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use new QUANTIZED_LINEAR_ADD1 operator

Browse source
This commit is contained in:
Jeff Bloomfield 2022-11-13 21:58:03 -08:00
parent 0080bb0331
commit 64e97eb10e
9 changed files with 160 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
onnxruntime/core/providers/dml/DmlExecutionProvider/src/External/DirectMLHelpers/ApiTraits.h vendored
View file

@ -2,6 +2,21 @@
// Licensed under the MIT License.
#pragma once
struct DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_DESC
{
const DML_TENSOR_DESC* ATensor;
const DML_TENSOR_DESC* AScaleTensor;
_Maybenull_ const DML_TENSOR_DESC* AZeroPointTensor;
const DML_TENSOR_DESC* BTensor;
_Maybenull_ const DML_TENSOR_DESC* BScaleTensor;
_Maybenull_ const DML_TENSOR_DESC* BZeroPointTensor;
const DML_TENSOR_DESC* OutputScaleTensor; // This is an input tensor
_Maybenull_ const DML_TENSOR_DESC* OutputZeroPointTensor; // This is an input tensor
const DML_TENSOR_DESC* OutputTensor;
};
const int DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1 = 0x8000000e;
namespace ApiTraits
{
template <typename T>
@ -1011,6 +1026,12 @@ struct OperatorDescTraits<DML_DIAGONAL_MATRIX1_OPERATOR_DESC>
static constexpr DML_OPERATOR_TYPE Type = DML_OPERATOR_DIAGONAL_MATRIX1;
};
template <>
struct OperatorDescTraits<DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_DESC>
{
static constexpr DML_OPERATOR_TYPE Type = (DML_OPERATOR_TYPE) DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1;
};
template <>
struct OperatorDescTraits<DML_ACTIVATION_ELU_OPERATOR_DESC>
{
@ -2139,6 +2160,11 @@ struct OperatorTypeTraits<(DML_OPERATOR_TYPE)DML_OPERATOR_ACTIVATION_GELU>
using DescType = DML_ACTIVATION_GELU_OPERATOR_DESC;
};
template <>
struct OperatorTypeTraits<(DML_OPERATOR_TYPE)DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1>
{
using DescType = DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_DESC;
};
// Calls a visitor functor, supplying an empty operator desc corresponding to the given DML_OPERATOR_TYPE as
// the first argument.
//
@ -2480,6 +2506,8 @@ auto OperatorTypeVisitor(DML_OPERATOR_TYPE type, Visitor&& visitor, Ts&&... args
return std::invoke(std::forward<Visitor>(visitor), DML_ACTIVATION_SHRINK_OPERATOR_DESC{}, std::forward<Ts>(args)...);
case DML_OPERATOR_ACTIVATION_GELU:
return std::invoke(std::forward<Visitor>(visitor), DML_ACTIVATION_GELU_OPERATOR_DESC{}, std::forward<Ts>(args)...);
case DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1:
return std::invoke(std::forward<Visitor>(visitor), DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_DESC{}, std::forward<Ts>(args)...);
default:
ORT_THROW_HR(E_INVALIDARG);
@ -2633,6 +2661,7 @@ inline gsl::czstring ToString(DML_OPERATOR_TYPE value)
case DML_OPERATOR_RESAMPLE2: return "DML_OPERATOR_RESAMPLE2";
case DML_OPERATOR_RESAMPLE_GRAD1: return "DML_OPERATOR_RESAMPLE_GRAD1";
case DML_OPERATOR_DIAGONAL_MATRIX1: return "DML_OPERATOR_DIAGONAL_MATRIX1";
case DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1: return "DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1";
default:
assert(false);
return "<unknown>";

19
onnxruntime/core/providers/dml/DmlExecutionProvider/src/External/DirectMLHelpers/DirectMLSchema.h vendored
View file

@ -2644,4 +2644,23 @@ constexpr DML_SCHEMA_FIELD DML_RNN_ZERO_OPERATOR_SCHEMA_FIELDS[3] {
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_OUTPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "OutputTensor", false },
};
constexpr DML_SCHEMA_FIELD DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA_FIELDS[9] {
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "ATensor", false },
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "AScaleTensor", false },
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "AZeroPointTensor", true },
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BTensor", false },
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BScaleTensor", true },
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "BZeroPointTensor", true },
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "OutputScaleTensor", false },
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_INPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "OutputZeroPointTensor", true },
DML_SCHEMA_FIELD { DML_SCHEMA_FIELD_KIND_OUTPUT_TENSOR, DML_SCHEMA_FIELD_TYPE_TENSOR_DESC, "OutputTensor", false },
};
constexpr DML_OPERATOR_SCHEMA DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA {
"DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1",
static_cast<DML_OPERATOR_TYPE>(DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1),
DML_SCHEMA_OPERATOR_SUPPORT_FLAG_NONE,
9,
DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA_FIELDS,
};
} // extern "C"

20
onnxruntime/core/providers/dml/DmlExecutionProvider/src/External/DirectMLHelpers/GeneratedSchemaHelpers.h vendored
View file

@ -1610,6 +1610,21 @@ inline std::vector<OperatorField> GetFields(const DML_ACTIVATION_GELU_OPERATOR_D
OperatorField(&DML_ACTIVATION_GELU_OPERATOR_SCHEMA.Fields[1], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.OutputTensor))),
};
}
inline std::vector<OperatorField> GetFields(const DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_DESC& desc)
{
return {
OperatorField(&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA.Fields[0], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.ATensor))),
OperatorField(&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA.Fields[1], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.AScaleTensor))),
OperatorField(&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA.Fields[2], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.AZeroPointTensor))),
OperatorField(&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA.Fields[3], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BTensor))),
OperatorField(&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA.Fields[4], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BScaleTensor))),
OperatorField(&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA.Fields[5], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.BZeroPointTensor))),
OperatorField(&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA.Fields[6], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.OutputScaleTensor))),
OperatorField(&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA.Fields[7], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.OutputZeroPointTensor))),
OperatorField(&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA.Fields[8], ToOperatorFieldType(static_cast<const DML_TENSOR_DESC*>(desc.OutputTensor))),
};
}
inline const DML_OPERATOR_SCHEMA& GetSchema(DML_OPERATOR_TYPE operatorType)
{
switch (operatorType)
@ -1777,6 +1792,7 @@ inline const DML_OPERATOR_SCHEMA& GetSchema(DML_OPERATOR_TYPE operatorType)
case DML_OPERATOR_ACTIVATION_THRESHOLDED_RELU: return DML_ACTIVATION_THRESHOLDED_RELU_OPERATOR_SCHEMA;
case DML_OPERATOR_ACTIVATION_SHRINK: return DML_ACTIVATION_SHRINK_OPERATOR_SCHEMA;
case DML_OPERATOR_ACTIVATION_GELU: return DML_ACTIVATION_GELU_OPERATOR_SCHEMA;
case DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1: return DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA;
default:
ORT_THROW_HR(E_INVALIDARG);
@ -2346,6 +2362,10 @@ inline AbstractOperatorDesc ConvertOperatorDesc(const DML_OPERATOR_DESC& opDesc)
return AbstractOperatorDesc(
&DML_DIAGONAL_MATRIX1_OPERATOR_SCHEMA,
GetFields(*static_cast<const DML_DIAGONAL_MATRIX1_OPERATOR_DESC*>(opDesc.Desc)));
case DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1:
return AbstractOperatorDesc(
&DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_SCHEMA,
GetFields(*static_cast<const DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_DESC*>(opDesc.Desc)));
case DML_OPERATOR_ACTIVATION_ELU:
return AbstractOperatorDesc(
&DML_ACTIVATION_ELU_OPERATOR_SCHEMA,

12
onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphDescBuilder.cpp
View file

@ -230,15 +230,11 @@ namespace Dml::GraphDescBuilder
{
ComPtr<IMLOperatorTensor> tensor = nullptr;
// Check whether this specific node requested support for constant CPU inputs
if (std::find(requiredConstantCpuInputs.begin(), requiredConstantCpuInputs.end(), inputIndex) != requiredConstantCpuInputs.end())
auto inputDefs = node.InputDefs();
if (inputIndex < inputDefs.size())
{
auto inputDefs = node.InputDefs();
if (inputIndex < inputDefs.size())
{
const onnxruntime::NodeArg* arg = inputDefs[inputIndex];
tensor = constantCpuGraphInputGetter(arg->Name());
}
const onnxruntime::NodeArg* arg = inputDefs[inputIndex];
tensor = constantCpuGraphInputGetter(arg->Name());
}
return tensor;

24
onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp
View file

@ -1153,6 +1153,30 @@ namespace Windows::AI::MachineLearning::Adapter
ORT_CATCH_RETURN
}
template <class NodeInfoImpl_t, class Base1_t, class Base2_t>
HRESULT STDMETHODCALLTYPE OpNodeInfoWrapper<NodeInfoImpl_t, Base1_t, Base2_t>::TryGetConstantInputTensor(uint32_t inputIndex, IMLOperatorTensor** tensor) const noexcept
{
ORT_TRY
{
ComPtr<IMLOperatorTensor> tensorWrapper = m_constantInputGetter(inputIndex);
if (tensorWrapper == nullptr)
{
assert(std::find(
m_requiredConstantCpuInputs.begin(),
m_requiredConstantCpuInputs.end(),
inputIndex) == m_requiredConstantCpuInputs.end());
return S_OK;
}
*tensor = tensorWrapper.Detach();
return S_OK;
}
ORT_CATCH_RETURN
}
HRESULT STDMETHODCALLTYPE OpKernelInfoWrapper::GetOutputTensorShape(uint32_t outputIndex, uint32_t dimensionCount, uint32_t* dimensions) const noexcept
{
ORT_TRY

5
onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h
View file

@ -239,6 +239,11 @@ class OpNodeInfoWrapper : public Base1_t, public Base2_t, public Closable
_Outptr_ IMLOperatorTensor** tensor
) const noexcept;
HRESULT STDMETHODCALLTYPE TryGetConstantInputTensor(
uint32_t inputIndex,
_Outptr_ IMLOperatorTensor** tensor
) const noexcept;
protected:
// Lifetime is managed by the caller and guaranteed to outlive this class
const onnxruntime::OpNodeProtoHelper<NodeInfoImpl_t>* m_impl = nullptr;

41
onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorQLinearAdd.cpp
View file

@ -16,14 +16,35 @@ class DmlOperatorQLinearAdd : public DmlOperator
IN_B_SCALE,
IN_B_ZERO_POINT,
IN_C_SCALE,
IN_C_ZERO_POINT
IN_C_ZERO_POINT,
IN_COUNT
};
public:
DmlOperatorQLinearAdd(const MLOperatorKernelCreationContext& kernelInfo)
: DmlOperator(kernelInfo)
{
DmlOperator::Initialize(kernelInfo);
std::vector<std::optional<uint32_t>> kernelInputIndices(8);
for (uint32_t i = 0; i < IN_COUNT; ++i)
{
kernelInputIndices[i] = i;
}
std::optional<MLOperatorTensor> constantAScaleTensor = kernelInfo.TryGetConstantInputTensor(IN_A_SCALE);
std::optional<MLOperatorTensor> constantBScaleTensor = kernelInfo.TryGetConstantInputTensor(IN_B_SCALE);
if (constantAScaleTensor && constantBScaleTensor)
{
if (constantAScaleTensor->GetUnalignedTensorByteSize() == constantBScaleTensor->GetUnalignedTensorByteSize() &&
!memcmp(constantAScaleTensor->GetByteData(), constantBScaleTensor->GetByteData(), constantAScaleTensor->GetUnalignedTensorByteSize()))
{
kernelInputIndices[IN_B_SCALE] = std::nullopt;
}
}
DmlOperator::Initialize(kernelInfo, kernelInputIndices);
std::vector<DimensionType> outputShape = kernelInfo.GetTensorShapeDescription().GetOutputTensorShape(0);
@ -36,7 +57,15 @@ public:
m_inputTensorDescs[IN_A_SCALE] = CreateTensorDescFromInput(kernelInfo, 1/*A Scale OnnxIndex*/, TensorAxis::DoNotCoerce, TensorAxis::W, TensorAxis::RightAligned, std::nullopt, dmlDimSize);
m_inputTensorDescs[IN_A_ZERO_POINT] = CreateTensorDescFromInput(kernelInfo, 2/*A Zero point OnnxIndex*/, TensorAxis::DoNotCoerce, TensorAxis::W, TensorAxis::RightAligned, std::nullopt, dmlDimSize);
m_inputTensorDescs[IN_B_SCALE] = CreateTensorDescFromInput(kernelInfo, 4/*B Scale OnnxIndex*/, TensorAxis::DoNotCoerce, TensorAxis::W, TensorAxis::RightAligned, std::nullopt, dmlDimSize);
if (kernelInputIndices[IN_B_SCALE])
{
m_inputTensorDescs[IN_B_SCALE] = CreateTensorDescFromInput(kernelInfo, 4/*B Scale OnnxIndex*/, TensorAxis::DoNotCoerce, TensorAxis::W, TensorAxis::RightAligned, std::nullopt, dmlDimSize);
}
else
{
m_inputTensorDescs[IN_B_SCALE] = TensorDesc();
}
m_inputTensorDescs[IN_B_ZERO_POINT] = CreateTensorDescFromInput(kernelInfo, 5/*B Zero point OnnxIndex*/, TensorAxis::DoNotCoerce, TensorAxis::W, TensorAxis::RightAligned, std::nullopt, dmlDimSize);
m_inputTensorDescs[IN_C_SCALE] = CreateTensorDescFromInput(kernelInfo, 6/*C Zero point OnnxIndex*/, TensorAxis::DoNotCoerce, TensorAxis::W, TensorAxis::RightAligned, std::nullopt, dmlDimSize);
@ -48,18 +77,18 @@ public:
std::vector<DML_TENSOR_DESC> inputDescs = GetDmlInputDescs();
std::vector<DML_TENSOR_DESC> outputDescs = GetDmlOutputDescs();
DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD_OPERATOR_DESC AddDesc = {};
DML_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1_OPERATOR_DESC AddDesc = {};
AddDesc.ATensor = &inputDescs[IN_A];
AddDesc.AScaleTensor = &inputDescs[IN_A_SCALE];
AddDesc.AZeroPointTensor = inputDescs[IN_A_ZERO_POINT].Desc != nullptr ? &inputDescs[IN_A_ZERO_POINT] : nullptr;
AddDesc.BTensor = &inputDescs[IN_B];
AddDesc.BScaleTensor = &inputDescs[IN_B_SCALE];
AddDesc.BScaleTensor = inputDescs[IN_B_SCALE].Desc != nullptr ? &inputDescs[IN_B_SCALE] : nullptr;
AddDesc.BZeroPointTensor = inputDescs[IN_B_ZERO_POINT].Desc != nullptr ? &inputDescs[IN_B_ZERO_POINT] : nullptr;
AddDesc.OutputScaleTensor = &inputDescs[IN_C_SCALE];
AddDesc.OutputZeroPointTensor = inputDescs[IN_C_ZERO_POINT].Desc != nullptr ? &inputDescs[IN_C_ZERO_POINT] : nullptr;
AddDesc.OutputTensor = &outputDescs[0];
DML_OPERATOR_DESC opDesc = { DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD, &AddDesc };
DML_OPERATOR_DESC opDesc = { (DML_OPERATOR_TYPE) DML_OPERATOR_ELEMENT_WISE_QUANTIZED_LINEAR_ADD1, &AddDesc };
SetDmlOperatorDesc(opDesc, kernelInfo);
}
};

14
onnxruntime/core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorHelper.h
View file

@ -12,6 +12,8 @@
#define ML_CHECK_BOOL(x) THROW_HR_IF(E_INVALIDARG, !(x))
#endif
#include <optional>
namespace onnxruntime
{
struct MLFloat16;
@ -603,6 +605,18 @@ public:
return MLOperatorTensor(tensor.Get());
}
std::optional<MLOperatorTensor> TryGetConstantInputTensor(uint32_t inputIndex) const
{
Microsoft::WRL::ComPtr<IMLOperatorTensor> tensor;
ORT_THROW_IF_FAILED(m_implPrivate->TryGetConstantInputTensor(inputIndex, &tensor));
if (tensor)
{
return MLOperatorTensor(tensor.Get());
}
return std::nullopt;
}
uint32_t GetInputTensorDimensionCount(uint32_t inputIndex) const
{
auto shapeDesc = GetTensorShapeDescription();

10
onnxruntime/core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorPrivate.h
View file

@ -41,6 +41,11 @@ IMLOperatorShapeInferenceContextPrivate : public IMLOperatorShapeInferenceContex
_Outptr_ IMLOperatorTensor** tensor
) const noexcept PURE;
STDMETHOD(TryGetConstantInputTensor)(
uint32_t inputIndex,
_Outptr_ IMLOperatorTensor** tensor
) const noexcept PURE;
//! Gets the number of dimensions of a tensor output of the operator.
STDMETHOD(GetSequenceInputInfo)(
uint32_t inputIndex,
@ -73,6 +78,11 @@ IMLOperatorKernelCreationContextPrivate : public IMLOperatorKernelCreationContex
_Outptr_ IMLOperatorTensor** tensor
) const noexcept PURE;
STDMETHOD(TryGetConstantInputTensor)(
uint32_t inputIndex,
_Outptr_ IMLOperatorTensor** tensor
) const noexcept PURE;
STDMETHOD_(bool, IsDmlGraphNode)() const noexcept PURE;
STDMETHOD(SetDmlOperator)(