From 3a867d83d5c8af4d308b7e4212f99d96613a7ae0 Mon Sep 17 00:00:00 2001 From: Dwayne Robinson Date: Wed, 18 May 2022 17:34:15 -0700 Subject: [PATCH] DirectML opset14 type updates for Add/Sub/Mul/Div and Relu/PRelu (#11560) * Add more type support for Add/Sub/Mul/Div and Relu/PRelu * Remove stale Remap64bitDmlDataTypeTo32bit --- .../DmlExecutionProvider/src/DmlCommon.cpp | 10 ---- .../dml/DmlExecutionProvider/src/DmlCommon.h | 1 - .../src/GraphPartitioner.cpp | 15 +---- .../src/GraphPartitioner.h | 4 +- .../src/GraphTransformer.cpp | 6 +- .../src/Operators/DmlOperatorCumSum.cpp | 3 +- .../src/Operators/OperatorRegistration.cpp | 60 +++++++++++++++---- .../src/Operators/OperatorUtility.cpp | 2 + .../DmlExecutionProvider/src/TensorDesc.cpp | 56 ----------------- .../dml/DmlExecutionProvider/src/TensorDesc.h | 1 - .../dml/OperatorAuthorHelper/OperatorHelper.h | 4 +- .../OperatorAuthorHelper/OperatorVersions.h | 11 ++++ 12 files changed, 71 insertions(+), 102 deletions(-) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/DmlCommon.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/DmlCommon.cpp index c4d3d60717..43204be90f 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/DmlCommon.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/DmlCommon.cpp @@ -30,16 +30,6 @@ DML_TENSOR_DATA_TYPE GetDmlDataTypeFromMlDataTypeNoThrow(MLOperatorTensorDataTyp }; } -DML_TENSOR_DATA_TYPE Remap64bitDmlDataTypeTo32bit(DML_TENSOR_DATA_TYPE dmlElementType) noexcept -{ - switch (dmlElementType) - { - case DML_TENSOR_DATA_TYPE_UINT64: return DML_TENSOR_DATA_TYPE_UINT32; break; - case DML_TENSOR_DATA_TYPE_INT64: return DML_TENSOR_DATA_TYPE_INT32; break; - default: return dmlElementType; - } -} - bool IsSigned(DML_TENSOR_DATA_TYPE dataType) { switch (dataType) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/DmlCommon.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/DmlCommon.h index e43b982364..0c3abcd255 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/DmlCommon.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/DmlCommon.h @@ -14,7 +14,6 @@ namespace Dml DML_TENSOR_DATA_TYPE GetDmlDataTypeFromMlDataType(MLOperatorTensorDataType tensorDataType); DML_TENSOR_DATA_TYPE GetDmlDataTypeFromMlDataTypeNoThrow(MLOperatorTensorDataType tensorDataType) noexcept; - DML_TENSOR_DATA_TYPE Remap64bitDmlDataTypeTo32bit(DML_TENSOR_DATA_TYPE dmlElementType) noexcept; MLOperatorTensorDataType GetMlDataTypeFromDmlDataType(DML_TENSOR_DATA_TYPE tensorDataType); size_t ComputeByteSizeFromDimensions(gsl::span dimensions, MLOperatorTensorDataType tensorDataType); size_t ComputeByteSizeFromTensor(IMLOperatorTensor& tensor); diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp index 9a9c4b81bb..ad52d4056c 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp @@ -164,14 +164,10 @@ namespace Dml bool DoesNodeContainSupportedDataTypes( const onnxruntime::Node& node, - bool allow64BitInputThroughStrides, - _In_opt_ const std::unordered_map* nodeNameToPartitionMap, // Only used when allow64BitInputThroughStrides is true _In_opt_ const InternalRegistrationInfo* regInfo, uint32_t supportedDeviceDataTypeMask // Each bit corresponds to each DML_TENSOR_DATA_TYPE. ) { - ORT_THROW_HR_IF(E_INVALIDARG, allow64BitInputThroughStrides && !nodeNameToPartitionMap); - std::vector constantCpuInputs; if (regInfo != nullptr) @@ -253,13 +249,9 @@ namespace Dml const onnxruntime::Node& node, const onnxruntime::KernelRegistry& registry, uint32_t supportedDeviceDataTypeMask, // Each bit corresponds to each DML_TENSOR_DATA_TYPE. - const InternalRegistrationInfoMap& internalRegInfoMap, - bool allow64BitInputThroughStrides, - _In_opt_ const std::unordered_map* nodeNameToPartitionMap + const InternalRegistrationInfoMap& internalRegInfoMap ) { - ORT_THROW_HR_IF(E_INVALIDARG, allow64BitInputThroughStrides && !nodeNameToPartitionMap); - const onnxruntime::KernelCreateInfo* createInfo; Status st = registry.TryFindKernel(node, onnxruntime::kDmlExecutionProvider, &createInfo); if (!st.IsOK()) @@ -279,7 +271,7 @@ namespace Dml } // Check whether the node uses any data types which are unsupported by the device. - if (!DoesNodeContainSupportedDataTypes(node, allow64BitInputThroughStrides, nodeNameToPartitionMap, internalRegInfo.get(), supportedDeviceDataTypeMask)) + if (!DoesNodeContainSupportedDataTypes(node, internalRegInfo.get(), supportedDeviceDataTypeMask)) { return false; } @@ -309,8 +301,7 @@ namespace Dml // registration. Determine if that registration supports usage as a graph node. for (auto registry : dmlRegistries) { - bool allow64BitInputThroughStrides = true; - if (IsNodeSupportedByDml(node, *registry, supportedDeviceDataTypeMask, internalRegInfoMap, allow64BitInputThroughStrides, nodeNameToPartitionMap)) + if (IsNodeSupportedByDml(node, *registry, supportedDeviceDataTypeMask, internalRegInfoMap)) { *isDmlNode = true; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.h index 1b2744ecb4..e0fd8af31d 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.h @@ -64,9 +64,7 @@ namespace Dml const onnxruntime::Node& node, const onnxruntime::KernelRegistry& registry, uint32_t supportedDeviceDataTypeMask, // Each bit corresponds to each DML_TENSOR_DATA_TYPE. - const Windows::AI::MachineLearning::Adapter::InternalRegistrationInfoMap& internalRegInfoMap, - bool allow64BitInputThroughStrides, - _In_opt_ const std::unordered_map* nodeNameToPartitionMap // Only used when allow64BitInputThroughStrides is true + const Windows::AI::MachineLearning::Adapter::InternalRegistrationInfoMap& internalRegInfoMap ); } // namespace Dml diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp index 54223e450f..dc8399059f 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp @@ -89,14 +89,12 @@ namespace Dml // We need to predict whether the nodes will be assigned to the DML transformer by Lotus, // which occurs in IExecutionProvider::GetCapability. - bool allow64BitInputThroughStrides = false; if (!IsNodeSupportedByDml( node, *registry, m_providerImpl->GetSupportedDeviceDataTypeMask(), - *m_providerImpl->GetInternalRegistrationInfoMap().get(), - allow64BitInputThroughStrides, - nullptr)) + *m_providerImpl->GetInternalRegistrationInfoMap().get() + )) { // Can't fuse nodes that don't belong to this execution provider continue; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCumSum.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCumSum.cpp index 2a80555583..8d42a9d38d 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCumSum.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCumSum.cpp @@ -51,6 +51,7 @@ public: } }; -DML_OP_DEFINE_CREATION_FUNCTION(CumSum, DmlOperatorCumSum); +DML_OP_DEFINE_CREATION_FUNCTION(CumSum11, DmlOperatorCumSum); +DML_OP_DEFINE_CREATION_FUNCTION(CumSum14, DmlOperatorCumSum); } // namespace Dml diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp index b072d1509b..700b6398c5 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp @@ -53,8 +53,8 @@ DEFINE_ENUM_FLAG_OPERATORS(Dml::SupportedTensorDataTypes); enum class DmlGraphSupport : uint32_t { - Supported = 0, - NotSupported = 1, + Supported = 0, + NotSupported = 1, }; DEFINE_ENUM_FLAG_OPERATORS(DmlGraphSupport); @@ -236,7 +236,8 @@ DML_OP_EXTERN_CREATION_FUNCTION(ConstantOfShape); DML_OP_EXTERN_CREATION_FUNCTION(IsInf); DML_OP_EXTERN_CREATION_FUNCTION(Mod); DML_OP_EXTERN_CREATION_FUNCTION(BitShift); -DML_OP_EXTERN_CREATION_FUNCTION(CumSum); +DML_OP_EXTERN_CREATION_FUNCTION(CumSum11); +DML_OP_EXTERN_CREATION_FUNCTION(CumSum14); DML_OP_EXTERN_CREATION_FUNCTION(GatherElements); DML_OP_EXTERN_CREATION_FUNCTION(GatherND); DML_OP_EXTERN_CREATION_FUNCTION(Range); @@ -278,6 +279,7 @@ constexpr static std::array supportedTypeListFloat1 constexpr static std::array supportedTypeListFloat16to32Ints32 = {SupportedTensorDataTypes::Float16to32 | SupportedTensorDataTypes::Int32 | SupportedTensorDataTypes::UInt32}; constexpr static std::array supportedTypeListFloat16to32Ints8to32 = {SupportedTensorDataTypes::Float16to32 | SupportedTensorDataTypes::Ints8Bit | SupportedTensorDataTypes::Ints16Bit | SupportedTensorDataTypes::Ints32Bit}; constexpr static std::array supportedTypeListFloat16to32Ints8to64 = {SupportedTensorDataTypes::Float16to32 | SupportedTensorDataTypes::Ints8Bit | SupportedTensorDataTypes::Ints16Bit | SupportedTensorDataTypes::Ints32Bit | SupportedTensorDataTypes::Ints64Bit}; +constexpr static std::array supportedTypeListFloat16to32SignedInts8to32 = {SupportedTensorDataTypes::Float16to32 | SupportedTensorDataTypes::Int8 | SupportedTensorDataTypes::Int16 | SupportedTensorDataTypes::Int32}; constexpr static std::array supportedTypeListFloat16to32Ints32to64 = {SupportedTensorDataTypes::Float16to32 | SupportedTensorDataTypes::Ints32Bit | SupportedTensorDataTypes::Ints64Bit}; constexpr static std::array supportedTypeListUInt8to64 = {SupportedTensorDataTypes::UInt8to64}; constexpr static std::array supportedTypeListNumericDefault = { SupportedTensorDataTypes::NumericDefault }; @@ -388,6 +390,10 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO( 7, InstanceNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, BatchNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 9, BatchNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, // v9 just removes 'spatial' attribute. + // TODO: Add additional type constraints in BatchNormalization-15, with scale and bias (T1) being different from input X (T). + // Add training-mode support to BatchNormalization-15 https://github.com/onnx/onnx/pull/3333 + // {REG_INFO( 14, BatchNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, // v9 just removes 'spatial' attribute. + // {REG_INFO( 15, BatchNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, // v9 just removes 'spatial' attribute. {REG_INFO( 7, LRN, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 13, LRN, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, MeanVarianceNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, @@ -395,8 +401,14 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO( 13, MeanVarianceNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, LpNormalization, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, RNN, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::NotSupported)}, + // TODO: Allow recurrent operations to be batchwise https://github.com/onnx/onnx/pull/3217 + // {REG_INFO( 14, RNN, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::NotSupported)}, {REG_INFO( 7, GRU, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::NotSupported)}, + // TODO: Allow recurrent operations to be batchwise https://github.com/onnx/onnx/pull/3217 + // {REG_INFO( 14, GRU, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::NotSupported)}, {REG_INFO( 7, LSTM, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::NotSupported)}, + // TODO: Allow recurrent operations to be batchwise https://github.com/onnx/onnx/pull/3217 + // {REG_INFO( 14, LSTM, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::NotSupported)}, {REG_INFO_MS( 1, ConvTransposeWithDynamicPads, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported, requiredConstantCpuInputs(2))}, // Data Reorganization Layers @@ -441,10 +453,13 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO( 11, ScatterND, typeNameListScatterGatherND, supportedTypeListScatterGatherND, DmlGraphSupport::Supported)}, {REG_INFO( 13, ScatterND, typeNameListScatterGatherND, supportedTypeListScatterGatherND, DmlGraphSupport::Supported)}, {REG_INFO( 9, EyeLike, typeNameListEyeLike, supportedTypeListScalars8to32, DmlGraphSupport::Supported)}, + // TODO: Add Trilu-14 to fill diagonal matrix https://github.com/onnx/onnx/pull/3291 + // {REG_INFO( 14, Trilu, typeNameListTrilu, supportedTypeListScalars8to32, DmlGraphSupport::Supported)}, // Data reorganization that merely changes the dimensions while keeping the data identical. {REG_INFO_ID( 7, Identity, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported)}, {REG_INFO_ID( 13, Identity, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported)}, + {REG_INFO_ID( 14, Identity, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported)}, {REG_INFO_ID( 7, Flatten, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported)}, {REG_INFO_ID( 9, Flatten, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported)}, {REG_INFO_ID( 11, Flatten, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported)}, @@ -457,6 +472,8 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO_ID( 13, Unsqueeze, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported, requiredConstantCpuInputs(1))}, {REG_INFO_ID( 7, Reshape, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported, requiredConstantCpuInputs(1))}, {REG_INFO_ID( 13, Reshape, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported, requiredConstantCpuInputs(1))}, + // TODO: Add allowzero attribute. + // {REG_INFO_ID(14, Reshape, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported, requiredConstantCpuInputs(1))}, // Elementwise {REG_INFO( 7, Sqrt, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, @@ -480,14 +497,18 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO_VER( 11, Clip, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported, requiredConstantCpuInputs(1,2))}, {REG_INFO_VER( 12, Clip, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported, requiredConstantCpuInputs(1,2))}, {REG_INFO_VER( 13, Clip, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported, requiredConstantCpuInputs(1,2))}, - {REG_INFO( 7, Add, typeNameListDefault, supportedTypeListFloat16to32Ints32to64, DmlGraphSupport::Supported)}, - {REG_INFO( 13, Add, typeNameListDefault, supportedTypeListFloat16to32Ints32to64, DmlGraphSupport::Supported)}, - {REG_INFO( 7, Sub, typeNameListDefault, supportedTypeListFloat16to32Ints32to64, DmlGraphSupport::Supported)}, - {REG_INFO( 13, Sub, typeNameListDefault, supportedTypeListFloat16to32Ints32to64, DmlGraphSupport::Supported)}, - {REG_INFO( 7, Mul, typeNameListDefault, supportedTypeListFloat16to32Ints32to64, DmlGraphSupport::Supported)}, - {REG_INFO( 13, Mul, typeNameListDefault, supportedTypeListFloat16to32Ints32to64, DmlGraphSupport::Supported)}, - {REG_INFO( 7, Div, typeNameListDefault, supportedTypeListFloat16to32Ints32, DmlGraphSupport::Supported)}, - {REG_INFO( 13, Div, typeNameListDefault, supportedTypeListFloat16to32Ints32, DmlGraphSupport::Supported)}, + {REG_INFO( 7, Add, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, + {REG_INFO( 13, Add, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, + {REG_INFO( 14, Add, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, + {REG_INFO( 7, Sub, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, + {REG_INFO( 13, Sub, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, + {REG_INFO( 14, Sub, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, + {REG_INFO( 7, Mul, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, + {REG_INFO( 13, Mul, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, + {REG_INFO( 14, Mul, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, + {REG_INFO( 7, Div, typeNameListDefault, supportedTypeListFloat16to32Ints8to32, DmlGraphSupport::Supported)}, + {REG_INFO( 13, Div, typeNameListDefault, supportedTypeListFloat16to32Ints8to32, DmlGraphSupport::Supported)}, + {REG_INFO( 14, Div, typeNameListDefault, supportedTypeListFloat16to32Ints8to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, Sum, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported, requiredConstantCpuInputs(), 2)}, {REG_INFO( 8, Sum, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported, requiredConstantCpuInputs(), 2)}, {REG_INFO( 13, Sum, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported, requiredConstantCpuInputs(), 2)}, @@ -598,6 +619,7 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO_VER( 10, Upsample, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported, requiredConstantCpuInputs(1) /*scales*/)}, {REG_INFO_VER( 10, Resize, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported, requiredConstantCpuInputs(1) /*scales*/)}, {REG_INFO_VER( 11, Resize, typeNameListTwo, supportedTypeListResize11, DmlGraphSupport::Supported, requiredConstantCpuInputs(1, 2, 3) /*roi, scales, sizes*/, std::nullopt, QueryResize)}, + // TODO: Resize-13 support nearest rounding mode attribute https://github.com/onnx/onnx/pull/3026 {REG_INFO_VER( 13, Resize, typeNameListTwo, supportedTypeListResize13, DmlGraphSupport::Supported, requiredConstantCpuInputs(1, 2, 3) /*roi, scales, sizes*/, std::nullopt, QueryResize)}, // Activation Functions @@ -609,9 +631,10 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO( 7, ScaledTanh, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, Relu, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 13, Relu, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, + {REG_INFO( 14, Relu, typeNameListDefault, supportedTypeListFloat16to32SignedInts8to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, LeakyRelu, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, PRelu, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, - {REG_INFO( 9, PRelu, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, + {REG_INFO( 9, PRelu, typeNameListDefault, supportedTypeListFloat16to32SignedInts8to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, ThresholdedRelu, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 10, ThresholdedRelu, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, Elu, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, @@ -619,10 +642,16 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO( 7, Selu, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, Softmax, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 11, Softmax, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, + // TODO: Update Softmax-13/LogSoftmax-13/Hardmax-13 family ops behavior to align with other frameworks https://github.com/onnx/onnx/pull/2879 + // {REG_INFO( 13, Softmax, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, LogSoftmax, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 11, LogSoftmax, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, + // TODO: Update Softmax-13/LogSoftmax-13/Hardmax-13 family ops behavior to align with other frameworks https://github.com/onnx/onnx/pull/2879 + // {REG_INFO( 13, LogSoftmax, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, Hardmax, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 11, Hardmax, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, + // TODO: Update Softmax-13/LogSoftmax-13/Hardmax-13 family ops behavior to align with other frameworks https://github.com/onnx/onnx/pull/2879 + // {REG_INFO( 13, Hardmax, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, Softsign, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, Softplus, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 7, ParametricSoftplus, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, @@ -637,6 +666,8 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO( 7, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported)}, {REG_INFO( 9, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported)}, {REG_INFO( 13, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported)}, + // TODO: Add CastLike-15 https://github.com/onnx/onnx/pull/3558 + // {REG_INFO( 15, CastLike, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported)}, {REG_INFO( 7, MemcpyFromHost, typeNameListDefault, supportedTypeListAll)}, {REG_INFO( 7, MemcpyToHost, typeNameListDefault, supportedTypeListAll)}, {REG_INFO_VER( 7, TopK, typeNameListTopK, supportedTypeListTopK, DmlGraphSupport::Supported)}, @@ -644,6 +675,8 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO_VER( 11, TopK, typeNameListTopK, supportedTypeListTopK, DmlGraphSupport::Supported, requiredConstantCpuInputs(1))}, {REG_INFO( 9, OneHot, typeNameListThree, supportedTypeListOneHot, DmlGraphSupport::Supported, requiredConstantCpuInputs(1))}, {REG_INFO( 11, OneHot, typeNameListThree, supportedTypeListOneHot, DmlGraphSupport::Supported, requiredConstantCpuInputs(1))}, + // Shape-1, Shape-13, Shape-15 rely on CPU. + // Size-1 relies on CPU. // Fused operators {REG_INFO_MSDML(1, FusedConv, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, @@ -662,7 +695,8 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO( 11, BitShift, typeNameListDefault, supportedTypeListUInt8to64, DmlGraphSupport::Supported)}, {REG_INFO( 11, Round, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 10, ReverseSequence, typeNameListDefault, supportedTypeListAllScalars, DmlGraphSupport::Supported)}, - {REG_INFO( 11, CumSum, typeNameListDefault, supportedTypeListFloat16to32Ints32to64, DmlGraphSupport::Supported, requiredConstantCpuInputs(1))}, + {REG_INFO_VER( 11, CumSum, typeNameListDefault, supportedTypeListFloat16to32Ints32to64, DmlGraphSupport::Supported, requiredConstantCpuInputs(1))}, + {REG_INFO_VER( 14, CumSum, typeNameListDefault, supportedTypeListFloat16to32Ints32to64, DmlGraphSupport::Supported, requiredConstantCpuInputs(1))}, {REG_INFO( 11, Range, typeNameListDefault, supportedTypeListRange, DmlGraphSupport::Supported, requiredConstantCpuInputs(0,1,2))}, {REG_INFO( 9, MaxUnpool, typeNameListTwo, supportedTypeListMaxUnpool, DmlGraphSupport::Supported, requiredConstantCpuInputs(2))}, diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp index b9302ed01e..2412a41d16 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp @@ -163,6 +163,7 @@ namespace Dml // The filter for activation functions maps to what DML's fused op internally fuses at the shader level. OperatorInfo{ "Add", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Add, {"Relu", "LeakyRelu"} }, OperatorInfo{ "Add", onnxruntime::kOnnxDomain, OnnxOperatorSet13::sc_sinceVer_Add, {"Relu", "LeakyRelu"} }, + OperatorInfo{ "Add", onnxruntime::kOnnxDomain, OnnxOperatorSet14::sc_sinceVer_Add, {"Relu", "LeakyRelu"} }, OperatorInfo{ "Sum", onnxruntime::kOnnxDomain, OnnxOperatorSet8::sc_sinceVer_Sum, {"Relu", "LeakyRelu"}, 2 }, OperatorInfo{ "Sum", onnxruntime::kOnnxDomain, OnnxOperatorSet13::sc_sinceVer_Sum, {"Relu", "LeakyRelu"}, 2 }, }; @@ -179,6 +180,7 @@ namespace Dml OperatorInfo{ "ScaledTanh", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_ScaledTanh }, OperatorInfo{ "Relu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Relu }, OperatorInfo{ "Relu", onnxruntime::kOnnxDomain, OnnxOperatorSet13::sc_sinceVer_Relu }, + OperatorInfo{ "Relu", onnxruntime::kOnnxDomain, OnnxOperatorSet14::sc_sinceVer_Relu }, OperatorInfo{ "LeakyRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_LeakyRelu }, OperatorInfo{ "PRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_PRelu }, OperatorInfo{ "PRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_PRelu }, diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/TensorDesc.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/TensorDesc.cpp index b8382d59b2..513bc125b9 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/TensorDesc.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/TensorDesc.cpp @@ -201,62 +201,6 @@ TensorDesc::TensorDesc( assert(m_bufferTensorDesc.TotalTensorSizeInBytes >= ComputeByteSizeFromDimensions(nonBroadcastDimensions, dataType)); } -void TensorDesc::Remap64bitDmlDataTypeTo32bit() -{ - if (m_bufferTensorDesc.DataType != DML_TENSOR_DATA_TYPE_UINT64 && - m_bufferTensorDesc.DataType != DML_TENSOR_DATA_TYPE_INT64) - { - return; // Nothing to do. - } - - uint64_t endPaddingInBytes = 0; - - // A workaround for older devices is to use strides to fake 64-bit memory access - // while only the lower 32 bits contains the data. This trick obviously doesn't - // work if the data element is genuine 64-bit. It also doesn't work if the data - // element is negative as the signed bit will be incorrectly interpreted. - m_bufferTensorDesc.DataType = Dml::Remap64bitDmlDataTypeTo32bit(m_bufferTensorDesc.DataType); - - // If the strides haven't been calculated yet, initialize them as packed. - if (m_bufferTensorDesc.Strides == nullptr) - { - uint32_t stride = 1; - for (int i = m_bufferTensorDesc.DimensionCount - 1; i >= 0; i--) - { - m_strides[i] = stride; - stride *= m_sizes[i]; - } - } - - // Double the stride values to emulate 64-bit integer support. - for (uint32_t i = 0; i < m_bufferTensorDesc.DimensionCount; ++i) - { - m_strides[i] *= 2; - } - - // The physical size of the tensor will have an extra 4 bytes at the end. - // DMLCalcBufferTensorSize calculates the minimum implied size, which is based on the last - // addressable element of the tensor plus the space for the last element. However, the size - // of the last element is now halved from 8 bytes to 4 bytes. - // - // Example: - // Original Tensor: size={2,3}, strides={3,1}, type=int64, size = (1+{1,2}*{3,1})*sizeof(int64) = 6 * 8 = 48 - // Emulated Tensor: size={2,3}, strides={6,2}, type=int32, size = (1+{1,2}*{6,2})*sizeof(int32) = 11 * 4 = 44 - // - // DirectML itself won't read/write the last 4 bytes, but we want the total size to be accurate - // so that the entire region can be zeroed. - endPaddingInBytes = sizeof(uint32_t); - - m_bufferTensorDesc.Strides = m_strides; - - m_bufferTensorDesc.TotalTensorSizeInBytes = DMLCalcBufferTensorSize( - m_bufferTensorDesc.DataType, - m_bufferTensorDesc.DimensionCount, - m_sizes, - m_strides - ) + endPaddingInBytes; -} - gsl::span TensorDesc::GetStrides() const { if (m_bufferTensorDesc.Strides == nullptr) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/TensorDesc.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/TensorDesc.h index b48725eb11..867b0c29c0 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/TensorDesc.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/TensorDesc.h @@ -37,7 +37,6 @@ namespace Dml inline DML_TENSOR_DATA_TYPE GetDmlDataType() const { return m_bufferTensorDesc.DataType; } inline MLOperatorTensorDataType GetMlOperatorDataType() const { return m_mlOperatorTensorDataType; } void ForceUnsignedDataType(); - void Remap64bitDmlDataTypeTo32bit(); inline bool IsValid() const { return m_tensorType != DML_TENSOR_TYPE_INVALID; } inline uint32_t GetDimensionCount() const { return m_bufferTensorDesc.DimensionCount; } diff --git a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.h b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.h index 7a3dd34760..609a0c33ec 100644 --- a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.h +++ b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.h @@ -1469,6 +1469,7 @@ using ShapeInferenceHelper_Gelu = GetOutputShapeAsInputShapeHelper; using ShapeInferenceHelper_Identity7 = GetOutputShapeAsInputShapeHelper; using ShapeInferenceHelper_Identity13 = GetOutputShapeAsInputShapeHelper; +using ShapeInferenceHelper_Identity14 = GetOutputShapeAsInputShapeHelper; using ShapeInferenceHelper_MatMul = MatMulHelper; using ShapeInferenceHelper_MatMulInteger = MatMulHelper; using ShapeInferenceHelper_QLinearMatMul = QLinearMatMulHelper; @@ -1489,7 +1490,8 @@ using ShapeInferenceHelper_RandomNormalLike = GetOutputShapeAsInputShapeHelper; using ShapeInferenceHelper_Multinomial = MultinomialHelper; using ShapeInferenceHelper_ReverseSequence = GetOutputShapeAsInputShapeHelper; -using ShapeInferenceHelper_CumSum = GetOutputShapeAsInputShapeHelper; +using ShapeInferenceHelper_CumSum11 = GetOutputShapeAsInputShapeHelper; +using ShapeInferenceHelper_CumSum14 = GetOutputShapeAsInputShapeHelper; using ShapeInferenceHelper_Range = RangeHelper; using ShapeInferenceHelper_FusedConv = ConvHelper; diff --git a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorVersions.h b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorVersions.h index 4ba4f944bd..e01fbb1ea4 100644 --- a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorVersions.h +++ b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorVersions.h @@ -338,6 +338,17 @@ namespace OperatorHelper static const int sc_sinceVer_ReduseSum = 13; } // namespace OnnxOperatorSet13 + namespace OnnxOperatorSet14 + { + static const int sc_sinceVer_Add = 14; + static const int sc_sinceVer_CumSum = 14; + static const int sc_sinceVer_Div = 14; + static const int sc_sinceVer_Identity = 14; + static const int sc_sinceVer_Mul = 14; + static const int sc_sinceVer_Relu = 14; + static const int sc_sinceVer_Sub = 14; + } // namespace OnnxOperatorSet14 + namespace MsftOperatorSet1 { static const int sc_sinceVer_FusedConv = 1;