From 915931384a1f78357b0408791ebcdafdb6da7f6c Mon Sep 17 00:00:00 2001 From: Adrian Tsai Date: Thu, 18 Mar 2021 00:42:32 +0000 Subject: [PATCH 1/5] Merged PR 5807585: Remove support for strided 64-bit emulation in DML's Cast kernel A model from one of our partners regressed with a failure to evaluate due to the addition of strided 64-bit emulation in the DML EP for the Cast operator. Specifically, the model uses a Cast from int32 to int64 to produce the input shape to a Reshape node. When supplied with a shape dimension of -1 (int32 0xffffffff), the strided emulation in Cast ends up producing an int64 result of 0x00000000ffffffff. This is then fed into the Reshape operator, where it produces an incorrect tensor shape and a failure during evaluation. Generally speaking we assume that using strided 64-bit emulation is safe if a node's inputs came from the DML EP itself. This isn't true in the general case for Cast, however - casting negative signed values can and will produce incorrect outputs with strided emulation. After this change, Cast nodes with 64-bit types will fall back to CPU unless running on a GPU that native supports 64-bit datatypes. Related work items: #31768166 --- .../src/Operators/DmlOperatorCast.cpp | 20 +------------------ .../src/Operators/OperatorRegistration.cpp | 4 ++-- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCast.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCast.cpp index 51f93d8683..33b39e6ad7 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCast.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCast.cpp @@ -13,17 +13,10 @@ public: DmlOperatorCast( const MLOperatorKernelCreationContext& kernelInfo - ) : DmlOperator(kernelInfo), - m_toDataType(static_cast(kernelInfo.GetAttribute(AttrName::To))) + ) : DmlOperator(kernelInfo) { Initialize(kernelInfo); - // Zero the output tensor's memory for 64-bit integer emulation with strides. - if (m_toDataType == MLOperatorTensorDataType::UInt64 || m_toDataType == MLOperatorTensorDataType::Int64) - { - m_zeroOperator = InitializeZeroInt64Tensor(m_outputTensorDescs[0].GetBufferSizeInBytes()); - } - std::vector inputDescs = GetDmlInputDescs(); std::vector outputDescs = GetDmlOutputDescs(); @@ -41,23 +34,12 @@ public: std::vector inputTensors = GetInputTensorsForExecute(kernelContext); std::vector outputTensors = GetOutputTensorsForExecute(kernelContext); - // Zero the output tensor's memory for 64-bit integer emulation with strides. - if (m_zeroOperator) - { - assert(m_toDataType == MLOperatorTensorDataType::UInt64 || m_toDataType == MLOperatorTensorDataType::Int64); - ExecuteZeroInt64Tensor(m_zeroOperator.Get(), outputTensors[0]); - } - THROW_IF_FAILED(m_executionProvider->ExecuteOperator( m_compiledOperator.Get(), m_persistentResourceBinding ? &*m_persistentResourceBinding : nullptr, gsl::make_span(inputTensors), gsl::make_span(outputTensors))); } - -private: - MLOperatorTensorDataType m_toDataType; - ComPtr m_zeroOperator; }; DML_OP_DEFINE_CREATION_FUNCTION(Cast, DmlOperatorCast); diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp index 3292e9d6fb..50897f1665 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp @@ -544,8 +544,8 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation // Uncategorized {REG_INFO( 7, MatMul, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 9, MatMul, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, - {REG_INFO( 7, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported|DmlGraphSupport::Prefer64BitTensorsDirectly|DmlGraphSupport::SupportedWith64BitTensorsVia32BitStrides)}, - {REG_INFO( 9, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported|DmlGraphSupport::Prefer64BitTensorsDirectly|DmlGraphSupport::SupportedWith64BitTensorsVia32BitStrides)}, + {REG_INFO( 7, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported)}, + {REG_INFO( 9, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported)}, {REG_INFO( 7, MemcpyFromHost, typeNameListDefault, supportedTypeListAll)}, {REG_INFO( 7, MemcpyToHost, typeNameListDefault, supportedTypeListAll)}, {REG_INFO_VER( 7, TopK, typeNameListTopK, supportedTypeListTopK, DmlGraphSupport::Supported|DmlGraphSupport::SupportedWith64BitTensorsVia32BitStrides)}, From f87527c0dfaec649cd4f4c4ee8de9a57e986770a Mon Sep 17 00:00:00 2001 From: Jeff Bloomfield Date: Wed, 31 Mar 2021 19:06:08 +0000 Subject: [PATCH 2/5] Merged PR 5861108: Allow nodes in DML graph partitions with empty shapes on constant CPU inputs Resize is spec'd to ignore the "roi" tensor in certain modes. For some reason, converters are specifying an arbitrary value for this tensor, even though it's optional. This makes the graph partitioner skip a check for empty shape dimensions for tensors such as this, which the DML kernel registers as consuming as CPU inputs. Otherwise, the node is not included in DML graph partitions, because the DML graph doesn't handle empty dimensions. Related work items: #32221164 --- .../dml/DmlExecutionProvider/src/GraphPartitioner.cpp | 4 ++-- .../dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp | 7 ++++--- .../dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp index 1340ff7c8a..32ba2abe50 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp @@ -477,9 +477,9 @@ namespace Dml std::optional requiredInputCount = internalRegInfo->graphNodeFactoryRegistration->requiredInputCount; if (requiredCpuInputsConstant && TryGetStaticInputShapes( node, graphNodeProperty.first->second.inputShapes) && - !ContainsEmptyDimensions(graphNodeProperty.first->second.inputShapes) && + !ContainsEmptyDimensions(graphNodeProperty.first->second.inputShapes, internalRegInfo->requiredConstantCpuInputs) && TryGetStaticOutputShapes(node, graphNodeProperty.first->second.outputShapes) && - !ContainsEmptyDimensions(graphNodeProperty.first->second.outputShapes) && + !ContainsEmptyDimensions(graphNodeProperty.first->second.outputShapes, internalRegInfo->requiredConstantCpuInputs) && (requiredInputCount == std::nullopt || *requiredInputCount == node.InputDefs().size())) { *isDmlGraphNode = true; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp index 47cbcae00f..d49bcfb370 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp @@ -1876,12 +1876,13 @@ bool TryGetStaticOutputShapes(const onnxruntime::Node& node, EdgeShapes& outputS return true; } -bool ContainsEmptyDimensions(const EdgeShapes& shapes) { +bool ContainsEmptyDimensions(const EdgeShapes& shapes, gsl::span ignoredShapeIndices) { for (size_t i = 0; i < shapes.EdgeCount(); i++) { const std::vector& shape = shapes.GetShape(i); - if (std::find(shape.begin(), shape.end(), 0) != shape.end()) { - return true; + if (std::find(shape.begin(), shape.end(), 0) != shape.end() && + std::find(ignoredShapeIndices.begin(), ignoredShapeIndices.end(), i) == ignoredShapeIndices.end()) { + return true; } } diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h index 0168da24ef..216faead4c 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h @@ -637,7 +637,7 @@ onnx::AttributeProto_AttributeType ToProto(MLOperatorAttributeType type); bool TryGetStaticInputShapes(const onnxruntime::Node& node, EdgeShapes& inputShapes); bool TryGetStaticOutputShapes(const onnxruntime::Node& node, EdgeShapes& outputShapes); -bool ContainsEmptyDimensions(const EdgeShapes& shapes); +bool ContainsEmptyDimensions(const EdgeShapes& shapes, gsl::span ignoredShapeIndices = gsl::span()); std::tuple, size_t> UnpackTensor(const onnx::TensorProto& initializer); } // namespace Windows::AI::MachineLearning::Adapter From e6f35cc132718a01dfe2337729044c276b1a7715 Mon Sep 17 00:00:00 2001 From: Jeff Bloomfield Date: Thu, 1 Apr 2021 00:24:38 +0000 Subject: [PATCH 3/5] Merged PR 5866812: Decompose unsupported QLinearSigmoid operation in DML EP Related work items: #32220862 --- .../src/GraphTransformer.cpp | 124 ++++++++++++++++-- .../src/GraphTransformer.h | 2 + 2 files changed, 118 insertions(+), 8 deletions(-) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp index 112e7279d0..220de8fb60 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp @@ -27,7 +27,8 @@ namespace Dml onnxruntime::common::Status GraphTransformer::ApplyImpl( onnxruntime::Graph& graph, bool& modified, - int graph_level, const onnxruntime::logging::Logger&) const { + int graph_level, const onnxruntime::logging::Logger&) const + { modified = false; // Perform fusion @@ -36,8 +37,13 @@ namespace Dml PerformOperatorFusion(&graph, &transformModifiedGraph); modified |= transformModifiedGraph; - if (modified) { - ORT_RETURN_IF_ERROR(graph.Resolve()); + transformModifiedGraph = false; + PerformQuantizedOperatorDecomposition(&graph, &transformModifiedGraph); + modified |= transformModifiedGraph; + + if (modified) + { + ORT_RETURN_IF_ERROR(graph.Resolve()); } } @@ -110,9 +116,10 @@ namespace Dml // We need to predict whether the nodes will be assigned to the DML transformer by Lotus, // which occurs in IExecutionProvider::GetCapability. - if (!onnxruntime::KernelRegistry::HasImplementationOf(*registry, outputNode, onnxruntime::kDmlExecutionProvider)) { - // Can't fuse nodes that don't belong to this execution provider - continue; + if (!onnxruntime::KernelRegistry::HasImplementationOf(*registry, outputNode, onnxruntime::kDmlExecutionProvider)) + { + // Can't fuse nodes that don't belong to this execution provider + continue; } if (outputNode.InputDefs().size() != 1) @@ -160,12 +167,14 @@ namespace Dml fusedNode.activationAttributes = activationNode.GetAttributes(); // Inputs to the fused node are the inputs to the fuseable node - for (const auto *input : fuseableNode.InputDefs()) { + for (const auto *input : fuseableNode.InputDefs()) + { fusedNode.inputs.push_back(graph->GetNodeArg(input->Name())); } // Outputs from the fused node are the outputs to the activation node - for (const auto *output : activationNode.OutputDefs()){ + for (const auto *output : activationNode.OutputDefs()) + { fusedNode.outputs.push_back(graph->GetNodeArg(output->Name())); } @@ -210,4 +219,103 @@ namespace Dml } } + // Converts certain QLinear operations unsupported by the DML API into a sequence of DeQuantizeLinear, 32-bit operator, QuantizeLinear + void GraphTransformer::PerformQuantizedOperatorDecomposition(onnxruntime::Graph* graph, bool* modified) const + { + struct NodeToAdd + { + std::string name; + std::string description; + std::string opType; + std::string domain; + onnxruntime::NodeAttributes attributes; + std::vector inputs; + std::vector outputs; + }; + + // Defer adding and removing nodes in the graph until after we're done iterating over it, because we can't mutate the + // graph while iterating over it + std::vector nodesToAdd; + std::vector nodesToRemove; + + for (auto& node : graph->Nodes()) + { + // For now, only QLinearSigmoid is handled + if (node.Domain() == onnxruntime::kMSDomain && + node.OpType() == "QLinearSigmoid") + { + // Intermediate node arg type proto with floating point format + onnx::TypeProto floatTensorProto; + floatTensorProto.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_FLOAT); + + // Add intermediate graph edges for the input and output of the FP32 sigmoid operator + auto* sigmoidInputArg = &graph->GetOrCreateNodeArg("decomposed_QLinearSigmoid_input_" + GetUniqueNodeName(&node), &floatTensorProto); + auto* sigmoidOutputArg = &graph->GetOrCreateNodeArg("decomposed_QLinearSigmoid_output_" + GetUniqueNodeName(&node), &floatTensorProto); + + { + NodeToAdd dequantizeNode; + dequantizeNode.name = "decomposed_QLinearSigmoid_DequantizeLinear_" + GetUniqueNodeName(&node); + dequantizeNode.description = ""; + dequantizeNode.opType = "DequantizeLinear"; + dequantizeNode.domain = ""; + + dequantizeNode.inputs.push_back(graph->GetNodeArg(node.InputDefs()[0]->Name())); + dequantizeNode.inputs.push_back(graph->GetNodeArg(node.InputDefs()[1]->Name())); + dequantizeNode.inputs.push_back(graph->GetNodeArg(node.InputDefs()[2]->Name())); + dequantizeNode.outputs.push_back(sigmoidInputArg); + + nodesToAdd.push_back(std::move(dequantizeNode)); + } + + { + NodeToAdd sigmoidNode; + sigmoidNode.name = "decomposed_QLinearSigmoid_Sigmoid_" + GetUniqueNodeName(&node); + sigmoidNode.description = ""; + sigmoidNode.opType = "Sigmoid"; + sigmoidNode.domain = ""; + sigmoidNode.inputs.push_back(sigmoidInputArg); + sigmoidNode.outputs.push_back(sigmoidOutputArg); + nodesToAdd.push_back(std::move(sigmoidNode)); + } + + { + NodeToAdd quantizeNode; + quantizeNode.name = "decomposed_QLinearSigmoid_QuantizeLinear_" + GetUniqueNodeName(&node); + quantizeNode.description = ""; + quantizeNode.opType = "QuantizeLinear"; + quantizeNode.domain = ""; + + quantizeNode.inputs.push_back(sigmoidOutputArg); + quantizeNode.inputs.push_back(graph->GetNodeArg(node.InputDefs()[3]->Name())); + quantizeNode.inputs.push_back(graph->GetNodeArg(node.InputDefs()[4]->Name())); + quantizeNode.outputs.push_back(graph->GetNodeArg(node.OutputDefs()[0]->Name())); + + nodesToAdd.push_back(std::move(quantizeNode)); + } + + nodesToRemove.push_back(node.Index()); + *modified = true; + } + } + + for (auto& nodeToAdd : nodesToAdd) + { + auto& node = graph->AddNode( + nodeToAdd.name, + nodeToAdd.opType, + nodeToAdd.description, + nodeToAdd.inputs, + nodeToAdd.outputs, + &nodeToAdd.attributes, + nodeToAdd.domain); + } + + for (const auto& nodeIndex : nodesToRemove) + { + onnxruntime::Node* node = graph->GetNode(nodeIndex); + onnxruntime::graph_utils::RemoveNodeOutputEdges(*graph, *node); + graph->RemoveNode(node->Index()); + } + } + } // namespace Dml diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h index af2cc456d5..a87c9b2314 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h @@ -28,6 +28,8 @@ namespace Dml private: void PerformOperatorFusion(onnxruntime::Graph* graph, bool* modified) const; + void PerformQuantizedOperatorDecomposition(onnxruntime::Graph* graph, bool* modified) const; + std::shared_ptr m_registry; uint32_t m_supportedDataTypeMask = 0; const ExecutionProviderImpl* m_providerImpl = nullptr; From 06a2b0401a9b6faa53ff94ce7d71259b8285b466 Mon Sep 17 00:00:00 2001 From: Dwayne Robinson Date: Fri, 2 Apr 2021 00:28:27 +0000 Subject: [PATCH 4/5] Merged PR 5873494: Resize support nearest_mode floor in DML EP Resize support nearest_mode floor in DML EP. Related work items: #32221069 --- .../src/Operators/DmlOperatorResize.cpp | 64 +++++++++++++++---- .../src/Operators/DmlOperatorRoiAlign.cpp | 5 +- .../src/Operators/OperatorUtility.cpp | 16 +++-- .../src/Operators/OperatorUtility.h | 8 ++- 4 files changed, 73 insertions(+), 20 deletions(-) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorResize.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorResize.cpp index a3547188f0..96c6557c01 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorResize.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorResize.cpp @@ -6,6 +6,24 @@ namespace Dml { +constexpr NameAndIndex coordinateTransformationModes[] = +{ + {"half_pixel", 0}, + {"pytorch_half_pixel", 1}, + {"align_corners", 2}, + {"asymmetric", 3}, + {"tf_half_pixel_for_nn", 4}, + {"tf_crop_and_resize", 5}, +}; + +constexpr NameAndIndex nearestNeighborRoundingModes[] = +{ + {"", 0}, + {"round_prefer_floor", 0}, + {"round_prefer_ceil", 1}, + {"floor", 2}, +}; + void ComputePixelOffsetsAndScales( const MLOperatorKernelCreationContext& kernelCreationContext, gsl::span regionOfInterest, // May be empty depending on mode. @@ -23,17 +41,12 @@ void ComputePixelOffsetsAndScales( assert(regionOfInterest.empty() || regionOfInterest.size() == inputDimensions.size() * 2); std::string coordinateTransformationMode = kernelCreationContext.GetOptionalAttribute(AttrName::CoordinateTransformationMode, "half_pixel"); - uint32_t coordinateTransformationModeValue = UINT32_MAX; - - const char* modes[] = { "half_pixel", "pytorch_half_pixel", "align_corners", "asymmetric", "tf_half_pixel_for_nn", "tf_crop_and_resize" }; - for (uint32_t i = 0; i < std::size(modes); ++i) + auto optionalCoordinateTransformationModeValue = TryMapStringToIndex(coordinateTransformationMode, coordinateTransformationModes); + if (!optionalCoordinateTransformationModeValue) { - if (strcmp(modes[i], coordinateTransformationMode.c_str()) == 0) - { - coordinateTransformationModeValue = i; - break; - } + ML_INVALID_ARGUMENT("Unsupported 'coordinate_transformation_mode'"); } + uint32_t coordinateTransformationModeValue = *optionalCoordinateTransformationModeValue; ML_CHECK_VALID_ARGUMENT( !regionOfInterest.empty() || coordinateTransformationModeValue != 5 /*tf_crop_and_resize*/, @@ -150,7 +163,7 @@ void ComputePixelOffsetsAndScales( break; default: - ML_INVALID_ARGUMENT("Unknown 'coordinate_transformation_mode'"); + assert(false); // TryMapStringToIndex would have already bailed above. } inputPixelOffsets[i] = inputPixelOffset; @@ -233,6 +246,34 @@ public: std::string mode = kernelCreationContext.GetOptionalAttribute(AttrName::Mode, "NEAREST"); DML_INTERPOLATION_MODE interpolationMode = Dml::MapStringToInteropolationMode(mode); + // DML's nearest neighbor mode uses round-halves-up (or round_prefer_ceil) via floor(input.x + 0.5). + // So to support floor, adjust the input by half a pixel. + // round_prefer_floor is not supported without an API extension, + // but existing code already default to treating it as round_prefer_ceil. + // So continue that. + if (interpolationMode == DML_INTERPOLATION_MODE_NEAREST_NEIGHBOR) + { + std::string nearestMode = kernelCreationContext.GetOptionalAttribute(AttrName::NearestMode, "round_prefer_floor"); + auto optionalNearestModeValue = TryMapStringToIndex(nearestMode, nearestNeighborRoundingModes); + if (optionalNearestModeValue) + { + switch (*optionalNearestModeValue) + { + case 0: // round_prefer_floor + case 1: // round_prefer_ceil + break; + case 2: // floor + for (auto& offset : inputPixelOffsets) + { + offset += 0.5; + } + break; + default: + assert(false); + } + } + } + // Create the operator description. std::vector inputDescs = GetDmlInputDescs(); std::vector outputDescs = GetDmlOutputDescs(); @@ -282,7 +323,8 @@ void CALLBACK QueryResize(IMLOperatorSupportQueryContextPrivate* context, bool* // DML's nearest neighbor mode uses half pixels rounded down. std::string nearestMode = attributes.GetOptionalAttribute(AttrName::NearestMode, "round_prefer_floor"); - if (nearestMode != "round_prefer_floor") + auto optionalNearestModeValue = TryMapStringToIndex(nearestMode, nearestNeighborRoundingModes); + if (!optionalNearestModeValue) { return; } diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorRoiAlign.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorRoiAlign.cpp index 75e7595a3a..533c894119 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorRoiAlign.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorRoiAlign.cpp @@ -31,10 +31,11 @@ public: {"avg", DML_REDUCE_FUNCTION_AVERAGE}, }; const std::string mode = kernelCreationContext.GetOptionalAttribute(AttrName::Mode, "avg"); - const auto reductionFunction = MapStringToIndex(mode, mapping); + const auto optionalReductionFunction = TryMapStringToIndex(mode, mapping); const float spatialScale = kernelCreationContext.GetOptionalAttribute(AttrName::SpatialScale, 1.0f); const int32_t samplesPerOutput = kernelCreationContext.GetOptionalAttribute(AttrName::SamplingRatio, 0u); ML_CHECK_VALID_ARGUMENT(samplesPerOutput >= 0, "sampling_ratio must be 0 or positive."); + ML_CHECK_VALID_ARGUMENT(!!optionalReductionFunction, "Unsupported RoiAlign mode."); DML_ROI_ALIGN_OPERATOR_DESC operatorDesc = {}; operatorDesc.InputTensor = &inputDescs[0]; @@ -46,7 +47,7 @@ public: operatorDesc.OutOfBoundsInputValue = 0.0f; // ONNX does not specify a value for input elements outside bounds. operatorDesc.MinimumSamplesPerOutput = (samplesPerOutput == 0) ? 1 : samplesPerOutput; operatorDesc.MaximumSamplesPerOutput = (samplesPerOutput == 0) ? UINT32_MAX : samplesPerOutput; - operatorDesc.ReductionFunction = reductionFunction; + operatorDesc.ReductionFunction = *optionalReductionFunction; operatorDesc.InterpolationMode = DML_INTERPOLATION_MODE_LINEAR; DML_OPERATOR_DESC opDesc = { DML_OPERATOR_ROI_ALIGN, &operatorDesc }; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp index afbe678b60..e8bd9bd952 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp @@ -359,7 +359,7 @@ namespace Dml } } - uint32_t MapStringToIndex(std::string_view mode, gsl::span nameAndIndexList) + std::optional TryMapStringToIndex(std::string_view mode, gsl::span nameAndIndexList) { for (auto& nameAndIndex : nameAndIndexList) { @@ -369,7 +369,7 @@ namespace Dml } } - ML_INVALID_ARGUMENT("Unknown mode value."); + return {}; } DML_INTERPOLATION_MODE MapStringToInteropolationMode(std::string_view mode) @@ -387,7 +387,11 @@ namespace Dml {"BILINEAR", DML_INTERPOLATION_MODE_LINEAR}, {"bilinear", DML_INTERPOLATION_MODE_LINEAR}, }; - return MapStringToIndex(mode, mapping); + if (auto index = TryMapStringToIndex(mode, mapping)) + { + return *index; + } + ML_INVALID_ARGUMENT("Unknown interpolation mode"); } DML_DEPTH_SPACE_ORDER MapStringToDepthSpaceMode(std::string_view mode) @@ -397,7 +401,11 @@ namespace Dml {"DCR", DML_DEPTH_SPACE_ORDER_DEPTH_COLUMN_ROW}, {"CRD", DML_DEPTH_SPACE_ORDER_COLUMN_ROW_DEPTH}, }; - return MapStringToIndex(mode, mapping); + if (auto index = TryMapStringToIndex(mode, mapping)) + { + return *index; + } + ML_INVALID_ARGUMENT("Unknown depth/space order"); } } // namespace Dml diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h index 027c2228bb..a638326b2a 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h @@ -64,12 +64,14 @@ namespace Dml }; template - T MapStringToIndex(std::string_view mode, gsl::span nameAndIndexList) + std::optional TryMapStringToIndex(std::string_view mode, gsl::span nameAndIndexList) { - return static_cast(MapStringToIndex(mode, nameAndIndexList)); + static_assert(sizeof(T) == sizeof(uint32_t)); + auto result = TryMapStringToIndex(mode, nameAndIndexList); + return *reinterpret_cast*>(std::addressof(result)); } - uint32_t MapStringToIndex(std::string_view mode, gsl::span nameAndIndexList); + std::optional TryMapStringToIndex(std::string_view mode, gsl::span nameAndIndexList); DML_INTERPOLATION_MODE MapStringToInteropolationMode(std::string_view mode); From 10e67b7340b9cc5f03feed5c9a0453ed1c4af967 Mon Sep 17 00:00:00 2001 From: Jeff Bloomfield Date: Mon, 12 Apr 2021 23:14:25 +0000 Subject: [PATCH 5/5] Merged PR 5918130: Add activation fusions missing in newer opsets Related work items: #32473540 --- .../src/Operators/OperatorRegistration.cpp | 6 +++ .../src/Operators/OperatorUtility.cpp | 44 +++++++++++++++++++ .../src/Operators/OperatorUtility.h | 7 +++ 3 files changed, 57 insertions(+) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp index 50897f1665..c2aac0bddd 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp @@ -620,6 +620,12 @@ void RegisterDmlOperators(IMLOperatorRegistry* registry) desc.typeConstraints = typeConstraints.data(); desc.typeConstraintCount = static_cast(typeConstraints.size()); +#if _DEBUG + // If some version of the operator is supported for fusion, check that each registered version is also supported. + // This ensures that table of operators and versions supporting fusion does not become stale as operator sets are added. + FusionHelpers::AssertFusableOperatorSupportsVersionIfExists(desc.name, desc.domain, desc.minimumOperatorSetVersion); +#endif + // edgeDescs will accumulate the edge descriptions across all type constraints. // The values of allowedTypeCount will indicate how many elements of edgeDescs // belong to each type constraint. diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp index e8bd9bd952..5e2cfd4ccc 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp @@ -143,12 +143,19 @@ namespace Dml static const OperatorInfo c_fusableOps[] = { OperatorInfo{ "Conv", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Conv }, + OperatorInfo{ "Conv", onnxruntime::kOnnxDomain, OnnxOperatorSet11::sc_sinceVer_Conv }, OperatorInfo{ "ConvTranspose", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_ConvTranspose }, + OperatorInfo{ "ConvTranspose", onnxruntime::kOnnxDomain, OnnxOperatorSet11::sc_sinceVer_ConvTranspose }, OperatorInfo{ "BatchNormalization", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_BatchNormalization }, + OperatorInfo{ "BatchNormalization", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_BatchNormalization }, OperatorInfo{ "InstanceNormalization", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_InstanceNormalization }, OperatorInfo{ "MeanVarianceNormalization", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_MeanVarianceNormalization }, + OperatorInfo{ "MeanVarianceNormalization", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_MeanVarianceNormalization }, OperatorInfo{ "Gemm", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Gemm }, + OperatorInfo{ "Gemm", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_Gemm }, + OperatorInfo{ "Gemm", onnxruntime::kOnnxDomain, OnnxOperatorSet11::sc_sinceVer_Gemm }, OperatorInfo{ "MatMul", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_MatMul }, + OperatorInfo{ "MatMul", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_MatMul }, // The filter for activation functions maps to what DML's fused op internally fuses at the shader level. OperatorInfo{ "Add", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Add, {"Relu", "LeakyRelu"} }, @@ -166,7 +173,9 @@ namespace Dml OperatorInfo{ "Relu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Relu }, OperatorInfo{ "LeakyRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_LeakyRelu }, OperatorInfo{ "PRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_PRelu }, + OperatorInfo{ "PRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_PRelu }, OperatorInfo{ "ThresholdedRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_ThresholdedRelu }, + OperatorInfo{ "ThresholdedRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet10::sc_sinceVer_ThresholdedRelu }, OperatorInfo{ "Elu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Elu }, OperatorInfo{ "Selu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Selu }, OperatorInfo{ "Softsign", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Softsign }, @@ -330,6 +339,41 @@ namespace Dml return std::string("fused_").append(name); } +#if _DEBUG + // This asserts that an exact match for the operator exists in the tables of fusable ops, if a prior version exists + void AssertFusableOperatorSupportsVersionIfExists( + std::string_view type, + std::string_view domain, + int sinceVersion) + { + for (const OperatorInfo& operatorInfo : c_fusableOps) + { + if (operatorInfo.type == type && operatorInfo.domain == domain && operatorInfo.sinceVersion < sinceVersion) + { + assert(std::end(c_fusableOps) != std::find( + std::begin(c_fusableOps), + std::end(c_fusableOps), + OperatorInfo{ type, domain, sinceVersion })); + + break; + } + } + + for (const OperatorInfo& operatorInfo : c_activationOps) + { + if (operatorInfo.type == type && operatorInfo.domain == domain && operatorInfo.sinceVersion < sinceVersion) + { + assert(std::end(c_activationOps) != std::find( + std::begin(c_activationOps), + std::end(c_activationOps), + OperatorInfo{ type, domain, sinceVersion })); + + break; + } + } + } +#endif + } // namespace FusionHelpers uint32_t GetDmlAdjustedAxis(int32_t onnxAxis, const MLOperatorKernelCreationContext& kernelCreationContext, uint32_t dmlDimCount) diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h index a638326b2a..a0e5d8aaeb 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h @@ -47,6 +47,13 @@ namespace Dml // produce the attribute for Activation in a fused Conv+Activation kernel. std::string GetFusedAttributeName(std::string_view name); +#if _DEBUG + void AssertFusableOperatorSupportsVersionIfExists( + std::string_view type, + std::string_view domain, + int sinceVersion); +#endif + } // namespace FusionHelpers // Given an axis in ONNX axis numbering, return the axis adjusted for DML based on how the sizes have been coerced.