diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp index 1340ff7c8a..32ba2abe50 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphPartitioner.cpp @@ -477,9 +477,9 @@ namespace Dml std::optional requiredInputCount = internalRegInfo->graphNodeFactoryRegistration->requiredInputCount; if (requiredCpuInputsConstant && TryGetStaticInputShapes( node, graphNodeProperty.first->second.inputShapes) && - !ContainsEmptyDimensions(graphNodeProperty.first->second.inputShapes) && + !ContainsEmptyDimensions(graphNodeProperty.first->second.inputShapes, internalRegInfo->requiredConstantCpuInputs) && TryGetStaticOutputShapes(node, graphNodeProperty.first->second.outputShapes) && - !ContainsEmptyDimensions(graphNodeProperty.first->second.outputShapes) && + !ContainsEmptyDimensions(graphNodeProperty.first->second.outputShapes, internalRegInfo->requiredConstantCpuInputs) && (requiredInputCount == std::nullopt || *requiredInputCount == node.InputDefs().size())) { *isDmlGraphNode = true; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp index 112e7279d0..220de8fb60 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.cpp @@ -27,7 +27,8 @@ namespace Dml onnxruntime::common::Status GraphTransformer::ApplyImpl( onnxruntime::Graph& graph, bool& modified, - int graph_level, const onnxruntime::logging::Logger&) const { + int graph_level, const onnxruntime::logging::Logger&) const + { modified = false; // Perform fusion @@ -36,8 +37,13 @@ namespace Dml PerformOperatorFusion(&graph, &transformModifiedGraph); modified |= transformModifiedGraph; - if (modified) { - ORT_RETURN_IF_ERROR(graph.Resolve()); + transformModifiedGraph = false; + PerformQuantizedOperatorDecomposition(&graph, &transformModifiedGraph); + modified |= transformModifiedGraph; + + if (modified) + { + ORT_RETURN_IF_ERROR(graph.Resolve()); } } @@ -110,9 +116,10 @@ namespace Dml // We need to predict whether the nodes will be assigned to the DML transformer by Lotus, // which occurs in IExecutionProvider::GetCapability. - if (!onnxruntime::KernelRegistry::HasImplementationOf(*registry, outputNode, onnxruntime::kDmlExecutionProvider)) { - // Can't fuse nodes that don't belong to this execution provider - continue; + if (!onnxruntime::KernelRegistry::HasImplementationOf(*registry, outputNode, onnxruntime::kDmlExecutionProvider)) + { + // Can't fuse nodes that don't belong to this execution provider + continue; } if (outputNode.InputDefs().size() != 1) @@ -160,12 +167,14 @@ namespace Dml fusedNode.activationAttributes = activationNode.GetAttributes(); // Inputs to the fused node are the inputs to the fuseable node - for (const auto *input : fuseableNode.InputDefs()) { + for (const auto *input : fuseableNode.InputDefs()) + { fusedNode.inputs.push_back(graph->GetNodeArg(input->Name())); } // Outputs from the fused node are the outputs to the activation node - for (const auto *output : activationNode.OutputDefs()){ + for (const auto *output : activationNode.OutputDefs()) + { fusedNode.outputs.push_back(graph->GetNodeArg(output->Name())); } @@ -210,4 +219,103 @@ namespace Dml } } + // Converts certain QLinear operations unsupported by the DML API into a sequence of DeQuantizeLinear, 32-bit operator, QuantizeLinear + void GraphTransformer::PerformQuantizedOperatorDecomposition(onnxruntime::Graph* graph, bool* modified) const + { + struct NodeToAdd + { + std::string name; + std::string description; + std::string opType; + std::string domain; + onnxruntime::NodeAttributes attributes; + std::vector inputs; + std::vector outputs; + }; + + // Defer adding and removing nodes in the graph until after we're done iterating over it, because we can't mutate the + // graph while iterating over it + std::vector nodesToAdd; + std::vector nodesToRemove; + + for (auto& node : graph->Nodes()) + { + // For now, only QLinearSigmoid is handled + if (node.Domain() == onnxruntime::kMSDomain && + node.OpType() == "QLinearSigmoid") + { + // Intermediate node arg type proto with floating point format + onnx::TypeProto floatTensorProto; + floatTensorProto.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_FLOAT); + + // Add intermediate graph edges for the input and output of the FP32 sigmoid operator + auto* sigmoidInputArg = &graph->GetOrCreateNodeArg("decomposed_QLinearSigmoid_input_" + GetUniqueNodeName(&node), &floatTensorProto); + auto* sigmoidOutputArg = &graph->GetOrCreateNodeArg("decomposed_QLinearSigmoid_output_" + GetUniqueNodeName(&node), &floatTensorProto); + + { + NodeToAdd dequantizeNode; + dequantizeNode.name = "decomposed_QLinearSigmoid_DequantizeLinear_" + GetUniqueNodeName(&node); + dequantizeNode.description = ""; + dequantizeNode.opType = "DequantizeLinear"; + dequantizeNode.domain = ""; + + dequantizeNode.inputs.push_back(graph->GetNodeArg(node.InputDefs()[0]->Name())); + dequantizeNode.inputs.push_back(graph->GetNodeArg(node.InputDefs()[1]->Name())); + dequantizeNode.inputs.push_back(graph->GetNodeArg(node.InputDefs()[2]->Name())); + dequantizeNode.outputs.push_back(sigmoidInputArg); + + nodesToAdd.push_back(std::move(dequantizeNode)); + } + + { + NodeToAdd sigmoidNode; + sigmoidNode.name = "decomposed_QLinearSigmoid_Sigmoid_" + GetUniqueNodeName(&node); + sigmoidNode.description = ""; + sigmoidNode.opType = "Sigmoid"; + sigmoidNode.domain = ""; + sigmoidNode.inputs.push_back(sigmoidInputArg); + sigmoidNode.outputs.push_back(sigmoidOutputArg); + nodesToAdd.push_back(std::move(sigmoidNode)); + } + + { + NodeToAdd quantizeNode; + quantizeNode.name = "decomposed_QLinearSigmoid_QuantizeLinear_" + GetUniqueNodeName(&node); + quantizeNode.description = ""; + quantizeNode.opType = "QuantizeLinear"; + quantizeNode.domain = ""; + + quantizeNode.inputs.push_back(sigmoidOutputArg); + quantizeNode.inputs.push_back(graph->GetNodeArg(node.InputDefs()[3]->Name())); + quantizeNode.inputs.push_back(graph->GetNodeArg(node.InputDefs()[4]->Name())); + quantizeNode.outputs.push_back(graph->GetNodeArg(node.OutputDefs()[0]->Name())); + + nodesToAdd.push_back(std::move(quantizeNode)); + } + + nodesToRemove.push_back(node.Index()); + *modified = true; + } + } + + for (auto& nodeToAdd : nodesToAdd) + { + auto& node = graph->AddNode( + nodeToAdd.name, + nodeToAdd.opType, + nodeToAdd.description, + nodeToAdd.inputs, + nodeToAdd.outputs, + &nodeToAdd.attributes, + nodeToAdd.domain); + } + + for (const auto& nodeIndex : nodesToRemove) + { + onnxruntime::Node* node = graph->GetNode(nodeIndex); + onnxruntime::graph_utils::RemoveNodeOutputEdges(*graph, *node); + graph->RemoveNode(node->Index()); + } + } + } // namespace Dml diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h index af2cc456d5..a87c9b2314 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h @@ -28,6 +28,8 @@ namespace Dml private: void PerformOperatorFusion(onnxruntime::Graph* graph, bool* modified) const; + void PerformQuantizedOperatorDecomposition(onnxruntime::Graph* graph, bool* modified) const; + std::shared_ptr m_registry; uint32_t m_supportedDataTypeMask = 0; const ExecutionProviderImpl* m_providerImpl = nullptr; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp index 47cbcae00f..d49bcfb370 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp @@ -1876,12 +1876,13 @@ bool TryGetStaticOutputShapes(const onnxruntime::Node& node, EdgeShapes& outputS return true; } -bool ContainsEmptyDimensions(const EdgeShapes& shapes) { +bool ContainsEmptyDimensions(const EdgeShapes& shapes, gsl::span ignoredShapeIndices) { for (size_t i = 0; i < shapes.EdgeCount(); i++) { const std::vector& shape = shapes.GetShape(i); - if (std::find(shape.begin(), shape.end(), 0) != shape.end()) { - return true; + if (std::find(shape.begin(), shape.end(), 0) != shape.end() && + std::find(ignoredShapeIndices.begin(), ignoredShapeIndices.end(), i) == ignoredShapeIndices.end()) { + return true; } } diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h index 0168da24ef..216faead4c 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.h @@ -637,7 +637,7 @@ onnx::AttributeProto_AttributeType ToProto(MLOperatorAttributeType type); bool TryGetStaticInputShapes(const onnxruntime::Node& node, EdgeShapes& inputShapes); bool TryGetStaticOutputShapes(const onnxruntime::Node& node, EdgeShapes& outputShapes); -bool ContainsEmptyDimensions(const EdgeShapes& shapes); +bool ContainsEmptyDimensions(const EdgeShapes& shapes, gsl::span ignoredShapeIndices = gsl::span()); std::tuple, size_t> UnpackTensor(const onnx::TensorProto& initializer); } // namespace Windows::AI::MachineLearning::Adapter diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCast.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCast.cpp index 51f93d8683..33b39e6ad7 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCast.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorCast.cpp @@ -13,17 +13,10 @@ public: DmlOperatorCast( const MLOperatorKernelCreationContext& kernelInfo - ) : DmlOperator(kernelInfo), - m_toDataType(static_cast(kernelInfo.GetAttribute(AttrName::To))) + ) : DmlOperator(kernelInfo) { Initialize(kernelInfo); - // Zero the output tensor's memory for 64-bit integer emulation with strides. - if (m_toDataType == MLOperatorTensorDataType::UInt64 || m_toDataType == MLOperatorTensorDataType::Int64) - { - m_zeroOperator = InitializeZeroInt64Tensor(m_outputTensorDescs[0].GetBufferSizeInBytes()); - } - std::vector inputDescs = GetDmlInputDescs(); std::vector outputDescs = GetDmlOutputDescs(); @@ -41,23 +34,12 @@ public: std::vector inputTensors = GetInputTensorsForExecute(kernelContext); std::vector outputTensors = GetOutputTensorsForExecute(kernelContext); - // Zero the output tensor's memory for 64-bit integer emulation with strides. - if (m_zeroOperator) - { - assert(m_toDataType == MLOperatorTensorDataType::UInt64 || m_toDataType == MLOperatorTensorDataType::Int64); - ExecuteZeroInt64Tensor(m_zeroOperator.Get(), outputTensors[0]); - } - THROW_IF_FAILED(m_executionProvider->ExecuteOperator( m_compiledOperator.Get(), m_persistentResourceBinding ? &*m_persistentResourceBinding : nullptr, gsl::make_span(inputTensors), gsl::make_span(outputTensors))); } - -private: - MLOperatorTensorDataType m_toDataType; - ComPtr m_zeroOperator; }; DML_OP_DEFINE_CREATION_FUNCTION(Cast, DmlOperatorCast); diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorResize.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorResize.cpp index a3547188f0..96c6557c01 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorResize.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorResize.cpp @@ -6,6 +6,24 @@ namespace Dml { +constexpr NameAndIndex coordinateTransformationModes[] = +{ + {"half_pixel", 0}, + {"pytorch_half_pixel", 1}, + {"align_corners", 2}, + {"asymmetric", 3}, + {"tf_half_pixel_for_nn", 4}, + {"tf_crop_and_resize", 5}, +}; + +constexpr NameAndIndex nearestNeighborRoundingModes[] = +{ + {"", 0}, + {"round_prefer_floor", 0}, + {"round_prefer_ceil", 1}, + {"floor", 2}, +}; + void ComputePixelOffsetsAndScales( const MLOperatorKernelCreationContext& kernelCreationContext, gsl::span regionOfInterest, // May be empty depending on mode. @@ -23,17 +41,12 @@ void ComputePixelOffsetsAndScales( assert(regionOfInterest.empty() || regionOfInterest.size() == inputDimensions.size() * 2); std::string coordinateTransformationMode = kernelCreationContext.GetOptionalAttribute(AttrName::CoordinateTransformationMode, "half_pixel"); - uint32_t coordinateTransformationModeValue = UINT32_MAX; - - const char* modes[] = { "half_pixel", "pytorch_half_pixel", "align_corners", "asymmetric", "tf_half_pixel_for_nn", "tf_crop_and_resize" }; - for (uint32_t i = 0; i < std::size(modes); ++i) + auto optionalCoordinateTransformationModeValue = TryMapStringToIndex(coordinateTransformationMode, coordinateTransformationModes); + if (!optionalCoordinateTransformationModeValue) { - if (strcmp(modes[i], coordinateTransformationMode.c_str()) == 0) - { - coordinateTransformationModeValue = i; - break; - } + ML_INVALID_ARGUMENT("Unsupported 'coordinate_transformation_mode'"); } + uint32_t coordinateTransformationModeValue = *optionalCoordinateTransformationModeValue; ML_CHECK_VALID_ARGUMENT( !regionOfInterest.empty() || coordinateTransformationModeValue != 5 /*tf_crop_and_resize*/, @@ -150,7 +163,7 @@ void ComputePixelOffsetsAndScales( break; default: - ML_INVALID_ARGUMENT("Unknown 'coordinate_transformation_mode'"); + assert(false); // TryMapStringToIndex would have already bailed above. } inputPixelOffsets[i] = inputPixelOffset; @@ -233,6 +246,34 @@ public: std::string mode = kernelCreationContext.GetOptionalAttribute(AttrName::Mode, "NEAREST"); DML_INTERPOLATION_MODE interpolationMode = Dml::MapStringToInteropolationMode(mode); + // DML's nearest neighbor mode uses round-halves-up (or round_prefer_ceil) via floor(input.x + 0.5). + // So to support floor, adjust the input by half a pixel. + // round_prefer_floor is not supported without an API extension, + // but existing code already default to treating it as round_prefer_ceil. + // So continue that. + if (interpolationMode == DML_INTERPOLATION_MODE_NEAREST_NEIGHBOR) + { + std::string nearestMode = kernelCreationContext.GetOptionalAttribute(AttrName::NearestMode, "round_prefer_floor"); + auto optionalNearestModeValue = TryMapStringToIndex(nearestMode, nearestNeighborRoundingModes); + if (optionalNearestModeValue) + { + switch (*optionalNearestModeValue) + { + case 0: // round_prefer_floor + case 1: // round_prefer_ceil + break; + case 2: // floor + for (auto& offset : inputPixelOffsets) + { + offset += 0.5; + } + break; + default: + assert(false); + } + } + } + // Create the operator description. std::vector inputDescs = GetDmlInputDescs(); std::vector outputDescs = GetDmlOutputDescs(); @@ -282,7 +323,8 @@ void CALLBACK QueryResize(IMLOperatorSupportQueryContextPrivate* context, bool* // DML's nearest neighbor mode uses half pixels rounded down. std::string nearestMode = attributes.GetOptionalAttribute(AttrName::NearestMode, "round_prefer_floor"); - if (nearestMode != "round_prefer_floor") + auto optionalNearestModeValue = TryMapStringToIndex(nearestMode, nearestNeighborRoundingModes); + if (!optionalNearestModeValue) { return; } diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorRoiAlign.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorRoiAlign.cpp index 75e7595a3a..533c894119 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorRoiAlign.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorRoiAlign.cpp @@ -31,10 +31,11 @@ public: {"avg", DML_REDUCE_FUNCTION_AVERAGE}, }; const std::string mode = kernelCreationContext.GetOptionalAttribute(AttrName::Mode, "avg"); - const auto reductionFunction = MapStringToIndex(mode, mapping); + const auto optionalReductionFunction = TryMapStringToIndex(mode, mapping); const float spatialScale = kernelCreationContext.GetOptionalAttribute(AttrName::SpatialScale, 1.0f); const int32_t samplesPerOutput = kernelCreationContext.GetOptionalAttribute(AttrName::SamplingRatio, 0u); ML_CHECK_VALID_ARGUMENT(samplesPerOutput >= 0, "sampling_ratio must be 0 or positive."); + ML_CHECK_VALID_ARGUMENT(!!optionalReductionFunction, "Unsupported RoiAlign mode."); DML_ROI_ALIGN_OPERATOR_DESC operatorDesc = {}; operatorDesc.InputTensor = &inputDescs[0]; @@ -46,7 +47,7 @@ public: operatorDesc.OutOfBoundsInputValue = 0.0f; // ONNX does not specify a value for input elements outside bounds. operatorDesc.MinimumSamplesPerOutput = (samplesPerOutput == 0) ? 1 : samplesPerOutput; operatorDesc.MaximumSamplesPerOutput = (samplesPerOutput == 0) ? UINT32_MAX : samplesPerOutput; - operatorDesc.ReductionFunction = reductionFunction; + operatorDesc.ReductionFunction = *optionalReductionFunction; operatorDesc.InterpolationMode = DML_INTERPOLATION_MODE_LINEAR; DML_OPERATOR_DESC opDesc = { DML_OPERATOR_ROI_ALIGN, &operatorDesc }; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp index 3292e9d6fb..c2aac0bddd 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp @@ -544,8 +544,8 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation // Uncategorized {REG_INFO( 7, MatMul, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, {REG_INFO( 9, MatMul, typeNameListDefault, supportedTypeListFloat16to32, DmlGraphSupport::Supported)}, - {REG_INFO( 7, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported|DmlGraphSupport::Prefer64BitTensorsDirectly|DmlGraphSupport::SupportedWith64BitTensorsVia32BitStrides)}, - {REG_INFO( 9, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported|DmlGraphSupport::Prefer64BitTensorsDirectly|DmlGraphSupport::SupportedWith64BitTensorsVia32BitStrides)}, + {REG_INFO( 7, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported)}, + {REG_INFO( 9, Cast, typeNameListTwo, supportedTypeListCast, DmlGraphSupport::Supported)}, {REG_INFO( 7, MemcpyFromHost, typeNameListDefault, supportedTypeListAll)}, {REG_INFO( 7, MemcpyToHost, typeNameListDefault, supportedTypeListAll)}, {REG_INFO_VER( 7, TopK, typeNameListTopK, supportedTypeListTopK, DmlGraphSupport::Supported|DmlGraphSupport::SupportedWith64BitTensorsVia32BitStrides)}, @@ -620,6 +620,12 @@ void RegisterDmlOperators(IMLOperatorRegistry* registry) desc.typeConstraints = typeConstraints.data(); desc.typeConstraintCount = static_cast(typeConstraints.size()); +#if _DEBUG + // If some version of the operator is supported for fusion, check that each registered version is also supported. + // This ensures that table of operators and versions supporting fusion does not become stale as operator sets are added. + FusionHelpers::AssertFusableOperatorSupportsVersionIfExists(desc.name, desc.domain, desc.minimumOperatorSetVersion); +#endif + // edgeDescs will accumulate the edge descriptions across all type constraints. // The values of allowedTypeCount will indicate how many elements of edgeDescs // belong to each type constraint. diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp index afbe678b60..5e2cfd4ccc 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.cpp @@ -143,12 +143,19 @@ namespace Dml static const OperatorInfo c_fusableOps[] = { OperatorInfo{ "Conv", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Conv }, + OperatorInfo{ "Conv", onnxruntime::kOnnxDomain, OnnxOperatorSet11::sc_sinceVer_Conv }, OperatorInfo{ "ConvTranspose", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_ConvTranspose }, + OperatorInfo{ "ConvTranspose", onnxruntime::kOnnxDomain, OnnxOperatorSet11::sc_sinceVer_ConvTranspose }, OperatorInfo{ "BatchNormalization", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_BatchNormalization }, + OperatorInfo{ "BatchNormalization", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_BatchNormalization }, OperatorInfo{ "InstanceNormalization", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_InstanceNormalization }, OperatorInfo{ "MeanVarianceNormalization", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_MeanVarianceNormalization }, + OperatorInfo{ "MeanVarianceNormalization", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_MeanVarianceNormalization }, OperatorInfo{ "Gemm", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Gemm }, + OperatorInfo{ "Gemm", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_Gemm }, + OperatorInfo{ "Gemm", onnxruntime::kOnnxDomain, OnnxOperatorSet11::sc_sinceVer_Gemm }, OperatorInfo{ "MatMul", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_MatMul }, + OperatorInfo{ "MatMul", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_MatMul }, // The filter for activation functions maps to what DML's fused op internally fuses at the shader level. OperatorInfo{ "Add", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Add, {"Relu", "LeakyRelu"} }, @@ -166,7 +173,9 @@ namespace Dml OperatorInfo{ "Relu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Relu }, OperatorInfo{ "LeakyRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_LeakyRelu }, OperatorInfo{ "PRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_PRelu }, + OperatorInfo{ "PRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet9::sc_sinceVer_PRelu }, OperatorInfo{ "ThresholdedRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_ThresholdedRelu }, + OperatorInfo{ "ThresholdedRelu", onnxruntime::kOnnxDomain, OnnxOperatorSet10::sc_sinceVer_ThresholdedRelu }, OperatorInfo{ "Elu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Elu }, OperatorInfo{ "Selu", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Selu }, OperatorInfo{ "Softsign", onnxruntime::kOnnxDomain, OnnxOperatorSet7::sc_sinceVer_Softsign }, @@ -330,6 +339,41 @@ namespace Dml return std::string("fused_").append(name); } +#if _DEBUG + // This asserts that an exact match for the operator exists in the tables of fusable ops, if a prior version exists + void AssertFusableOperatorSupportsVersionIfExists( + std::string_view type, + std::string_view domain, + int sinceVersion) + { + for (const OperatorInfo& operatorInfo : c_fusableOps) + { + if (operatorInfo.type == type && operatorInfo.domain == domain && operatorInfo.sinceVersion < sinceVersion) + { + assert(std::end(c_fusableOps) != std::find( + std::begin(c_fusableOps), + std::end(c_fusableOps), + OperatorInfo{ type, domain, sinceVersion })); + + break; + } + } + + for (const OperatorInfo& operatorInfo : c_activationOps) + { + if (operatorInfo.type == type && operatorInfo.domain == domain && operatorInfo.sinceVersion < sinceVersion) + { + assert(std::end(c_activationOps) != std::find( + std::begin(c_activationOps), + std::end(c_activationOps), + OperatorInfo{ type, domain, sinceVersion })); + + break; + } + } + } +#endif + } // namespace FusionHelpers uint32_t GetDmlAdjustedAxis(int32_t onnxAxis, const MLOperatorKernelCreationContext& kernelCreationContext, uint32_t dmlDimCount) @@ -359,7 +403,7 @@ namespace Dml } } - uint32_t MapStringToIndex(std::string_view mode, gsl::span nameAndIndexList) + std::optional TryMapStringToIndex(std::string_view mode, gsl::span nameAndIndexList) { for (auto& nameAndIndex : nameAndIndexList) { @@ -369,7 +413,7 @@ namespace Dml } } - ML_INVALID_ARGUMENT("Unknown mode value."); + return {}; } DML_INTERPOLATION_MODE MapStringToInteropolationMode(std::string_view mode) @@ -387,7 +431,11 @@ namespace Dml {"BILINEAR", DML_INTERPOLATION_MODE_LINEAR}, {"bilinear", DML_INTERPOLATION_MODE_LINEAR}, }; - return MapStringToIndex(mode, mapping); + if (auto index = TryMapStringToIndex(mode, mapping)) + { + return *index; + } + ML_INVALID_ARGUMENT("Unknown interpolation mode"); } DML_DEPTH_SPACE_ORDER MapStringToDepthSpaceMode(std::string_view mode) @@ -397,7 +445,11 @@ namespace Dml {"DCR", DML_DEPTH_SPACE_ORDER_DEPTH_COLUMN_ROW}, {"CRD", DML_DEPTH_SPACE_ORDER_COLUMN_ROW_DEPTH}, }; - return MapStringToIndex(mode, mapping); + if (auto index = TryMapStringToIndex(mode, mapping)) + { + return *index; + } + ML_INVALID_ARGUMENT("Unknown depth/space order"); } } // namespace Dml diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h index 027c2228bb..a0e5d8aaeb 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorUtility.h @@ -47,6 +47,13 @@ namespace Dml // produce the attribute for Activation in a fused Conv+Activation kernel. std::string GetFusedAttributeName(std::string_view name); +#if _DEBUG + void AssertFusableOperatorSupportsVersionIfExists( + std::string_view type, + std::string_view domain, + int sinceVersion); +#endif + } // namespace FusionHelpers // Given an axis in ONNX axis numbering, return the axis adjusted for DML based on how the sizes have been coerced. @@ -64,12 +71,14 @@ namespace Dml }; template - T MapStringToIndex(std::string_view mode, gsl::span nameAndIndexList) + std::optional TryMapStringToIndex(std::string_view mode, gsl::span nameAndIndexList) { - return static_cast(MapStringToIndex(mode, nameAndIndexList)); + static_assert(sizeof(T) == sizeof(uint32_t)); + auto result = TryMapStringToIndex(mode, nameAndIndexList); + return *reinterpret_cast*>(std::addressof(result)); } - uint32_t MapStringToIndex(std::string_view mode, gsl::span nameAndIndexList); + std::optional TryMapStringToIndex(std::string_view mode, gsl::span nameAndIndexList); DML_INTERPOLATION_MODE MapStringToInteropolationMode(std::string_view mode);