From dc87691000ceca1b42e3c46b1ce9066a739505ed Mon Sep 17 00:00:00 2001 From: Justin Stoecker Date: Fri, 24 Mar 2023 13:50:17 -0700 Subject: [PATCH] Enable DML graph fusion independently of graph optimization level (#15172) ### Description Apply the DML graph fusion transformer optimization independently of ORT graph optimization level. ### Motivation and Context The DML graph fusion transformer is not a graph optimizer in the normal sense: it isn't optimizing the ONNX graph structure, but rather fusing nodes into what will later become a single IDMLCompiledOperator (using IDMLDevice1::CompileGraph). This transformer can't be done ahead of time (hence why it's disabled if saving an optimized model), but it's also gated by the ORT graph optimization level; this makes it impossible to preoptimize ONNX models ("offline mode") and then later disable graph optimizations for better startup performance ("online mode") while benefiting from DML graph fusion. --- .../dml/dml_session_options_config_keys.h | 23 +++++++++++++++++++ onnxruntime/core/session/inference_session.cc | 10 ++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 onnxruntime/core/providers/dml/dml_session_options_config_keys.h diff --git a/onnxruntime/core/providers/dml/dml_session_options_config_keys.h b/onnxruntime/core/providers/dml/dml_session_options_config_keys.h new file mode 100644 index 0000000000..d11fa7516e --- /dev/null +++ b/onnxruntime/core/providers/dml/dml_session_options_config_keys.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +/* + * This file defines SessionOptions Config Keys and format of the Config Values. + * + * The Naming Convention for a SessionOptions Config Key, + * "[Area][.[SubArea1].[SubArea2]...].[Keyname]" + * Such as "ep.cuda.use_arena" + * The Config Key cannot be empty + * The maximum length of the Config Key is 128 + * + * The string format of a SessionOptions Config Value is defined individually for each Config. + * The maximum length of the Config Value is 1024 + */ + +// Influences whether or not the DirectML graph fusion transformer is allowed. +// "0": not disabled (allowed). Graph fusion will be used if the session is not configured to save an optimized model. +// "1": disabled (disallowed). Graph fusion will never be used. +// The default value is "0" +static const char* const kOrtSessionOptionsConfigDisableDmlGraphFusion = "ep.dml.disable_graph_fusion"; diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index e07e6a2d6b..2c5d9f0919 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -51,6 +51,7 @@ #ifdef USE_DML // TODO: This is necessary for the workaround in TransformGraph #include "core/providers/dml/DmlExecutionProvider/src/DmlGraphFusionTransformer.h" #include "core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h" +#include "core/providers/dml/dml_session_options_config_keys.h" #endif #include "core/session/environment.h" #include "core/session/IOBinding.h" @@ -1023,7 +1024,7 @@ Status InferenceSession::LoadOrtModelWithLoader(std::function load_ort "The ORT format model version [", fbs_ort_model_version->string_view(), "] is not supported in this build ", ORT_VERSION, ". ", kOrtFormatVersion5BreakingChangeNote); -#else // ^^ defined(ORT_MINIMAL_BUILD) ^^ / vv !defined(ORT_MINIMAL_BUILD) vv +#else // ^^ defined(ORT_MINIMAL_BUILD) ^^ / vv !defined(ORT_MINIMAL_BUILD) vv const auto has_saved_runtime_optimizations = [](const fbs::InferenceSession& fbs_session) -> bool { if (const auto* fbs_model = fbs_session.model()) { if (const auto* fbs_graph = fbs_model->graph()) { @@ -1384,8 +1385,13 @@ common::Status InferenceSession::Initialize() { #ifdef USE_DML if (execution_providers_.Get(kDmlExecutionProvider)) { + // DML graph fusion is an important runtime optimization that cannot be done ahead of time; it must be disabled + // when running in "offline mode" and saving an optimized model to disk. To support users that want to optimize + // models offline, and then disable graph optimizations when running "online", this transformer ignores the ORT + // graph optimization level and is generally always applied. bool dml_graph_fusion_enabled = session_options_.optimized_model_filepath.empty() && - session_options_.graph_optimization_level >= TransformerLevel::Level3; + session_options_.config_options.GetConfigOrDefault(kOrtSessionOptionsConfigDisableDmlGraphFusion, "0") == "0"; + if (dml_graph_fusion_enabled) { std::unique_ptr dmlGraphFusionTransformer = std::make_unique("DmlGraphFusionTransformer", execution_providers_.Get(kDmlExecutionProvider));