Helper for compiling EP to generate deterministic unique ids for use in MetaDef names (#6156)

* Create a helper for generating unique ids that can be used by an EP that creates compiled nodes and needs ids to be deterministic for a model when used in multiple sessions.

Added to IExecutionProvider as this can potentially be used by all compiling EPs and is more robust than a simplistic counter (although EP implementer is free to choose either approach).

* Restructure the helper so it can be called across the EP bridge.
Add ability to call id generation helper from EP bridge
  - convert DNNL EP to use helper to validate
Address issue where a new Model may be loaded into the same address as a previous one.
  - hash the bytes in the Graph instance (1728 bytes currently) to use as the key to the full hash for the model
Add lock around id generation to ensure no issues if multiple sessions partitions graphs at exactly the same time.
  - Extremely unlikely but would be hard to debug and the locking cost is not an issue as it's only incurred during graph partitioning and not execution.

include/onnxruntime/core/framework/execution_provider.h

@@ -48,7 +48,12 @@ struct NodeComputeInfo {
 
 class IExecutionProvider {
  protected:
-  IExecutionProvider(const std::string& type) : type_{type} {}
+  IExecutionProvider(const std::string& type, bool use_metadef_id_creator = false)
+      : type_{type} {
+    if (use_metadef_id_creator) {
+      metadef_id_generator_ = onnxruntime::make_unique<ModelMetadefIdGenerator>();
+    }
+  }
 
  public:
   virtual ~IExecutionProvider() = default;
@@ -234,6 +239,19 @@ class IExecutionProvider {
     return logger_;
   }
 
+  /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. 
+   The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models.
+   @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph.
+   @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. 
+                          This is created using the model path if available, 
+                          or the model input names and the output names from all nodes in the main graph.
+   @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches
+            compiled kernels, so the name must be unique and deterministic across models and sessions.
+            NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and 
+			      virtual, and ModelMetadefIdGenerator but be defined in the header as well.
+   */
+  virtual int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) const;
+
  private:
   const std::string type_;
   AllocatorMap allocators_;
@@ -243,5 +261,18 @@ class IExecutionProvider {
   // convenience list of the allocators so GetAllocatorList doesn't have to build a new vector each time
   // contains the same instances as allocators_
   std::vector<AllocatorPtr> allocator_list_;
+
+  // helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across
+  // multiple sessions.
+  class ModelMetadefIdGenerator {
+   public:
+    int GenerateId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash);
+
+   private:
+    std::unordered_map<uint64_t, int64_t> main_graph_hash_;  // map graph instance hash to model contents hash
+    std::unordered_map<int64_t, int> model_metadef_id_;      // current unique id for model
+  };
+
+  std::unique_ptr<ModelMetadefIdGenerator> metadef_id_generator_;
 };
 }  // namespace onnxruntime

onnxruntime/core/framework/execution_provider.cc

@@ -4,9 +4,10 @@
 
 #include "core/graph/graph_viewer.h"
 #include "core/framework/compute_capability.h"
-#include "core/framework/kernel_registry_manager.h"
-#include "core/framework/op_kernel.h"
 #include "core/framework/kernel_registry.h"
+#include "core/framework/kernel_registry_manager.h"
+#include "core/framework/murmurhash3.h"
+#include "core/framework/op_kernel.h"
 
 namespace onnxruntime {
 
@@ -92,4 +93,76 @@ common::Status IExecutionProvider::Compile(const std::vector<FusedNodeAndGraph>&
 }
 #endif
 
+int IExecutionProvider::ModelMetadefIdGenerator::GenerateId(const onnxruntime::GraphViewer& graph_viewer,
+                                                            uint64_t& model_hash) {
+  model_hash = 0;
+
+  // find the top level graph
+  const Graph* cur_graph = &graph_viewer.GetGraph();
+  while (cur_graph->IsSubgraph()) {
+    cur_graph = cur_graph->ParentGraph();
+  }
+
+  uint32_t instance_hash[4] = {0, 0, 0, 0};
+
+  const Graph& main_graph = *cur_graph;
+
+  // hash the bytes in the Graph instance. we can't just use the address as a new Graph instance may use
+  // the same memory (unit tests prove this can occur). the raw bytes of the Graph instance should be a unique
+  // fingerprint for the instance that can use used as the key to the hash of the model path/contents.
+  MurmurHash3::x86_128(&main_graph, gsl::narrow_cast<int32_t>(sizeof(Graph)), instance_hash[0], &instance_hash);
+  uint64_t graph_instance_hash = instance_hash[0] | (uint64_t(instance_hash[1]) << 32);
+
+  // if we've already hashed this main graph instance use the cached value
+  auto entry = main_graph_hash_.find(graph_instance_hash);
+  if (entry != main_graph_hash_.cend()) {
+    model_hash = entry->second;
+  } else {
+    uint32_t hash[4] = {0, 0, 0, 0};
+
+    // prefer path the model was loaded from
+    // this may not be available if the model was loaded from a stream or in-memory bytes
+    const auto& model_path_str = main_graph.ModelPath().ToPathString();
+    if (!model_path_str.empty()) {
+      MurmurHash3::x86_128(model_path_str.data(), gsl::narrow_cast<int32_t>(model_path_str.size()), hash[0], &hash);
+    } else {
+      auto hash_str = [&hash](const std::string& str) {
+        MurmurHash3::x86_128(str.data(), gsl::narrow_cast<int32_t>(str.size()), hash[0], &hash);
+      };
+
+      // fingerprint the main graph by hashing graph inputs and the ordered outputs from each node
+      for (const auto* node_arg : main_graph.GetInputsIncludingInitializers()) {
+        hash_str(node_arg->Name());
+      }
+
+      // note: process nodes in order defined in model to be deterministic
+      for (const auto& node : main_graph.Nodes()) {
+        for (const auto* node_arg : node.OutputDefs()) {
+          if (node_arg->Exists()) {
+            hash_str(node_arg->Name());
+          }
+        }
+      }
+    }
+
+    model_hash = hash[0] | (uint64_t(hash[1]) << 32);
+
+    main_graph_hash_[graph_instance_hash] = model_hash;
+  }
+
+  // return the current unique id, and increment to update
+  return model_metadef_id_[model_hash]++;
+}
+
+int IExecutionProvider::GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) const {
+  ORT_ENFORCE(metadef_id_generator_,
+              "IExecutionProvider constructor must be called with true for use_metadef_id_creator");
+
+  // if the EP is shared across multiple sessions there's a very small potential for concurrency issues.
+  // use a lock when generating an id to be paranoid
+  static OrtMutex mutex;
+  std::lock_guard<OrtMutex> lock(mutex);
+  return metadef_id_generator_->GenerateId(graph_viewer, model_hash);
+}
+
 }  // namespace onnxruntime

onnxruntime/core/framework/provider_bridge_ort.cc

@@ -217,6 +217,10 @@ struct ProviderHostImpl : ProviderHost {
     return p->IExecutionProvider::Compile(fused_nodes_and_graphs, node_compute_funcs);
   }
 
+  int IExecutionProvider__GenerateMetaDefId(const IExecutionProvider* p, const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) override {
+    return p->IExecutionProvider::GenerateMetaDefId(graph_viewer, model_hash);
+  }
+
   // Status
   std::string Status__ToString(const Status* p) override { return p->ToString(); }
 

onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc

@@ -17,7 +17,7 @@ constexpr const char* DNNL = "Dnnl";
 constexpr const char* DNNL_CPU = "DnnlCpu";
 
 DNNLExecutionProvider::DNNLExecutionProvider(const DNNLExecutionProviderInfo& info)
-    : IExecutionProvider{onnxruntime::kDnnlExecutionProvider} {
+    : IExecutionProvider{onnxruntime::kDnnlExecutionProvider, true} {
   AllocatorCreationInfo default_memory_info(
       {[](int) {
         return onnxruntime::CreateCPUAllocator(OrtMemoryInfo(DNNL, OrtAllocatorType::OrtDeviceAllocator));
@@ -356,8 +356,9 @@ void DNNLExecutionProvider::CreateMetaDef(const GraphViewer& graph_viewer,
                                           std::vector<std::unique_ptr<ComputeCapability>>& result) const {
   std::string graph_fused_nodes;
   std::string node_list;
-  std::string subgraph_id = std::to_string(subgraph_index_);
-  subgraph_index_++;
+  uint64_t model_hash = 0;
+  int id = GenerateMetaDefId(graph_viewer, model_hash);
+  std::string subgraph_id = std::to_string(model_hash) + "_" + std::to_string(id);
 
   // This is a list of initializers that subgraph considers as constants.
   // Example weights, reshape shape etc.
@@ -377,7 +378,7 @@ void DNNLExecutionProvider::CreateMetaDef(const GraphViewer& graph_viewer,
 
   auto meta_def = ::onnxruntime::IndexedSubGraph_MetaDef::Create();
   meta_def->attributes()["initializers"] = *initializers;
-  meta_def->name() = "DnnlCustomOp" + std::to_string(subgraph_index_);
+  meta_def->name() = "DnnlCustomOp_" + subgraph_id;
   meta_def->domain() = kMSDomain;
   meta_def->since_version() = 1;
   meta_def->status() = ONNX_NAMESPACE::EXPERIMENTAL;

onnxruntime/core/providers/dnnl/dnnl_execution_provider.h

@@ -165,8 +165,6 @@ class DNNLExecutionProvider : public IExecutionProvider {
   }
 
  private:
-  mutable int subgraph_index_ = 0;
-
   // supported Dnnl Operators
   std::set<std::string> dnnl_ops_ = {"Conv", "BatchNormalization", "Relu", "Sum",
                                      "AveragePool", "GlobalMaxPool", "GlobalAveragePool", "MaxPool", "LRN"};

onnxruntime/core/providers/shared_library/provider_bridge_provider.cc

@@ -134,6 +134,10 @@ common::Status IExecutionProvider::Compile(const std::vector<FusedNodeAndGraph>&
   return g_host->IExecutionProvider__Compile(this, fused_nodes_and_graphs, node_compute_funcs);
 }
 
+int IExecutionProvider::GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) const {
+  return g_host->IExecutionProvider__GenerateMetaDefId(this, graph_viewer, model_hash);
+}
+
 #ifdef USE_TENSORRT
 std::unique_ptr<IAllocator> CreateCUDAAllocator(int16_t device_id, const char* name) {
   return g_host->CreateCUDAAllocator(device_id, name);

onnxruntime/core/providers/shared_library/provider_interfaces.h

@@ -166,6 +166,8 @@ struct ProviderHost {
   virtual common::Status IExecutionProvider__Compile(IExecutionProvider* p, const std::vector<onnxruntime::Node*>& fused_nodes, std::string& dll_path) = 0;
   virtual common::Status IExecutionProvider__Compile(IExecutionProvider* p, const std::vector<IExecutionProvider::FusedNodeAndGraph>& fused_nodes_and_graphs, std::vector<NodeComputeInfo>& node_compute_funcs) = 0;
 
+  virtual int IExecutionProvider__GenerateMetaDefId(const IExecutionProvider* p, const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) = 0;
+
   // Status
   virtual std::string Status__ToString(const Status* p) = 0;
 

onnxruntime/test/framework/execution_provider_test.cc

@@ -0,0 +1,94 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/framework/execution_provider.h"
+#include "core/graph/model.h"
+#include "test_utils.h"
+#include "test/test_environment.h"
+#include "test/util/include/asserts.h"
+
+#include "gtest/gtest.h"
+
+#include <fstream>
+
+namespace onnxruntime {
+namespace test {
+
+class TestEP : public IExecutionProvider {
+  static constexpr const char* kEPType = "TestEP";
+
+ public:
+  TestEP() : IExecutionProvider{kEPType, true} {}
+
+  int GetId(const GraphViewer& viewer, uint64_t& model_hash) {
+    return GenerateMetaDefId(viewer, model_hash);
+  }
+};
+
+TEST(ExecutionProviderTest, MetadefIdGeneratorUsingModelPath) {
+  TestEP ep;
+
+  auto test_model = [&ep](const std::basic_string<ORTCHAR_T>& model_path) {
+    std::shared_ptr<Model> model;
+    ASSERT_TRUE(Model::Load(model_path, model, nullptr, DefaultLoggingManager().DefaultLogger()).IsOK());
+
+    Graph& graph = model->MainGraph();
+    GraphViewer viewer(graph);
+
+    // check for stable non-zero model_hash, and incrementing id.
+    uint64_t model_hash;
+    int id = ep.GetId(viewer, model_hash);
+    ASSERT_EQ(id, 0);
+    ASSERT_NE(model_hash, 0);
+
+    for (int i = 1; i < 4; ++i) {
+      uint64_t cur_model_hash;
+      int cur_id = ep.GetId(viewer, cur_model_hash);
+      ASSERT_EQ(cur_id, i);
+      ASSERT_EQ(cur_model_hash, model_hash);
+    }
+  };
+
+  test_model(ORT_TSTR("testdata/mnist.onnx"));
+  // load a new model instance and check it has a separate scope for the generated ids
+  test_model(ORT_TSTR("testdata/ort_github_issue_4031.onnx"));
+}
+
+// test when the model hash is created by hashing the contents of the main graph instead of the model path
+TEST(ExecutionProviderTest, MetadefIdGeneratorUsingModelHashing) {
+  TestEP ep;
+
+  auto model_path = ORT_TSTR("testdata/mnist.onnx");
+
+  std::shared_ptr<Model> model;
+  ASSERT_TRUE(Model::Load(model_path, model, nullptr, DefaultLoggingManager().DefaultLogger()).IsOK());
+
+  Graph& graph = model->MainGraph();
+  GraphViewer viewer(graph);
+
+  // get the hash for the model when loaded from file
+  uint64_t model_hash;
+  int id = ep.GetId(viewer, model_hash);
+  ASSERT_EQ(id, 0);
+  ASSERT_NE(model_hash, 0);
+
+  // now load the model from bytes and check the hash differs
+  std::ifstream model_file_stream(model_path, std::ios::in | std::ios::binary);
+
+  std::shared_ptr<Model> model2;
+  ONNX_NAMESPACE::ModelProto model_proto;
+  ASSERT_STATUS_OK(Model::Load(model_file_stream, &model_proto));
+  ASSERT_STATUS_OK(Model::Load(std::move(model_proto), PathString(), model2, nullptr,
+                               DefaultLoggingManager().DefaultLogger()));
+
+  Graph& graph2 = model2->MainGraph();
+  GraphViewer viewer2(graph2);
+
+  uint64_t model_hash2;
+  int id2 = ep.GetId(viewer2, model_hash2);
+  ASSERT_EQ(id2, 0) << "Id for new model should always start at zero";
+  ASSERT_NE(model_hash, model_hash2) << "Hash from model path should differ from hash based on model contents";
+}
+
+}  // namespace test
+}  // namespace onnxruntime

onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc

@@ -18,7 +18,7 @@ namespace onnxruntime {
 constexpr const char* INTERNAL_TESTING_EP = "InternalTestingEP";
 
 InternalTestingExecutionProvider::InternalTestingExecutionProvider(const std::unordered_set<std::string>& ops)
-    : IExecutionProvider{utils::kInternalTestingExecutionProvider},
+    : IExecutionProvider{utils::kInternalTestingExecutionProvider, true},
       ops_{ops} {
   // TODO: Allocation planner calls GetAllocator for the individual EP. It would be better if it goes through
   // the session state to get the allocator so it's per-device (or for the allocation planner to try the EP first
@@ -129,9 +129,11 @@ InternalTestingExecutionProvider::GetCapability(const onnxruntime::GraphViewer&
     }
 
     // Assign inputs and outputs to subgraph's meta_def
+    uint64_t model_hash;
+    int metadef_id = GenerateMetaDefId(graph_viewer, model_hash);
     auto meta_def = onnxruntime::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>();
-    meta_def->name = "InternalTestingEP_" + std::to_string(metadef_id_++);
-    meta_def->domain = kMSDomain;
+    meta_def->name = "InternalTestingEP_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id);
+    meta_def->domain = "InternalTesting";
     meta_def->since_version = 1;
     meta_def->status = ONNX_NAMESPACE::EXPERIMENTAL;
 

onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h

@@ -24,8 +24,5 @@ class InternalTestingExecutionProvider : public IExecutionProvider {
 
  private:
   const std::unordered_set<std::string> ops_;
-
-  // unique counter to name each fused kernel across the entire model
-  mutable int metadef_id_{0};
 };
 }  // namespace onnxruntime

onnxruntime/test/providers/internal_testing/internal_testing_tests.cc

@@ -175,25 +175,23 @@ TEST(InternalTestingEP, TestLoadOrtModelWithReducedOpCoverage) {
   const auto& func_mgr = session->GetSessionState().GetFuncMgr();
   NodeComputeInfo* compute_func = nullptr;
 
+  // the generated op type should have a hash for the model based on the model path
+  const std::string expected_op_type_prefix = "InternalTestingEP_9611636968429821767_";
+  int compiled_node_num = 0;
+
   for (const auto& node : graph.Nodes()) {
     EXPECT_EQ(supported_ops.count(node.OpType()), size_t(0))
         << "Nodes with supported op types should have been replaced. Node with type " << node.OpType() << " was not.";
     if (node.GetExecutionProviderType() == utils::kInternalTestingExecutionProvider) {
       EXPECT_STATUS_OK(func_mgr.GetFuncs(node.Name(), compute_func));
       EXPECT_NE(compute_func, nullptr);
+      EXPECT_EQ(node.OpType(), expected_op_type_prefix + std::to_string(compiled_node_num++));
     }
   }
 
   ExecuteMnist(*session, enable_custom_ep);
 }
 
-TEST(InternalTestingEP, TestMinimalRegistrationOfEPwithGetCapability) {
-  // TODO: In a full build we want to be able to call GetCapability for the NNAPI EP and produce an ORT format model
-  // with nodes correctly preserved. That requires being able to do a minimal registration of that EP where
-  // GetCapability is fully implemented, but Compile is a stub that just throws NOT_IMPLEMENTED if someone attempts
-  // to execute a model in that InferenceSession.
-}
-
 // count nodes assigned to the test EP and make sure they all have valid compute funcs
 static int CountAndValidateAssignedNodes(const Graph& current_graph,
                                          const std::unordered_set<std::string>& supported_ops,