diff --git a/include/onnxruntime/core/framework/execution_provider.h b/include/onnxruntime/core/framework/execution_provider.h index bc48f27ecc..3c877f2bf0 100644 --- a/include/onnxruntime/core/framework/execution_provider.h +++ b/include/onnxruntime/core/framework/execution_provider.h @@ -48,7 +48,12 @@ struct NodeComputeInfo { class IExecutionProvider { protected: - IExecutionProvider(const std::string& type) : type_{type} {} + IExecutionProvider(const std::string& type, bool use_metadef_id_creator = false) + : type_{type} { + if (use_metadef_id_creator) { + metadef_id_generator_ = onnxruntime::make_unique(); + } + } public: virtual ~IExecutionProvider() = default; @@ -234,6 +239,19 @@ class IExecutionProvider { return logger_; } + /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance. + The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models. + @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph. + @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model. + This is created using the model path if available, + or the model input names and the output names from all nodes in the main graph. + @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches + compiled kernels, so the name must be unique and deterministic across models and sessions. + NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and + virtual, and ModelMetadefIdGenerator but be defined in the header as well. + */ + virtual int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) const; + private: const std::string type_; AllocatorMap allocators_; @@ -243,5 +261,18 @@ class IExecutionProvider { // convenience list of the allocators so GetAllocatorList doesn't have to build a new vector each time // contains the same instances as allocators_ std::vector allocator_list_; + + // helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across + // multiple sessions. + class ModelMetadefIdGenerator { + public: + int GenerateId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash); + + private: + std::unordered_map main_graph_hash_; // map graph instance hash to model contents hash + std::unordered_map model_metadef_id_; // current unique id for model + }; + + std::unique_ptr metadef_id_generator_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/framework/execution_provider.cc b/onnxruntime/core/framework/execution_provider.cc index be5cf81301..27ab81bdae 100644 --- a/onnxruntime/core/framework/execution_provider.cc +++ b/onnxruntime/core/framework/execution_provider.cc @@ -4,9 +4,10 @@ #include "core/graph/graph_viewer.h" #include "core/framework/compute_capability.h" -#include "core/framework/kernel_registry_manager.h" -#include "core/framework/op_kernel.h" #include "core/framework/kernel_registry.h" +#include "core/framework/kernel_registry_manager.h" +#include "core/framework/murmurhash3.h" +#include "core/framework/op_kernel.h" namespace onnxruntime { @@ -92,4 +93,76 @@ common::Status IExecutionProvider::Compile(const std::vector& } #endif +int IExecutionProvider::ModelMetadefIdGenerator::GenerateId(const onnxruntime::GraphViewer& graph_viewer, + uint64_t& model_hash) { + model_hash = 0; + + // find the top level graph + const Graph* cur_graph = &graph_viewer.GetGraph(); + while (cur_graph->IsSubgraph()) { + cur_graph = cur_graph->ParentGraph(); + } + + uint32_t instance_hash[4] = {0, 0, 0, 0}; + + const Graph& main_graph = *cur_graph; + + // hash the bytes in the Graph instance. we can't just use the address as a new Graph instance may use + // the same memory (unit tests prove this can occur). the raw bytes of the Graph instance should be a unique + // fingerprint for the instance that can use used as the key to the hash of the model path/contents. + MurmurHash3::x86_128(&main_graph, gsl::narrow_cast(sizeof(Graph)), instance_hash[0], &instance_hash); + uint64_t graph_instance_hash = instance_hash[0] | (uint64_t(instance_hash[1]) << 32); + + // if we've already hashed this main graph instance use the cached value + auto entry = main_graph_hash_.find(graph_instance_hash); + if (entry != main_graph_hash_.cend()) { + model_hash = entry->second; + } else { + uint32_t hash[4] = {0, 0, 0, 0}; + + // prefer path the model was loaded from + // this may not be available if the model was loaded from a stream or in-memory bytes + const auto& model_path_str = main_graph.ModelPath().ToPathString(); + if (!model_path_str.empty()) { + MurmurHash3::x86_128(model_path_str.data(), gsl::narrow_cast(model_path_str.size()), hash[0], &hash); + } else { + auto hash_str = [&hash](const std::string& str) { + MurmurHash3::x86_128(str.data(), gsl::narrow_cast(str.size()), hash[0], &hash); + }; + + // fingerprint the main graph by hashing graph inputs and the ordered outputs from each node + for (const auto* node_arg : main_graph.GetInputsIncludingInitializers()) { + hash_str(node_arg->Name()); + } + + // note: process nodes in order defined in model to be deterministic + for (const auto& node : main_graph.Nodes()) { + for (const auto* node_arg : node.OutputDefs()) { + if (node_arg->Exists()) { + hash_str(node_arg->Name()); + } + } + } + } + + model_hash = hash[0] | (uint64_t(hash[1]) << 32); + + main_graph_hash_[graph_instance_hash] = model_hash; + } + + // return the current unique id, and increment to update + return model_metadef_id_[model_hash]++; +} + +int IExecutionProvider::GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) const { + ORT_ENFORCE(metadef_id_generator_, + "IExecutionProvider constructor must be called with true for use_metadef_id_creator"); + + // if the EP is shared across multiple sessions there's a very small potential for concurrency issues. + // use a lock when generating an id to be paranoid + static OrtMutex mutex; + std::lock_guard lock(mutex); + return metadef_id_generator_->GenerateId(graph_viewer, model_hash); +} + } // namespace onnxruntime diff --git a/onnxruntime/core/framework/provider_bridge_ort.cc b/onnxruntime/core/framework/provider_bridge_ort.cc index 25002c7f0d..ad881f72a6 100644 --- a/onnxruntime/core/framework/provider_bridge_ort.cc +++ b/onnxruntime/core/framework/provider_bridge_ort.cc @@ -217,6 +217,10 @@ struct ProviderHostImpl : ProviderHost { return p->IExecutionProvider::Compile(fused_nodes_and_graphs, node_compute_funcs); } + int IExecutionProvider__GenerateMetaDefId(const IExecutionProvider* p, const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) override { + return p->IExecutionProvider::GenerateMetaDefId(graph_viewer, model_hash); + } + // Status std::string Status__ToString(const Status* p) override { return p->ToString(); } diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc index 78cabe717f..d064f1429b 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.cc @@ -17,7 +17,7 @@ constexpr const char* DNNL = "Dnnl"; constexpr const char* DNNL_CPU = "DnnlCpu"; DNNLExecutionProvider::DNNLExecutionProvider(const DNNLExecutionProviderInfo& info) - : IExecutionProvider{onnxruntime::kDnnlExecutionProvider} { + : IExecutionProvider{onnxruntime::kDnnlExecutionProvider, true} { AllocatorCreationInfo default_memory_info( {[](int) { return onnxruntime::CreateCPUAllocator(OrtMemoryInfo(DNNL, OrtAllocatorType::OrtDeviceAllocator)); @@ -356,8 +356,9 @@ void DNNLExecutionProvider::CreateMetaDef(const GraphViewer& graph_viewer, std::vector>& result) const { std::string graph_fused_nodes; std::string node_list; - std::string subgraph_id = std::to_string(subgraph_index_); - subgraph_index_++; + uint64_t model_hash = 0; + int id = GenerateMetaDefId(graph_viewer, model_hash); + std::string subgraph_id = std::to_string(model_hash) + "_" + std::to_string(id); // This is a list of initializers that subgraph considers as constants. // Example weights, reshape shape etc. @@ -377,7 +378,7 @@ void DNNLExecutionProvider::CreateMetaDef(const GraphViewer& graph_viewer, auto meta_def = ::onnxruntime::IndexedSubGraph_MetaDef::Create(); meta_def->attributes()["initializers"] = *initializers; - meta_def->name() = "DnnlCustomOp" + std::to_string(subgraph_index_); + meta_def->name() = "DnnlCustomOp_" + subgraph_id; meta_def->domain() = kMSDomain; meta_def->since_version() = 1; meta_def->status() = ONNX_NAMESPACE::EXPERIMENTAL; diff --git a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h index 5ceca87f10..6dcc0cbc6a 100644 --- a/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h +++ b/onnxruntime/core/providers/dnnl/dnnl_execution_provider.h @@ -165,8 +165,6 @@ class DNNLExecutionProvider : public IExecutionProvider { } private: - mutable int subgraph_index_ = 0; - // supported Dnnl Operators std::set dnnl_ops_ = {"Conv", "BatchNormalization", "Relu", "Sum", "AveragePool", "GlobalMaxPool", "GlobalAveragePool", "MaxPool", "LRN"}; diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc index 68ed1a3ff5..f8fd2071c5 100644 --- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc +++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc @@ -134,6 +134,10 @@ common::Status IExecutionProvider::Compile(const std::vector& return g_host->IExecutionProvider__Compile(this, fused_nodes_and_graphs, node_compute_funcs); } +int IExecutionProvider::GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) const { + return g_host->IExecutionProvider__GenerateMetaDefId(this, graph_viewer, model_hash); +} + #ifdef USE_TENSORRT std::unique_ptr CreateCUDAAllocator(int16_t device_id, const char* name) { return g_host->CreateCUDAAllocator(device_id, name); diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 3da1a05790..8b5ea5af8d 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -166,6 +166,8 @@ struct ProviderHost { virtual common::Status IExecutionProvider__Compile(IExecutionProvider* p, const std::vector& fused_nodes, std::string& dll_path) = 0; virtual common::Status IExecutionProvider__Compile(IExecutionProvider* p, const std::vector& fused_nodes_and_graphs, std::vector& node_compute_funcs) = 0; + virtual int IExecutionProvider__GenerateMetaDefId(const IExecutionProvider* p, const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) = 0; + // Status virtual std::string Status__ToString(const Status* p) = 0; diff --git a/onnxruntime/test/framework/execution_provider_test.cc b/onnxruntime/test/framework/execution_provider_test.cc new file mode 100644 index 0000000000..10125a1d81 --- /dev/null +++ b/onnxruntime/test/framework/execution_provider_test.cc @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/framework/execution_provider.h" +#include "core/graph/model.h" +#include "test_utils.h" +#include "test/test_environment.h" +#include "test/util/include/asserts.h" + +#include "gtest/gtest.h" + +#include + +namespace onnxruntime { +namespace test { + +class TestEP : public IExecutionProvider { + static constexpr const char* kEPType = "TestEP"; + + public: + TestEP() : IExecutionProvider{kEPType, true} {} + + int GetId(const GraphViewer& viewer, uint64_t& model_hash) { + return GenerateMetaDefId(viewer, model_hash); + } +}; + +TEST(ExecutionProviderTest, MetadefIdGeneratorUsingModelPath) { + TestEP ep; + + auto test_model = [&ep](const std::basic_string& model_path) { + std::shared_ptr model; + ASSERT_TRUE(Model::Load(model_path, model, nullptr, DefaultLoggingManager().DefaultLogger()).IsOK()); + + Graph& graph = model->MainGraph(); + GraphViewer viewer(graph); + + // check for stable non-zero model_hash, and incrementing id. + uint64_t model_hash; + int id = ep.GetId(viewer, model_hash); + ASSERT_EQ(id, 0); + ASSERT_NE(model_hash, 0); + + for (int i = 1; i < 4; ++i) { + uint64_t cur_model_hash; + int cur_id = ep.GetId(viewer, cur_model_hash); + ASSERT_EQ(cur_id, i); + ASSERT_EQ(cur_model_hash, model_hash); + } + }; + + test_model(ORT_TSTR("testdata/mnist.onnx")); + // load a new model instance and check it has a separate scope for the generated ids + test_model(ORT_TSTR("testdata/ort_github_issue_4031.onnx")); +} + +// test when the model hash is created by hashing the contents of the main graph instead of the model path +TEST(ExecutionProviderTest, MetadefIdGeneratorUsingModelHashing) { + TestEP ep; + + auto model_path = ORT_TSTR("testdata/mnist.onnx"); + + std::shared_ptr model; + ASSERT_TRUE(Model::Load(model_path, model, nullptr, DefaultLoggingManager().DefaultLogger()).IsOK()); + + Graph& graph = model->MainGraph(); + GraphViewer viewer(graph); + + // get the hash for the model when loaded from file + uint64_t model_hash; + int id = ep.GetId(viewer, model_hash); + ASSERT_EQ(id, 0); + ASSERT_NE(model_hash, 0); + + // now load the model from bytes and check the hash differs + std::ifstream model_file_stream(model_path, std::ios::in | std::ios::binary); + + std::shared_ptr model2; + ONNX_NAMESPACE::ModelProto model_proto; + ASSERT_STATUS_OK(Model::Load(model_file_stream, &model_proto)); + ASSERT_STATUS_OK(Model::Load(std::move(model_proto), PathString(), model2, nullptr, + DefaultLoggingManager().DefaultLogger())); + + Graph& graph2 = model2->MainGraph(); + GraphViewer viewer2(graph2); + + uint64_t model_hash2; + int id2 = ep.GetId(viewer2, model_hash2); + ASSERT_EQ(id2, 0) << "Id for new model should always start at zero"; + ASSERT_NE(model_hash, model_hash2) << "Hash from model path should differ from hash based on model contents"; +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc index 3c02d26cf6..2a987c3f29 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.cc @@ -18,7 +18,7 @@ namespace onnxruntime { constexpr const char* INTERNAL_TESTING_EP = "InternalTestingEP"; InternalTestingExecutionProvider::InternalTestingExecutionProvider(const std::unordered_set& ops) - : IExecutionProvider{utils::kInternalTestingExecutionProvider}, + : IExecutionProvider{utils::kInternalTestingExecutionProvider, true}, ops_{ops} { // TODO: Allocation planner calls GetAllocator for the individual EP. It would be better if it goes through // the session state to get the allocator so it's per-device (or for the allocation planner to try the EP first @@ -129,9 +129,11 @@ InternalTestingExecutionProvider::GetCapability(const onnxruntime::GraphViewer& } // Assign inputs and outputs to subgraph's meta_def + uint64_t model_hash; + int metadef_id = GenerateMetaDefId(graph_viewer, model_hash); auto meta_def = onnxruntime::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); - meta_def->name = "InternalTestingEP_" + std::to_string(metadef_id_++); - meta_def->domain = kMSDomain; + meta_def->name = "InternalTestingEP_" + std::to_string(model_hash) + "_" + std::to_string(metadef_id); + meta_def->domain = "InternalTesting"; meta_def->since_version = 1; meta_def->status = ONNX_NAMESPACE::EXPERIMENTAL; diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h index a10ee1bd90..938a91ffc3 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h +++ b/onnxruntime/test/providers/internal_testing/internal_testing_execution_provider.h @@ -24,8 +24,5 @@ class InternalTestingExecutionProvider : public IExecutionProvider { private: const std::unordered_set ops_; - - // unique counter to name each fused kernel across the entire model - mutable int metadef_id_{0}; }; } // namespace onnxruntime diff --git a/onnxruntime/test/providers/internal_testing/internal_testing_tests.cc b/onnxruntime/test/providers/internal_testing/internal_testing_tests.cc index 56b4252808..d752d57959 100644 --- a/onnxruntime/test/providers/internal_testing/internal_testing_tests.cc +++ b/onnxruntime/test/providers/internal_testing/internal_testing_tests.cc @@ -175,25 +175,23 @@ TEST(InternalTestingEP, TestLoadOrtModelWithReducedOpCoverage) { const auto& func_mgr = session->GetSessionState().GetFuncMgr(); NodeComputeInfo* compute_func = nullptr; + // the generated op type should have a hash for the model based on the model path + const std::string expected_op_type_prefix = "InternalTestingEP_9611636968429821767_"; + int compiled_node_num = 0; + for (const auto& node : graph.Nodes()) { EXPECT_EQ(supported_ops.count(node.OpType()), size_t(0)) << "Nodes with supported op types should have been replaced. Node with type " << node.OpType() << " was not."; if (node.GetExecutionProviderType() == utils::kInternalTestingExecutionProvider) { EXPECT_STATUS_OK(func_mgr.GetFuncs(node.Name(), compute_func)); EXPECT_NE(compute_func, nullptr); + EXPECT_EQ(node.OpType(), expected_op_type_prefix + std::to_string(compiled_node_num++)); } } ExecuteMnist(*session, enable_custom_ep); } -TEST(InternalTestingEP, TestMinimalRegistrationOfEPwithGetCapability) { - // TODO: In a full build we want to be able to call GetCapability for the NNAPI EP and produce an ORT format model - // with nodes correctly preserved. That requires being able to do a minimal registration of that EP where - // GetCapability is fully implemented, but Compile is a stub that just throws NOT_IMPLEMENTED if someone attempts - // to execute a model in that InferenceSession. -} - // count nodes assigned to the test EP and make sure they all have valid compute funcs static int CountAndValidateAssignedNodes(const Graph& current_graph, const std::unordered_set& supported_ops,