From 76dbd88526dcb177a7b73ac0476e030b695952dc Mon Sep 17 00:00:00 2001 From: stevenlix <38092805+stevenlix@users.noreply.github.com> Date: Tue, 26 Jan 2021 10:41:31 -0800 Subject: [PATCH] Expose graph ModelPath to TensorRT shared library (#6353) * Update graph_viewer.cc * Update tensorrt_execution_provider.cc * Update graph_viewer.h * Update tensorrt_execution_provider.cc * Update tensorrt_execution_provider.cc * Update provider_api.h * Update provider_bridge_ort.cc * Update provider_interfaces.h * Update provider_interfaces.h * expose GraphViewer ModelPath API to TRT shared lib * add modelpath to compile * update * add model_path to onnx tensorrt parser * use GenerateMetaDefId to generate unique TRT kernel name * use GenerateMetaDefId to generate unique TRT engine name * fix issue * Update tensorrt_execution_provider.cc * remove GetVecHash * Update tensorrt_execution_provider.h * convert wchar_t to char for tensorrt parser * update tensorrt parser to include latest changes * fix issues * Update tensorrt_execution_provider.cc * merge trt parser latest change * add PROVIDER_DISALLOW_ALL(Path) --- .gitmodules | 7 ++-- cmake/external/onnx-tensorrt | 2 +- .../core/framework/provider_bridge_ort.cc | 5 +++ .../providers/shared_library/provider_api.h | 2 ++ .../shared_library/provider_interfaces.h | 13 ++++++- .../tensorrt/tensorrt_execution_provider.cc | 36 +++++++++---------- .../tensorrt/tensorrt_execution_provider.h | 2 +- 7 files changed, 43 insertions(+), 24 deletions(-) diff --git a/.gitmodules b/.gitmodules index b7397054f0..9d68a7c10b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -56,9 +56,10 @@ [submodule "cmake/external/SafeInt/safeint"] path = cmake/external/SafeInt/safeint url = https://github.com/dcleblanc/SafeInt.git -[submodule "cmake/external/onnx-tensorrt"] - path = cmake/external/onnx-tensorrt - url = https://github.com/onnx/onnx-tensorrt.git [submodule "cmake/external/optional-lite"] path = cmake/external/optional-lite url = https://github.com/martinmoene/optional-lite.git +[submodule "cmake/external/onnx-tensorrt"] + path = cmake/external/onnx-tensorrt + url = https://github.com/onnx/onnx-tensorrt.git + branch = 7.1 diff --git a/cmake/external/onnx-tensorrt b/cmake/external/onnx-tensorrt index a3a4e38b2d..b3eda616d3 160000 --- a/cmake/external/onnx-tensorrt +++ b/cmake/external/onnx-tensorrt @@ -1 +1 @@ -Subproject commit a3a4e38b2dfa7a62b6dcae33c0d1678b3bb5ef2a +Subproject commit b3eda616d3bb60dcd40a142e4a0a2ad95a7aa166 diff --git a/onnxruntime/core/framework/provider_bridge_ort.cc b/onnxruntime/core/framework/provider_bridge_ort.cc index ad881f72a6..9e8e55182f 100644 --- a/onnxruntime/core/framework/provider_bridge_ort.cc +++ b/onnxruntime/core/framework/provider_bridge_ort.cc @@ -514,6 +514,7 @@ struct ProviderHostImpl : ProviderHost { } const std::string& GraphViewer__Name(const GraphViewer* p) noexcept override { return p->Name(); } + const Path& GraphViewer__ModelPath(const GraphViewer* p) noexcept override { return p->ModelPath(); } const Node* GraphViewer__GetNode(const GraphViewer* p, NodeIndex node_index) override { return p->GetNode(node_index); } const NodeArg* GraphViewer__GetNodeArg(const GraphViewer* p, const std::string& name) override { return p->GetNodeArg(name); } @@ -535,6 +536,10 @@ struct ProviderHostImpl : ProviderHost { const std::vector& GraphViewer__GetNodesInTopologicalOrder(const GraphViewer* p) override { return p->GetNodesInTopologicalOrder(); } const std::vector& GraphViewer__GetInputsIncludingInitializers(const GraphViewer* p) noexcept override { return p->GetInputsIncludingInitializers(); } + //Path + //Gets a string representation of the path. + PathString Path__ToPathString(const Path* p) noexcept override { return p->ToPathString(); } + // Provider_OpKernel_Base const OpKernelInfo& Provider_OpKernel_Base__GetInfo(const Provider_OpKernel_Base* p) override { return reinterpret_cast(p)->Info(); } diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 0d0d24086e..c32e171e83 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -21,6 +21,7 @@ #include "core/framework/allocatormgr.h" #include "core/framework/tensor_shape.h" #include "core/providers/providers.h" +#include "core/common/path_string.h" namespace onnxruntime { namespace logging { @@ -73,6 +74,7 @@ struct Function; struct Graph; struct GraphViewer; struct Model; +struct Path; struct Node; struct NodeArg; struct NodeAttributes; diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 8b5ea5af8d..49bc2d8d11 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -22,7 +22,6 @@ using DataType = const std::string*; } // namespace ONNX_NAMESPACE namespace onnxruntime { - // These types don't directly map to internal types struct Provider_KernelCreateInfo; struct Provider_OpKernel_Base; @@ -436,6 +435,7 @@ struct ProviderHost { virtual std::unique_ptr GraphViewer__CreateModel(const GraphViewer* p, const logging::Logger& logger) = 0; virtual const std::string& GraphViewer__Name(const GraphViewer* p) noexcept = 0; + virtual const Path& GraphViewer__ModelPath(const GraphViewer* p) noexcept = 0; virtual const Node* GraphViewer__GetNode(const GraphViewer* p, NodeIndex node_index) = 0; virtual const NodeArg* GraphViewer__GetNodeArg(const GraphViewer* p, const std::string& name) = 0; @@ -456,6 +456,9 @@ struct ProviderHost { virtual const std::vector& GraphViewer__GetNodesInTopologicalOrder(const GraphViewer* p) = 0; virtual const std::vector& GraphViewer__GetInputsIncludingInitializers(const GraphViewer* p) noexcept = 0; + // Path + virtual PathString Path__ToPathString(const Path* p) noexcept = 0; + // Provider_OpKernel_Base virtual const OpKernelInfo& Provider_OpKernel_Base__GetInfo(const Provider_OpKernel_Base* p) = 0; @@ -974,6 +977,7 @@ struct GraphViewer { std::unique_ptr CreateModel(const logging::Logger& logger) const { return g_host->GraphViewer__CreateModel(this, logger); } const std::string& Name() const noexcept { return g_host->GraphViewer__Name(this); } + const Path& ModelPath() const noexcept { return g_host->GraphViewer__ModelPath(this); } const Node* GetNode(NodeIndex node_index) const { return g_host->GraphViewer__GetNode(this, node_index); } const NodeArg* GetNodeArg(const std::string& name) const { return g_host->GraphViewer__GetNodeArg(this, name); } @@ -1000,6 +1004,13 @@ struct GraphViewer { GraphViewer(const GraphViewer&) = delete; void operator=(const GraphViewer&) = delete; }; + +struct Path { + PathString ToPathString() const noexcept { return g_host->Path__ToPathString(this); } + + PROVIDER_DISALLOW_ALL(Path) +}; + #endif struct Provider_OpKernel_Base { diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 993530a0a3..48a509c68c 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -41,14 +41,6 @@ std::string GetCachePath(const std::string& root, const std::string& name) { } } -std::string GetVecHash(const std::string& vec) { - std::size_t ret = vec.size(); - for (auto i : vec) { - ret ^= static_cast(i) + 0x9e3779b9 + (ret << 6) + (ret >> 2); - } - return std::to_string(ret); -} - float ConvertSinglePrecisionIEEE754ToFloat(unsigned long input) { int s = (input >> 31) & 0x01; int e = ((input & 0x7f800000) >> 23) - 127; @@ -259,7 +251,6 @@ bool SetDynamicRange(nvinfer1::INetworkDefinition& network, std::unordered_map TensorrtExecutionProvider::GetSubGraph(SubGraph outputs.insert(std::pair(it->second, it->first)); } - // Assign inputs and outputs to subgraph's meta_def + // Generate unique kernel name for TRT subgraph + uint64_t model_hash = 0; + int id = GenerateMetaDefId(graph, model_hash); + std::string subgraph_id = std::to_string(model_hash) + "_" + std::to_string(id); auto meta_def = IndexedSubGraph_MetaDef::Create(); const std::string graph_type = graph.IsSubgraph() ? "subgraph" : "graph"; - meta_def->name() = "TRTKernel_" + graph_type + "_" + graph.Name() + "_" + std::to_string(subgraph_id_++); - meta_def->domain() = kMSDomain; + meta_def->name() = "TRTKernel_" + graph_type + "_" + graph.Name() + "_" + subgraph_id; + // Assign inputs and outputs to subgraph's meta_def for (const auto& input : inputs) { if (input.second->Exists()) { meta_def->inputs().push_back(input.second->Name()); @@ -618,6 +612,7 @@ std::unique_ptr TensorrtExecutionProvider::GetSubGraph(SubGraph } } + meta_def->domain() = kMSDomain; meta_def->since_version() = 1; sub_graph->SetMetaDef(std::move(meta_def)); @@ -742,7 +737,7 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect auto trt_network = tensorrt_ptr::unique_pointer(trt_builder->createNetworkV2(explicitBatch)); auto trt_parser = tensorrt_ptr::unique_pointer(nvonnxparser::createParser(*trt_network, trt_logger)); - trt_parser->supportsModel(string_buf.data(), string_buf.size(), parser_nodes_list); + trt_parser->supportsModel(string_buf.data(), string_buf.size(), parser_nodes_list, model_path_); SubGraphCollection_t next_nodes_list; const std::vector& subgraph_node_index = graph_viewer->GetNodesInTopologicalOrder(); @@ -875,6 +870,14 @@ void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t& std::vector> TensorrtExecutionProvider::GetCapability(const GraphViewer& graph, const std::vector& /*kernel_registries*/) const { + // Get ModelPath + const auto& path_string = graph.ModelPath().ToPathString(); +#ifdef _WIN32 + wcstombs(model_path_, path_string.c_str(), sizeof(model_path_)); +#else + strcpy(model_path_, path_string.c_str()); +#endif + // Get supported node list from TensorRT parser const int number_of_ort_nodes = graph.NumberOfNodes(); std::vector nodes_vector(number_of_ort_nodes); @@ -948,7 +951,6 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector& fuse auto graph_body_viewer = graph_body.CreateGraphViewer(); auto model = graph_body_viewer->CreateModel(*GetLogger()); auto model_proto = model->ToProto(); - *model_proto->mutable_graph() = *graph_body.ToGraphProto(); model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); std::string string_buf; @@ -966,7 +968,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector& fuse auto trt_network = tensorrt_ptr::unique_pointer(trt_builder->createNetworkV2(explicitBatch)); auto trt_config = tensorrt_ptr::unique_pointer(trt_builder->createBuilderConfig()); auto trt_parser = tensorrt_ptr::unique_pointer(nvonnxparser::createParser(*trt_network, trt_logger)); - trt_parser->parse(string_buf.data(), string_buf.size()); + trt_parser->parse(string_buf.data(), string_buf.size(), model_path_); trt_config->setMaxWorkspaceSize(max_workspace_size_); int num_inputs = trt_network->getNbInputs(); @@ -1037,8 +1039,6 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector& fuse trt_node_name_with_precision += "_int8"; LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] INT8 mode is enabled"; } - int num_nodes = graph_body_viewer->NumberOfNodes(); - trt_node_name_with_precision += "_" + GetVecHash(trt_node_name_with_precision + std::to_string(num_nodes)); // Build TRT engine here if the graph doesn't have dynamic shape input. Otherwise engine will // be built at runtime diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h index 08acd01bae..87dc5f3b59 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h @@ -129,7 +129,7 @@ class TensorrtExecutionProvider : public IExecutionProvider { OrtMutex tensorrt_mu_; int device_id_; AllocatorPtr allocator_; - mutable int subgraph_id_ = 0; + mutable char model_path_[4096]; // Reserved for max path length std::unordered_map> parsers_; std::unordered_map> engines_;