mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-11 17:48:34 +00:00
Clean up TensorRT deprecations, warnings, unbounded string copy (#14148)
### Description - Updates deprecated use of `nvinfer1::___::destroy()` by using a `std::unique_ptr<>` instead of our own smart pointer that calls `destroy`. See [TensorRT deprecation list](https://docs.nvidia.com/deeplearning/tensorrt/api/c_api/deprecated.html#:~:text=Deprecated%20prior%20to%20TensorRT%208.0%20and%20will%20be,noexcept%20Use%20addMatrixMultiply%20instead.%20Deprecated%20in%20TensorRT%208.4.) and search for `destroy`. - Fixes warnings regarding uninitialized member variables. - Fixes bugs in TensorRT model ID generation: - Potential segfault when model path only has a root component. - Unbounded string copy for non-Windows builds. ### Motivation and Context Clean up
This commit is contained in:
Adrian Lizarraga
GitHub
parent
60290393f3
commit
30b9f5dde1
6 changed files with 71 additions and 70 deletions
|
|
@ -702,4 +702,9 @@ void MurmurHash3::x86_128(const void* key, int len, uint32_t seed, void* out) {
|
|||
return g_host->MurmurHash3__x86_128(key, len, seed, out);
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
std::string ToUTF8String(const std::wstring& s) {
|
||||
return g_host->ToUTF8String(s);
|
||||
}
|
||||
#endif
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -885,6 +885,10 @@ struct ProviderHost {
|
|||
|
||||
virtual void MurmurHash3__x86_128(const void* key, int len, uint32_t seed, void* out) = 0;
|
||||
|
||||
#ifdef _WIN32
|
||||
virtual std::string ToUTF8String(const std::wstring& s) = 0;
|
||||
#endif
|
||||
|
||||
virtual ProviderHostCPU& GetProviderHostCPU() = 0;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -278,7 +278,7 @@ std::unique_lock<OrtMutex> TensorrtExecutionProvider::GetApiLock() const {
|
|||
#ifdef ORT_TENSORRT_PLACEHOLDER_BUILDER
|
||||
// instantiate global unused builder object which keeps the TRT kernel library in memory
|
||||
// so that subsequent builders avoid the expensive load / unload process.
|
||||
auto const placeholder = tensorrt_ptr::unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(GetTensorrtLogger()));
|
||||
auto const placeholder = std::unique_ptr<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(GetTensorrtLogger()));
|
||||
#endif
|
||||
|
||||
TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProviderInfo& info)
|
||||
|
|
@ -446,7 +446,7 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
|
|||
}
|
||||
{
|
||||
auto lock = GetApiLock();
|
||||
runtime_ = tensorrt_ptr::unique_pointer<nvinfer1::IRuntime>(nvinfer1::createInferRuntime(GetTensorrtLogger()));
|
||||
runtime_ = std::unique_ptr<nvinfer1::IRuntime>(nvinfer1::createInferRuntime(GetTensorrtLogger()));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -921,9 +921,9 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect
|
|||
// Get supported node list recursively
|
||||
SubGraphCollection_t parser_nodes_list;
|
||||
TensorrtLogger& trt_logger = GetTensorrtLogger();
|
||||
auto trt_builder = tensorrt_ptr::unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
|
||||
auto trt_builder = std::unique_ptr<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
|
||||
const auto explicitBatch = 1U << static_cast<uint32_t>(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
|
||||
auto trt_network = tensorrt_ptr::unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetworkV2(explicitBatch));
|
||||
auto trt_network = std::unique_ptr<nvinfer1::INetworkDefinition>(trt_builder->createNetworkV2(explicitBatch));
|
||||
|
||||
auto trt_parser = tensorrt_ptr::unique_pointer<nvonnxparser::IParser>(nvonnxparser::createParser(*trt_network, trt_logger));
|
||||
trt_parser->supportsModel(string_buf.data(), string_buf.size(), parser_nodes_list, model_path_);
|
||||
|
|
@ -1253,10 +1253,10 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
}
|
||||
|
||||
TensorrtLogger& trt_logger = GetTensorrtLogger();
|
||||
auto trt_builder = tensorrt_ptr::unique_pointer<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
|
||||
auto trt_builder = std::unique_ptr<nvinfer1::IBuilder>(nvinfer1::createInferBuilder(trt_logger));
|
||||
const auto explicitBatch = 1U << static_cast<uint32_t>(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
|
||||
auto trt_network = tensorrt_ptr::unique_pointer<nvinfer1::INetworkDefinition>(trt_builder->createNetworkV2(explicitBatch));
|
||||
auto trt_config = tensorrt_ptr::unique_pointer<nvinfer1::IBuilderConfig>(trt_builder->createBuilderConfig());
|
||||
auto trt_network = std::unique_ptr<nvinfer1::INetworkDefinition>(trt_builder->createNetworkV2(explicitBatch));
|
||||
auto trt_config = std::unique_ptr<nvinfer1::IBuilderConfig>(trt_builder->createBuilderConfig());
|
||||
auto trt_parser = tensorrt_ptr::unique_pointer<nvonnxparser::IParser>(nvonnxparser::createParser(*trt_network, trt_logger));
|
||||
trt_parser->parse(string_buf.data(), string_buf.size(), model_path_);
|
||||
trt_config->setMaxWorkspaceSize(max_workspace_size_);
|
||||
|
|
@ -1368,8 +1368,8 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
|
||||
// Build TRT engine here if the graph doesn't have dynamic shape input. Otherwise engine will
|
||||
// be built at runtime
|
||||
tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine> trt_engine;
|
||||
tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext> trt_context;
|
||||
std::unique_ptr<nvinfer1::ICudaEngine> trt_engine;
|
||||
std::unique_ptr<nvinfer1::IExecutionContext> trt_context;
|
||||
if (!has_dynamic_shape) {
|
||||
const std::string cache_path = GetCachePath(cache_path_, trt_node_name_with_precision);
|
||||
const std::string engine_cache_path = cache_path + ".engine";
|
||||
|
|
@ -1384,7 +1384,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
engine_file.seekg(0, std::ios::beg);
|
||||
std::unique_ptr<char[]> engine_buf{new char[engine_size]};
|
||||
engine_file.read((char*)engine_buf.get(), engine_size);
|
||||
trt_engine = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(runtime_->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr));
|
||||
trt_engine = std::unique_ptr<nvinfer1::ICudaEngine>(runtime_->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr));
|
||||
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] DeSerialized " + engine_cache_path;
|
||||
if (trt_engine == nullptr) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
|
||||
|
|
@ -1403,7 +1403,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
"TensorRT EP could not call engine decryption function decrypt");
|
||||
}
|
||||
// Deserialize engine
|
||||
trt_engine = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(runtime_->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr));
|
||||
trt_engine = std::unique_ptr<nvinfer1::ICudaEngine>(runtime_->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr));
|
||||
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] DeSerialized " + engine_cache_path;
|
||||
if (trt_engine == nullptr) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
|
||||
|
|
@ -1420,13 +1420,13 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
}
|
||||
|
||||
// Build engine
|
||||
trt_engine = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(trt_builder->buildEngineWithConfig(*trt_network, *trt_config));
|
||||
trt_engine = std::unique_ptr<nvinfer1::ICudaEngine>(trt_builder->buildEngineWithConfig(*trt_network, *trt_config));
|
||||
if (trt_engine == nullptr) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
|
||||
"TensorRT EP could not build engine for fused node: " + fused_node.Name());
|
||||
}
|
||||
if (engine_cache_enable_) {
|
||||
nvinfer1::IHostMemory* serializedModel = trt_engine->serialize();
|
||||
std::unique_ptr<nvinfer1::IHostMemory> serializedModel(trt_engine->serialize());
|
||||
size_t engine_size = serializedModel->size();
|
||||
if (engine_decryption_enable_) {
|
||||
// Encrypt engine
|
||||
|
|
@ -1438,7 +1438,6 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
std::ofstream file(engine_cache_path, std::ios::binary | std::ios::out);
|
||||
file.write(reinterpret_cast<char*>(serializedModel->data()), engine_size);
|
||||
}
|
||||
serializedModel->destroy();
|
||||
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Serialized " + engine_cache_path;
|
||||
}
|
||||
}
|
||||
|
|
@ -1451,9 +1450,9 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
max_ctx_mem_size_ = mem_size;
|
||||
context_memory_ = IAllocator::MakeUniquePtr<void>(allocator_, max_ctx_mem_size_);
|
||||
}
|
||||
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContextWithoutDeviceMemory());
|
||||
trt_context = std::unique_ptr<nvinfer1::IExecutionContext>(trt_engine->createExecutionContextWithoutDeviceMemory());
|
||||
} else {
|
||||
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
|
||||
trt_context = std::unique_ptr<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
|
||||
}
|
||||
if (trt_context == nullptr) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
|
||||
|
|
@ -1561,7 +1560,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
engine_file.seekg(0, std::ios::beg);
|
||||
std::unique_ptr<char[]> engine_buf{new char[engine_size]};
|
||||
engine_file.read((char*)engine_buf.get(), engine_size);
|
||||
*(trt_state->engine) = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(
|
||||
*(trt_state->engine) = std::unique_ptr<nvinfer1::ICudaEngine>(
|
||||
trt_state->runtime->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr));
|
||||
if (trt_state->engine == nullptr) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Build Engine.");
|
||||
|
|
@ -1569,10 +1568,10 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] DeSerialized " + engine_cache_path;
|
||||
trt_engine = trt_state->engine->get();
|
||||
if (trt_state->context_memory_sharing_enable) {
|
||||
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
|
||||
*(trt_state->context) = std::unique_ptr<nvinfer1::IExecutionContext>(
|
||||
trt_state->engine->get()->createExecutionContextWithoutDeviceMemory());
|
||||
} else {
|
||||
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
|
||||
*(trt_state->context) = std::unique_ptr<nvinfer1::IExecutionContext>(
|
||||
trt_state->engine->get()->createExecutionContext());
|
||||
}
|
||||
if (trt_state->context == nullptr) {
|
||||
|
|
@ -1596,7 +1595,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
// Deserialize engine
|
||||
trt_state->context->reset();
|
||||
trt_state->engine->reset();
|
||||
*(trt_state->engine) = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(trt_state->runtime->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr));
|
||||
*(trt_state->engine) = std::unique_ptr<nvinfer1::ICudaEngine>(trt_state->runtime->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr));
|
||||
if (trt_state->engine == nullptr) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
|
||||
"TensorRT EP could not deserialize engine from encrypted cache: " + engine_cache_path);
|
||||
|
|
@ -1604,10 +1603,10 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] DeSerialized " + engine_cache_path;
|
||||
trt_engine = trt_state->engine->get();
|
||||
if (trt_state->context_memory_sharing_enable) {
|
||||
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
|
||||
*(trt_state->context) = std::unique_ptr<nvinfer1::IExecutionContext>(
|
||||
trt_state->engine->get()->createExecutionContextWithoutDeviceMemory());
|
||||
} else {
|
||||
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
|
||||
*(trt_state->context) = std::unique_ptr<nvinfer1::IExecutionContext>(
|
||||
trt_state->engine->get()->createExecutionContext());
|
||||
}
|
||||
if (trt_state->context == nullptr) {
|
||||
|
|
@ -1751,7 +1750,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
if (engine_update) {
|
||||
trt_state->context->reset();
|
||||
trt_state->engine->reset();
|
||||
auto trt_config = tensorrt_ptr::unique_pointer<nvinfer1::IBuilderConfig>(trt_builder->createBuilderConfig());
|
||||
auto trt_config = std::unique_ptr<nvinfer1::IBuilderConfig>(trt_builder->createBuilderConfig());
|
||||
trt_config->setMaxWorkspaceSize(*(trt_state->max_workspace_size_ptr));
|
||||
trt_config->addOptimizationProfile(*trt_profile);
|
||||
|
||||
|
|
@ -1783,7 +1782,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
// Build engine
|
||||
{
|
||||
auto lock = GetApiLock();
|
||||
*(trt_state->engine) = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(
|
||||
*(trt_state->engine) = std::unique_ptr<nvinfer1::ICudaEngine>(
|
||||
trt_builder->buildEngineWithConfig(*trt_state->network->get(), *trt_config));
|
||||
}
|
||||
if (trt_state->engine == nullptr) {
|
||||
|
|
@ -1796,7 +1795,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Serialized " + profile_cache_path;
|
||||
|
||||
// Serialize engine
|
||||
nvinfer1::IHostMemory* serializedModel = trt_engine->serialize();
|
||||
std::unique_ptr<nvinfer1::IHostMemory> serializedModel(trt_engine->serialize());
|
||||
size_t engine_size = serializedModel->size();
|
||||
if (trt_state->engine_decryption_enable) {
|
||||
// Encrypt engine
|
||||
|
|
@ -1808,15 +1807,14 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
|
|||
std::ofstream file(engine_cache_path, std::ios::binary | std::ios::out);
|
||||
file.write(reinterpret_cast<char*>(serializedModel->data()), engine_size);
|
||||
}
|
||||
serializedModel->destroy();
|
||||
}
|
||||
|
||||
// Build context
|
||||
if (trt_state->context_memory_sharing_enable) {
|
||||
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
|
||||
*(trt_state->context) = std::unique_ptr<nvinfer1::IExecutionContext>(
|
||||
trt_state->engine->get()->createExecutionContextWithoutDeviceMemory());
|
||||
} else {
|
||||
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
|
||||
*(trt_state->context) = std::unique_ptr<nvinfer1::IExecutionContext>(
|
||||
trt_state->engine->get()->createExecutionContext());
|
||||
}
|
||||
if (trt_state->context == nullptr) {
|
||||
|
|
|
|||
|
|
@ -87,33 +87,33 @@ struct TensorrtFuncState {
|
|||
DestroyFunc test_release_func = nullptr;
|
||||
AllocatorHandle allocator = nullptr;
|
||||
tensorrt_ptr::unique_pointer<nvonnxparser::IParser>* parser = nullptr;
|
||||
tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>* engine = nullptr;
|
||||
tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>* context = nullptr;
|
||||
tensorrt_ptr::unique_pointer<nvinfer1::IBuilder>* builder = nullptr;
|
||||
tensorrt_ptr::unique_pointer<nvinfer1::INetworkDefinition>* network = nullptr;
|
||||
std::unique_ptr<nvinfer1::ICudaEngine>* engine = nullptr;
|
||||
std::unique_ptr<nvinfer1::IExecutionContext>* context = nullptr;
|
||||
std::unique_ptr<nvinfer1::IBuilder>* builder = nullptr;
|
||||
std::unique_ptr<nvinfer1::INetworkDefinition>* network = nullptr;
|
||||
std::vector<std::unordered_map<std::string, size_t>> input_info;
|
||||
std::vector<std::unordered_map<std::string, size_t>> output_info;
|
||||
std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>> input_shape_ranges;
|
||||
OrtMutex* tensorrt_mu_ptr = nullptr;
|
||||
bool fp16_enable;
|
||||
bool int8_enable;
|
||||
bool int8_calibration_cache_available;
|
||||
bool dla_enable;
|
||||
int dla_core;
|
||||
bool fp16_enable = false;
|
||||
bool int8_enable = false;
|
||||
bool int8_calibration_cache_available = false;
|
||||
bool dla_enable = false;
|
||||
int dla_core = 0;
|
||||
size_t* max_workspace_size_ptr = nullptr;
|
||||
std::string trt_node_name_with_precision;
|
||||
bool engine_cache_enable;
|
||||
bool engine_cache_enable = false;
|
||||
std::string engine_cache_path;
|
||||
nvinfer1::IRuntime* runtime = nullptr;
|
||||
nvinfer1::IOptimizationProfile* trt_profile = nullptr;
|
||||
AllocatorPtr scratch_allocator;
|
||||
bool context_memory_sharing_enable;
|
||||
bool context_memory_sharing_enable = false;
|
||||
size_t* max_context_mem_size_ptr = nullptr;
|
||||
IAllocatorUniquePtr<void>* context_memory = nullptr;
|
||||
std::unordered_map<std::string, float> dynamic_range_map;
|
||||
bool engine_decryption_enable;
|
||||
int (*engine_decryption)(const char*, char*, size_t*);
|
||||
int (*engine_encryption)(const char*, char*, size_t);
|
||||
bool engine_decryption_enable = false;
|
||||
int (*engine_decryption)(const char*, char*, size_t*) = nullptr;
|
||||
int (*engine_encryption)(const char*, char*, size_t) = nullptr;
|
||||
};
|
||||
|
||||
// Logical device representation.
|
||||
|
|
@ -164,7 +164,7 @@ class TensorrtExecutionProvider : public IExecutionProvider {
|
|||
bool dump_subgraphs_ = false;
|
||||
bool engine_cache_enable_ = false;
|
||||
std::string cache_path_, engine_decryption_lib_path_;
|
||||
tensorrt_ptr::unique_pointer<nvinfer1::IRuntime> runtime_ = nullptr;
|
||||
std::unique_ptr<nvinfer1::IRuntime> runtime_ = nullptr;
|
||||
OrtMutex tensorrt_mu_;
|
||||
int device_id_;
|
||||
AllocatorPtr allocator_;
|
||||
|
|
@ -172,17 +172,17 @@ class TensorrtExecutionProvider : public IExecutionProvider {
|
|||
bool layer_norm_fp32_fallback_ = false;
|
||||
size_t max_ctx_mem_size_ = 0;
|
||||
IAllocatorUniquePtr<void> context_memory_ = nullptr;
|
||||
mutable char model_path_[4096]; // Reserved for max path length
|
||||
mutable char model_path_[4096] = {}; // Reserved for max path length
|
||||
bool engine_decryption_enable_ = false;
|
||||
int (*engine_decryption_)(const char*, char*, size_t*);
|
||||
int (*engine_encryption_)(const char*, char*, size_t);
|
||||
int (*engine_decryption_)(const char*, char*, size_t*) = nullptr;
|
||||
int (*engine_encryption_)(const char*, char*, size_t) = nullptr;
|
||||
|
||||
std::unordered_set<std::string> control_flow_op_set_ = {"If", "Loop", "Scan"};
|
||||
std::unordered_map<std::string, tensorrt_ptr::unique_pointer<nvonnxparser::IParser>> parsers_;
|
||||
std::unordered_map<std::string, tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>> engines_;
|
||||
std::unordered_map<std::string, tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>> contexts_;
|
||||
std::unordered_map<std::string, tensorrt_ptr::unique_pointer<nvinfer1::IBuilder>> builders_;
|
||||
std::unordered_map<std::string, tensorrt_ptr::unique_pointer<nvinfer1::INetworkDefinition>> networks_;
|
||||
std::unordered_map<std::string, std::unique_ptr<nvinfer1::ICudaEngine>> engines_;
|
||||
std::unordered_map<std::string, std::unique_ptr<nvinfer1::IExecutionContext>> contexts_;
|
||||
std::unordered_map<std::string, std::unique_ptr<nvinfer1::IBuilder>> builders_;
|
||||
std::unordered_map<std::string, std::unique_ptr<nvinfer1::INetworkDefinition>> networks_;
|
||||
std::unordered_map<std::string, std::vector<std::unordered_map<std::string, size_t>>> input_info_;
|
||||
std::unordered_map<std::string, std::vector<std::unordered_map<std::string, size_t>>> output_info_;
|
||||
std::unordered_map<std::string, std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>>> input_shape_ranges_;
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include "ort_trt_int8_cal_table.fbs.h"
|
||||
#include <NvInferVersion.h>
|
||||
#include "core/providers/cuda/cuda_pch.h"
|
||||
#include "core/common/path_string.h"
|
||||
#include "core/framework/murmurhash3.h"
|
||||
|
||||
namespace fs = std::experimental::filesystem;
|
||||
|
|
@ -215,24 +216,18 @@ HashValue TRTGenerateId(const GraphViewer& graph_viewer) {
|
|||
MurmurHash3::x86_128(str.data(), gsl::narrow_cast<int32_t>(str.size()), hash[0], &hash);
|
||||
};
|
||||
|
||||
// Use model name instead of path to avoid cache regeneration if path changes
|
||||
const auto& model_path = main_graph.ModelPath();
|
||||
if (!model_path.IsEmpty()) {
|
||||
// Get model name
|
||||
PathString path_string = model_path.GetComponents().back();
|
||||
char arr[256];
|
||||
#ifdef _WIN32
|
||||
wcstombs_s(nullptr, arr, sizeof(arr), path_string.c_str(), sizeof(arr));
|
||||
#else
|
||||
strcpy(arr, path_string.c_str());
|
||||
#endif
|
||||
std::string model_name(arr);
|
||||
// Use the model's file name instead of the entire path to avoid cache regeneration if path changes
|
||||
const auto& model_path_components = main_graph.ModelPath().GetComponents();
|
||||
|
||||
if (!model_path_components.empty()) {
|
||||
std::string model_name = PathToUTF8String(model_path_components.back());
|
||||
|
||||
LOGS_DEFAULT(INFO) << "[TensorRT EP] Model name is " << model_name;
|
||||
// Ensure enough characters are hashed in case model names are too short
|
||||
int32_t model_name_length = gsl::narrow_cast<int32_t>(model_name.size());
|
||||
constexpr int32_t hash_string_length = 500;
|
||||
const size_t model_name_length = model_name.size();
|
||||
constexpr size_t hash_string_length = 500;
|
||||
std::string repeat_model_name = model_name;
|
||||
for (int i = model_name_length; i > 0 && i < hash_string_length; i += model_name_length) {
|
||||
for (size_t i = model_name_length; i > 0 && i < hash_string_length; i += model_name_length) {
|
||||
repeat_model_name += model_name;
|
||||
}
|
||||
hash_str(repeat_model_name);
|
||||
|
|
@ -240,11 +235,6 @@ HashValue TRTGenerateId(const GraphViewer& graph_viewer) {
|
|||
LOGS_DEFAULT(INFO) << "[TensorRT EP] Model path is empty";
|
||||
}
|
||||
|
||||
// fingerprint the main graph by hashing graph inputs
|
||||
for (const auto* node_arg : main_graph.GetInputsIncludingInitializers()) {
|
||||
hash_str(node_arg->Name());
|
||||
}
|
||||
|
||||
// fingerprint current graph by hashing graph inputs
|
||||
for (const auto* node_arg : graph_viewer.GetInputsIncludingInitializers()) {
|
||||
hash_str(node_arg->Name());
|
||||
|
|
|
|||
|
|
@ -1033,6 +1033,10 @@ struct ProviderHostImpl : ProviderHost {
|
|||
MurmurHash3::x86_128(key, len, seed, out);
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
std::string ToUTF8String(const std::wstring& s) override { return onnxruntime::ToUTF8String(s); }
|
||||
#endif
|
||||
|
||||
ProviderHostCPU& GetProviderHostCPU() override { return onnxruntime::GetProviderHostCPU(); }
|
||||
} provider_host_;
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
|
|
|
|||
Loading…
Reference in a new issue