saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Solve name conflict in TensorRT engine caching (#5128)

Browse source 
* fix hash conflict

* Add verbose for engine deserialization and destroy old engine memory if new engine is generated

* update parser

* Update tensorrt_execution_provider.cc

* use a better hash algorithm

* Update tensorrt_execution_provider.cc
This commit is contained in:
stevenlix 2020-09-11 09:12:56 -07:00 committed by GitHub
parent 51f3d3af72
commit c794c88ae0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 9 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.gitmodules vendored
View file

@ -56,12 +56,12 @@
[submodule "cmake/external/libprotobuf-mutator"]
path = cmake/external/libprotobuf-mutator
url = https://github.com/google/libprotobuf-mutator.git
[submodule "cmake/external/onnx-tensorrt"]
path = cmake/external/onnx-tensorrt
url = https://github.com/onnx/onnx-tensorrt.git
[submodule "cmake/external/flatbuffers"]
path = cmake/external/flatbuffers
url = https://github.com/google/flatbuffers.git
[submodule "cmake/external/SafeInt/safeint"]
path = cmake/external/SafeInt/safeint
url = https://github.com/gwang-msft/SafeInt.git
[submodule "cmake/external/onnx-tensorrt"]
path = cmake/external/onnx-tensorrt
url = https://github.com/onnx/onnx-tensorrt.git

2
cmake/external/onnx-tensorrt vendored

@ -1 +1 @@
Subproject commit 088554a5fbee9ba183c05c09c1abe986034e9208
Subproject commit fbebb144744b3be8defecd8478d74940056df305

13
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -47,9 +47,9 @@ std::string GetEnginePath(const ::std::string& root, const std::string& name) {
}
std::string GetVecHash(const ::std::vector<int>& vec) {
std::size_t ret = 0;
std::size_t ret = vec.size();
for (auto& i : vec) {
ret ^= std::hash<uint32_t>()(i);
ret ^= i + 0x9e3779b9 + (ret << 6) + (ret >> 2);
}
return std::to_string(ret);
}
@ -1026,22 +1026,22 @@ common::Status TensorrtExecutionProvider::Provider_Compile(const std::vector<onn
std::string trt_node_name_with_precision_shape = trt_state->trt_node_name_with_precision + "_" + GetVecHash(input_shapes);
std::string cached_path = GetEnginePath(trt_state->engine_cache_path, trt_node_name_with_precision_shape);
std::ifstream plan_file(cached_path, std::ios::binary | std::ios::in);
trt_state->context->reset();
trt_state->engine->reset();
if (plan_file && trt_state->engine_cache_enable) {
plan_file.seekg(0, std::ios::end);
int engine_size = plan_file.tellg();
plan_file.seekg(0, std::ios::beg);
std::unique_ptr<char[]> engine_buf{new char[engine_size]};
plan_file.read((char*)engine_buf.get(), engine_size);
auto runtime_ = trt_state->runtime;
trt_state->engine->reset();
*(trt_state->engine) = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(
runtime_->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr));
if (trt_state->engine->get() == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Build Engine.");
}
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] DeSerialized " + cached_path;
trt_engine = trt_state->engine->get();
} else {
auto trt_config = tensorrt_ptr::unique_pointer<nvinfer1::IBuilderConfig>(trt_builder->createBuilderConfig());
trt_config->setMaxWorkspaceSize(*(trt_state->max_workspace_size_ptr));
@ -1049,8 +1049,6 @@ common::Status TensorrtExecutionProvider::Provider_Compile(const std::vector<onn
if (*(trt_state->fp16_enable_ptr) && trt_builder->platformHasFastFp16()) {
trt_config->setFlag(nvinfer1::BuilderFlag::kFP16);
}
trt_state->context->reset();
trt_state->engine->reset();
*(trt_state->engine) = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(
trt_builder->buildEngineWithConfig(*trt_state->network, *trt_config));
@ -1066,7 +1064,6 @@ common::Status TensorrtExecutionProvider::Provider_Compile(const std::vector<onn
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Serialized " + cached_path;
}
}
trt_state->context->reset();
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
trt_state->engine->get()->createExecutionContext());
if (trt_state->context->get() == nullptr) {