saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-06-29 03:30:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

TensorRT EP engine cache serialization/deserialization refactor (#11045)

Browse source 
* Code refactor

* fix bug

* modify comment

* modify test for the new ORT TRT cache behavior

* update comment

* rename variable

* fix bug for not having trt context

* Custom parameters (#10964)

* get inputs independently for trtexec

* track one process only

* remove engine and profile files

* change time to commit time

* add runtime option for io binding

* move to commit date

* fixes

* add option for graph optimization

* cleanup docker script

* note second time creation

* allow for parameters to be configured from pipeline at runtime

* uncomment

* include optional arguments at runtime

* post second session creation

* update cmake version

* Revert "update cmake version"

This reverts commit 09a1364eae68610724c8e90eeea777b7ee03f74b.

* Move data format import

* Perf FasterRCNN + MaskRCNN (#11102)

* add faster mask

* fix paths

* add a test scenario that - if engine cache is present, trt ep should load the engine cache and run inference

* Revert "Merge branch 'trt_cache_refactor' of https://github.com/microsoft/onnxruntime into trt_cache_refactor"

This reverts commit 8edc574de1ea6055534f33a57b9365c721c2eb29, reversing
changes made to 0c92e5b2b1d453527001fe731ed4ccfc79e6adad.

Co-authored-by: Olivia Jain <oljain@microsoft.com>
This commit is contained in:
Chi Lo 2022-04-26 11:07:48 -07:00 committed by GitHub
parent 81d78706fe
commit 0292356bd7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 284 additions and 214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

200
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -1074,6 +1074,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
std::unordered_map<std::string, std::unordered_map<size_t, std::pair<int64_t, int64_t>>> input_shape_ranges;
std::unordered_map<std::string, size_t> output_indexes(num_outputs);
std::unordered_map<std::string, size_t> output_types(num_outputs);
bool update_engine_cache = false;
// Initialize shape range for dynamic shape tensors
bool has_dynamic_shape = false;
@ -1158,15 +1159,15 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
}
}
// Build TRT engine here if the graph doesn't have dynamic shape input. Otherwise engine will
// be built at runtime
// If engine cache enable is set,
// load and deserialize TRT engine cache regardless of the graph has dynamic shape input or not
tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine> trt_engine;
tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext> trt_context;
if (!has_dynamic_shape) {
if (engine_cache_enable_) {
const std::string cache_path = GetCachePath(cache_path_, trt_node_name_with_precision);
const std::string engine_cache_path = cache_path + ".engine";
std::ifstream engine_file(engine_cache_path, std::ios::binary | std::ios::in);
if (engine_cache_enable_ && engine_file) {
if (engine_file) {
engine_file.seekg(0, std::ios::end);
size_t engine_size = engine_file.tellg();
engine_file.seekg(0, std::ios::beg);
@ -1178,7 +1179,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not deserialize engine from cache: " + engine_cache_path);
}
} else if (engine_decryption_enable_ && engine_cache_enable_ && !engine_file) {
} else if (engine_decryption_enable_ && !engine_file) {
// Decrypt engine
size_t engine_size = 0;
if (!engine_decryption_(engine_cache_path.c_str(), nullptr, &engine_size)) {
@ -1197,7 +1198,34 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not deserialize engine from encrypted cache: " + engine_cache_path);
}
} else {
}
if (trt_engine != nullptr) {
// Build context
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
if (trt_context == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not build execution context for fused node: " + fused_node->Name());
}
}
// If graph has dynamic shape input,
// load and deserialize TRT engine profile cache
if (has_dynamic_shape) {
const std::string profile_cache_path = cache_path + ".profile";
std::ifstream profile_file(profile_cache_path, std::ios::binary | std::ios::in);
if (profile_file) {
input_shape_ranges = DeserializeProfile(profile_file);
}
}
}
// If (1) engine cache enable is not set or (2) first time enable engine cache and no engine cache is present,
// build TRT engine here if the graph doesn't have dynamic shape input. Otherwise engine will
// be built at runtime
if (!has_dynamic_shape) {
if (trt_engine == nullptr) {
// Set INT8 per tensor dynamic range
if (int8_enable_ && trt_builder->platformHasFastInt8() && int8_calibration_cache_available_) {
trt_config->setInt8Calibrator(nullptr);
@ -1216,29 +1244,16 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not build engine for fused node: " + fused_node->Name());
}
if (engine_cache_enable_) {
nvinfer1::IHostMemory* serializedModel = trt_engine->serialize();
size_t engine_size = serializedModel->size();
if (engine_decryption_enable_) {
// Encrypt engine
if (!engine_encryption_(engine_cache_path.c_str(), reinterpret_cast<char*>(serializedModel->data()), engine_size)) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not call engine encryption function encrypt");
}
} else {
std::ofstream file(engine_cache_path, std::ios::binary | std::ios::out);
file.write(reinterpret_cast<char*>(serializedModel->data()), engine_size);
}
serializedModel->destroy();
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Serialized " + engine_cache_path;
}
}
// Build context
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
if (trt_context == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not build execution context for fused node: " + fused_node->Name());
if (engine_cache_enable_)
update_engine_cache = true;
// Build context
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
if (trt_context == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not build execution context for fused node: " + fused_node->Name());
}
}
}
@ -1285,16 +1300,52 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
&networks_[context->node_name], input_info_[context->node_name], output_info_[context->node_name],
input_shape_ranges_[context->node_name], &tensorrt_mu_, fp16_enable_, int8_enable_, int8_calibration_cache_available_,
dla_enable_, dla_core_, &max_workspace_size_, trt_node_name_with_precision, engine_cache_enable_, cache_path_,
runtime_.get(), nullptr, allocator_, dynamic_range_map, engine_decryption_enable_, engine_decryption_, engine_encryption_};
runtime_.get(), nullptr, allocator_, dynamic_range_map, engine_decryption_enable_, engine_decryption_, engine_encryption_,
update_engine_cache};
*state = p.release();
return 0;
};
// Release function state
compute_info.release_state_func = [](FunctionState state) {
if (state)
if (state) {
// Serialize and save engine to cache
//
// Note: only save engine to file if engine cache enable is set and engine is being updated due to input shape changed
// or engine file is not previously existed
TensorrtFuncState* trt_state = reinterpret_cast<TensorrtFuncState*>(state);
if (trt_state->update_engine_cache) {
// Serialize engine
const std::string cache_path = GetCachePath(trt_state->engine_cache_path, trt_state->trt_node_name_with_precision);
const std::string engine_cache_path = cache_path + ".engine";
nvinfer1::IHostMemory* serializedModel = trt_state->engine->get()->serialize();
size_t engine_size = serializedModel->size();
if (trt_state->engine_decryption_enable) {
// Encrypt engine
if (!trt_state->engine_encryption(engine_cache_path.c_str(), reinterpret_cast<char*>(serializedModel->data()), engine_size)) {
delete static_cast<TensorrtFuncState*>(state);
ORT_THROW_IF_ERROR(ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not call engine encryption function encrypt"));
}
} else {
std::ofstream file(engine_cache_path, std::ios::binary | std::ios::out);
file.write(reinterpret_cast<char*>(serializedModel->data()), engine_size);
}
serializedModel->destroy();
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Serialized " + engine_cache_path;
// Serialize engine profile if needed
if (!trt_state->input_shape_ranges.empty()) {
const std::string profile_cache_path = cache_path + ".profile";
SerializeProfile(profile_cache_path, trt_state->input_shape_ranges);
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Serialized " + profile_cache_path;
}
}
delete static_cast<TensorrtFuncState*>(state);
}
};
// Create compute function
compute_info.compute_func = [this](FunctionState state, const OrtCustomOpApi* api, OrtKernelContext* context) {
Ort::CustomOpApi ort{*api};
@ -1317,71 +1368,6 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
cudaStream_t stream = static_cast<cudaStream_t>(this->GetComputeStream());
// Load serialized engine
const std::string cache_path = GetCachePath(trt_state->engine_cache_path, trt_state->trt_node_name_with_precision);
const std::string engine_cache_path = cache_path + ".engine";
const std::string profile_cache_path = cache_path + ".profile";
if (trt_state->engine_cache_enable && trt_engine == nullptr) {
std::ifstream engine_file(engine_cache_path, std::ios::binary | std::ios::in);
std::ifstream profile_file(profile_cache_path, std::ios::binary | std::ios::in);
if (engine_file && profile_file) {
// Deserialize profile
shape_ranges = DeserializeProfile(profile_file);
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] DeSerialized " + profile_cache_path;
// Deserialize engine
trt_state->context->reset();
trt_state->engine->reset();
engine_file.seekg(0, std::ios::end);
size_t engine_size = engine_file.tellg();
engine_file.seekg(0, std::ios::beg);
std::unique_ptr<char[]> engine_buf{new char[engine_size]};
engine_file.read((char*)engine_buf.get(), engine_size);
*(trt_state->engine) = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(
trt_state->runtime->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr));
if (trt_state->engine == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Build Engine.");
}
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] DeSerialized " + engine_cache_path;
trt_engine = trt_state->engine->get();
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
trt_state->engine->get()->createExecutionContext());
if (trt_state->context == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP failed to create context.");
}
trt_context = trt_state->context->get();
} else if (trt_state->engine_decryption_enable && !engine_file && profile_file) {
shape_ranges = DeserializeProfile(profile_file);
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] DeSerialized " + profile_cache_path;
// Decrypt engine
size_t engine_size = 0;
if (!trt_state->engine_decryption(engine_cache_path.c_str(), nullptr, &engine_size)) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not get engine buffer size");
}
std::unique_ptr<char[]> engine_buf{new char[engine_size]};
if (!trt_state->engine_decryption(engine_cache_path.c_str(), &engine_buf[0], &engine_size)) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not call engine decryption function decrypt");
}
// Deserialize engine
trt_state->context->reset();
trt_state->engine->reset();
*(trt_state->engine) = tensorrt_ptr::unique_pointer<nvinfer1::ICudaEngine>(trt_state->runtime->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr));
if (trt_state->engine == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not deserialize engine from encrypted cache: " + engine_cache_path);
}
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] DeSerialized " + engine_cache_path;
trt_engine = trt_state->engine->get();
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
trt_state->engine->get()->createExecutionContext());
if (trt_state->context == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP failed to create context.");
}
trt_context = trt_state->context->get();
}
}
for (int i = 0, end = num_inputs; i < end; ++i) {
auto input = trt_state->network->get()->getInput(i);
const std::string& input_name = input->getName();
@ -1558,26 +1544,6 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Build Engine.");
}
trt_engine = trt_state->engine->get();
if (trt_state->engine_cache_enable) {
// Serialize engine profile
SerializeProfile(profile_cache_path, shape_ranges);
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Serialized " + profile_cache_path;
// Serialize engine
nvinfer1::IHostMemory* serializedModel = trt_engine->serialize();
size_t engine_size = serializedModel->size();
if (trt_state->engine_decryption_enable) {
// Encrypt engine
if (!trt_state->engine_encryption(engine_cache_path.c_str(), reinterpret_cast<char*>(serializedModel->data()), engine_size)) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not call engine encryption function encrypt");
}
} else {
std::ofstream file(engine_cache_path, std::ios::binary | std::ios::out);
file.write(reinterpret_cast<char*>(serializedModel->data()), engine_size);
}
serializedModel->destroy();
}
// Build context
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
@ -1586,6 +1552,10 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<Node*>& fuse
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP failed to create context.");
}
trt_context = trt_state->context->get();
if (trt_state->engine_cache_enable)
trt_state->update_engine_cache = true;
trt_state->input_shape_ranges = shape_ranges;
}
// Get input and output binding names

3
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
View file

@ -107,6 +107,9 @@ struct TensorrtFuncState {
bool engine_decryption_enable;
int (*engine_decryption)(const char*, char*, size_t*);
int (*engine_encryption)(const char*, char*, size_t);
// If sub-graph has dynamic input shape and the shape range changes, or the first time writing out engine cache, this flag is set to true and engine cache will be saved. Otherwise the flag is false.
// Note: For dynamic input shape, if update_engine_cache flag is true, profile cache will be saved as well.
bool update_engine_cache;
};
// Logical device representation.

295
onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc
View file

@ -245,6 +245,8 @@ void RunWithOneSessionMultiThreadsInference(std::string model_name, std::string
th.join();
}
void CreateAndRunInferenceSession() {}
TEST(TensorrtExecutionProviderTest, MultiThreadsTestWithOneSessionSingleThreadInference) {
std::vector<std::thread> threads;
std::string model_name = "trt_execution_provider_multithreading_test.onnx";
@ -292,38 +294,51 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) {
CreateBaseModel(model_name, cache_type + "cachingtest", dims);
SessionOptions so;
so.session_logid = "TensorrtExecutionProvider" + cache_type + "cacheTest";
RunOptions run_options;
run_options.run_tag = so.session_logid;
InferenceSession session_object{so, GetEnvironment()};
auto allocator_manager = session_object.GetAllocatorManager();
auto cuda_provider = DefaultCudaExecutionProvider();
cuda_provider->RegisterAllocator(allocator_manager);
auto cpu_allocator = cuda_provider->GetAllocator(0, OrtMemTypeCPU);
std::vector<int64_t> dims_mul_x = {1, 3, 2};
std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
OrtValue ml_value_x;
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_x);
OrtValue ml_value_y;
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_y);
OrtValue ml_value_z;
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_z);
NameMLValMap feeds;
feeds.insert(std::make_pair("X", ml_value_x));
feeds.insert(std::make_pair("Y", ml_value_y));
feeds.insert(std::make_pair("Z", ml_value_z));
/* If cache_type is "engine", following code will test the functionality of engine and optimization profile of ORT TRT, including:
* - engine cache serialization/de-serialization
* - profile cache serialization/de-serialization
* - engine/profile cache should be updated when the input shape changes
* - min/max shape ranges of dynamic shape dimensions saved in profile cache
* - if engine cache is present, trt ep should load the engine cache and run inference
* - read corrupted profile cache #TODO
*/
// prepare outputs
std::vector<std::string> output_names;
output_names.push_back("M");
std::vector<OrtValue> fetches;
/*
* First inference run
*/
{
SessionOptions so;
so.session_logid = "TensorrtExecutionProvider" + cache_type + "cacheTest";
RunOptions run_options;
run_options.run_tag = so.session_logid;
InferenceSession session_object{so, GetEnvironment()};
auto allocator_manager = session_object.GetAllocatorManager();
auto cuda_provider = DefaultCudaExecutionProvider();
cuda_provider->RegisterAllocator(allocator_manager);
auto cpu_allocator = cuda_provider->GetAllocator(0, OrtMemTypeCPU);
std::vector<int64_t> dims_mul_x = {1, 3, 2};
std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
OrtValue ml_value_x;
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_x);
OrtValue ml_value_y;
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_y);
OrtValue ml_value_z;
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_z);
NameMLValMap feeds;
feeds.insert(std::make_pair("X", ml_value_x));
feeds.insert(std::make_pair("Y", ml_value_y));
feeds.insert(std::make_pair("Z", ml_value_z));
// prepare expected inputs and outputs
std::vector<int64_t> expected_dims_mul_m = {1, 3, 2};
std::vector<float> expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f};
// prepare outputs
std::vector<std::string> output_names;
output_names.push_back("M");
std::vector<OrtValue> fetches;
OrtTensorRTProviderOptionsV2 params{
// prepare expected inputs and outputs
std::vector<int64_t> expected_dims_mul_m = {1, 3, 2};
std::vector<float> expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f};
OrtTensorRTProviderOptionsV2 params{
0,
0,
nullptr,
@ -343,35 +358,40 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) {
nullptr,
0};
if (cache_type.compare("engine") == 0) {
params.trt_engine_cache_enable = 1;
std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
auto status = session_object.Load(model_name);
ASSERT_TRUE(status.IsOK());
status = session_object.Initialize();
ASSERT_TRUE(status.IsOK());
// run inference
// TRT engine will be created and cached
// TRT profile will be created and cached only for dynamic input shape
// Data in profile,
// X: 1, 3, 3, 2, 2, 2
// Y: 1, 3, 3, 2, 2, 2
// Z: 1, 3, 3, 2, 2, 2
status = session_object.Run(run_options, feeds, output_names, &fetches);
ASSERT_TRUE(status.IsOK());
VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
} else if (cache_type.compare("timing") == 0) {
// add test code here for timing cache
}
} // end of first inference run scope
/* Validate engine cache counts and engine profile content after first inference run.
*
* Note: Cache won't be saved to file until destructor of inference session is called,
* to be more specific, cache is saved at FunctionKernel's destructor (the release_state_func will be called).
* At this point, all the cache are saved becasue inference run scope ends.
*
*/
if (cache_type.compare("engine") == 0) {
/* Following code block tests the functionality of engine and optimization profile of ORT TRT, including:
* - engine cache serialization/de-serialization
* - profile cache serialization/de-serialization
* - engine/profile cache should be updated when the input shape changes
* - min/max shape ranges of dynamic shape dimensions saved in profile cache
* - read corrupted profile cache #TODO
*
*/
params.trt_engine_cache_enable = 1;
std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
auto status = session_object.Load(model_name);
ASSERT_TRUE(status.IsOK());
status = session_object.Initialize();
ASSERT_TRUE(status.IsOK());
// run inference
// TRT engine will be created and cached
// TRT profile will be created and cached only for dynamic input shape
// Data in profile,
// X: 1, 3, 3, 2, 2, 2
// Y: 1, 3, 3, 2, 2, 2
// Z: 1, 3, 3, 2, 2, 2
status = session_object.Run(run_options, feeds, output_names, &fetches);
ASSERT_TRUE(status.IsOK());
VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
ASSERT_TRUE(IsCacheExistedByType("./", ".engine"));
std::vector<fs::path> profile_files;
@ -387,8 +407,13 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) {
std::ifstream profile_file(profile_files[0], std::ios::binary | std::ios::in);
auto shape_ranges = DeserializeProfile(profile_file);
// Data in profile,
// X: 1, 3, 3, 2, 2, 2
// Y: 1, 3, 3, 2, 2, 2
// Z: 1, 3, 3, 2, 2, 2
// check min/max shape ranges of dynamic shape dimensions
for(auto it = shape_ranges.cbegin(); it != shape_ranges.cend(); ++it) {
for (auto it = shape_ranges.cbegin(); it != shape_ranges.cend(); ++it) {
auto ranges = it->second;
for (auto it2 = ranges.cbegin(); it2 != ranges.cend(); ++it2) {
if (it2->first == 1) {
@ -401,60 +426,132 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) {
}
}
}
}
// another inference run with input shape {1, 1, 6}
// TRT engine and profile will be updated
// Data in profile,
// X: 1, 1, 3, 2, 2, 6
// Y: 1, 1, 3, 2, 2, 6
// Z: 1, 1, 3, 2, 2, 6
dims_mul_x = {1, 1, 6};
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_x);
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_y);
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_z);
feeds.clear();
feeds.insert(std::make_pair("X", ml_value_x));
feeds.insert(std::make_pair("Y", ml_value_y));
feeds.insert(std::make_pair("Z", ml_value_z));
// prepare outputs
fetches.clear();
for (int i = 0; i < 2; ++i) {
/*
* Second/Third inference run
*/
{
SessionOptions so;
so.session_logid = "TensorrtExecutionProvider" + cache_type + "cacheTest";
RunOptions run_options;
run_options.run_tag = so.session_logid;
InferenceSession session_object{so, GetEnvironment()};
auto allocator_manager = session_object.GetAllocatorManager();
auto cuda_provider = DefaultCudaExecutionProvider();
cuda_provider->RegisterAllocator(allocator_manager);
auto cpu_allocator = cuda_provider->GetAllocator(0, OrtMemTypeCPU);
std::vector<int64_t> dims_mul_x = {1, 1, 6};
std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
OrtValue ml_value_x;
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_x);
OrtValue ml_value_y;
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_y);
OrtValue ml_value_z;
CreateMLValue<float>(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_z);
NameMLValMap feeds;
feeds.insert(std::make_pair("X", ml_value_x));
feeds.insert(std::make_pair("Y", ml_value_y));
feeds.insert(std::make_pair("Z", ml_value_z));
// prepare expected inputs and outputs
expected_dims_mul_m = {1, 1, 6};
// prepare outputs
std::vector<std::string> output_names;
output_names.push_back("M");
std::vector<OrtValue> fetches;
status = session_object.Run(run_options, feeds, output_names, &fetches);
// prepare expected inputs and outputs
std::vector<int64_t> expected_dims_mul_m = {1, 1, 6};
std::vector<float> expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f};
if (input_type.compare("static") == 0) {
// Can't run inference since input shape changes but the engine is built with static input
ASSERT_FALSE(status.IsOK());
} else {
ASSERT_TRUE(status.IsOK());
VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
OrtTensorRTProviderOptionsV2 params{
0,
0,
nullptr,
1000,
1,
1 << 30,
0,
0,
nullptr,
0,
0,
0,
0,
0,
nullptr,
0,
nullptr,
0};
profile_files = GetCachesByType("./", ".profile");
ASSERT_EQ(profile_files.size(), 1);
std::ifstream profile_file2(profile_files[0], std::ios::binary | std::ios::in);
auto shape_ranges2 = DeserializeProfile(profile_file2);
if (cache_type.compare("engine") == 0) {
params.trt_engine_cache_enable = 1;
std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
auto status = session_object.Load(model_name);
ASSERT_TRUE(status.IsOK());
status = session_object.Initialize();
ASSERT_TRUE(status.IsOK());
// check min/max shape ranges of dynamic shape dimensions
for(auto it = shape_ranges2.cbegin(); it != shape_ranges2.cend(); ++it) {
auto ranges = it->second;
for (auto it2 = ranges.cbegin(); it2 != ranges.cend(); ++it2) {
if (it2->first == 1) {
ASSERT_EQ(it2->second.first, 1);
ASSERT_EQ(it2->second.second, 3);
} else if (it2->first == 2) {
ASSERT_EQ(it2->second.first, 2);
ASSERT_EQ(it2->second.second, 6);
// another inference run with input shape {1, 1, 6}
// TRT engine and profile will be updated
// Data in profile,
// X: 1, 1, 3, 2, 2, 6
// Y: 1, 1, 3, 2, 2, 6
// Z: 1, 1, 3, 2, 2, 6
status = session_object.Run(run_options, feeds, output_names, &fetches);
if (input_type.compare("static") == 0) {
// Can't run inference since input shape changes but the engine is built with static input
ASSERT_FALSE(status.IsOK());
} else {
ASSERT_TRUE(status.IsOK());
VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
}
}
} // end of second/third inference run scope
/* Validate engine cache counts and engine profile content after second/third inference run.
*
* Note: Cache won't be saved to file until destructor of inference session is called,
* to be more specific, cache is saved at FunctionKernel's destructor (the release_state_func will be called).
* At this point, all the cache are saved becasue inference run scope ends.
*
*/
if (cache_type.compare("engine") == 0) {
ASSERT_TRUE(IsCacheExistedByType("./", ".engine"));
std::vector<fs::path> profile_files;
// profile cache only being generated for dynamic input shape
if (input_type.compare("static") == 0) {
ASSERT_TRUE(!IsCacheExistedByType("./", ".profile"));
} else {
ASSERT_TRUE(IsCacheExistedByType("./", ".profile"));
profile_files = GetCachesByType("./", ".profile");
ASSERT_EQ(profile_files.size(), 1);
std::ifstream profile_file2(profile_files[0], std::ios::binary | std::ios::in);
auto shape_ranges2 = DeserializeProfile(profile_file2);
// check min/max shape ranges of dynamic shape dimensions
for (auto it = shape_ranges2.cbegin(); it != shape_ranges2.cend(); ++it) {
auto ranges = it->second;
for (auto it2 = ranges.cbegin(); it2 != ranges.cend(); ++it2) {
if (it2->first == 1) {
ASSERT_EQ(it2->second.first, 1);
ASSERT_EQ(it2->second.second, 3);
} else if (it2->first == 2) {
ASSERT_EQ(it2->second.first, 2);
ASSERT_EQ(it2->second.second, 6);
}
}
}
}
}
} else if (cache_type.compare("timing") == 0) {
// add test code here
}
// clean up caches
RemoveCachesByType("./", ".engine");
RemoveCachesByType("./", ".profile");