saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix TRT timing cache test (#15588)

Browse source 
TRT EP test for timing cache has wrong logic where it enables timing
cache for both sessions to compare the trt engine build time, that's why
CI got some intermittent failures.

This PR disabled the timing cache test for comparing the engine build
time between enabling/disabling timing cache until we find a model that
can benefit from timing cache.
This commit is contained in:
Chi Lo 2023-04-25 10:20:26 -07:00 committed by GitHub
parent d0c3f92ec6
commit e1755541cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 29 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc
View file

@ -595,18 +595,18 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) {
}
}
} else if (cache_type.compare("timing") == 0) {
// add test code here
/* Following code block tests the functionality of engine and optimization profile of ORT TRT, including:
/* Following code block tests the functionality of timing cache, including:
* - timing cache cache serialization/de-serialization
* - benefir of usign a timing cache no matter if dynamic / static input
* - TODO: benefir of usign a timing cache no matter if dynamic / static input
*/
uint64_t compilation_without_cache_ms, compilation_with_cache_ms;
// Temporarily disable comparing the engine build time until we find the model that can benefit from timing cache to get engine build time reduced.
// uint64_t compilation_without_cache_ms, compilation_with_cache_ms;
// First session is created with TRT EP with timing cache enabled
params.trt_timing_cache_enable = 1;
// std::chrono
{
auto start = chrono::steady_clock::now();
// auto start = chrono::steady_clock::now();
std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
auto status = session_object.Load(model_name);
@ -614,46 +614,39 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) {
status = session_object.Initialize();
ASSERT_TRUE(status.IsOK());
// run inference
// TRT timing cache should be created under the situation of non-dynamic/dynamic shape input
status = session_object.Run(run_options, feeds, output_names, &fetches);
auto end = chrono::steady_clock::now();
// auto end = chrono::steady_clock::now();
ASSERT_TRUE(status.IsOK());
VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
ASSERT_TRUE(IsCacheExistedByType("./", ".timing"));
compilation_without_cache_ms = chrono::duration_cast<chrono::microseconds>(end - start).count();
// compilation_with_cache_ms = chrono::duration_cast<chrono::microseconds>(end - start).count();
}
// get new session and reinitialize model
// second same inference should resuse the cache and therefore have a faster build
if (input_type.compare("static") == 0) {
// Second session is created with TRT EP without timing cache enabled
params.trt_timing_cache_enable = 0;
{
InferenceSession session_object_new{so, GetEnvironment()};
{
InferenceSession session_object_new{so, GetEnvironment()};
{
auto start = chrono::steady_clock::now();
std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
EXPECT_TRUE(session_object_new.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
auto status = session_object_new.Load(model_name);
ASSERT_TRUE(status.IsOK());
status = session_object_new.Initialize();
ASSERT_TRUE(status.IsOK());
// auto start = chrono::steady_clock::now();
std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
EXPECT_TRUE(session_object_new.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
auto status = session_object_new.Load(model_name);
ASSERT_TRUE(status.IsOK());
status = session_object_new.Initialize();
ASSERT_TRUE(status.IsOK());
// run inference
// TRT timing cache should be created under the situation of non-dynamic/dynamic shape input
status = session_object_new.Run(run_options, feeds, output_names, &fetches);
// TODO narrow down actual compilation section
auto end = chrono::steady_clock::now();
status = session_object_new.Run(run_options, feeds, output_names, &fetches);
// TODO narrow down actual compilation section
// auto end = chrono::steady_clock::now();
ASSERT_TRUE(status.IsOK());
VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
ASSERT_TRUE(IsCacheExistedByType("./", ".timing"));
compilation_with_cache_ms = chrono::duration_cast<chrono::microseconds>(end - start).count();
}
ASSERT_TRUE(status.IsOK());
VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
// compilation_without_cache_ms = chrono::duration_cast<chrono::microseconds>(end - start).count();
}
ASSERT_TRUE(compilation_with_cache_ms <= compilation_without_cache_ms);
} else {
// TODO test dynamic shapes
}
// Temporarily disable comparing the engine build time until we find the model that can benefit from timing cache to get engine build time reduced.
// ASSERT_TRUE(compilation_with_cache_ms <= compilation_without_cache_ms);
}
// clean up caches