diff --git a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc index 779c724fad..93fb2fe901 100644 --- a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc +++ b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc @@ -134,21 +134,22 @@ void CreateBaseModel(const PathString& model_name, * All three inputs have the same dimensions. * output: "M" * - * "X" "Y" + * "X" "Y" * \ / - * "Z" Add + * "Z" Add * \ / - * Add - * / - * "A" NonZero (This node will be placed on CUDA EP) - * \ | + * Add + * / + * "A" NonZero (This node will be placed on CUDA EP) + * \ / * Add * | * "M" */ void CreateParititionedModel(const PathString& model_name, std::string graph_name, - std::vector dims) { + std::vector dims, + std::vector dims2) { onnxruntime::Model model(graph_name, false, DefaultLoggingManager().DefaultLogger()); auto& graph = model.MainGraph(); std::vector inputs; @@ -162,6 +163,13 @@ void CreateParititionedModel(const PathString& model_name, float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dim); } + // INT tensor + ONNX_NAMESPACE::TypeProto int_tensor; + int_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT64); + for (auto dim : dims2) { + int_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dim); + } + auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor); auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor); inputs.push_back(&input_arg_1); @@ -182,8 +190,6 @@ void CreateParititionedModel(const PathString& model_name, inputs.clear(); inputs.push_back(&output_arg_2); - ONNX_NAMESPACE::TypeProto int_tensor; - int_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT64); auto& output_arg_3 = graph.GetOrCreateNodeArg("node_3_out_1", &int_tensor); outputs.clear(); outputs.push_back(&output_arg_3); @@ -203,6 +209,7 @@ void CreateParititionedModel(const PathString& model_name, status = onnxruntime::Model::Save(model, model_name); } + std::vector ReadFileFromDisk(const PathString& path) { std::fstream file(path.c_str(), std::fstream::binary | std::fstream::in | std::fstream::ate); std::vector file_bytes; @@ -465,8 +472,11 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { std::string model_name_str = "EPContextNode_test.onnx"; PathString model_name = ToPathString(model_name_str); std::string graph_name = "EPContextNode_test"; + std::string ctx_model_path = "EP_Context_model.onnx"; std::string sess_log_id = "EPContextNode_test"; std::vector dims = {1, 3, 2}; + + remove(ctx_model_path.c_str()); // remove the context model file generated by previous test CreateBaseModel(model_name, graph_name, dims); SessionOptions so; @@ -513,14 +523,14 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { * context model "EP_Context_model.onnx" should be created in current directory */ (void)so.config_options.AddConfigEntry("ep.context_enable", "1"); - (void)so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx"); + (void)so.config_options.AddConfigEntry("ep.context_file_path", ctx_model_path.c_str()); (void)so.config_options.AddConfigEntry("ep.context_embed_mode", "0"); InferenceSession session_object{so, GetEnvironment()}; // Need to set corresponding trt params since options merging logic in privider_bridge_ort is not called in unit test OrtTensorRTProviderOptionsV2 params; params.trt_engine_cache_enable = 1; params.trt_dump_ep_context_model = 1; - params.trt_ep_context_file_path = "EP_Context_model.onnx"; + params.trt_ep_context_file_path = ctx_model_path.c_str(); params.trt_ep_context_embed_mode = 0; params.trt_engine_cache_prefix = "TRTEP_Cache_Test"; std::unique_ptr execution_provider = TensorrtExecutionProviderWithOptions(¶ms); @@ -532,7 +542,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { // Engine cache with params.trt_engine_cache_prefix prefix should be created ASSERT_TRUE(HasCacheFileWithPrefix(params.trt_engine_cache_prefix)); // EP_Context_model.onnx should be created - ASSERT_TRUE(HasCacheFileWithPrefix("EP_Context_model.onnx")); + ASSERT_TRUE(HasCacheFileWithPrefix(ctx_model_path)); /* * Test case 1.2: Run the dumped context model, context saved in engine cache @@ -546,7 +556,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { */ InferenceSession session_object2{so, GetEnvironment()}; OrtTensorRTProviderOptionsV2 params2; - PathString ctx_model_name = ToPathString("EP_Context_model.onnx"); + PathString ctx_model_name = ToPathString(ctx_model_path); execution_provider = TensorrtExecutionProviderWithOptions(¶ms2); EXPECT_TRUE(session_object2.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); status = session_object2.Load(ctx_model_name); @@ -629,12 +639,13 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { /* * Test case 3.1: Dump context model with embed_model = 1 */ - (void)so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model_emb.onnx"); + const std::string ctx_model_path_emb = "EP_Context_model_emb.onnx"; + (void)so.config_options.AddConfigEntry("ep.context_file_path", ctx_model_path_emb.c_str()); (void)so.config_options.AddConfigEntry("ep.context_embed_mode", "1"); InferenceSession session_object5{so, GetEnvironment()}; OrtTensorRTProviderOptionsV2 params5; params5.trt_dump_ep_context_model = 1; - params5.trt_ep_context_file_path = "EP_Context_model_emb.onnx"; + params5.trt_ep_context_file_path = ctx_model_path_emb.c_str(); params5.trt_ep_context_embed_mode = 1; execution_provider = TensorrtExecutionProviderWithOptions(¶ms5); EXPECT_TRUE(session_object5.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); @@ -642,7 +653,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { ASSERT_TRUE(status.IsOK()); status = session_object5.Initialize(); ASSERT_TRUE(status.IsOK()); - ASSERT_TRUE(HasCacheFileWithPrefix("EP_Context_model_emb.onnx")); + ASSERT_TRUE(HasCacheFileWithPrefix(ctx_model_path_emb)); /* * Test case 3.2: Run context model with embed_model = 1 @@ -724,12 +735,14 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { } TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { - std::string model_name_str = "EPContextNode_test.onnx"; + std::string model_name_str = "EPContextNode_test_multi.onnx"; PathString model_name = ToPathString(model_name_str); std::string graph_name = "EPContextNode_test"; + std::string ctx_model_path = "EP_Context_model_multi.onnx"; std::string sess_log_id = "EPContextNode_test"; std::vector dims = {1, 3, 2}; - CreateParititionedModel(model_name, graph_name, dims); + std::vector dims2 = {3, 6}; + CreateParititionedModel(model_name, graph_name, dims, dims2); SessionOptions so; so.session_logid = sess_log_id; @@ -740,7 +753,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { std::vector dims_mul_x = {1, 3, 2}; std::vector dims_mul_a = {3, 6}; std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; - std::vector values_mul_a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17}; + std::vector values_mul_a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17}; OrtValue ml_value_x; CreateMLValue(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_x); OrtValue ml_value_y; @@ -748,7 +761,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { OrtValue ml_value_z; CreateMLValue(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_z); OrtValue ml_value_a; - CreateMLValue(cpu_allocator, dims_mul_x, values_mul_a, &ml_value_a); + CreateMLValue(cpu_allocator, dims_mul_a, values_mul_a, &ml_value_a); NameMLValMap feeds; feeds.insert(std::make_pair("X", ml_value_x)); feeds.insert(std::make_pair("Y", ml_value_y)); @@ -780,14 +793,14 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { * context model "EP_Context_model.onnx" should be created in current directory */ (void)so.config_options.AddConfigEntry("ep.context_enable", "1"); - (void)so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx"); + (void)so.config_options.AddConfigEntry("ep.context_file_path", ctx_model_path.c_str()); (void)so.config_options.AddConfigEntry("ep.context_embed_mode", "0"); InferenceSession session_object{so, GetEnvironment()}; // Need to set corresponding trt params since options merging logic in privider_bridge_ort is not called in unit test OrtTensorRTProviderOptionsV2 params; params.trt_engine_cache_enable = 1; params.trt_dump_ep_context_model = 1; - params.trt_ep_context_file_path = "EP_Context_model.onnx"; + params.trt_ep_context_file_path = ctx_model_path.c_str(); params.trt_ep_context_embed_mode = 0; std::unique_ptr execution_provider = TensorrtExecutionProviderWithOptions(¶ms); EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); @@ -798,7 +811,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { // Engine cache TensorrtExecutionProvider_*.engine should be created ASSERT_TRUE(HasCacheFileWithPrefix("TensorrtExecutionProvider")); // EP_Context_model.onnx should be created - ASSERT_TRUE(HasCacheFileWithPrefix("EP_Context_model.onnx")); + ASSERT_TRUE(HasCacheFileWithPrefix(ctx_model_path)); /* * Test case 1.2: Run the dumped context model, context saved in engine cache @@ -812,10 +825,10 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { */ InferenceSession session_object2{so, GetEnvironment()}; OrtTensorRTProviderOptionsV2 params2; - PathString ctx_model_name = ToPathString("EP_Context_model.onnx"); + PathString ctx_model_name = ToPathString(ctx_model_path); execution_provider = TensorrtExecutionProviderWithOptions(¶ms2); EXPECT_TRUE(session_object2.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); - status = session_object2.Load(ctx_model_name); + auto status = session_object2.Load(ctx_model_name); ASSERT_TRUE(status.IsOK()); status = session_object2.Initialize(); ASSERT_TRUE(status.IsOK()); @@ -869,7 +882,8 @@ TEST(TensorrtExecutionProviderTest, TRTPluginsCustomOpTest) { } TEST_P(TensorrtExecutionProviderCacheTest, Run) { - remove("EP_Context_model.onnx"); // remove the context model file if it exists + std::string ctx_model_path = "EP_Context_model.onnx"; + remove(ctx_model_path.c_str()); // remove the context model file generated by previous test // GetParam() returns the parameter of following format: // ##cache type##_##input shape type## std::string param = GetParam(); @@ -930,14 +944,14 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) { * */ (void)so.config_options.AddConfigEntry("ep.context_enable", "1"); - (void)so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx"); + (void)so.config_options.AddConfigEntry("ep.context_file_path", ctx_model_path.c_str()); (void)so.config_options.AddConfigEntry("ep.context_embed_mode", "0"); InferenceSession session_object{so, GetEnvironment()}; params.trt_engine_cache_enable = 1; params.trt_engine_cache_prefix = "TRTEP_Cache_Test"; params.trt_dump_ep_context_model = 1; - params.trt_ep_context_file_path = "EP_Context_model.onnx"; + params.trt_ep_context_file_path = ctx_model_path.c_str(); std::unique_ptr execution_provider = TensorrtExecutionProviderWithOptions(¶ms); EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); auto status = session_object.Load(model_name); @@ -1039,7 +1053,7 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) { } } - remove("EP_Context_model.onnx"); // remove the context model file generated earlier + remove(ctx_model_path.c_str()); // remove the context model file generated by previous test // Test explicit min/max/opt profile shapes // create another session object with TRT EP provider options: