From bf7c0df5a0d1192cffbd19b9118798170a700ab0 Mon Sep 17 00:00:00 2001 From: Jingyan Wang Date: Tue, 7 Jan 2025 19:20:21 +0000 Subject: [PATCH] Unit test and perftest fix --- .../tensorrt/tensorrt_execution_provider.cc | 51 +- onnxruntime/test/perftest/ort_test_session.cc | 20 +- .../providers/tensorrt/tensorrt_basic_test.cc | 507 ++++++++++++++---- 3 files changed, 445 insertions(+), 133 deletions(-) diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 6df42d5959..baf79c972c 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -2488,36 +2488,61 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph, result.push_back(ComputeCapability::Create(std::move(sub_graph))); return result; } else { - SubGraphCollection_t supported_node_vectors = { - {{0}, true}, - {{1}, true}, - {{2}, true}, - {{3}, true}, - {{4}, true}, - }; + const size_t number_of_ort_nodes = graph.NumberOfNodes(); + SubGraphCollection_t supported_node_vectors; + std::vector subgraph_indices; + const std::vector& node_index = graph.GetNodesInTopologicalOrder(1 /*priority-based topological sort*/); + for (size_t i = 0; i < number_of_ort_nodes; i++) { + const auto& node = graph.GetNode(node_index[i]); + const bool is_context_node = node && !node->OpType().empty() && node->OpType() == "EPContext"; + if (is_context_node) { + // Add previous nonempty subgraph + if (subgraph_indices.size() > 0) { + supported_node_vectors.emplace_back(subgraph_indices, true); + } + // Add epcontext node, which is always just 1 node + supported_node_vectors.emplace_back(std::vector{i}, true); + subgraph_indices = {}; + } else { + subgraph_indices.emplace_back(i); + } + if (i == number_of_ort_nodes - 1 && !is_context_node) { + supported_node_vectors.emplace_back(subgraph_indices, true); + } + } + // SubGraphCollection_t supported_node_vectors = { + // {{0}, true}, + // {{1}, true}, + // {{2}, true}, + // {{3}, true}, + // {{4}, true}, + // }; + std::ostringstream oss; for (auto supported_node_vector: supported_node_vectors) { auto subgraph_idx = supported_node_vector.first[0]; std::unique_ptr sub_graph = GetSubGraph(supported_node_vector, graph, TRTGenerateId(graph), subgraph_idx); // Print supported_node_vector of std::pair, bool> - std::ostringstream oss; - oss << "First: ["; + + oss << "supported_node_vector={"; for (auto item: supported_node_vector.first) { oss << item << ", "; } - LOGS_DEFAULT(VERBOSE) << "*#* supported_node_vector=" << oss.str() << "], Second (bool): " << supported_node_vector.second; - oss.str(""); + oss << "}, " << supported_node_vector.second << " "; + oss << "Subgraph nodes=["; // Print sub_graph nodes of type std::unique_ptr -> Nodes() const auto nodes = sub_graph->Nodes(); //std::vector Nodes() for (const auto node: nodes) { oss << node << ", "; } - // Log the formatted string - LOGS_DEFAULT(VERBOSE) << "*#* sub_graph: Nodes=[" << oss.str() << "]"; + oss << "] \n"; + // End of print result.push_back(ComputeCapability::Create(std::move(sub_graph))); } + // Log the formatted string + LOGS_DEFAULT(VERBOSE) << "*#* " << oss.str(); return result; } diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 8f2e5282ed..81392518d1 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -57,6 +57,16 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device : rand_engine_(rd()), input_names_(m.GetInputCount()), input_names_str_(m.GetInputCount()), input_length_(m.GetInputCount()) { Ort::SessionOptions session_options; + // Set any extra session configuration entries provided by the user via command-line arguments. + // + // Some session config entries can also be set via dedicated command-line options. + // If the user uses multiple command-line options to set the same session config entry, + // we'll print a warning. Note that the dedicated command-line options will take precedence. + const auto& user_session_configs = performance_test_config.run_config.session_config_entries; + for (auto& it : user_session_configs) { + session_options.AddConfigEntry(it.first.c_str(), it.second.c_str()); + } + provider_name_ = performance_test_config.machine_config.provider_type_name; if (provider_name_ == onnxruntime::kDnnlExecutionProvider) { #ifdef USE_DNNL @@ -650,16 +660,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); session_options.DisableMemPattern(); session_options.SetExecutionMode(performance_test_config.run_config.execution_mode); - // Set any extra session configuration entries provided by the user via command-line arguments. - // - // Some session config entries can also be set via dedicated command-line options. - // If the user uses multiple command-line options to set the same session config entry, - // we'll print a warning. Note that the dedicated command-line options will take precedence. - const auto& user_session_configs = performance_test_config.run_config.session_config_entries; - for (auto& it : user_session_configs) { - session_options.AddConfigEntry(it.first.c_str(), it.second.c_str()); - } - auto warn_dup_config_entry = [&user_session_configs](const char* key) -> void { if (user_session_configs.find(key) != user_session_configs.end()) { fprintf(stderr, "[WARNING]: Trying to set session config entry '%s' via multiple command-line options\n", key); diff --git a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc index b4199548ae..3355d91a43 100644 --- a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc +++ b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc @@ -122,6 +122,81 @@ void CreateBaseModel(const PathString& model_name, status = onnxruntime::Model::Save(model, model_name); } +/** + * Create a model that would be partitioned to run on different EP and multiple EP context nodes with dynamic or non-dynamic input shape. That w + * \param model_name - model name + * \param graph_name - graph name + * \param dims - input dimensions + * \param add_non_zero_node - add NonZero node which makes the whole model partition into TRT EP and CUDA EP subgraphs. + * + * input: "X", "Y" and "Z" + * you can specify input dimensions, for example (1, 3, 2), (1, 2) or (1, -1, -1)). Note: -1 means the dimension is dynamic. + * All three inputs have the same dimensions. + * output: "M" + * + * "X" "Y" + * \ / + * "Z" Add + * \ / + * Add + * / + * NonZero (This node will be placed on CUDA EP) + * | + * Add + * | + * "M" + */ +// void CreateParititionedModel(const PathString& model_name, +// std::string graph_name, +// std::vector dims) { +// onnxruntime::Model model(graph_name, false, DefaultLoggingManager().DefaultLogger()); +// auto& graph = model.MainGraph(); +// std::vector inputs; +// std::vector outputs; + +// // FLOAT tensor +// ONNX_NAMESPACE::TypeProto float_tensor; +// float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + +// for (auto dim : dims) { +// float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dim); +// } + +// auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor); +// auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor); +// inputs.push_back(&input_arg_1); +// inputs.push_back(&input_arg_2); +// auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &float_tensor); +// outputs.push_back(&output_arg); +// graph.AddNode("node_1", "Add", "node 1.", inputs, outputs); + +// auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &float_tensor); +// inputs.clear(); +// inputs.push_back(&output_arg); +// inputs.push_back(&input_arg_3); + +// auto& output_arg_2 = graph.GetOrCreateNodeArg("node_2_out_1", &float_tensor); +// outputs.clear(); +// outputs.push_back(&output_arg_2); +// graph.AddNode("node_2", "Add", "node 2.", inputs, outputs); + +// inputs.clear(); +// inputs.push_back(&output_arg_2); +// ONNX_NAMESPACE::TypeProto int_tensor; +// int_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT64); +// auto& output_arg_3 = graph.GetOrCreateNodeArg("M", &int_tensor); +// outputs.clear(); +// outputs.push_back(&output_arg_3); +// graph.AddNode("node_3", "NonZero", "node 3.", inputs, outputs); + + + + +// auto status = graph.Resolve(); +// ASSERT_TRUE(status.IsOK()); +// status = onnxruntime::Model::Save(model, model_name); +// } + std::vector ReadFileFromDisk(const PathString& path) { std::fstream file(path.c_str(), std::fstream::binary | std::fstream::in | std::fstream::ate); std::vector file_bytes; @@ -376,6 +451,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { PathString model_name = ToPathString(model_name_str); std::string graph_name = "EPContextNode_test"; std::string sess_log_id = "EPContextNode_test"; + // std::string ctx_model_str = "EP_Context_model.onnx"; std::vector dims = {1, 3, 2}; CreateBaseModel(model_name, graph_name, dims); @@ -383,6 +459,9 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { so.session_logid = sess_log_id; RunOptions run_options; run_options.run_tag = so.session_logid; + so.config_options.AddConfigEntry("ep.context_enable", "1"); + so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx"); + so.config_options.AddConfigEntry("ep.context_embed_mode", "0"); // Default 1 InferenceSession session_object{so, GetEnvironment()}; auto cuda_provider = DefaultCudaExecutionProvider(); auto cpu_allocator = cuda_provider->CreatePreferredAllocators()[1]; @@ -415,22 +494,36 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { * * expected result => * context model "EP_Context_model.onnx" should be created in current directory - * + * --- New UT + * session options => + * ep.context_enable = "1" + * ep.context_file_path = "EP_Context_model.onnx" + * ep.context_embed_mode = "0" + * + * provider options => + * trt_engine_cache_enable = 1 + * + * expected result => + * context model "EP_Context_model.onnx" should be created in current directory */ OrtTensorRTProviderOptionsV2 params; params.trt_engine_cache_enable = 1; params.trt_dump_ep_context_model = 1; - params.trt_ep_context_file_path = "EP_Context_model.onnx"; + // params.trt_ep_context_file_path = "EP_Context_model.onnx"; std::unique_ptr execution_provider = TensorrtExecutionProviderWithOptions(¶ms); EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); auto status = session_object.Load(model_name); ASSERT_TRUE(status.IsOK()); status = session_object.Initialize(); ASSERT_TRUE(status.IsOK()); - ASSERT_TRUE(HasCacheFileWithPrefix(params.trt_ep_context_file_path)); + // ASSERT_TRUE(HasCacheFileWithPrefix(params.trt_ep_context_file_path)); + // Engine cache TensorrtExecutionProvider_*.engine should be created + ASSERT_TRUE(HasCacheFileWithPrefix("TensorrtExecutionProvider_")); + // EP_Context_model.onnx should be created + ASSERT_TRUE(HasCacheFileWithPrefix("EP_Context_model.onnx")); /* - * Test case 2: Dump context model + * Test case 2: Dump context model to directory * * provider options=> * trt_engine_cache_prefix = "TRT_engine_cache" @@ -440,27 +533,49 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { * expected result => * engine cache "./context_model_folder/engine_cache_folder/TRT_engine_cache...engine" should be created * context model "./context_model_folder/EPContextNode_test_ctx.onnx" should be created + * --- New UT + * session options => + * ep.context_enable = "1" + * ep.context_file_path = "context_model_folder" + * ep.context_embed_mode = "0" + * + * provider options => + * trt_engine_cache_enable = 1 + * + * expected result => + * engine cache "" + * context model "EP_Context_model.onnx" should be created in current directory */ + // std::string ctx_model_folder = "context_model_folder"; + so.config_options.AddConfigEntry("ep.context_file_path", "context_model_folder/EPContextNode_test_ctx.onnx"); InferenceSession session_object2{so, GetEnvironment()}; OrtTensorRTProviderOptionsV2 params2; params2.trt_engine_cache_enable = 1; params2.trt_dump_ep_context_model = 1; - params2.trt_engine_cache_prefix = "TRT_engine_cache"; - params2.trt_engine_cache_path = "engine_cache_folder"; // due to dump_ep_context_model = 1, the new cache path is ./context_model_folder/engine_cache_folder - params2.trt_ep_context_file_path = "context_model_folder"; + // params2.trt_engine_cache_prefix = "TRT_engine_cache"; + // params2.trt_engine_cache_path = "engine_cache_folder"; // due to dump_ep_context_model = 1, the new cache path is ./context_model_folder/engine_cache_folder + // params2.trt_ep_context_file_path = "context_model_folder"; + params2.trt_engine_cache_path = "context_model_folder"; execution_provider = TensorrtExecutionProviderWithOptions(¶ms2); EXPECT_TRUE(session_object2.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); status = session_object2.Load(model_name); ASSERT_TRUE(status.IsOK()); status = session_object2.Initialize(); ASSERT_TRUE(status.IsOK()); - auto new_engine_cache_path = std::filesystem::path(params2.trt_ep_context_file_path).append(params2.trt_engine_cache_path).string(); + // auto new_engine_cache_path = std::filesystem::path(params2.trt_ep_context_file_path).append(params2.trt_engine_cache_path).string(); // Test engine cache path: // "./context_model_folder/engine_cache_folder/TRT_engine_cache...engine" should be created - ASSERT_TRUE(HasCacheFileWithPrefix(params2.trt_engine_cache_prefix, new_engine_cache_path)); + // ASSERT_TRUE(HasCacheFileWithPrefix(params2.trt_engine_cache_prefix, new_engine_cache_path)); + // // Test context model path: + // // "./context_model_folder/EPContextNode_test_ctx.onnx" should be created + // ASSERT_TRUE(HasCacheFileWithPrefix("EPContextNode_test_ctx.onnx", params2.trt_ep_context_file_path)); + + // Test engine cache path: + // Engine cache ./context_model_folder/TensorrtExecutionProvider_...engine" should be created + ASSERT_TRUE(HasCacheFileWithPrefix("TensorrtExecutionProvider_", "context_model_folder")); // Test context model path: - // "./context_model_folder/EPContextNode_test_ctx.onnx" should be created - ASSERT_TRUE(HasCacheFileWithPrefix("EPContextNode_test_ctx.onnx", params2.trt_ep_context_file_path)); + // onnx model file ./context_model_folder/EPContextNode_test_ctx.onnx should be created + ASSERT_TRUE(HasCacheFileWithPrefix("EPContextNode_test_ctx.onnx", "context_model_folder")); /* * Test case 3: Run the dumped context model @@ -472,15 +587,19 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { * and the "ep_cache_context" attribute node of the context model should point to that. * */ + so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx"); + // so.session_log_verbosity_level = 0; + std::cout << "*#* so" << so << std::endl; InferenceSession session_object3{so, GetEnvironment()}; OrtTensorRTProviderOptionsV2 params3; - PathString ctx_model_name = ToPathString(params.trt_ep_context_file_path); - params3.trt_engine_cache_enable = 1; + PathString ctx_model_name = ToPathString("EP_Context_model.onnx"); + // params3.trt_engine_cache_enable = 1; execution_provider = TensorrtExecutionProviderWithOptions(¶ms3); EXPECT_TRUE(session_object3.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); status = session_object3.Load(ctx_model_name); ASSERT_TRUE(status.IsOK()); status = session_object3.Initialize(); + std::cout << status.ErrorMessage() << std::endl; ASSERT_TRUE(status.IsOK()); // run inference // TRT engine will be created and cached @@ -491,16 +610,18 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { // Z: 1, 3, 3, 2, 2, 2 RunSession(session_object3, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); - /* - * Test case 4: Run the dumped context model - * - * context model path = "./context_model_folder/EPContextNode_test_ctx.onnx" (created from case 2) - * - * expected result=> - * engine cache path is "./context_model_folder/engine_cache_folder/xxxxx.engine" - * and the "ep_cache_context" attribute node of the context model should point to "engine_cache_folder/xxxxx.engine". - * - */ + // /* + // * Test case 4: Run the dumped context model + // * + // * context model path = "./context_model_folder/EPContextNode_test_ctx.onnx" (created from case 2) + // * + // * expected result=> + // * engine cache path is "./context_model_folder/engine_cache_folder/xxxxx.engine" + // * and the "ep_cache_context" attribute node of the context model should point to "engine_cache_folder/xxxxx.engine". + // * + // */ + so.config_options.AddConfigEntry("ep.context_file_path", "context_model_folder/EPContextNode_test_ctx.onnx"); + so.session_log_severity_level = 0; InferenceSession session_object4{so, GetEnvironment()}; OrtTensorRTProviderOptionsV2 params4; ctx_model_name = ToPathString("./context_model_folder/EPContextNode_test_ctx.onnx"); @@ -519,99 +640,265 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { // Z: 1, 3, 3, 2, 2, 2 RunSession(session_object4, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); - /* - * Test case 5: Dump context model with embed_model = 1 - */ - InferenceSession session_object5{so, GetEnvironment()}; - OrtTensorRTProviderOptionsV2 params5; - params5.trt_dump_ep_context_model = 1; - params5.trt_ep_context_embed_mode = 1; - params5.trt_ep_context_file_path = "EP_Context_model_2.onnx"; - execution_provider = TensorrtExecutionProviderWithOptions(¶ms5); - EXPECT_TRUE(session_object5.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); - status = session_object5.Load(model_name); - ASSERT_TRUE(status.IsOK()); - status = session_object5.Initialize(); - ASSERT_TRUE(status.IsOK()); + // /* + // * Test case 5: Dump context model with embed_model = 1 + // */ + // InferenceSession session_object5{so, GetEnvironment()}; + // OrtTensorRTProviderOptionsV2 params5; + // params5.trt_dump_ep_context_model = 1; + // params5.trt_ep_context_embed_mode = 1; + // params5.trt_ep_context_file_path = "EP_Context_model_2.onnx"; + // execution_provider = TensorrtExecutionProviderWithOptions(¶ms5); + // EXPECT_TRUE(session_object5.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); + // status = session_object5.Load(model_name); + // ASSERT_TRUE(status.IsOK()); + // status = session_object5.Initialize(); + // ASSERT_TRUE(status.IsOK()); - /* - * Test case 6: Run context model with embed_model = 1 (created from case 5) - */ - InferenceSession session_object6{so, GetEnvironment()}; - OrtTensorRTProviderOptionsV2 params6; - params6.trt_ep_context_embed_mode = 1; - ctx_model_name = ToPathString(params5.trt_ep_context_file_path); - execution_provider = TensorrtExecutionProviderWithOptions(¶ms6); - EXPECT_TRUE(session_object6.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); - status = session_object6.Load(ctx_model_name); - ASSERT_TRUE(status.IsOK()); - status = session_object6.Initialize(); - ASSERT_TRUE(status.IsOK()); - // run inference - // TRT engine will be created and cached - // TRT profile will be created and cached only for dynamic input shape - // Data in profile, - // X: 1, 3, 3, 2, 2, 2 - // Y: 1, 3, 3, 2, 2, 2 - // Z: 1, 3, 3, 2, 2, 2 - RunSession(session_object6, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); + // /* + // * Test case 6: Run context model with embed_model = 1 (created from case 5) + // */ + // InferenceSession session_object6{so, GetEnvironment()}; + // OrtTensorRTProviderOptionsV2 params6; + // params6.trt_ep_context_embed_mode = 1; + // ctx_model_name = ToPathString(params5.trt_ep_context_file_path); + // execution_provider = TensorrtExecutionProviderWithOptions(¶ms6); + // EXPECT_TRUE(session_object6.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); + // status = session_object6.Load(ctx_model_name); + // ASSERT_TRUE(status.IsOK()); + // status = session_object6.Initialize(); + // ASSERT_TRUE(status.IsOK()); + // // run inference + // // TRT engine will be created and cached + // // TRT profile will be created and cached only for dynamic input shape + // // Data in profile, + // // X: 1, 3, 3, 2, 2, 2 + // // Y: 1, 3, 3, 2, 2, 2 + // // Z: 1, 3, 3, 2, 2, 2 + // RunSession(session_object6, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); - /* - * Test case 7: Run context model with ONNX in memory - */ - auto model_bytes = ReadFileFromDisk(model_name); - std::string ctx_model_name_str = "EP_Context_model_weight_stripped.onnx"; - ctx_model_name = ToPathString(ctx_model_name_str); - InferenceSession session_object7{so, GetEnvironment()}; - OrtTensorRTProviderOptionsV2 params7; - params7.trt_dump_ep_context_model = 1; - params7.trt_ep_context_embed_mode = 1; - params7.trt_weight_stripped_engine_enable = 1; - params7.trt_ep_context_file_path = ctx_model_name_str.c_str(); - execution_provider = TensorrtExecutionProviderWithOptions(¶ms7); - EXPECT_TRUE(session_object7.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); - status = session_object7.Load(model_bytes.data(), static_cast(model_bytes.size())); - ASSERT_TRUE(status.IsOK()); - status = session_object7.Initialize(); - std::cerr << status.ErrorMessage(); - ASSERT_TRUE(status.IsOK()); - RunSession(session_object7, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); + // /* + // * Test case 7: Run context model with ONNX in memory + // */ + // auto model_bytes = ReadFileFromDisk(model_name); + // std::string ctx_model_name_str = "EP_Context_model_weight_stripped.onnx"; + // ctx_model_name = ToPathString(ctx_model_name_str); + // InferenceSession session_object7{so, GetEnvironment()}; + // OrtTensorRTProviderOptionsV2 params7; + // params7.trt_dump_ep_context_model = 1; + // params7.trt_ep_context_embed_mode = 1; + // params7.trt_weight_stripped_engine_enable = 1; + // params7.trt_ep_context_file_path = ctx_model_name_str.c_str(); + // execution_provider = TensorrtExecutionProviderWithOptions(¶ms7); + // EXPECT_TRUE(session_object7.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); + // status = session_object7.Load(model_bytes.data(), static_cast(model_bytes.size())); + // ASSERT_TRUE(status.IsOK()); + // status = session_object7.Initialize(); + // std::cerr << status.ErrorMessage(); + // ASSERT_TRUE(status.IsOK()); + // RunSession(session_object7, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); - /* - * Test case 7: Refit weightless context model with ONNX in memory - */ - auto ctx_model_bytes = ReadFileFromDisk(ctx_model_name); - InferenceSession session_object8{so, GetEnvironment()}; - OrtTensorRTProviderOptionsV2 params8; - params8.trt_weight_stripped_engine_enable = 1; - params8.trt_onnx_bytestream = model_bytes.data(); - params8.trt_onnx_bytestream_size = model_bytes.size(); - execution_provider = TensorrtExecutionProviderWithOptions(¶ms8); - EXPECT_TRUE(session_object8.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); - status = session_object8.Load(ctx_model_bytes.data(), static_cast(ctx_model_bytes.size())); - std::cerr << status.ErrorMessage(); - ASSERT_TRUE(status.IsOK()); - status = session_object8.Initialize(); - std::cerr << status.ErrorMessage(); - ASSERT_TRUE(status.IsOK()); - RunSession(session_object8, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); + // /* + // * Test case 7: Refit weightless context model with ONNX in memory + // */ + // auto ctx_model_bytes = ReadFileFromDisk(ctx_model_name); + // InferenceSession session_object8{so, GetEnvironment()}; + // OrtTensorRTProviderOptionsV2 params8; + // params8.trt_weight_stripped_engine_enable = 1; + // params8.trt_onnx_bytestream = model_bytes.data(); + // params8.trt_onnx_bytestream_size = model_bytes.size(); + // execution_provider = TensorrtExecutionProviderWithOptions(¶ms8); + // EXPECT_TRUE(session_object8.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); + // status = session_object8.Load(ctx_model_bytes.data(), static_cast(ctx_model_bytes.size())); + // std::cerr << status.ErrorMessage(); + // ASSERT_TRUE(status.IsOK()); + // status = session_object8.Initialize(); + // std::cerr << status.ErrorMessage(); + // ASSERT_TRUE(status.IsOK()); + // RunSession(session_object8, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); - /* - * Test case 7: Refit weightless context model with ONNX from disk - */ - InferenceSession session_object9{so, GetEnvironment()}; - OrtTensorRTProviderOptionsV2 params9; - params9.trt_weight_stripped_engine_enable = 1; - params9.trt_onnx_model_folder_path = model_name_str.c_str(); - execution_provider = TensorrtExecutionProviderWithOptions(¶ms9); - EXPECT_TRUE(session_object9.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); - status = session_object9.Load(ctx_model_bytes.data(), static_cast(ctx_model_bytes.size())); - ASSERT_TRUE(status.IsOK()); - status = session_object9.Initialize(); - ASSERT_TRUE(status.IsOK()); - RunSession(session_object9, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); + // /* + // * Test case 7: Refit weightless context model with ONNX from disk + // */ + // InferenceSession session_object9{so, GetEnvironment()}; + // OrtTensorRTProviderOptionsV2 params9; + // params9.trt_weight_stripped_engine_enable = 1; + // params9.trt_onnx_model_folder_path = model_name_str.c_str(); + // execution_provider = TensorrtExecutionProviderWithOptions(¶ms9); + // EXPECT_TRUE(session_object9.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); + // status = session_object9.Load(ctx_model_bytes.data(), static_cast(ctx_model_bytes.size())); + // ASSERT_TRUE(status.IsOK()); + // status = session_object9.Initialize(); + // ASSERT_TRUE(status.IsOK()); + // RunSession(session_object9, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); } +// TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { +// std::string model_name_str = "EPContextNode_test.onnx"; +// PathString model_name = ToPathString(model_name_str); +// std::string graph_name = "EPContextNode_test"; +// std::string sess_log_id = "EPContextNode_test"; +// // std::string ctx_model_str = "EP_Context_model.onnx"; +// std::vector dims = {1, 3, 2}; +// CreateParititionedModel(model_name, graph_name, dims); + +// SessionOptions so; +// so.session_logid = sess_log_id; +// RunOptions run_options; +// run_options.run_tag = so.session_logid; +// so.config_options.AddConfigEntry("ep.context_enable", "1"); +// so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx"); +// so.config_options.AddConfigEntry("ep.context_embed_mode", "0"); // Default 1 +// InferenceSession session_object{so, GetEnvironment()}; +// auto cuda_provider = DefaultCudaExecutionProvider(); +// auto cpu_allocator = cuda_provider->CreatePreferredAllocators()[1]; +// std::vector dims_mul_x = {1, 3, 2}; +// std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; +// OrtValue ml_value_x; +// CreateMLValue(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_x); +// OrtValue ml_value_y; +// CreateMLValue(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_y); +// OrtValue ml_value_z; +// CreateMLValue(cpu_allocator, dims_mul_x, values_mul_x, &ml_value_z); +// NameMLValMap feeds; +// feeds.insert(std::make_pair("X", ml_value_x)); +// feeds.insert(std::make_pair("Y", ml_value_y)); +// feeds.insert(std::make_pair("Z", ml_value_z)); + +// // prepare outputs +// std::vector output_names; +// output_names.push_back("M"); + +// // prepare expected inputs and outputs +// std::vector expected_dims_mul_m = {1, 3, 2}; +// std::vector expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f}; + +// /* +// * Test case 1: Dump context model +// * +// * provider options=> +// * trt_ep_context_file_path = "EP_Context_model.onnx" +// * +// * expected result => +// * context model "EP_Context_model.onnx" should be created in current directory +// * --- New UT +// * session options => +// * ep.context_enable = "1" +// * ep.context_file_path = "EP_Context_model.onnx" +// * ep.context_embed_mode = "0" +// * +// * provider options => +// * trt_engine_cache_enable = 1 +// * +// * expected result => +// * context model "EP_Context_model.onnx" should be created in current directory +// */ +// OrtTensorRTProviderOptionsV2 params; +// params.trt_engine_cache_enable = 1; +// params.trt_dump_ep_context_model = 1; +// // params.trt_ep_context_file_path = "EP_Context_model.onnx"; +// std::unique_ptr execution_provider = TensorrtExecutionProviderWithOptions(¶ms); +// EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); +// auto status = session_object.Load(model_name); +// ASSERT_TRUE(status.IsOK()); +// status = session_object.Initialize(); +// ASSERT_TRUE(status.IsOK()); +// // ASSERT_TRUE(HasCacheFileWithPrefix(params.trt_ep_context_file_path)); +// // Engine cache TensorrtExecutionProvider_*.engine should be created +// ASSERT_TRUE(HasCacheFileWithPrefix("TensorrtExecutionProvider_")); +// // EP_Context_model.onnx should be created +// ASSERT_TRUE(HasCacheFileWithPrefix("EP_Context_model.onnx")); + +// /* +// * Test case 2: Dump context model to directory +// * +// * provider options=> +// * trt_engine_cache_prefix = "TRT_engine_cache" +// * trt_ep_context_file_path = "context_model_folder" +// * trt_engine_cache_path = "engine_cache_folder" +// * +// * expected result => +// * engine cache "./context_model_folder/engine_cache_folder/TRT_engine_cache...engine" should be created +// * context model "./context_model_folder/EPContextNode_test_ctx.onnx" should be created +// * --- New UT +// * session options => +// * ep.context_enable = "1" +// * ep.context_file_path = "context_model_folder" +// * ep.context_embed_mode = "0" +// * +// * provider options => +// * trt_engine_cache_enable = 1 +// * +// * expected result => +// * engine cache "" +// * context model "EP_Context_model.onnx" should be created in current directory +// */ +// // std::string ctx_model_folder = "context_model_folder"; +// so.config_options.AddConfigEntry("ep.context_file_path", "context_model_folder/EPContextNode_test_ctx.onnx"); +// InferenceSession session_object2{so, GetEnvironment()}; +// OrtTensorRTProviderOptionsV2 params2; +// params2.trt_engine_cache_enable = 1; +// params2.trt_dump_ep_context_model = 1; +// // params2.trt_engine_cache_prefix = "TRT_engine_cache"; +// // params2.trt_engine_cache_path = "engine_cache_folder"; // due to dump_ep_context_model = 1, the new cache path is ./context_model_folder/engine_cache_folder +// // params2.trt_ep_context_file_path = "context_model_folder"; +// params2.trt_engine_cache_path = "context_model_folder"; +// execution_provider = TensorrtExecutionProviderWithOptions(¶ms2); +// EXPECT_TRUE(session_object2.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); +// status = session_object2.Load(model_name); +// ASSERT_TRUE(status.IsOK()); +// status = session_object2.Initialize(); +// ASSERT_TRUE(status.IsOK()); +// // auto new_engine_cache_path = std::filesystem::path(params2.trt_ep_context_file_path).append(params2.trt_engine_cache_path).string(); +// // Test engine cache path: +// // "./context_model_folder/engine_cache_folder/TRT_engine_cache...engine" should be created +// // ASSERT_TRUE(HasCacheFileWithPrefix(params2.trt_engine_cache_prefix, new_engine_cache_path)); +// // // Test context model path: +// // // "./context_model_folder/EPContextNode_test_ctx.onnx" should be created +// // ASSERT_TRUE(HasCacheFileWithPrefix("EPContextNode_test_ctx.onnx", params2.trt_ep_context_file_path)); + +// // Test engine cache path: +// // Engine cache ./context_model_folder/TensorrtExecutionProvider_...engine" should be created +// ASSERT_TRUE(HasCacheFileWithPrefix("TensorrtExecutionProvider_", "context_model_folder")); +// // Test context model path: +// // onnx model file ./context_model_folder/EPContextNode_test_ctx.onnx should be created +// ASSERT_TRUE(HasCacheFileWithPrefix("EPContextNode_test_ctx.onnx", "context_model_folder")); + +// /* +// * Test case 3: Run the dumped context model +// * +// * context model path = "./EP_Context_model.onnx" (created from case 1) +// * +// * expected result=> +// * engine cache is also in the same current dirctory as "./xxxxx.engine" +// * and the "ep_cache_context" attribute node of the context model should point to that. +// * +// */ +// so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx"); +// // so.session_log_verbosity_level = 0; +// so.session_log_severity_level = 0; +// std::cout << "*#* so" << so << std::endl; +// InferenceSession session_object3{so, GetEnvironment()}; +// OrtTensorRTProviderOptionsV2 params3; +// PathString ctx_model_name = ToPathString("EP_Context_model.onnx"); +// // params3.trt_engine_cache_enable = 1; +// execution_provider = TensorrtExecutionProviderWithOptions(¶ms3); +// EXPECT_TRUE(session_object3.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); +// status = session_object3.Load(ctx_model_name); +// ASSERT_TRUE(status.IsOK()); +// status = session_object3.Initialize(); +// std::cout << status.ErrorMessage() << std::endl; +// ASSERT_TRUE(status.IsOK()); +// // run inference +// // TRT engine will be created and cached +// // TRT profile will be created and cached only for dynamic input shape +// // Data in profile, +// // X: 1, 3, 3, 2, 2, 2 +// // Y: 1, 3, 3, 2, 2, 2 +// // Z: 1, 3, 3, 2, 2, 2 +// RunSession(session_object3, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); +// } + TEST(TensorrtExecutionProviderTest, ExcludeOpsTest) { /* The mnist.onnx looks like this: * Conv