diff --git a/include/onnxruntime/core/framework/run_options.h b/include/onnxruntime/core/framework/run_options.h index 038b99f72c..f10f161ebe 100644 --- a/include/onnxruntime/core/framework/run_options.h +++ b/include/onnxruntime/core/framework/run_options.h @@ -15,11 +15,6 @@ struct OrtRunOptions { unsigned run_log_verbosity_level = 0; ///< applies to a particular Run() invocation std::string run_tag; ///< to identify logs generated by a particular Run() invocation - /** - Create and/or use cached info for the combination of feed and output names used in the call to Run. - */ - bool cache_feeds_fetches_info = false; - /// set to 'true' to terminate any currently executing Run() calls that are using this /// OrtRunOptions instance. the individual calls will exit gracefully and return an error status. bool terminate = false; diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 2ebfbfd9a2..fe06045c73 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -269,11 +269,9 @@ ORT_API(OrtRunOptions*, OrtCreateRunOptions); ORT_API_STATUS(OrtRunOptionsSetRunLogVerbosityLevel, _In_ OrtRunOptions*, unsigned int); ORT_API_STATUS(OrtRunOptionsSetRunTag, _In_ OrtRunOptions*, _In_ const char* run_tag); -ORT_API(void, OrtRunOptionsSetCacheFeedsFetchesInfoEnabled, _In_ OrtRunOptions* options, int bool_value); ORT_API(unsigned int, OrtRunOptionsGetRunLogVerbosityLevel, _In_ OrtRunOptions*); ORT_API(const char*, OrtRunOptionsGetRunTag, _In_ OrtRunOptions*); -ORT_API(int, OrtRunOptionsGetCacheFeedsFetchesInfoEnabled, _In_ OrtRunOptions*); // Set a flag so that any running OrtRun* calls that are using this instance of OrtRunOptions // will exit as soon as possible if the flag is true. diff --git a/onnxruntime/core/framework/feeds_fetches_manager.h b/onnxruntime/core/framework/feeds_fetches_manager.h index fe921aadc4..9554534232 100644 --- a/onnxruntime/core/framework/feeds_fetches_manager.h +++ b/onnxruntime/core/framework/feeds_fetches_manager.h @@ -27,8 +27,8 @@ struct DeviceCopyChecks { struct FeedsFetchesInfo { FeedsFetchesInfo() = default; - FeedsFetchesInfo(std::vector& feed_names_in, - std::vector& output_names_in) + FeedsFetchesInfo(const std::vector& feed_names_in, + const std::vector& output_names_in) : feed_names{feed_names_in}, output_names{output_names_in} {} static Status MapNamesToMLValueIdxs(const std::vector& names, diff --git a/onnxruntime/core/framework/run_options.cc b/onnxruntime/core/framework/run_options.cc index bf7a05903a..d1b6ef5d51 100644 --- a/onnxruntime/core/framework/run_options.cc +++ b/onnxruntime/core/framework/run_options.cc @@ -22,10 +22,6 @@ ORT_API_STATUS_IMPL(OrtRunOptionsSetRunTag, _In_ OrtRunOptions* options, _In_ co return nullptr; } -ORT_API(void, OrtRunOptionsSetCacheFeedsFetchesInfoEnabled, _In_ OrtRunOptions* options, int bool_value) { - options->cache_feeds_fetches_info = bool_value != 0; -} - ORT_API(unsigned int, OrtRunOptionsGetRunLogVerbosityLevel, _In_ OrtRunOptions* options) { return options->run_log_verbosity_level; } @@ -34,10 +30,6 @@ ORT_API(const char*, OrtRunOptionsGetRunTag, _In_ OrtRunOptions* options) { return options->run_tag.c_str(); } -ORT_API(int, OrtRunOptionsGetCacheFeedsFetchesInfoEnabled, _In_ OrtRunOptions* options) { - return options->cache_feeds_fetches_info; -} - ORT_API(void, OrtRunOptionsSetTerminate, _In_ OrtRunOptions* options, bool value) { options->terminate = value; } diff --git a/onnxruntime/core/framework/session_state.cc b/onnxruntime/core/framework/session_state.cc index e8b116d283..5be86cd841 100644 --- a/onnxruntime/core/framework/session_state.cc +++ b/onnxruntime/core/framework/session_state.cc @@ -211,66 +211,4 @@ const NodeIndexInfo& SessionState::GetNodeIndexInfo() const { ORT_ENFORCE(node_index_info_, "CalculateNodeIndexInfo must be called prior to GetExecutionInfo."); return *node_index_info_; } - -// use a cheap way of matching first. if we have multiple entries with this key, we will do the more expensive -// check of the individual feed/output names -static int MakeFeedsFetchesManagerCacheKey(const std::vector& feed_names, - const std::vector& output_names) { - return static_cast(feed_names.size()) << 16 | static_cast(output_names.size()); -}; - -const FeedsFetchesManager* SessionState::GetFeedsFetchesManager(const std::vector& feed_names, - const std::vector& output_names) const { - int key = MakeFeedsFetchesManagerCacheKey(feed_names, output_names); - auto num_matches = cached_feeds_fetches_managers_.count(key); - - const FeedsFetchesManager* manager = nullptr; - - if (num_matches > 0) { - auto begin_end_pair = cached_feeds_fetches_managers_.equal_range(key); - auto iter = begin_end_pair.first; - auto end = begin_end_pair.second; - - while (iter != end) { - auto& ffi = iter->second->GetFeedsFetchesInfo(); - auto check = [](const std::vector& input, const std::vector& existing) { - for (size_t i = 0, end = input.size(); i < end; ++i) { - if (input[i] != existing[i]) { - return false; - } - } - - return true; - }; - - if (check(feed_names, ffi.feed_names) && check(output_names, ffi.output_names)) { - manager = iter->second.get(); - break; - } - - ++iter; - } - } - - return manager; -} - -Status SessionState::CacheFeedsFetchesManager(const std::vector& feed_names, - const std::vector& output_names, - std::unique_ptr manager) { - int key = MakeFeedsFetchesManagerCacheKey(feed_names, output_names); - - auto num_matches = cached_feeds_fetches_managers_.count(key); - - if (num_matches) { - // be paranoid and make sure we're not attempting to insert a duplicate entry. - // if so it would imply that there is probably a concurrency issue in InferenceSession::Run. - ORT_ENFORCE(GetFeedsFetchesManager(feed_names, output_names) == nullptr, "Existing FeedsFetchesManager found."); - } - - cached_feeds_fetches_managers_.emplace(key, std::move(manager)); - - return Status::OK(); -} - } // namespace onnxruntime diff --git a/onnxruntime/core/framework/session_state.h b/onnxruntime/core/framework/session_state.h index b4b3dbdfcd..7fff7a6bd7 100644 --- a/onnxruntime/core/framework/session_state.h +++ b/onnxruntime/core/framework/session_state.h @@ -187,13 +187,6 @@ class SessionState { void CalculateNodeIndexInfo(); const NodeIndexInfo& GetNodeIndexInfo() const; - const FeedsFetchesManager* GetFeedsFetchesManager(const std::vector& feed_names, - const std::vector& output_names) const; - - Status CacheFeedsFetchesManager(const std::vector& feed_names, - const std::vector& output_names, - std::unique_ptr manager); - private: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SessionState); diff --git a/onnxruntime/core/framework/utils.cc b/onnxruntime/core/framework/utils.cc index c84cc33217..1da0704ffa 100644 --- a/onnxruntime/core/framework/utils.cc +++ b/onnxruntime/core/framework/utils.cc @@ -245,20 +245,6 @@ static common::Status SetupFetchesForExecute(const SessionState& session_state, new_fetches.resize(num_outputs); - if (copy_to_new_fetches_cached_values && !copy_to_new_fetches_cached_values->empty()) { - // use the cached values - ORT_ENFORCE(copy_to_new_fetches_cached_values->size() == num_outputs); - - auto& copy = *copy_to_new_fetches_cached_values; - for (size_t i = 0; i < num_outputs; ++i) { - if (copy[i]) { - new_fetches[i] = fetches[i]; - } - } - - return Status::OK(); - } - // track which fetches can be copied to new_fetches and used directly in the execution. std::vector local_can_copy_flags(num_outputs, false); @@ -354,15 +340,6 @@ static common::Status CopyOutputsAcrossDevices(const SessionState& session_state needed_copy = false; auto num_outputs = fetches.size(); - // used the cached copy logic if available - if (copiers && !copiers->empty()) { - for (size_t idx = 0; idx < num_outputs; ++idx) { - ORT_RETURN_IF_ERROR(CopyMLValue((*copiers)[idx], fetches[idx], user_fetches[idx])); - } - - return Status::OK(); - } - if (copiers) { // resize so we have default values and only need to update an entry if there's a device copy required. copiers->resize(num_outputs); diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt index 0d11befcb5..4f8aaaca4a 100644 --- a/onnxruntime/core/providers/cpu/symbols.txt +++ b/onnxruntime/core/providers/cpu/symbols.txt @@ -57,10 +57,8 @@ OrtReleaseTensorTypeAndShapeInfo OrtReleaseTypeInfo OrtReleaseValue OrtRun -OrtRunOptionsGetCacheFeedsFetchesInfoEnabled OrtRunOptionsGetRunLogVerbosityLevel OrtRunOptionsGetRunTag -OrtRunOptionsSetCacheFeedsFetchesInfoEnabled OrtRunOptionsSetRunLogVerbosityLevel OrtRunOptionsSetRunTag OrtRunOptionsSetTerminate diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index a925d52ff2..562e56fbbb 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -575,68 +575,22 @@ class InferenceSession::Impl { Status retval = Status::OK(); try { - // use cached info if available, otherwise create a FeedsFetchesManager and update it in the call to ExecuteGraph - std::unique_ptr local_ffm; - FeedsFetchesManager* feeds_fetches_manager = nullptr; - const FeedsFetchesManager* cached_feeds_fetches_manager = nullptr; - - // lambda to construct so that we can call it under the lock if we're caching this, or outside of the lock - // if we're not. - auto create_feeds_fetches_manager = [&]() { - ORT_RETURN_IF_ERROR(ValidateInputs(feed_names, feeds)); - - // if the output vector is non-empty, ensure that its the same size as the output_names - ORT_RETURN_IF_ERROR(ValidateOutputs(output_names, p_fetches)); - - auto status = FeedsFetchesManager::Create(feed_names, output_names, session_state_.GetMLValueNameIdxMap(), - local_ffm); - ORT_RETURN_IF_ERROR(status); - feeds_fetches_manager = local_ffm.get(); - - if (run_options.cache_feeds_fetches_info) { - session_state_.CacheFeedsFetchesManager(feed_names, output_names, std::move(local_ffm)); - } - - return Status::OK(); - }; - { std::lock_guard l(session_mutex_); if (!is_inited_) { LOGS(*session_logger_, ERROR) << "Session was not initialized"; retval = Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized."); } - - if (run_options.cache_feeds_fetches_info) { - cached_feeds_fetches_manager = session_state_.GetFeedsFetchesManager(feed_names, output_names); - if (!cached_feeds_fetches_manager) { - // create the instance under the lock as we add it to SessionState and don't want concurrent calls to Run - // to clash with each other - ORT_RETURN_IF_ERROR(create_feeds_fetches_manager()); - } - } } - if (!run_options.cache_feeds_fetches_info) { - // if we're not creating/using cached info, create an instance for this run - ORT_RETURN_IF_ERROR(create_feeds_fetches_manager()); - } else if (cached_feeds_fetches_manager) { - // make sure that if we didn't create the FeedsFetchesManager it has been fully initialized by the - // successful completion of a call to Run. this is primarily to detect concurrent calls to Run - // prior to the initial call completing. we could do something more complicated to handle failure on the - // initial call if a real need to do so is proven. - if (cached_feeds_fetches_manager->GetDeviceCopyChecks().status == DeviceCopyCheck::Unknown) { - return ORT_MAKE_STATUS( - ONNXRUNTIME, FAIL, - "Existing cached information was found but was not fully initialized. " - "If caching is enabled, the first call to Run must successfully complete to fully initialize the " - "cache information. Once it is fully initialized, Run calls can be made in parallel. " - "If the first call to Run failed and you wish to use cached information, you will need to create a new " - "InferenceSession."); - } + ORT_RETURN_IF_ERROR(ValidateInputs(feed_names, feeds)); - LOGS(*session_logger_, INFO) << "Skipped validation of inputs and outputs as cached information was found"; - } + // if the output vector is non-empty, ensure that its the same size as the output_names + ORT_RETURN_IF_ERROR(ValidateOutputs(output_names, p_fetches)); + + FeedsFetchesInfo info(feed_names, output_names); + ORT_RETURN_IF_ERROR(info.SetMLValueIdxs(session_state_.GetMLValueNameIdxMap())); + FeedsFetchesManager feeds_fetches_manager{std::move(info)}; if (!run_options.run_tag.empty()) { LOGS(*session_logger_, INFO) << "Running with tag: " << run_options.run_tag; @@ -657,19 +611,12 @@ class InferenceSession::Impl { ORT_CHECK_AND_SET_RETVAL(xp->OnRunStart()); } - if (cached_feeds_fetches_manager) { - // used the const cached_feeds_fetches_manager to execute the graph - ORT_CHECK_AND_SET_RETVAL( - utils::ExecuteGraphWithCachedInfo(session_state_, *cached_feeds_fetches_manager, feeds, *p_fetches, {}, - session_options_.enable_sequential_execution, run_options.terminate, - run_logger)); - } else { - // execute the graph and update feeds_fetches_manager - ORT_CHECK_AND_SET_RETVAL( - utils::ExecuteGraph(session_state_, *feeds_fetches_manager, feeds, *p_fetches, {}, - session_options_.enable_sequential_execution, run_options.terminate, run_logger, - run_options.cache_feeds_fetches_info)); - } + // execute the graph + ORT_CHECK_AND_SET_RETVAL( + utils::ExecuteGraph(session_state_, feeds_fetches_manager, feeds, *p_fetches, {}, + session_options_.enable_sequential_execution, run_options.terminate, run_logger, + false)); + } catch (const std::exception& e) { retval = Status(common::ONNXRUNTIME, common::FAIL, e.what()); } catch (...) { diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc index c252488a00..4bed856cb2 100644 --- a/onnxruntime/test/framework/inference_session_test.cc +++ b/onnxruntime/test/framework/inference_session_test.cc @@ -1189,7 +1189,6 @@ TEST(InferenceSessionTests, TestTruncatedSequence) { RunOptions run_options; run_options.run_tag = "one session/one tag"; - run_options.cache_feeds_fetches_info = true; // caching should handle the truncated and non-truncated cases std::vector X_dims = {5, 1, 3}; std::vector X = {0.5488135f, 0.71518934f, 0.60276335f, @@ -1332,7 +1331,6 @@ TEST(InferenceSessionTests, TestCopyToFromDevices) { // Now run RunOptions run_options; run_options.run_tag = "run:" + std::to_string(run_num); - run_options.cache_feeds_fetches_info = true; common::Status st = session_object.Run(run_options, feed_names, feeds, output_names, &fetches); ASSERT_TRUE(st.IsOK()) << st.ErrorMessage(); @@ -1345,42 +1343,5 @@ TEST(InferenceSessionTests, TestCopyToFromDevices) { run_test(run_number++); } -// Make sure we don't match the wrong cache entry -TEST(InferenceSessionTests, TestCacheMatching) { - SessionOptions so; - - so.session_logid = "InferenceSessionTests.TestCacheMatching"; - - InferenceSession session_object{so, &DefaultLoggingManager()}; - ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK()); - ASSERT_TRUE(session_object.Initialize().IsOK()); - - RunOptions run_options; - run_options.run_tag = "one session/one tag"; - run_options.cache_feeds_fetches_info = true; - - // run once with valid names. this will cache an entry with input name of 'X' and output name of 'Y' - RunModel(session_object, run_options); - - // run again with an invalid output name. if we match the cache, we don't do name validation so this would - // not error out as it would use the valid output name information from the cache. - std::vector dims_mul_x = {3, 2}; - std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; - MLValue ml_value; - CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, - &ml_value); - NameMLValMap feeds; - feeds.insert(std::make_pair("X", ml_value)); - - std::vector output_names; - output_names.push_back("Y_invalid"); - std::vector fetches; - - common::Status status = session_object.Run(run_options, feeds, output_names, &fetches); - - EXPECT_FALSE(status.IsOK()); - EXPECT_THAT(status.ErrorMessage(), testing::HasSubstr("Invalid Output Names: Y_invalid")); -} - } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index 42d5331834..ea95721962 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -50,7 +50,7 @@ Status PerformanceRunner::Run() { Status PerformanceRunner::RunOneIteration(bool isWarmup) { auto start = std::chrono::high_resolution_clock::now(); OrtRunOptions run_options; - run_options.cache_feeds_fetches_info = true; + ORT_THROW_ON_ERROR(OrtRun(session_object_, nullptr, input_names_.data(), input_values_.data(), input_names_.size(), output_names_raw_ptr.data(), output_names_raw_ptr.size(), output_values_.data())); auto end = std::chrono::high_resolution_clock::now(); diff --git a/onnxruntime/test/shared_lib/test_run_options.cc b/onnxruntime/test/shared_lib/test_run_options.cc index 9b0848dd7c..ea20ada5f1 100644 --- a/onnxruntime/test/shared_lib/test_run_options.cc +++ b/onnxruntime/test/shared_lib/test_run_options.cc @@ -12,7 +12,4 @@ TEST_F(CApiTest, run_options) { ASSERT_EQ(OrtRunOptionsSetRunTag(options.get(), "abc"), nullptr); ASSERT_STREQ(OrtRunOptionsGetRunTag(options.get()), "abc"); ASSERT_EQ(OrtRunOptionsGetRunLogVerbosityLevel(options.get()), unsigned(1)); - ASSERT_EQ(OrtRunOptionsGetCacheFeedsFetchesInfoEnabled(options.get()), int(0)); - OrtRunOptionsSetCacheFeedsFetchesInfoEnabled(options.get(), 3); // any non-zero int should convert to true - ASSERT_EQ(OrtRunOptionsGetCacheFeedsFetchesInfoEnabled(options.get()), int(true)); }