saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-06 04:28:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove caching from InferenceSession::Run (#547)

Browse source 
* Remove caching from InferenceSession::Run

* Fix automatic merge of one file

* trigger rerunning checks
This commit is contained in:
Scott McKay 2019-03-06 14:29:42 -08:00 committed by GitHub
parent 54eeb7394a
commit 0e65bfe7ae
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 16 additions and 220 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
include/onnxruntime/core/framework/run_options.h
View file

@ -15,11 +15,6 @@ struct OrtRunOptions {
unsigned run_log_verbosity_level = 0; ///< applies to a particular Run() invocation
std::string run_tag; ///< to identify logs generated by a particular Run() invocation
/**
Create and/or use cached info for the combination of feed and output names used in the call to Run.
*/
bool cache_feeds_fetches_info = false;
/// set to 'true' to terminate any currently executing Run() calls that are using this
/// OrtRunOptions instance. the individual calls will exit gracefully and return an error status.
bool terminate = false;

2
include/onnxruntime/core/session/onnxruntime_c_api.h
View file

@ -269,11 +269,9 @@ ORT_API(OrtRunOptions*, OrtCreateRunOptions);
ORT_API_STATUS(OrtRunOptionsSetRunLogVerbosityLevel, _In_ OrtRunOptions*, unsigned int);
ORT_API_STATUS(OrtRunOptionsSetRunTag, _In_ OrtRunOptions*, _In_ const char* run_tag);
ORT_API(void, OrtRunOptionsSetCacheFeedsFetchesInfoEnabled, _In_ OrtRunOptions* options, int bool_value);
ORT_API(unsigned int, OrtRunOptionsGetRunLogVerbosityLevel, _In_ OrtRunOptions*);
ORT_API(const char*, OrtRunOptionsGetRunTag, _In_ OrtRunOptions*);
ORT_API(int, OrtRunOptionsGetCacheFeedsFetchesInfoEnabled, _In_ OrtRunOptions*);
// Set a flag so that any running OrtRun* calls that are using this instance of OrtRunOptions
// will exit as soon as possible if the flag is true.

4
onnxruntime/core/framework/feeds_fetches_manager.h
View file

@ -27,8 +27,8 @@ struct DeviceCopyChecks {
struct FeedsFetchesInfo {
FeedsFetchesInfo() = default;
FeedsFetchesInfo(std::vector<std::string>& feed_names_in,
std::vector<std::string>& output_names_in)
FeedsFetchesInfo(const std::vector<std::string>& feed_names_in,
const std::vector<std::string>& output_names_in)
: feed_names{feed_names_in}, output_names{output_names_in} {}
static Status MapNamesToMLValueIdxs(const std::vector<std::string>& names,

8
onnxruntime/core/framework/run_options.cc
View file

@ -22,10 +22,6 @@ ORT_API_STATUS_IMPL(OrtRunOptionsSetRunTag, _In_ OrtRunOptions* options, _In_ co
return nullptr;
}
ORT_API(void, OrtRunOptionsSetCacheFeedsFetchesInfoEnabled, _In_ OrtRunOptions* options, int bool_value) {
options->cache_feeds_fetches_info = bool_value != 0;
}
ORT_API(unsigned int, OrtRunOptionsGetRunLogVerbosityLevel, _In_ OrtRunOptions* options) {
return options->run_log_verbosity_level;
}
@ -34,10 +30,6 @@ ORT_API(const char*, OrtRunOptionsGetRunTag, _In_ OrtRunOptions* options) {
return options->run_tag.c_str();
}
ORT_API(int, OrtRunOptionsGetCacheFeedsFetchesInfoEnabled, _In_ OrtRunOptions* options) {
return options->cache_feeds_fetches_info;
}
ORT_API(void, OrtRunOptionsSetTerminate, _In_ OrtRunOptions* options, bool value) {
options->terminate = value;
}

62
onnxruntime/core/framework/session_state.cc
View file

@ -211,66 +211,4 @@ const NodeIndexInfo& SessionState::GetNodeIndexInfo() const {
ORT_ENFORCE(node_index_info_, "CalculateNodeIndexInfo must be called prior to GetExecutionInfo.");
return *node_index_info_;
}
// use a cheap way of matching first. if we have multiple entries with this key, we will do the more expensive
// check of the individual feed/output names
static int MakeFeedsFetchesManagerCacheKey(const std::vector<std::string>& feed_names,
const std::vector<std::string>& output_names) {
return static_cast<int>(feed_names.size()) << 16 | static_cast<int>(output_names.size());
};
const FeedsFetchesManager* SessionState::GetFeedsFetchesManager(const std::vector<std::string>& feed_names,
const std::vector<std::string>& output_names) const {
int key = MakeFeedsFetchesManagerCacheKey(feed_names, output_names);
auto num_matches = cached_feeds_fetches_managers_.count(key);
const FeedsFetchesManager* manager = nullptr;
if (num_matches > 0) {
auto begin_end_pair = cached_feeds_fetches_managers_.equal_range(key);
auto iter = begin_end_pair.first;
auto end = begin_end_pair.second;
while (iter != end) {
auto& ffi = iter->second->GetFeedsFetchesInfo();
auto check = [](const std::vector<std::string>& input, const std::vector<std::string>& existing) {
for (size_t i = 0, end = input.size(); i < end; ++i) {
if (input[i] != existing[i]) {
return false;
}
}
return true;
};
if (check(feed_names, ffi.feed_names) && check(output_names, ffi.output_names)) {
manager = iter->second.get();
break;
}
++iter;
}
}
return manager;
}
Status SessionState::CacheFeedsFetchesManager(const std::vector<std::string>& feed_names,
const std::vector<std::string>& output_names,
std::unique_ptr<FeedsFetchesManager> manager) {
int key = MakeFeedsFetchesManagerCacheKey(feed_names, output_names);
auto num_matches = cached_feeds_fetches_managers_.count(key);
if (num_matches) {
// be paranoid and make sure we're not attempting to insert a duplicate entry.
// if so it would imply that there is probably a concurrency issue in InferenceSession::Run.
ORT_ENFORCE(GetFeedsFetchesManager(feed_names, output_names) == nullptr, "Existing FeedsFetchesManager found.");
}
cached_feeds_fetches_managers_.emplace(key, std::move(manager));
return Status::OK();
}
} // namespace onnxruntime

7
onnxruntime/core/framework/session_state.h
View file

@ -187,13 +187,6 @@ class SessionState {
void CalculateNodeIndexInfo();
const NodeIndexInfo& GetNodeIndexInfo() const;
const FeedsFetchesManager* GetFeedsFetchesManager(const std::vector<std::string>& feed_names,
const std::vector<std::string>& output_names) const;
Status CacheFeedsFetchesManager(const std::vector<std::string>& feed_names,
const std::vector<std::string>& output_names,
std::unique_ptr<FeedsFetchesManager> manager);
private:
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SessionState);

23
onnxruntime/core/framework/utils.cc
View file

@ -245,20 +245,6 @@ static common::Status SetupFetchesForExecute(const SessionState& session_state,
new_fetches.resize(num_outputs);
if (copy_to_new_fetches_cached_values && !copy_to_new_fetches_cached_values->empty()) {
// use the cached values
ORT_ENFORCE(copy_to_new_fetches_cached_values->size() == num_outputs);
auto& copy = *copy_to_new_fetches_cached_values;
for (size_t i = 0; i < num_outputs; ++i) {
if (copy[i]) {
new_fetches[i] = fetches[i];
}
}
return Status::OK();
}
// track which fetches can be copied to new_fetches and used directly in the execution.
std::vector<bool> local_can_copy_flags(num_outputs, false);
@ -354,15 +340,6 @@ static common::Status CopyOutputsAcrossDevices(const SessionState& session_state
needed_copy = false;
auto num_outputs = fetches.size();
// used the cached copy logic if available
if (copiers && !copiers->empty()) {
for (size_t idx = 0; idx < num_outputs; ++idx) {
ORT_RETURN_IF_ERROR(CopyMLValue((*copiers)[idx], fetches[idx], user_fetches[idx]));
}
return Status::OK();
}
if (copiers) {
// resize so we have default values and only need to update an entry if there's a device copy required.
copiers->resize(num_outputs);

2
onnxruntime/core/providers/cpu/symbols.txt
View file

@ -57,10 +57,8 @@ OrtReleaseTensorTypeAndShapeInfo
OrtReleaseTypeInfo
OrtReleaseValue
OrtRun
OrtRunOptionsGetCacheFeedsFetchesInfoEnabled
OrtRunOptionsGetRunLogVerbosityLevel
OrtRunOptionsGetRunTag
OrtRunOptionsSetCacheFeedsFetchesInfoEnabled
OrtRunOptionsSetRunLogVerbosityLevel
OrtRunOptionsSetRunTag
OrtRunOptionsSetTerminate

79
onnxruntime/core/session/inference_session.cc
View file

@ -575,68 +575,22 @@ class InferenceSession::Impl {
Status retval = Status::OK();
try {
// use cached info if available, otherwise create a FeedsFetchesManager and update it in the call to ExecuteGraph
std::unique_ptr<FeedsFetchesManager> local_ffm;
FeedsFetchesManager* feeds_fetches_manager = nullptr;
const FeedsFetchesManager* cached_feeds_fetches_manager = nullptr;
// lambda to construct so that we can call it under the lock if we're caching this, or outside of the lock
// if we're not.
auto create_feeds_fetches_manager = [&]() {
ORT_RETURN_IF_ERROR(ValidateInputs(feed_names, feeds));
// if the output vector is non-empty, ensure that its the same size as the output_names
ORT_RETURN_IF_ERROR(ValidateOutputs(output_names, p_fetches));
auto status = FeedsFetchesManager::Create(feed_names, output_names, session_state_.GetMLValueNameIdxMap(),
local_ffm);
ORT_RETURN_IF_ERROR(status);
feeds_fetches_manager = local_ffm.get();
if (run_options.cache_feeds_fetches_info) {
session_state_.CacheFeedsFetchesManager(feed_names, output_names, std::move(local_ffm));
}
return Status::OK();
};
{
std::lock_guard<onnxruntime::OrtMutex> l(session_mutex_);
if (!is_inited_) {
LOGS(*session_logger_, ERROR) << "Session was not initialized";
retval = Status(common::ONNXRUNTIME, common::FAIL, "Session not initialized.");
}
if (run_options.cache_feeds_fetches_info) {
cached_feeds_fetches_manager = session_state_.GetFeedsFetchesManager(feed_names, output_names);
if (!cached_feeds_fetches_manager) {
// create the instance under the lock as we add it to SessionState and don't want concurrent calls to Run
// to clash with each other
ORT_RETURN_IF_ERROR(create_feeds_fetches_manager());
}
}
}
if (!run_options.cache_feeds_fetches_info) {
// if we're not creating/using cached info, create an instance for this run
ORT_RETURN_IF_ERROR(create_feeds_fetches_manager());
} else if (cached_feeds_fetches_manager) {
// make sure that if we didn't create the FeedsFetchesManager it has been fully initialized by the
// successful completion of a call to Run. this is primarily to detect concurrent calls to Run
// prior to the initial call completing. we could do something more complicated to handle failure on the
// initial call if a real need to do so is proven.
if (cached_feeds_fetches_manager->GetDeviceCopyChecks().status == DeviceCopyCheck::Unknown) {
return ORT_MAKE_STATUS(
ONNXRUNTIME, FAIL,
"Existing cached information was found but was not fully initialized. "
"If caching is enabled, the first call to Run must successfully complete to fully initialize the "
"cache information. Once it is fully initialized, Run calls can be made in parallel. "
"If the first call to Run failed and you wish to use cached information, you will need to create a new "
"InferenceSession.");
}
ORT_RETURN_IF_ERROR(ValidateInputs(feed_names, feeds));
LOGS(*session_logger_, INFO) << "Skipped validation of inputs and outputs as cached information was found";
}
// if the output vector is non-empty, ensure that its the same size as the output_names
ORT_RETURN_IF_ERROR(ValidateOutputs(output_names, p_fetches));
FeedsFetchesInfo info(feed_names, output_names);
ORT_RETURN_IF_ERROR(info.SetMLValueIdxs(session_state_.GetMLValueNameIdxMap()));
FeedsFetchesManager feeds_fetches_manager{std::move(info)};
if (!run_options.run_tag.empty()) {
LOGS(*session_logger_, INFO) << "Running with tag: " << run_options.run_tag;
@ -657,19 +611,12 @@ class InferenceSession::Impl {
ORT_CHECK_AND_SET_RETVAL(xp->OnRunStart());
}
if (cached_feeds_fetches_manager) {
// used the const cached_feeds_fetches_manager to execute the graph
ORT_CHECK_AND_SET_RETVAL(
utils::ExecuteGraphWithCachedInfo(session_state_, *cached_feeds_fetches_manager, feeds, *p_fetches, {},
session_options_.enable_sequential_execution, run_options.terminate,
run_logger));
} else {
// execute the graph and update feeds_fetches_manager
ORT_CHECK_AND_SET_RETVAL(
utils::ExecuteGraph(session_state_, *feeds_fetches_manager, feeds, *p_fetches, {},
session_options_.enable_sequential_execution, run_options.terminate, run_logger,
run_options.cache_feeds_fetches_info));
}
// execute the graph
ORT_CHECK_AND_SET_RETVAL(
utils::ExecuteGraph(session_state_, feeds_fetches_manager, feeds, *p_fetches, {},
session_options_.enable_sequential_execution, run_options.terminate, run_logger,
false));
} catch (const std::exception& e) {
retval = Status(common::ONNXRUNTIME, common::FAIL, e.what());
} catch (...) {

39
onnxruntime/test/framework/inference_session_test.cc
View file

@ -1189,7 +1189,6 @@ TEST(InferenceSessionTests, TestTruncatedSequence) {
RunOptions run_options;
run_options.run_tag = "one session/one tag";
run_options.cache_feeds_fetches_info = true; // caching should handle the truncated and non-truncated cases
std::vector<int64_t> X_dims = {5, 1, 3};
std::vector<float> X = {0.5488135f, 0.71518934f, 0.60276335f,
@ -1332,7 +1331,6 @@ TEST(InferenceSessionTests, TestCopyToFromDevices) {
// Now run
RunOptions run_options;
run_options.run_tag = "run:" + std::to_string(run_num);
run_options.cache_feeds_fetches_info = true;
common::Status st = session_object.Run(run_options, feed_names, feeds, output_names, &fetches);
ASSERT_TRUE(st.IsOK()) << st.ErrorMessage();
@ -1345,42 +1343,5 @@ TEST(InferenceSessionTests, TestCopyToFromDevices) {
run_test(run_number++);
}
// Make sure we don't match the wrong cache entry
TEST(InferenceSessionTests, TestCacheMatching) {
SessionOptions so;
so.session_logid = "InferenceSessionTests.TestCacheMatching";
InferenceSession session_object{so, &DefaultLoggingManager()};
ASSERT_TRUE(session_object.Load(MODEL_URI).IsOK());
ASSERT_TRUE(session_object.Initialize().IsOK());
RunOptions run_options;
run_options.run_tag = "one session/one tag";
run_options.cache_feeds_fetches_info = true;
// run once with valid names. this will cache an entry with input name of 'X' and output name of 'Y'
RunModel(session_object, run_options);
// run again with an invalid output name. if we match the cache, we don't do name validation so this would
// not error out as it would use the valid output name information from the cache.
std::vector<int64_t> dims_mul_x = {3, 2};
std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
MLValue ml_value;
CreateMLValue<float>(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x,
&ml_value);
NameMLValMap feeds;
feeds.insert(std::make_pair("X", ml_value));
std::vector<std::string> output_names;
output_names.push_back("Y_invalid");
std::vector<MLValue> fetches;
common::Status status = session_object.Run(run_options, feeds, output_names, &fetches);
EXPECT_FALSE(status.IsOK());
EXPECT_THAT(status.ErrorMessage(), testing::HasSubstr("Invalid Output Names: Y_invalid"));
}
} // namespace test
} // namespace onnxruntime

2
onnxruntime/test/perftest/performance_runner.cc
View file

@ -50,7 +50,7 @@ Status PerformanceRunner::Run() {
Status PerformanceRunner::RunOneIteration(bool isWarmup) {
auto start = std::chrono::high_resolution_clock::now();
OrtRunOptions run_options;
run_options.cache_feeds_fetches_info = true;
ORT_THROW_ON_ERROR(OrtRun(session_object_, nullptr, input_names_.data(), input_values_.data(), input_names_.size(),
output_names_raw_ptr.data(), output_names_raw_ptr.size(), output_values_.data()));
auto end = std::chrono::high_resolution_clock::now();

3
onnxruntime/test/shared_lib/test_run_options.cc
View file

@ -12,7 +12,4 @@ TEST_F(CApiTest, run_options) {
ASSERT_EQ(OrtRunOptionsSetRunTag(options.get(), "abc"), nullptr);
ASSERT_STREQ(OrtRunOptionsGetRunTag(options.get()), "abc");
ASSERT_EQ(OrtRunOptionsGetRunLogVerbosityLevel(options.get()), unsigned(1));
ASSERT_EQ(OrtRunOptionsGetCacheFeedsFetchesInfoEnabled(options.get()), int(0));
OrtRunOptionsSetCacheFeedsFetchesInfoEnabled(options.get(), 3); // any non-zero int should convert to true
ASSERT_EQ(OrtRunOptionsGetCacheFeedsFetchesInfoEnabled(options.get()), int(true));
}