saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-06-04 23:59:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Resort to sequential execution if the inter op thread pool ptr is nullptr; (#2023)

Browse source
This commit is contained in:
Pranav Sharma 2019-10-06 16:08:41 -07:00 committed by GitHub
parent 544e53e24e
commit 4cdb95e436
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 59 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
onnxruntime/core/framework/utils.cc
View file

@ -113,8 +113,8 @@ common::Status AllocateHelper(const IExecutionProvider& execution_provider, cons
}
std::unique_ptr<Tensor> p_tensor = onnxruntime::make_unique<Tensor>(fetched_tensor.DataType(),
fetched_tensor.Shape(),
allocator);
fetched_tensor.Shape(),
allocator);
output_mlvalue.Init(p_tensor.release(),
DataTypeImpl::GetType<Tensor>(),
DataTypeImpl::GetType<Tensor>()->GetDeleteFunc());
@ -439,7 +439,13 @@ static common::Status ExecuteGraphImpl(const SessionState& session_state,
if (sequential_execution) {
p_exec = std::unique_ptr<IExecutor>(new SequentialExecutor(terminate_flag));
} else {
p_exec = std::unique_ptr<IExecutor>(new ParallelExecutor(session_state, terminate_flag));
auto* p_inter_op_thread_pool = session_state.GetInterOpThreadPool();
if (!p_inter_op_thread_pool) {
LOGS(logger, WARNING) << "Only one thread was configured for parallel execution. Hence will use sequential execution.";
p_exec = std::unique_ptr<IExecutor>(new SequentialExecutor(terminate_flag));
} else {
p_exec = std::unique_ptr<IExecutor>(new ParallelExecutor(session_state, terminate_flag));
}
}
const auto& feeds_fetches_info = feeds_fetches_manager.GetFeedsFetchesInfo();

26
onnxruntime/test/framework/parallel_executor_test.cc
View file

@ -5,6 +5,7 @@
#include "core/framework/op_kernel.h"
#include "test/providers/provider_test_utils.h"
#include "test_utils.h"
#include "core/session/inference_session.h"
#include "gtest/gtest.h"
@ -76,7 +77,7 @@ struct TestOp {
};
// test that the status from TestOp is correctly returned from InferenceSession::Run
TEST(ParallelExecutor, DISABLED_TestStatusPropagation) {
TEST(ParallelExecutor, TestStatusPropagation) {
auto registry = std::make_shared<CustomRegistry>();
std::vector<OpSchema> schemas{TestOp::OpSchema()};
Status status;
@ -113,5 +114,28 @@ TEST(ParallelExecutor, DISABLED_TestStatusPropagation) {
tester.Run(OpTester::ExpectResult::kExpectFailure, "Throwing as action was 2", {kTensorrtExecutionProvider}, nullptr, nullptr, false);
}
}
TEST(ParallelExecutor, TestNullInterOpThreadPool) {
auto registry = std::make_shared<CustomRegistry>();
std::vector<OpSchema> schemas{TestOp::OpSchema()};
Status status;
ASSERT_TRUE((status = registry->RegisterOpSet(schemas, TestOp::OpDomain, 10, 11)).IsOK()) << status;
KernelCreateFn kernel_create_fn = [](const OpKernelInfo& info) { return new typename TestOp::OpKernelImpl(info); };
auto kernel_def = TestOp::KernelDef();
ASSERT_TRUE((status = registry->RegisterCustomKernel(kernel_def, kernel_create_fn)).IsOK()) << status;
OpTester tester{"TestOp", 10, TestOp::OpDomain};
tester.AddCustomOpRegistry(registry);
tester.AddInput<int64_t>("action", {1}, {/*success*/ 0});
tester.AddOutput<int64_t>("action_out", {1}, {0});
// TensorRT doesn't handle a custom op. Possibly it should, but that would be a separate PR
onnxruntime::SessionOptions so;
so.session_logid = "TestOp";
so.session_log_verbosity_level = 1;
so.enable_sequential_execution = false;
so.inter_op_num_threads = 1;
tester.Run(so, OpTester::ExpectResult::kExpectSuccess, {}, {kTensorrtExecutionProvider}, nullptr, nullptr);
}
} // namespace test
} // namespace onnxruntime

24
onnxruntime/test/providers/provider_test_utils.cc
View file

@ -214,8 +214,8 @@ void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, OrtValu
void Check(const OpTester::Data& expected_data, OrtValue& ort_value, const std::string& provider_type) {
#ifdef MICROSOFT_AUTOML
CheckDispatch<dtf::TimePoint,VectorMapStringToFloat, VectorMapInt64ToFloat>(expected_data.data_.Type(), expected_data, ort_value,
provider_type);
CheckDispatch<dtf::TimePoint, VectorMapStringToFloat, VectorMapInt64ToFloat>(expected_data.data_.Type(), expected_data, ort_value,
provider_type);
#else
CheckDispatch<VectorMapStringToFloat, VectorMapInt64ToFloat>(expected_data.data_.Type(), expected_data, ort_value,
provider_type);
@ -333,7 +333,7 @@ std::unique_ptr<onnxruntime::Model> OpTester::BuildGraph() {
std::unordered_map<std::string, int> domain_to_version;
domain_to_version[domain_] = opset_version_;
auto p_model = onnxruntime::make_unique<onnxruntime::Model>("test", false, ModelMetaData(),
custom_schema_registries_, domain_to_version);
custom_schema_registries_, domain_to_version);
onnxruntime::Graph& graph = p_model->MainGraph();
AddNodes(graph, node_input_defs, output_defs, add_attribute_funcs_);
@ -445,6 +445,19 @@ void OpTester::Run(ExpectResult expect_result,
const RunOptions* run_options,
std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers,
bool sequential_execution) {
SessionOptions so;
so.session_logid = op_;
so.session_log_verbosity_level = 1;
so.enable_sequential_execution = sequential_execution;
Run(so, expect_result, expected_failure_string, excluded_provider_types, run_options, execution_providers);
}
void OpTester::Run(const SessionOptions& so,
ExpectResult expect_result,
const std::string& expected_failure_string,
const std::unordered_set<std::string>& excluded_provider_types,
const RunOptions* run_options,
std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers) {
try {
#ifndef NDEBUG
run_called_ = true;
@ -483,11 +496,6 @@ void OpTester::Run(ExpectResult expect_result,
std::vector<std::string> output_names;
FillFeedsAndOutputNames(feeds, output_names);
// Run the model
SessionOptions so;
so.session_logid = op_;
so.session_log_verbosity_level = 1;
so.enable_sequential_execution = sequential_execution;
static const std::string all_provider_types[] = {
kCpuExecutionProvider,
kCudaExecutionProvider,

11
onnxruntime/test/providers/provider_test_utils.h
View file

@ -24,6 +24,7 @@
namespace onnxruntime {
class InferenceSession;
struct SessionOptions;
namespace test {
// unfortunately std::optional is in C++17 so use a miniversion of it
@ -313,13 +314,19 @@ class OpTester {
std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers = nullptr,
bool sequential_execution = true);
void Run(const SessionOptions& session_options,
ExpectResult expect_result = ExpectResult::kExpectSuccess,
const std::string& expected_failure_string = "",
const std::unordered_set<std::string>& excluded_provider_types = {},
const RunOptions* run_options = nullptr,
std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers = nullptr);
struct Data {
onnxruntime::NodeArg def_;
OrtValue data_;
optional<float> relative_error_;
optional<float> absolute_error_;
Data(onnxruntime::NodeArg&& def, OrtValue&& data, optional<float>&& rel, optional<float>&& abs) :
def_(std::move(def)), data_(std::move(data)), relative_error_(std::move(rel)), absolute_error_(abs) {}
Data(onnxruntime::NodeArg&& def, OrtValue&& data, optional<float>&& rel, optional<float>&& abs) : def_(std::move(def)), data_(std::move(data)), relative_error_(std::move(rel)), absolute_error_(abs) {}
Data(Data&&) = default;
Data& operator=(Data&&) = default;
};