mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-28 20:11:22 +00:00
Maajid/multi threading 2 (#5568)
* Enabled multi-threading for OpenVino EP ->Enabled support for concurrent_session_runs *Run UEP using concurrent_session_runs > 1 *Enabled support for ORT_PARALLEL ExecutionMode ->Documentation Added for Enabling MultiThreading Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com> * Minor Fixes added *Configure the value of nireq during Runtime *Documentation typos rectified and details added for Multi_Threaded Inference Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com> * Some checks added for this fix *Added checks to invalidate wrong nireq value and assigned it to default value of 8 *Added new config options for enable_vpu_fast_compile which were changed w.r.t OpenVINO_2021.1 Release Signed-off-by: MaajidKhan <n.maajidkhan@gmail.com>
This commit is contained in:
Maajid khan
GitHub
parent
b851973f22
commit
ddf83d1ace
11 changed files with 153 additions and 68 deletions
|
|
@ -15,7 +15,7 @@ Key-Value pairs for config options can be set using the Session.set_providers AP
|
|||
|
||||
```
|
||||
session = onnxruntime.InferenceSession(<path_to_model_file>, options)
|
||||
session.set_providers(['OpenVINOExecutionProviders'], [{Key1 : Value1, Key2 : Value2, ...}])
|
||||
session.set_providers(['OpenVINOExecutionProvider'], [{Key1 : Value1, Key2 : Value2, ...}])
|
||||
```
|
||||
*Note that this causes the InferenceSession to be re-initialized, which may cause model recompilation and hardware re-initialization*
|
||||
|
||||
|
|
@ -27,7 +27,7 @@ std::string settings_str;
|
|||
settings_str.append("Key1|Value1\n");
|
||||
settings_str.append("Key2|Value2\n");
|
||||
settings_str.append("Key3|Value3\n");
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str));
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str.c_str()));
|
||||
```
|
||||
|
||||
### Available configuration options
|
||||
|
|
@ -38,6 +38,7 @@ The following table lists all the available configuratoin optoins and the Key-Va
|
|||
| device_type | string | CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VAD-M_FP16, VAD-F_FP32 | string | Overrides the accelerator hardware type and precision with these values at runtime. If this option is not explicitly set, default hardware and precision specified during build time is used. |
|
||||
| device_id | string | Any valid OpenVINO device ID | string | Selects a particular hardware device for inference. The list of valid OpenVINO device ID's available on a platform can be obtained either by Python API (`onnxruntime.capi._pybind_state.get_available_openvino_device_ids()`) or by [OpenVINO C/C++ API](https://docs.openvinotoolkit.org/latest/classInferenceEngine_1_1Core.html#acb212aa879e1234f51b845d2befae41c). If this option is not explicitly set, an arbitrary free device will be automatically selected by OpenVINO runtime.|
|
||||
| enable_vpu_fast_compile | string | True/False | boolean | This option is only available for MYRIAD_FP16 VPU devices. During initialization of the VPU device with compiled model, Fast-compile may be optionally enabled to speeds up the model's compilation to VPU device specific format. This in-turn speeds up model initialization time. However, enabling this option may slowdown inference due to some of the optimizations not being fully applied, so caution is to be exercised while enabling this option. |
|
||||
| num_of_threads | string | Any unsigned positive number other than 0 | size_t | Overrides the accelerator default value of number of threads with this value at runtime. If this option is not explicitly set, default value of 8 is used during build time. |
|
||||
|
||||
## Other configuration settings
|
||||
### Onnxruntime Graph Optimization level
|
||||
|
|
@ -76,7 +77,7 @@ Append the settings string "device_type|<hardware_option>\n" to the EP settings
|
|||
std::string settings_str;
|
||||
...
|
||||
settings_str.append("device_type|CPU_FP32\n");
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str));
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str.c_str()));
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -86,7 +87,7 @@ Append the settings string "device_id|<device_id>\n" to the EP settings string,
|
|||
std::string settings_str;
|
||||
...
|
||||
settings_str.append("device_id|<device_id>\n");
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str));
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str.c_str()));
|
||||
```
|
||||
|
||||
## ONNX Layers supported using OpenVINO
|
||||
|
|
@ -246,3 +247,7 @@ Below topologies from ONNX open model zoo are fully supported on OpenVINO Execut
|
|||
## CSharp API
|
||||
|
||||
To use csharp api for openvino execution provider create a custom nuget package. Follow the instructions [here](../../BUILD.md##build-nuget-packages) to install prerequisites for nuget creation. Once prerequisites are installed follow the instructions to [build openvino](../../BUILD.md#openvino) and add an extra flag `--build_nuget` to create nuget packages. Two nuget packages will be created Microsoft.ML.OnnxRuntime.Managed and Microsoft.ML.OnnxRuntime.Openvino.
|
||||
|
||||
## Multi-threading for OpenVINO EP
|
||||
|
||||
OpenVINO Execution Provider enables thread-safe deep learning inference
|
||||
|
|
@ -45,6 +45,16 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
std::map<std::string, std::string> config;
|
||||
if(global_context_.device_type == "MYRIAD"){
|
||||
|
||||
#if defined(OPENVINO_2021_1)
|
||||
if(subgraph_context_.set_vpu_config) {
|
||||
config["MYRIAD_DETECT_NETWORK_BATCH"] = CONFIG_VALUE(NO);
|
||||
}
|
||||
|
||||
if(global_context_.enable_vpu_fast_compile) {
|
||||
config["MYRIAD_HW_INJECT_STAGES"] = CONFIG_VALUE(NO);
|
||||
config["MYRIAD_COPY_OPTIMIZATION"] = CONFIG_VALUE(NO);
|
||||
}
|
||||
#else
|
||||
if(subgraph_context_.set_vpu_config) {
|
||||
config["VPU_DETECT_NETWORK_BATCH"] = CONFIG_VALUE(NO);
|
||||
}
|
||||
|
|
@ -53,6 +63,7 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
config["VPU_HW_INJECT_STAGES"] = CONFIG_VALUE(NO);
|
||||
config["VPU_COPY_OPTIMIZATION"] = CONFIG_VALUE(NO);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
std::string& hw_target = (global_context_.device_id != "") ? global_context_.device_id : global_context_.device_type;
|
||||
try {
|
||||
|
|
@ -64,20 +75,20 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
}
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin";
|
||||
|
||||
// Create infer request
|
||||
try {
|
||||
infer_request_ = exe_network.CreateInferRequestPtr();
|
||||
} catch (InferenceEngine::details::InferenceEngineException e) {
|
||||
ORT_THROW(log_tag + " Exception while creating InferRequest object: " + e.what());
|
||||
} catch (...) {
|
||||
ORT_THROW(log_tag + "Exception while creating InferRequest object");
|
||||
//Initializing the infer_requests_ pool with 8 infer_request's (creating infer_request)
|
||||
size_t nireq = global_context_.num_of_threads;
|
||||
LOGS_DEFAULT(INFO) << "The value of nireq being used is: " << nireq;
|
||||
#ifndef NDEBUG
|
||||
if (openvino_ep::backend_utils::IsDebugEnabled()) {
|
||||
std::cout << "The value of nireq being used is: " << nireq << std::endl;
|
||||
}
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Infer request created";
|
||||
#endif
|
||||
inferRequestsQueue_ = std::unique_ptr<InferRequestsQueue>(new InferRequestsQueue(exe_network, nireq));
|
||||
}
|
||||
|
||||
// Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on
|
||||
// an Infer Request indexed by infer_req_idx
|
||||
void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context) {
|
||||
void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, std::shared_ptr<InferenceEngine::InferRequest> infer_request) {
|
||||
|
||||
auto graph_input_info = ie_cnn_network_->getInputsInfo();
|
||||
|
||||
|
|
@ -88,7 +99,7 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext*
|
|||
InferenceEngine::Blob::Ptr graph_input_blob;
|
||||
std::string input_name = input_info_iter->first;
|
||||
try {
|
||||
graph_input_blob = infer_request_->GetBlob(input_name);
|
||||
graph_input_blob = infer_request->GetBlob(input_name);
|
||||
|
||||
} catch (InferenceEngine::details::InferenceEngineException e) {
|
||||
ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name + e.what());
|
||||
|
|
@ -101,7 +112,7 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext*
|
|||
}
|
||||
// Start Async inference
|
||||
try {
|
||||
infer_request_->StartAsync();
|
||||
infer_request->StartAsync();
|
||||
} catch (InferenceEngine::details::InferenceEngineException e) {
|
||||
ORT_THROW(log_tag + " Couldn't start Inference: " + e.what());
|
||||
} catch (...) {
|
||||
|
|
@ -111,10 +122,10 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext*
|
|||
|
||||
// Wait for asynchronous inference completion on an Infer Request object indexed by infer_req_idx
|
||||
// and copy the results into a slice location within the batched output buffer indexed by batch_slice_idx
|
||||
void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context) {
|
||||
void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, std::shared_ptr<InferenceEngine::InferRequest> infer_request) {
|
||||
// Wait for Async inference completion
|
||||
try {
|
||||
infer_request_->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY);
|
||||
infer_request->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY);
|
||||
} catch (InferenceEngine::details::InferenceEngineException e) {
|
||||
ORT_THROW(log_tag + " Exception with completing Inference: " + e.what());
|
||||
} catch (...) {
|
||||
|
|
@ -128,14 +139,14 @@ void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContex
|
|||
InferenceEngine::Blob::Ptr graph_output_blob;
|
||||
auto output_name = output_info_iter->first;
|
||||
try {
|
||||
graph_output_blob = infer_request_->GetBlob(output_name);
|
||||
graph_output_blob = infer_request->GetBlob(output_name);
|
||||
} catch (InferenceEngine::details::InferenceEngineException e) {
|
||||
ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name + e.what());
|
||||
} catch (...) {
|
||||
ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name);
|
||||
}
|
||||
size_t batch_size = 1;
|
||||
auto output_tensor = GetOutputTensor(ort, context, batch_size, infer_request_, output_name, subgraph_context_.output_names);
|
||||
auto output_tensor = GetOutputTensor(ort, context, batch_size, infer_request, output_name, subgraph_context_.output_names);
|
||||
auto precision = output_info_iter->second->getPrecision();
|
||||
|
||||
size_t batch_slice = 0;
|
||||
|
|
@ -156,11 +167,17 @@ void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContex
|
|||
|
||||
void BasicBackend::Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) {
|
||||
// Preliminary Thread safety mechanism
|
||||
// Currently allows only one Infer execution at a time
|
||||
// currently allows a maximum of 8 Infer request's to paralelly execute at the same time
|
||||
|
||||
//Requesting for an idle infer_request from a pool of infer_requests_
|
||||
std::shared_ptr<InferenceEngine::InferRequest> infer_request = inferRequestsQueue_->getIdleRequest();
|
||||
if (!infer_request) {
|
||||
LOGS_DEFAULT(INFO) << "No idle Infer Requests found from the infer_requests_ pool!";
|
||||
THROW_IE_EXCEPTION << "No idle Infer Requests!";
|
||||
}
|
||||
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Running graph " << subgraph_context_.subgraph_name;
|
||||
LOGS_DEFAULT(INFO) << log_tag << "In Infer";
|
||||
std::lock_guard<std::mutex> lock(compute_lock_);
|
||||
|
||||
if(subgraph_context_.is_constant){
|
||||
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1)
|
||||
|
|
@ -173,11 +190,18 @@ void BasicBackend::Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) {
|
|||
#endif
|
||||
}
|
||||
else{
|
||||
StartAsyncInference(ort, context);
|
||||
CompleteAsyncInference(ort, context);
|
||||
StartAsyncInference(ort, context, infer_request);
|
||||
CompleteAsyncInference(ort, context, infer_request);
|
||||
}
|
||||
// Get Output tensors
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Inference successful";
|
||||
//Once the inference is completed, the infer_request becomes free and is placed back into pool of infer_requests_
|
||||
inferRequestsQueue_->putIdleRequest(infer_request);
|
||||
#ifndef NDEBUG
|
||||
if (openvino_ep::backend_utils::IsDebugEnabled()) {
|
||||
inferRequestsQueue_->printstatus(); //Printing the elements of infer_requests_ vector pool only in debug mode
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace openvino_ep
|
||||
|
|
|
|||
|
|
@ -10,9 +10,15 @@
|
|||
#include "core/providers/openvino/contexts.h"
|
||||
#include "core/providers/openvino/ibackend.h"
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <condition_variable>
|
||||
#include <mutex>
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace openvino_ep {
|
||||
|
||||
class InferRequestsQueue;
|
||||
class BasicBackend : public IBackend {
|
||||
public:
|
||||
BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
||||
|
|
@ -22,16 +28,63 @@ class BasicBackend : public IBackend {
|
|||
void Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) override;
|
||||
|
||||
private:
|
||||
void StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context);
|
||||
void StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, std::shared_ptr<InferenceEngine::InferRequest> infer_request);
|
||||
|
||||
void CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context);
|
||||
void CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, std::shared_ptr<InferenceEngine::InferRequest> infer_request);
|
||||
|
||||
GlobalContext& global_context_;
|
||||
SubGraphContext subgraph_context_;
|
||||
mutable std::mutex compute_lock_;
|
||||
std::shared_ptr<InferenceEngine::CNNNetwork> ie_cnn_network_;
|
||||
std::map<std::string, std::shared_ptr<ngraph::Node>> const_outputs_map_;
|
||||
InferenceEngine::InferRequest::Ptr infer_request_;
|
||||
std::unique_ptr<InferRequestsQueue> inferRequestsQueue_;
|
||||
};
|
||||
|
||||
class InferRequestsQueue {
|
||||
public:
|
||||
InferRequestsQueue(InferenceEngine::ExecutableNetwork& net, size_t nireq) {
|
||||
InferenceEngine::InferRequest::Ptr infer_request;
|
||||
for (size_t id = 0; id < nireq; id++) {
|
||||
infer_request = net.CreateInferRequestPtr();
|
||||
infer_requests_.push_back(infer_request);
|
||||
}
|
||||
}
|
||||
|
||||
~InferRequestsQueue() {
|
||||
// clearing out the infer_requests_ vector pool in the class's destructor
|
||||
for(auto& pointer : infer_requests_) {
|
||||
pointer = nullptr;
|
||||
}
|
||||
infer_requests_.erase(std::remove(infer_requests_.begin(), infer_requests_.end(), nullptr), infer_requests_.end());
|
||||
}
|
||||
|
||||
void printstatus() {
|
||||
std::cout << "printing elements of the vector (infer_requests_): " << std::endl;
|
||||
for (auto i = infer_requests_.begin(); i != infer_requests_.end(); ++i) {
|
||||
std::cout << *i << " ";
|
||||
}
|
||||
std::cout << '\n';
|
||||
}
|
||||
|
||||
void putIdleRequest(InferenceEngine::InferRequest::Ptr infer_request) {
|
||||
std::unique_lock<std::mutex> lock(_mutex);
|
||||
infer_requests_.push_back(infer_request);
|
||||
_cv.notify_one();
|
||||
}
|
||||
|
||||
InferenceEngine::InferRequest::Ptr getIdleRequest() {
|
||||
std::unique_lock<std::mutex> lock(_mutex);
|
||||
_cv.wait(lock, [this]{ return infer_requests_.size() > 0; });
|
||||
auto request = infer_requests_.at(0);
|
||||
infer_requests_.erase(infer_requests_.begin());
|
||||
return request;
|
||||
}
|
||||
|
||||
private:
|
||||
std::mutex _mutex;
|
||||
std::condition_variable _cv;
|
||||
std::vector<InferenceEngine::InferRequest::Ptr> infer_requests_;
|
||||
};
|
||||
|
||||
} // namespace openvino_ep
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ struct GlobalContext {
|
|||
InferenceEngine::Core ie_core;
|
||||
bool is_wholly_supported_graph = false;
|
||||
bool enable_vpu_fast_compile = false;
|
||||
size_t num_of_threads;
|
||||
std::string device_type;
|
||||
std::string precision_str;
|
||||
std::string device_id;
|
||||
|
|
|
|||
|
|
@ -24,6 +24,13 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv
|
|||
openvino_ep::BackendManager::GetGlobalContext().device_type = info.device_type_;
|
||||
openvino_ep::BackendManager::GetGlobalContext().precision_str = info.precision_;
|
||||
openvino_ep::BackendManager::GetGlobalContext().enable_vpu_fast_compile = info.enable_vpu_fast_compile_;
|
||||
if((int)info.num_of_threads_ <= 0) {
|
||||
openvino_ep::BackendManager::GetGlobalContext().num_of_threads = 8;
|
||||
}
|
||||
else {
|
||||
openvino_ep::BackendManager::GetGlobalContext().num_of_threads = info.num_of_threads_;
|
||||
}
|
||||
openvino_ep::BackendManager::GetGlobalContext().num_of_threads = info.num_of_threads_;
|
||||
if(info.device_id_ != "") {
|
||||
bool device_found = false;
|
||||
auto available_devices = openvino_ep::BackendManager::GetGlobalContext().ie_core.GetAvailableDevices();
|
||||
|
|
|
|||
|
|
@ -19,9 +19,10 @@ struct OpenVINOExecutionProviderInfo {
|
|||
std::string precision_;
|
||||
bool enable_vpu_fast_compile_;
|
||||
std::string device_id_;
|
||||
size_t num_of_threads_;
|
||||
|
||||
explicit OpenVINOExecutionProviderInfo(std::string dev_type, bool enable_vpu_fast_compile, std::string dev_id)
|
||||
: enable_vpu_fast_compile_(enable_vpu_fast_compile), device_id_(dev_id) {
|
||||
explicit OpenVINOExecutionProviderInfo(std::string dev_type, bool enable_vpu_fast_compile, std::string dev_id, size_t num_of_threads)
|
||||
: enable_vpu_fast_compile_(enable_vpu_fast_compile), device_id_(dev_id), num_of_threads_(num_of_threads) {
|
||||
|
||||
if (dev_type == "") {
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP]"
|
||||
|
|
@ -70,7 +71,7 @@ struct OpenVINOExecutionProviderInfo {
|
|||
<< "Choosing Device: " << device_type_ << " , Precision: " << precision_;
|
||||
}
|
||||
OpenVINOExecutionProviderInfo() {
|
||||
OpenVINOExecutionProviderInfo("", false, "");
|
||||
OpenVINOExecutionProviderInfo("", false, "", 0);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@
|
|||
namespace onnxruntime {
|
||||
struct OpenVINOProviderFactory : IExecutionProviderFactory {
|
||||
OpenVINOProviderFactory(const char* device_type, bool enable_vpu_fast_compile,
|
||||
const char* device_id)
|
||||
: enable_vpu_fast_compile_(enable_vpu_fast_compile) {
|
||||
const char* device_id, size_t num_of_threads)
|
||||
: enable_vpu_fast_compile_(enable_vpu_fast_compile), num_of_threads_(num_of_threads) {
|
||||
device_type_ = (device_type == nullptr) ? "" : device_type;
|
||||
device_id_ = (device_id == nullptr) ? "" : device_id;
|
||||
}
|
||||
|
|
@ -22,16 +22,17 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
|
|||
std::string device_type_;
|
||||
bool enable_vpu_fast_compile_;
|
||||
std::string device_id_;
|
||||
size_t num_of_threads_;
|
||||
};
|
||||
|
||||
std::unique_ptr<IExecutionProvider> OpenVINOProviderFactory::CreateProvider() {
|
||||
OpenVINOExecutionProviderInfo info(device_type_, enable_vpu_fast_compile_, device_id_);
|
||||
OpenVINOExecutionProviderInfo info(device_type_, enable_vpu_fast_compile_, device_id_, num_of_threads_);
|
||||
return std::make_unique<OpenVINOExecutionProvider>(info);
|
||||
}
|
||||
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_OpenVINO(
|
||||
const char* device_type, bool enable_vpu_fast_compile, const char* device_id) {
|
||||
return std::make_shared<onnxruntime::OpenVINOProviderFactory>(device_type, enable_vpu_fast_compile, device_id);
|
||||
const char* device_type, bool enable_vpu_fast_compile, const char* device_id, size_t num_of_threads) {
|
||||
return std::make_shared<onnxruntime::OpenVINOProviderFactory>(device_type, enable_vpu_fast_compile, device_id, num_of_threads);
|
||||
}
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -39,16 +40,17 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_OpenVI
|
|||
ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_OpenVINO,
|
||||
_In_ OrtSessionOptions* options, _In_ const char* device_type) {
|
||||
options->provider_factories.push_back(
|
||||
onnxruntime::CreateExecutionProviderFactory_OpenVINO(device_type, false, ""));
|
||||
onnxruntime::CreateExecutionProviderFactory_OpenVINO(device_type, false, "", 8));
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProviderEx_OpenVINOEP,
|
||||
ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO,
|
||||
_In_ OrtSessionOptions* options, _In_ const char* settings_str) {
|
||||
|
||||
std::string device_type = "";
|
||||
bool enable_vpu_fast_compile = false;
|
||||
std::string device_id = "";
|
||||
size_t num_of_threads = 8;
|
||||
|
||||
// Parse settings string
|
||||
std::stringstream iss;
|
||||
|
|
@ -74,6 +76,14 @@ ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProviderEx_OpenVINOEP,
|
|||
}
|
||||
} else if(key == "device_id") {
|
||||
device_id = value;
|
||||
} else if (key == "num_of_threads") {
|
||||
size_t n_t = std::stoi(value);
|
||||
if((int)n_t <= 0) {
|
||||
num_of_threads = 8;
|
||||
}
|
||||
else {
|
||||
num_of_threads = n_t;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -81,6 +91,7 @@ ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProviderEx_OpenVINOEP,
|
|||
options->provider_factories.push_back(
|
||||
onnxruntime::CreateExecutionProviderFactory_OpenVINO(device_type.c_str(),
|
||||
enable_vpu_fast_compile,
|
||||
device_id.c_str()));
|
||||
device_id.c_str(),
|
||||
num_of_threads));
|
||||
return nullptr;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -193,7 +193,8 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Dnnl(i
|
|||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_NGraph(const char* ng_backend_type);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_OpenVINO(const char* device_type,
|
||||
bool enable_vpu_fast_compile,
|
||||
const char* device_id);
|
||||
const char* device_id,
|
||||
size_t num_of_threads);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nuphar(bool, const char*);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_VITISAI(const char* backend_type, int device_id);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ACL(int use_arena);
|
||||
|
|
@ -646,6 +647,7 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
|
|||
} else if (type == kOpenVINOExecutionProvider) {
|
||||
#ifdef USE_OPENVINO
|
||||
bool enable_vpu_fast_compile = false;
|
||||
size_t num_of_threads = 8;
|
||||
std::string openvino_device_id;
|
||||
auto it = provider_options_map.find(type);
|
||||
if (it != provider_options_map.end()) {
|
||||
|
|
@ -661,8 +663,12 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
|
|||
ORT_THROW("Invalid value passed for enable_vpu_fast_compile: ", option.second);
|
||||
}
|
||||
|
||||
} else if (option.first == "device_id")
|
||||
} else if (option.first == "device_id") {
|
||||
openvino_device_id = option.second;
|
||||
}
|
||||
else if (option.first == "num_of_threads") {
|
||||
num_of_threads = std::stoi(option.second);
|
||||
}
|
||||
else {
|
||||
ORT_THROW("Invalid OpenVINO EP option: ", option.first);
|
||||
}
|
||||
|
|
@ -670,7 +676,8 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
|
|||
}
|
||||
RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_OpenVINO(openvino_device_type.c_str(),
|
||||
enable_vpu_fast_compile,
|
||||
openvino_device_id.c_str()));
|
||||
openvino_device_id.c_str(),
|
||||
num_of_threads));
|
||||
// Reset global variables config to avoid it being accidentally passed on to the next session
|
||||
openvino_device_type.clear();
|
||||
#endif
|
||||
|
|
@ -867,7 +874,7 @@ void addGlobalMethods(py::module& m, const Environment& env) {
|
|||
onnxruntime::CreateExecutionProviderFactory_NGraph("CPU"),
|
||||
#endif
|
||||
#ifdef USE_OPENVINO
|
||||
onnxruntime::CreateExecutionProviderFactory_OpenVINO(openvino_device_type, false, ""),
|
||||
onnxruntime::CreateExecutionProviderFactory_OpenVINO(openvino_device_type, false, "", 8),
|
||||
#endif
|
||||
#ifdef USE_TENSORRT
|
||||
onnxruntime::CreateExecutionProviderFactory_Tensorrt(0),
|
||||
|
|
|
|||
|
|
@ -317,20 +317,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
#ifdef USE_OPENVINO
|
||||
//Setting default optimization level for OpenVINO can be overriden with -o option
|
||||
sf.SetGraphOptimizationLevel(ORT_DISABLE_ALL);
|
||||
if (p_models != 1) {
|
||||
fprintf(stderr, "OpenVINO doesn't support more than 1 model running simultaneously default value of 1 will be set \n");
|
||||
p_models = 1;
|
||||
}
|
||||
if (concurrent_session_runs != 1) {
|
||||
fprintf(stderr, "OpenVINO doesn't support more than 1 session running simultaneously default value of 1 will be set \n");
|
||||
concurrent_session_runs = 1;
|
||||
}
|
||||
if (execution_mode == ExecutionMode::ORT_PARALLEL) {
|
||||
fprintf(stderr, "OpenVINO doesn't support parallel executor switching to sequential executor\n");
|
||||
sf.SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL);
|
||||
}
|
||||
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_OpenVINO(sf, ""));
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, ""));
|
||||
#else
|
||||
fprintf(stderr, "OpenVINO is not supported in this build");
|
||||
return -1;
|
||||
|
|
|
|||
|
|
@ -41,17 +41,6 @@ int real_main(int argc, char* argv[]) {
|
|||
if (failed)
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (test_config.machine_config.provider_type_name == onnxruntime::kOpenVINOExecutionProvider) {
|
||||
if (test_config.run_config.concurrent_session_runs != 1) {
|
||||
fprintf(stderr, "OpenVINO doesn't support more than 1 session running simultaneously default value of 1 will be set \n");
|
||||
test_config.run_config.concurrent_session_runs = 1;
|
||||
}
|
||||
if (test_config.run_config.execution_mode == ExecutionMode::ORT_PARALLEL) {
|
||||
fprintf(stderr, "OpenVINO doesn't support parallel executor using sequential executor\n");
|
||||
test_config.run_config.execution_mode = ExecutionMode::ORT_SEQUENTIAL;
|
||||
}
|
||||
}
|
||||
std::random_device rd;
|
||||
perftest::PerformanceRunner perf_runner(env, test_config, rd);
|
||||
auto status = perf_runner.Run();
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CUDA(O
|
|||
bool do_copy_in_default_stream = true);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Dnnl(int use_arena);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_NGraph(const char* ng_backend_type);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_OpenVINO(const char* device_type, bool enable_vpu_fast_compile, const char* device_id);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_OpenVINO(const char* device_type, bool enable_vpu_fast_compile, const char* device_id, size_t num_of_threads);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nuphar(bool, const char*);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nnapi();
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Rknpu();
|
||||
|
|
@ -50,7 +50,7 @@ std::unique_ptr<IExecutionProvider> DefaultMIGraphXExecutionProvider() {
|
|||
|
||||
std::unique_ptr<IExecutionProvider> DefaultOpenVINOExecutionProvider() {
|
||||
#ifdef USE_OPENVINO
|
||||
return CreateExecutionProviderFactory_OpenVINO("", false, "")->CreateProvider();
|
||||
return CreateExecutionProviderFactory_OpenVINO("", false, "", 8)->CreateProvider();
|
||||
#else
|
||||
return nullptr;
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in a new issue