diff --git a/docs/execution_providers/OpenVINO-ExecutionProvider.md b/docs/execution_providers/OpenVINO-ExecutionProvider.md index e409cae97a..d45b662620 100644 --- a/docs/execution_providers/OpenVINO-ExecutionProvider.md +++ b/docs/execution_providers/OpenVINO-ExecutionProvider.md @@ -15,7 +15,7 @@ Key-Value pairs for config options can be set using the Session.set_providers AP ``` session = onnxruntime.InferenceSession(, options) -session.set_providers(['OpenVINOExecutionProviders'], [{Key1 : Value1, Key2 : Value2, ...}]) +session.set_providers(['OpenVINOExecutionProvider'], [{Key1 : Value1, Key2 : Value2, ...}]) ``` *Note that this causes the InferenceSession to be re-initialized, which may cause model recompilation and hardware re-initialization* @@ -27,7 +27,7 @@ std::string settings_str; settings_str.append("Key1|Value1\n"); settings_str.append("Key2|Value2\n"); settings_str.append("Key3|Value3\n"); -Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str)); +Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str.c_str())); ``` ### Available configuration options @@ -38,6 +38,7 @@ The following table lists all the available configuratoin optoins and the Key-Va | device_type | string | CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VAD-M_FP16, VAD-F_FP32 | string | Overrides the accelerator hardware type and precision with these values at runtime. If this option is not explicitly set, default hardware and precision specified during build time is used. | | device_id | string | Any valid OpenVINO device ID | string | Selects a particular hardware device for inference. The list of valid OpenVINO device ID's available on a platform can be obtained either by Python API (`onnxruntime.capi._pybind_state.get_available_openvino_device_ids()`) or by [OpenVINO C/C++ API](https://docs.openvinotoolkit.org/latest/classInferenceEngine_1_1Core.html#acb212aa879e1234f51b845d2befae41c). If this option is not explicitly set, an arbitrary free device will be automatically selected by OpenVINO runtime.| | enable_vpu_fast_compile | string | True/False | boolean | This option is only available for MYRIAD_FP16 VPU devices. During initialization of the VPU device with compiled model, Fast-compile may be optionally enabled to speeds up the model's compilation to VPU device specific format. This in-turn speeds up model initialization time. However, enabling this option may slowdown inference due to some of the optimizations not being fully applied, so caution is to be exercised while enabling this option. | +| num_of_threads | string | Any unsigned positive number other than 0 | size_t | Overrides the accelerator default value of number of threads with this value at runtime. If this option is not explicitly set, default value of 8 is used during build time. | ## Other configuration settings ### Onnxruntime Graph Optimization level @@ -76,7 +77,7 @@ Append the settings string "device_type|\n" to the EP settings std::string settings_str; ... settings_str.append("device_type|CPU_FP32\n"); -Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str)); +Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str.c_str())); ``` @@ -86,7 +87,7 @@ Append the settings string "device_id|\n" to the EP settings string, std::string settings_str; ... settings_str.append("device_id|\n"); -Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str)); +Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, settings_str.c_str())); ``` ## ONNX Layers supported using OpenVINO @@ -246,3 +247,7 @@ Below topologies from ONNX open model zoo are fully supported on OpenVINO Execut ## CSharp API To use csharp api for openvino execution provider create a custom nuget package. Follow the instructions [here](../../BUILD.md##build-nuget-packages) to install prerequisites for nuget creation. Once prerequisites are installed follow the instructions to [build openvino](../../BUILD.md#openvino) and add an extra flag `--build_nuget` to create nuget packages. Two nuget packages will be created Microsoft.ML.OnnxRuntime.Managed and Microsoft.ML.OnnxRuntime.Openvino. + +## Multi-threading for OpenVINO EP + +OpenVINO Execution Provider enables thread-safe deep learning inference \ No newline at end of file diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index d89536e688..88858b0e06 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -45,6 +45,16 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, std::map config; if(global_context_.device_type == "MYRIAD"){ +#if defined(OPENVINO_2021_1) + if(subgraph_context_.set_vpu_config) { + config["MYRIAD_DETECT_NETWORK_BATCH"] = CONFIG_VALUE(NO); + } + + if(global_context_.enable_vpu_fast_compile) { + config["MYRIAD_HW_INJECT_STAGES"] = CONFIG_VALUE(NO); + config["MYRIAD_COPY_OPTIMIZATION"] = CONFIG_VALUE(NO); + } +#else if(subgraph_context_.set_vpu_config) { config["VPU_DETECT_NETWORK_BATCH"] = CONFIG_VALUE(NO); } @@ -53,6 +63,7 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, config["VPU_HW_INJECT_STAGES"] = CONFIG_VALUE(NO); config["VPU_COPY_OPTIMIZATION"] = CONFIG_VALUE(NO); } +#endif } std::string& hw_target = (global_context_.device_id != "") ? global_context_.device_id : global_context_.device_type; try { @@ -64,20 +75,20 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, } LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin"; - // Create infer request - try { - infer_request_ = exe_network.CreateInferRequestPtr(); - } catch (InferenceEngine::details::InferenceEngineException e) { - ORT_THROW(log_tag + " Exception while creating InferRequest object: " + e.what()); - } catch (...) { - ORT_THROW(log_tag + "Exception while creating InferRequest object"); + //Initializing the infer_requests_ pool with 8 infer_request's (creating infer_request) + size_t nireq = global_context_.num_of_threads; + LOGS_DEFAULT(INFO) << "The value of nireq being used is: " << nireq; +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "The value of nireq being used is: " << nireq << std::endl; } - LOGS_DEFAULT(INFO) << log_tag << "Infer request created"; +#endif + inferRequestsQueue_ = std::unique_ptr(new InferRequestsQueue(exe_network, nireq)); } // Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on // an Infer Request indexed by infer_req_idx -void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context) { +void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, std::shared_ptr infer_request) { auto graph_input_info = ie_cnn_network_->getInputsInfo(); @@ -88,7 +99,7 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* InferenceEngine::Blob::Ptr graph_input_blob; std::string input_name = input_info_iter->first; try { - graph_input_blob = infer_request_->GetBlob(input_name); + graph_input_blob = infer_request->GetBlob(input_name); } catch (InferenceEngine::details::InferenceEngineException e) { ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name + e.what()); @@ -101,7 +112,7 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* } // Start Async inference try { - infer_request_->StartAsync(); + infer_request->StartAsync(); } catch (InferenceEngine::details::InferenceEngineException e) { ORT_THROW(log_tag + " Couldn't start Inference: " + e.what()); } catch (...) { @@ -111,10 +122,10 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* // Wait for asynchronous inference completion on an Infer Request object indexed by infer_req_idx // and copy the results into a slice location within the batched output buffer indexed by batch_slice_idx -void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context) { +void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, std::shared_ptr infer_request) { // Wait for Async inference completion try { - infer_request_->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY); + infer_request->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY); } catch (InferenceEngine::details::InferenceEngineException e) { ORT_THROW(log_tag + " Exception with completing Inference: " + e.what()); } catch (...) { @@ -128,14 +139,14 @@ void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContex InferenceEngine::Blob::Ptr graph_output_blob; auto output_name = output_info_iter->first; try { - graph_output_blob = infer_request_->GetBlob(output_name); + graph_output_blob = infer_request->GetBlob(output_name); } catch (InferenceEngine::details::InferenceEngineException e) { ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name + e.what()); } catch (...) { ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name); } size_t batch_size = 1; - auto output_tensor = GetOutputTensor(ort, context, batch_size, infer_request_, output_name, subgraph_context_.output_names); + auto output_tensor = GetOutputTensor(ort, context, batch_size, infer_request, output_name, subgraph_context_.output_names); auto precision = output_info_iter->second->getPrecision(); size_t batch_slice = 0; @@ -156,11 +167,17 @@ void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContex void BasicBackend::Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) { // Preliminary Thread safety mechanism - // Currently allows only one Infer execution at a time + // currently allows a maximum of 8 Infer request's to paralelly execute at the same time + + //Requesting for an idle infer_request from a pool of infer_requests_ + std::shared_ptr infer_request = inferRequestsQueue_->getIdleRequest(); + if (!infer_request) { + LOGS_DEFAULT(INFO) << "No idle Infer Requests found from the infer_requests_ pool!"; + THROW_IE_EXCEPTION << "No idle Infer Requests!"; + } LOGS_DEFAULT(INFO) << log_tag << "Running graph " << subgraph_context_.subgraph_name; LOGS_DEFAULT(INFO) << log_tag << "In Infer"; - std::lock_guard lock(compute_lock_); if(subgraph_context_.is_constant){ #if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) @@ -173,11 +190,18 @@ void BasicBackend::Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) { #endif } else{ - StartAsyncInference(ort, context); - CompleteAsyncInference(ort, context); + StartAsyncInference(ort, context, infer_request); + CompleteAsyncInference(ort, context, infer_request); } // Get Output tensors LOGS_DEFAULT(INFO) << log_tag << "Inference successful"; + //Once the inference is completed, the infer_request becomes free and is placed back into pool of infer_requests_ + inferRequestsQueue_->putIdleRequest(infer_request); +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + inferRequestsQueue_->printstatus(); //Printing the elements of infer_requests_ vector pool only in debug mode + } +#endif } } // namespace openvino_ep diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index 8782f51271..d119b798f7 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -10,9 +10,15 @@ #include "core/providers/openvino/contexts.h" #include "core/providers/openvino/ibackend.h" +#include +#include +#include +#include + namespace onnxruntime { namespace openvino_ep { +class InferRequestsQueue; class BasicBackend : public IBackend { public: BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, @@ -22,16 +28,63 @@ class BasicBackend : public IBackend { void Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) override; private: - void StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context); + void StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, std::shared_ptr infer_request); - void CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context); + void CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, std::shared_ptr infer_request); GlobalContext& global_context_; SubGraphContext subgraph_context_; mutable std::mutex compute_lock_; std::shared_ptr ie_cnn_network_; std::map> const_outputs_map_; - InferenceEngine::InferRequest::Ptr infer_request_; + std::unique_ptr inferRequestsQueue_; }; + +class InferRequestsQueue { +public: +InferRequestsQueue(InferenceEngine::ExecutableNetwork& net, size_t nireq) { + InferenceEngine::InferRequest::Ptr infer_request; + for (size_t id = 0; id < nireq; id++) { + infer_request = net.CreateInferRequestPtr(); + infer_requests_.push_back(infer_request); + } + } + +~InferRequestsQueue() { + // clearing out the infer_requests_ vector pool in the class's destructor + for(auto& pointer : infer_requests_) { + pointer = nullptr; + } + infer_requests_.erase(std::remove(infer_requests_.begin(), infer_requests_.end(), nullptr), infer_requests_.end()); +} + +void printstatus() { + std::cout << "printing elements of the vector (infer_requests_): " << std::endl; + for (auto i = infer_requests_.begin(); i != infer_requests_.end(); ++i) { + std::cout << *i << " "; + } + std::cout << '\n'; +} + +void putIdleRequest(InferenceEngine::InferRequest::Ptr infer_request) { + std::unique_lock lock(_mutex); + infer_requests_.push_back(infer_request); + _cv.notify_one(); +} + +InferenceEngine::InferRequest::Ptr getIdleRequest() { + std::unique_lock lock(_mutex); + _cv.wait(lock, [this]{ return infer_requests_.size() > 0; }); + auto request = infer_requests_.at(0); + infer_requests_.erase(infer_requests_.begin()); + return request; +} + +private: +std::mutex _mutex; +std::condition_variable _cv; +std::vector infer_requests_; +}; + } // namespace openvino_ep } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 3ddb4b3e98..efda6d66c4 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -13,6 +13,7 @@ struct GlobalContext { InferenceEngine::Core ie_core; bool is_wholly_supported_graph = false; bool enable_vpu_fast_compile = false; + size_t num_of_threads; std::string device_type; std::string precision_str; std::string device_id; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 4bcf42f7ad..94377d22bf 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -24,6 +24,13 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv openvino_ep::BackendManager::GetGlobalContext().device_type = info.device_type_; openvino_ep::BackendManager::GetGlobalContext().precision_str = info.precision_; openvino_ep::BackendManager::GetGlobalContext().enable_vpu_fast_compile = info.enable_vpu_fast_compile_; + if((int)info.num_of_threads_ <= 0) { + openvino_ep::BackendManager::GetGlobalContext().num_of_threads = 8; + } + else { + openvino_ep::BackendManager::GetGlobalContext().num_of_threads = info.num_of_threads_; + } + openvino_ep::BackendManager::GetGlobalContext().num_of_threads = info.num_of_threads_; if(info.device_id_ != "") { bool device_found = false; auto available_devices = openvino_ep::BackendManager::GetGlobalContext().ie_core.GetAvailableDevices(); diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 5e4755ac25..2ef6ac8cb5 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -19,9 +19,10 @@ struct OpenVINOExecutionProviderInfo { std::string precision_; bool enable_vpu_fast_compile_; std::string device_id_; + size_t num_of_threads_; - explicit OpenVINOExecutionProviderInfo(std::string dev_type, bool enable_vpu_fast_compile, std::string dev_id) - : enable_vpu_fast_compile_(enable_vpu_fast_compile), device_id_(dev_id) { + explicit OpenVINOExecutionProviderInfo(std::string dev_type, bool enable_vpu_fast_compile, std::string dev_id, size_t num_of_threads) + : enable_vpu_fast_compile_(enable_vpu_fast_compile), device_id_(dev_id), num_of_threads_(num_of_threads) { if (dev_type == "") { LOGS_DEFAULT(INFO) << "[OpenVINO-EP]" @@ -70,7 +71,7 @@ struct OpenVINOExecutionProviderInfo { << "Choosing Device: " << device_type_ << " , Precision: " << precision_; } OpenVINOExecutionProviderInfo() { - OpenVINOExecutionProviderInfo("", false, ""); + OpenVINOExecutionProviderInfo("", false, "", 0); } }; diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 379afad430..99ce594d53 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -8,8 +8,8 @@ namespace onnxruntime { struct OpenVINOProviderFactory : IExecutionProviderFactory { OpenVINOProviderFactory(const char* device_type, bool enable_vpu_fast_compile, - const char* device_id) - : enable_vpu_fast_compile_(enable_vpu_fast_compile) { + const char* device_id, size_t num_of_threads) + : enable_vpu_fast_compile_(enable_vpu_fast_compile), num_of_threads_(num_of_threads) { device_type_ = (device_type == nullptr) ? "" : device_type; device_id_ = (device_id == nullptr) ? "" : device_id; } @@ -22,16 +22,17 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { std::string device_type_; bool enable_vpu_fast_compile_; std::string device_id_; + size_t num_of_threads_; }; std::unique_ptr OpenVINOProviderFactory::CreateProvider() { - OpenVINOExecutionProviderInfo info(device_type_, enable_vpu_fast_compile_, device_id_); + OpenVINOExecutionProviderInfo info(device_type_, enable_vpu_fast_compile_, device_id_, num_of_threads_); return std::make_unique(info); } std::shared_ptr CreateExecutionProviderFactory_OpenVINO( - const char* device_type, bool enable_vpu_fast_compile, const char* device_id) { - return std::make_shared(device_type, enable_vpu_fast_compile, device_id); + const char* device_type, bool enable_vpu_fast_compile, const char* device_id, size_t num_of_threads) { + return std::make_shared(device_type, enable_vpu_fast_compile, device_id, num_of_threads); } } // namespace onnxruntime @@ -39,16 +40,17 @@ std::shared_ptr CreateExecutionProviderFactory_OpenVI ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_OpenVINO, _In_ OrtSessionOptions* options, _In_ const char* device_type) { options->provider_factories.push_back( - onnxruntime::CreateExecutionProviderFactory_OpenVINO(device_type, false, "")); + onnxruntime::CreateExecutionProviderFactory_OpenVINO(device_type, false, "", 8)); return nullptr; } -ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProviderEx_OpenVINOEP, +ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO, _In_ OrtSessionOptions* options, _In_ const char* settings_str) { std::string device_type = ""; bool enable_vpu_fast_compile = false; std::string device_id = ""; + size_t num_of_threads = 8; // Parse settings string std::stringstream iss; @@ -74,6 +76,14 @@ ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProviderEx_OpenVINOEP, } } else if(key == "device_id") { device_id = value; + } else if (key == "num_of_threads") { + size_t n_t = std::stoi(value); + if((int)n_t <= 0) { + num_of_threads = 8; + } + else { + num_of_threads = n_t; + } } } @@ -81,6 +91,7 @@ ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProviderEx_OpenVINOEP, options->provider_factories.push_back( onnxruntime::CreateExecutionProviderFactory_OpenVINO(device_type.c_str(), enable_vpu_fast_compile, - device_id.c_str())); + device_id.c_str(), + num_of_threads)); return nullptr; } diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 793277d949..8c1757cc8b 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -193,7 +193,8 @@ std::shared_ptr CreateExecutionProviderFactory_Dnnl(i std::shared_ptr CreateExecutionProviderFactory_NGraph(const char* ng_backend_type); std::shared_ptr CreateExecutionProviderFactory_OpenVINO(const char* device_type, bool enable_vpu_fast_compile, - const char* device_id); + const char* device_id, + size_t num_of_threads); std::shared_ptr CreateExecutionProviderFactory_Nuphar(bool, const char*); std::shared_ptr CreateExecutionProviderFactory_VITISAI(const char* backend_type, int device_id); std::shared_ptr CreateExecutionProviderFactory_ACL(int use_arena); @@ -646,6 +647,7 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector } else if (type == kOpenVINOExecutionProvider) { #ifdef USE_OPENVINO bool enable_vpu_fast_compile = false; + size_t num_of_threads = 8; std::string openvino_device_id; auto it = provider_options_map.find(type); if (it != provider_options_map.end()) { @@ -661,8 +663,12 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector ORT_THROW("Invalid value passed for enable_vpu_fast_compile: ", option.second); } - } else if (option.first == "device_id") + } else if (option.first == "device_id") { openvino_device_id = option.second; + } + else if (option.first == "num_of_threads") { + num_of_threads = std::stoi(option.second); + } else { ORT_THROW("Invalid OpenVINO EP option: ", option.first); } @@ -670,7 +676,8 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector } RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_OpenVINO(openvino_device_type.c_str(), enable_vpu_fast_compile, - openvino_device_id.c_str())); + openvino_device_id.c_str(), + num_of_threads)); // Reset global variables config to avoid it being accidentally passed on to the next session openvino_device_type.clear(); #endif @@ -867,7 +874,7 @@ void addGlobalMethods(py::module& m, const Environment& env) { onnxruntime::CreateExecutionProviderFactory_NGraph("CPU"), #endif #ifdef USE_OPENVINO - onnxruntime::CreateExecutionProviderFactory_OpenVINO(openvino_device_type, false, ""), + onnxruntime::CreateExecutionProviderFactory_OpenVINO(openvino_device_type, false, "", 8), #endif #ifdef USE_TENSORRT onnxruntime::CreateExecutionProviderFactory_Tensorrt(0), diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index ab05d8b43e..f67b54d607 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -317,20 +317,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) { #ifdef USE_OPENVINO //Setting default optimization level for OpenVINO can be overriden with -o option sf.SetGraphOptimizationLevel(ORT_DISABLE_ALL); - if (p_models != 1) { - fprintf(stderr, "OpenVINO doesn't support more than 1 model running simultaneously default value of 1 will be set \n"); - p_models = 1; - } - if (concurrent_session_runs != 1) { - fprintf(stderr, "OpenVINO doesn't support more than 1 session running simultaneously default value of 1 will be set \n"); - concurrent_session_runs = 1; - } - if (execution_mode == ExecutionMode::ORT_PARALLEL) { - fprintf(stderr, "OpenVINO doesn't support parallel executor switching to sequential executor\n"); - sf.SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL); - } - - Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_OpenVINO(sf, "")); + Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProviderEx_OpenVINO(sf, "")); #else fprintf(stderr, "OpenVINO is not supported in this build"); return -1; diff --git a/onnxruntime/test/perftest/main.cc b/onnxruntime/test/perftest/main.cc index 1f7d29315a..36f08167c2 100644 --- a/onnxruntime/test/perftest/main.cc +++ b/onnxruntime/test/perftest/main.cc @@ -41,17 +41,6 @@ int real_main(int argc, char* argv[]) { if (failed) return -1; } - - if (test_config.machine_config.provider_type_name == onnxruntime::kOpenVINOExecutionProvider) { - if (test_config.run_config.concurrent_session_runs != 1) { - fprintf(stderr, "OpenVINO doesn't support more than 1 session running simultaneously default value of 1 will be set \n"); - test_config.run_config.concurrent_session_runs = 1; - } - if (test_config.run_config.execution_mode == ExecutionMode::ORT_PARALLEL) { - fprintf(stderr, "OpenVINO doesn't support parallel executor using sequential executor\n"); - test_config.run_config.execution_mode = ExecutionMode::ORT_SEQUENTIAL; - } - } std::random_device rd; perftest::PerformanceRunner perf_runner(env, test_config, rd); auto status = perf_runner.Run(); diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 324c3dcd48..7278de502c 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -17,7 +17,7 @@ std::shared_ptr CreateExecutionProviderFactory_CUDA(O bool do_copy_in_default_stream = true); std::shared_ptr CreateExecutionProviderFactory_Dnnl(int use_arena); std::shared_ptr CreateExecutionProviderFactory_NGraph(const char* ng_backend_type); -std::shared_ptr CreateExecutionProviderFactory_OpenVINO(const char* device_type, bool enable_vpu_fast_compile, const char* device_id); +std::shared_ptr CreateExecutionProviderFactory_OpenVINO(const char* device_type, bool enable_vpu_fast_compile, const char* device_id, size_t num_of_threads); std::shared_ptr CreateExecutionProviderFactory_Nuphar(bool, const char*); std::shared_ptr CreateExecutionProviderFactory_Nnapi(); std::shared_ptr CreateExecutionProviderFactory_Rknpu(); @@ -50,7 +50,7 @@ std::unique_ptr DefaultMIGraphXExecutionProvider() { std::unique_ptr DefaultOpenVINOExecutionProvider() { #ifdef USE_OPENVINO - return CreateExecutionProviderFactory_OpenVINO("", false, "")->CreateProvider(); + return CreateExecutionProviderFactory_OpenVINO("", false, "", 8)->CreateProvider(); #else return nullptr; #endif