mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Build OnnxRT with Openvino EP on Windows (#1865)
* Avoid variable length stack array variables for VC++ compatibility Use dynamically allocated arrays or vectors instead. * windows enabling * openvino windows build * Update build instructions * resolve conflicts for PR * remove debug messages from cmake * PR fix for window support
This commit is contained in:
avidiyal
jywu-msft
parent
1f4190de3c
commit
ca89387817
5 changed files with 86 additions and 39 deletions
34
BUILD.md
34
BUILD.md
|
|
@ -219,26 +219,46 @@ ONNX Runtime supports OpenVINO Execution Provider to enable deep learning infere
|
|||
|
||||
The OpenVINO Execution Provider can be built using the following commands:
|
||||
|
||||
- Currently supports and validated on two versions of OpenVINO: OpenVINO 2018 R5.0.1 and OpenVINO 2019 R1.1(Recommended). Install the OpenVINO release along with its dependencies from ([https://software.intel.com/en-us/openvino-toolkit](https://software.intel.com/en-us/openvino-toolkit)).
|
||||
- Currently supports and validated on two versions of OpenVINO: OpenVINO 2018 R5.0.1 and OpenVINO 2019 R1.1(Recommended). Install the OpenVINO release along with its dependencies from ([https://software.intel.com/en-us/openvino-toolkit](https://software.intel.com/en-us/openvino-toolkit)).For windows, please download 2019 R1.1 windows installer
|
||||
|
||||
- Install the model optimizer prerequisites for ONNX by running
|
||||
<code><openvino_install_dir>/deployment_tools/model_optimizer/install_prerequisites/install_prerequisites_onnx.sh</code>
|
||||
|
||||
- Initialize the OpenVINO environment by running the setupvars.sh in <code>\<openvino\_install\_directory\>\/bin</code> using the below command:
|
||||
For Linux:
|
||||
|
||||
<code>source setupvars.sh</code>
|
||||
<code><openvino_install_dir>/deployment_tools/model_optimizer/install_prerequisites/install_prerequisites_onnx.sh</code>
|
||||
|
||||
- To configure Intel<sup>®</sup> Processor Graphics(GPU), please follow the installation steps from (https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_linux.html#additional-GPU-steps)
|
||||
For Windows:
|
||||
|
||||
- To configure Intel<sup>®</sup> Movidius<sup>TM</sup> USB, please follow the getting started guide from (https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_linux.html#additional-NCS-steps)
|
||||
<code><openvino_install_dir>/deployment_tools/model_optimizer/install_prerequisites/install_prerequisites_onnx.bat</code>
|
||||
|
||||
- To configure Intel<sup>®</sup> Vision Accelerator Design based on 8 Movidius<sup>TM</sup> MyriadX VPUs, please follow the configuration guide from (https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_linux.html#install-VPU)
|
||||
- Initialize the OpenVINO environment by running the setupvars in <code>\<openvino\_install\_directory\>\/bin</code> using the below command:
|
||||
|
||||
<code>source setupvars.sh (Linux)</code>
|
||||
|
||||
<code>setupvars.bat (Windows)</code>
|
||||
|
||||
- To configure Intel<sup>®</sup> Processor Graphics(GPU), please follow the installation steps from (https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_linux.html#additional-GPU-steps (Linux))
|
||||
(https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_windows.html#Install-GPU (Windows))
|
||||
|
||||
- To configure Intel<sup>®</sup> Movidius<sup>TM</sup> USB, please follow the getting started guide from (https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_linux.html#additional-NCS-steps (Linux))
|
||||
(https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_windows.html#usb-myriad (Windows))
|
||||
|
||||
- To configure Intel<sup>®</sup> Vision Accelerator Design based on 8 Movidius<sup>TM</sup> MyriadX VPUs, please follow the configuration guide from (https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_linux.html#install-VPU (Linux))
|
||||
(https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_windows.html#hddl-myriad (Windows))
|
||||
|
||||
|
||||
- Build ONNX Runtime using the below command.
|
||||
|
||||
For Linux:
|
||||
|
||||
<code>./build.sh --config RelWithDebInfo --use_openvino <hardware_option> </code>
|
||||
|
||||
For Windows:
|
||||
|
||||
<code> build.bat --config RelWithDebInfo --use_openvino <hardware_option> </code>
|
||||
|
||||
*Note: The default Windows CMake Generator is Visual Studio 2017, but you can also use the newer Visual Studio 2019 by passing `--cmake_generator "Visual Studio 16 2019"` to build.bat.*
|
||||
|
||||
<code>--use_openvino</code>: Builds the OpenVINO Execution Provider in ONNX Runtime.
|
||||
|
||||
<code><hardware_option></code>: Specifies the hardware target for building OpenVINO Execution Provider. Below are the options for different Intel target devices.
|
||||
|
|
|
|||
|
|
@ -518,10 +518,6 @@ endif()
|
|||
|
||||
if(onnxruntime_USE_OPENVINO)
|
||||
|
||||
if(WIN32)
|
||||
message(FATAL_ERROR "Building on Windows using OpenVINO execution provider is not supported")
|
||||
endif()
|
||||
|
||||
add_definitions(-DUSE_OPENVINO=1)
|
||||
|
||||
if (onnxruntime_USE_OPENVINO_SOURCE)
|
||||
|
|
|
|||
|
|
@ -374,9 +374,16 @@ if (onnxruntime_USE_OPENVINO_BINARY)
|
|||
message($ENV{INTEL_CVSDK_DIR})
|
||||
set(OPENVINO_INCLUDE_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/include)
|
||||
set(OPENVINO_TBB_INCLUDE_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/tbb/include)
|
||||
if(WIN32)
|
||||
set(OPENVINO_LIB_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/lib/intel64/Release)
|
||||
set(OPENVINO_TBB_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/lib/intel64/Release)
|
||||
set(OPENVINO_MKL_TINY_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/bin/intel64/Release)
|
||||
|
||||
else()
|
||||
set(OPENVINO_LIB_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/lib/intel64/)
|
||||
set(OPENVINO_TBB_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/tbb/lib)
|
||||
set(OPENVINO_MKL_TINY_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/mkltiny_lnx/lib)
|
||||
endif()
|
||||
endif()
|
||||
if ($ENV{INTEL_CVSDK_DIR} MATCHES "2018.5")
|
||||
set(OPENVINO_INCLUDE_DIR $ENV{INTEL_CVSDK_DIR}/deployment_tools/inference_engine/include)
|
||||
|
|
@ -390,12 +397,29 @@ endif()
|
|||
onnxruntime_add_include_to_target(onnxruntime_providers_openvino gsl onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf)
|
||||
add_dependencies(onnxruntime_providers_openvino ${onnxruntime_EXTERNAL_DEPENDENCIES})
|
||||
set_target_properties(onnxruntime_providers_openvino PROPERTIES FOLDER "ONNXRuntime")
|
||||
target_include_directories(onnxruntime_providers_openvino SYSTEM PUBLIC ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${OPENVINO_INCLUDE_DIR} ${OPENVINO_TBB_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS})
|
||||
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/openvino DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
|
||||
set_target_properties(onnxruntime_providers_openvino PROPERTIES LINKER_LANGUAGE CXX)
|
||||
if (WIN32)
|
||||
target_include_directories(onnxruntime_providers_openvino SYSTEM PUBLIC ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${OPENVINO_INCLUDE_DIR} ${OPENVINO_TBB_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS} ${PYTHONPATH})
|
||||
#${pybind11_INCLUDE_DIRS}
|
||||
else()
|
||||
target_include_directories(onnxruntime_providers_openvino SYSTEM PUBLIC ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${OPENVINO_INCLUDE_DIR} ${OPENVINO_TBB_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS})
|
||||
endif()
|
||||
|
||||
if ($ENV{INTEL_CVSDK_DIR} MATCHES "2019.1")
|
||||
link_directories(onnxruntime_providers_openvino ${OPENVINO_LIB_DIR} ${OPENVINO_TBB_DIR} ${OPENVINO_MKL_TINY_DIR})
|
||||
target_link_libraries(onnxruntime_providers_openvino -linference_engine -ltbb ${PYTHON_LIBRARIES})
|
||||
if (WIN32)
|
||||
|
||||
string(REPLACE "include" "libs" PYTHON_LIB ${PYTHON_INCLUDE_DIRS})
|
||||
set(PYTHON_LIBRARIES ${PYTHON_LIB})
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996 /wd4244 /wd4267 /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456 /w")
|
||||
link_directories(onnxruntime_providers_openvino -linference_engine ${PYTHON_LIBRARIES} ${OPENVINO_LIB_DIR} ${OPENVINO_TBB_DIR} ${OPENVINO_MKL_TINY_DIR} ${PYTHONPATH})
|
||||
target_link_libraries(onnxruntime_providers_openvino $ENV{INTEL_CVSDK_DIR}/deployment_tools/inference_engine/lib/intel64/Release/inference_engine.lib)
|
||||
file(COPY ${onnxruntime_providers_openvino_py_srcs} DESTINATION ${onnxruntime_BINARY_DIR}/${CMAKE_BUILD_TYPE})
|
||||
else()
|
||||
link_directories(onnxruntime_providers_openvino ${OPENVINO_LIB_DIR} ${OPENVINO_TBB_DIR} ${OPENVINO_MKL_TINY_DIR})
|
||||
target_link_libraries(onnxruntime_providers_openvino -linference_engine -ltbb ${PYTHON_LIBRARIES})
|
||||
file(COPY ${onnxruntime_providers_openvino_py_srcs} DESTINATION ${onnxruntime_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
if ($ENV{INTEL_CVSDK_DIR} MATCHES "2018.5")
|
||||
link_directories(onnxruntime_providers_openvino ${OPENVINO_LIB_DIR})
|
||||
|
|
|
|||
|
|
@ -11,6 +11,13 @@
|
|||
#include <Python.h>
|
||||
|
||||
#include <inference_engine.hpp>
|
||||
//MSVC does not allow initialization of a typedef
|
||||
|
||||
#ifdef OPTIONAL
|
||||
#undef OPTIONAL
|
||||
|
||||
#endif
|
||||
|
||||
#include <ie_builders.hpp>
|
||||
|
||||
#include "core/graph/graph.h"
|
||||
|
|
@ -99,12 +106,9 @@ OpenVINOGraph::OpenVINOGraph(const onnxruntime::Node* fused_node) {
|
|||
// Create hardware specific OpenVINO network representation
|
||||
GetExecutableHandle(openvino_network_);
|
||||
|
||||
std::vector<std::string> plugin_path = GetEnvLdLibraryPath();
|
||||
plugin_path.push_back("");
|
||||
plugin_ = InferenceEngine::PluginDispatcher(
|
||||
plugin_path)
|
||||
.getPluginByDevice(device_id_);
|
||||
|
||||
|
||||
plugin_ = InferenceEngine::PluginDispatcher().getPluginByDevice(device_id_);
|
||||
//Loading model to the plugin
|
||||
InferenceEngine::ExecutableNetwork exeNetwork = plugin_.LoadNetwork(*openvino_network_, {});
|
||||
|
||||
|
|
@ -140,7 +144,11 @@ void OpenVINOGraph::ConvertONNXModelToOpenVINOIR(const std::string& onnx_model,
|
|||
}
|
||||
|
||||
PyObject* pModule = NULL;
|
||||
pModule = PyImport_ImportModule("openvino_mo");
|
||||
PyObject* pName;
|
||||
pName = PyUnicode_FromString("openvino_mo");
|
||||
|
||||
|
||||
pModule = PyImport_Import(pName);
|
||||
if (pModule == NULL) {
|
||||
throw "Python module import failure";
|
||||
}
|
||||
|
|
@ -326,7 +334,7 @@ size_t OpenVINOGraph::DeduceBatchSize(Ort::CustomOpApi ort, const OrtValue* inpu
|
|||
|
||||
// Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on
|
||||
// an Infer Request indexed by infer_req_idx
|
||||
void OpenVINOGraph::StartAsyncInference(Ort::CustomOpApi ort, const OrtValue* input_tensors[],
|
||||
void OpenVINOGraph::StartAsyncInference(Ort::CustomOpApi ort, std::vector<const OrtValue*> input_tensors,
|
||||
size_t batch_slice_idx,
|
||||
size_t infer_req_idx) {
|
||||
auto infer_request = infer_requests_[infer_req_idx];
|
||||
|
|
@ -354,7 +362,7 @@ void OpenVINOGraph::StartAsyncInference(Ort::CustomOpApi ort, const OrtValue* in
|
|||
|
||||
// Wait for asynchronous inference completion on an Infer Request object indexed by infer_req_idx
|
||||
// and copy the results into a slice location within the batched output buffer indexed by batch_slice_idx
|
||||
void OpenVINOGraph::CompleteAsyncInference(Ort::CustomOpApi ort, OrtValue* output_tensors[],
|
||||
void OpenVINOGraph::CompleteAsyncInference(Ort::CustomOpApi ort, std::vector<OrtValue*> output_tensors,
|
||||
size_t batch_slice_idx,
|
||||
size_t infer_req_idx) {
|
||||
auto infer_request = infer_requests_[infer_req_idx];
|
||||
|
|
@ -380,15 +388,18 @@ void OpenVINOGraph::CompleteAsyncInference(Ort::CustomOpApi ort, OrtValue* outpu
|
|||
}
|
||||
}
|
||||
|
||||
void OpenVINOGraph::GetInputTensors(Ort::CustomOpApi ort, OrtKernelContext* context, const OrtValue* input_tensors[]) {
|
||||
std::vector<const OrtValue*> OpenVINOGraph::GetInputTensors(Ort::CustomOpApi ort, OrtKernelContext* context) {
|
||||
std::vector<const OrtValue*> input_tensors;
|
||||
size_t input_count = openvino_network_->getInputsInfo().size();
|
||||
|
||||
for (size_t i = 0; i < input_count; i++) {
|
||||
input_tensors[i] = ort.KernelContext_GetInput(context, input_indexes_[i]);
|
||||
input_tensors.push_back(ort.KernelContext_GetInput(context, input_indexes_[i]));
|
||||
}
|
||||
return input_tensors;
|
||||
}
|
||||
|
||||
void OpenVINOGraph::GetOutputTensors(Ort::CustomOpApi ort, OrtKernelContext* context, OrtValue* output_tensors[], size_t batch_size) {
|
||||
std::vector<OrtValue*> OpenVINOGraph::GetOutputTensors(Ort::CustomOpApi ort, OrtKernelContext* context, size_t batch_size) {
|
||||
std::vector<OrtValue*> output_tensors;
|
||||
auto graph_output_info = openvino_network_->getOutputsInfo();
|
||||
|
||||
// All infer_requests process identical tensor slices from the batch.
|
||||
|
|
@ -407,13 +418,15 @@ void OpenVINOGraph::GetOutputTensors(Ort::CustomOpApi ort, OrtKernelContext* con
|
|||
}
|
||||
|
||||
size_t num_dims = graph_output_dims.size();
|
||||
int64_t output_shape[num_dims];
|
||||
auto output_shape = new int64_t[num_dims];
|
||||
for (size_t j = 0; j < num_dims; j++) {
|
||||
output_shape[j] = static_cast<int64_t>(graph_output_dims[j]);
|
||||
}
|
||||
|
||||
output_tensors[i] = ort.KernelContext_GetOutput(context, i, output_shape, num_dims);
|
||||
output_tensors.push_back(ort.KernelContext_GetOutput(context, i, output_shape, num_dims));
|
||||
delete output_shape;
|
||||
}
|
||||
return output_tensors;
|
||||
}
|
||||
|
||||
void OpenVINOGraph::Infer(Ort::CustomOpApi ort, OrtKernelContext* context) {
|
||||
|
|
@ -423,13 +436,7 @@ void OpenVINOGraph::Infer(Ort::CustomOpApi ort, OrtKernelContext* context) {
|
|||
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Starting inference";
|
||||
|
||||
// Get Input and Output tensors
|
||||
size_t input_count = openvino_network_->getInputsInfo().size();
|
||||
size_t output_count = openvino_network_->getOutputsInfo().size();
|
||||
const OrtValue* input_tensors[input_count];
|
||||
OrtValue* output_tensors[output_count];
|
||||
|
||||
GetInputTensors(ort, context, input_tensors);
|
||||
auto input_tensors = GetInputTensors(ort, context);
|
||||
|
||||
// Calculate the batch_size from the input tensor shape.
|
||||
auto batch_size = DeduceBatchSize(ort, input_tensors[0],
|
||||
|
|
@ -438,7 +445,7 @@ void OpenVINOGraph::Infer(Ort::CustomOpApi ort, OrtKernelContext* context) {
|
|||
size_t full_parallel_runs = batch_size / num_inf_reqs_;
|
||||
size_t remainder_parallel_runs = batch_size % num_inf_reqs_;
|
||||
|
||||
GetOutputTensors(ort, context, output_tensors, batch_size);
|
||||
auto output_tensors = GetOutputTensors(ort, context, batch_size);
|
||||
|
||||
// Distribute the batched inputs among available Infer Requests
|
||||
// for parallel inference.
|
||||
|
|
|
|||
|
|
@ -40,13 +40,13 @@ class OpenVINOGraph {
|
|||
size_t DeduceBatchSize(Ort::CustomOpApi ort, const OrtValue* input_tensor,
|
||||
InferenceEngine::SizeVector graph_dims);
|
||||
|
||||
void GetInputTensors(Ort::CustomOpApi ort, OrtKernelContext* context, const OrtValue* input_tensors[]);
|
||||
std::vector<const OrtValue*> GetInputTensors(Ort::CustomOpApi ort, OrtKernelContext* context);
|
||||
|
||||
void GetOutputTensors(Ort::CustomOpApi ort, OrtKernelContext* context, OrtValue* output_tensors[], size_t batch_size);
|
||||
std::vector<OrtValue*> GetOutputTensors(Ort::CustomOpApi ort, OrtKernelContext* context, size_t batch_size);
|
||||
|
||||
void StartAsyncInference(Ort::CustomOpApi ort, const OrtValue* input_tensors[], size_t batch_slice_idx, size_t infer_req_idx);
|
||||
void StartAsyncInference(Ort::CustomOpApi ort, std::vector<const OrtValue*> input_tensors, size_t batch_slice_idx, size_t infer_req_idx);
|
||||
|
||||
void CompleteAsyncInference(Ort::CustomOpApi ort, OrtValue* output_tensors[], size_t batch_slice_idx, size_t infer_req_idx);
|
||||
void CompleteAsyncInference(Ort::CustomOpApi ort, std::vector<OrtValue*> output_tensors, size_t batch_slice_idx, size_t infer_req_idx);
|
||||
|
||||
std::vector<std::string> GetEnvLdLibraryPath() const;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue