mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-14 20:48:00 +00:00
Updating EP docs with Onnxruntime API calls (#5503)
* updating examples with current api calls * Fixing capitalization in api calls, adding RKNPU update * Correcting nuphar and rknpu ep api calls * Include creating session in readme
This commit is contained in:
Olivia Jain
GitHub
parent
0b59004666
commit
1e4b259d28
10 changed files with 40 additions and 96 deletions
|
|
@ -7,19 +7,11 @@ For build instructions, please see the [BUILD page](../../BUILD.md#ARM-Compute-L
|
|||
|
||||
### Using the ACL execution provider
|
||||
#### C/C++
|
||||
To use ACL as execution provider for inferencing, please register it as below.
|
||||
```
|
||||
string log_id = "Foo";
|
||||
auto logging_manager = std::make_unique<LoggingManager>
|
||||
(std::unique_ptr<ISink>{new CLogSink{}},
|
||||
static_cast<Severity>(lm_info.default_warning_level),
|
||||
false,
|
||||
LoggingManager::InstanceType::Default,
|
||||
&log_id)
|
||||
Environment::Create(std::move(logging_manager), env)
|
||||
InferenceSession session_object{so, env};
|
||||
session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::ACLExecutionProvider>());
|
||||
status = session_object.Load(model_file_name);
|
||||
Ort::Env env = Ort::Env{ORT_LOGGING_LEVEL_ERROR, "Default"};
|
||||
Ort::SessionOptions sf;
|
||||
bool enable_cpu_mem_arena = true;
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_ACL(sf, enable_cpu_mem_arena));
|
||||
```
|
||||
The C API details are [here](../C_API.md#c-api).
|
||||
|
||||
|
|
|
|||
|
|
@ -9,17 +9,10 @@ For build instructions, please see the [BUILD page](../../BUILD.md#ArmNN).
|
|||
#### C/C++
|
||||
To use ArmNN as execution provider for inferencing, please register it as below.
|
||||
```
|
||||
string log_id = "Foo";
|
||||
auto logging_manager = std::make_unique<LoggingManager>
|
||||
(std::unique_ptr<ISink>{new CLogSink{}},
|
||||
static_cast<Severity>(lm_info.default_warning_level),
|
||||
false,
|
||||
LoggingManager::InstanceType::Default,
|
||||
&log_id)
|
||||
Environment::Create(std::move(logging_manager), env)
|
||||
InferenceSession session_object{so, env};
|
||||
session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::ArmNNExecutionProvider>());
|
||||
status = session_object.Load(model_file_name);
|
||||
Ort::Env env = Ort::Env{ORT_LOGGING_LEVEL_ERROR, "Default"};
|
||||
Ort::SessionOptions sf;
|
||||
bool enable_cpu_mem_arena = true;
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_ArmNN(sf, enable_cpu_mem_arena));
|
||||
```
|
||||
The C API details are [here](../C_API.md#c-api).
|
||||
|
||||
|
|
|
|||
|
|
@ -21,17 +21,10 @@ For build instructions, please see the [BUILD page](../../BUILD.md#dnnl-and-mklm
|
|||
### C/C++
|
||||
The DNNLExecutionProvider execution provider needs to be registered with ONNX Runtime to enable in the inference session.
|
||||
```
|
||||
string log_id = "Foo";
|
||||
auto logging_manager = std::make_unique<LoggingManager>
|
||||
(std::unique_ptr<ISink>{new CLogSink{}},
|
||||
static_cast<Severity>(lm_info.default_warning_level),
|
||||
false,
|
||||
LoggingManager::InstanceType::Default,
|
||||
&log_id)
|
||||
Environment::Create(std::move(logging_manager), env)
|
||||
InferenceSession session_object{so,env};
|
||||
session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime:: DNNLExecutionProvider >());
|
||||
status = session_object.Load(model_file_name);
|
||||
Ort::Env env = Ort::Env{ORT_LOGGING_LEVEL_ERROR, "Default"};
|
||||
Ort::SessionOptions sf;
|
||||
bool enable_cpu_mem_arena = true;
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Dnnl(sf, enable_cpu_mem_arena));
|
||||
```
|
||||
The C API details are [here](../C_API.md#c-api).
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
DirectML is a high-performance, hardware-accelerated DirectX 12 library for machine learning on Windows. DirectML provides GPU acceleration for common machine learning tasks across a broad range of supported hardware and drivers.
|
||||
|
||||
When used standalone, the DirectML API is a low-level DirectX 12 library and is suitable for high-performance, low-latency applications such as frameworks, games, and other real-time applications. The seamless interoperability of DirectML with Direct3D 12 as well as its low overhead and conformance across hardware makes DirectML ideal for accelerating machine learning when both high performance is desired, and the reliability and predictabiltiy of results across hardware is critical.
|
||||
When used standalone, the DirectML API is a low-level DirectX 12 library and is suitable for high-performance, low-latency applications such as frameworks, games, and other real-time applications. The seamless interoperability of DirectML with Direct3D 12 as well as its low overhead and conformance across hardware makes DirectML ideal for accelerating machine learning when both high performance is desired, and the reliability and predictability of results across hardware is critical.
|
||||
|
||||
The *DirectML Execution Provider* is an optional component of ONNX Runtime that uses DirectML to accelerate inference of ONNX models. The DirectML execution provider is capable of greatly improving evaluation time of models using commodity GPU hardware, without sacrificing broad hardware support or requiring vendor-specific extensions to be installed.
|
||||
|
||||
|
|
|
|||
|
|
@ -7,19 +7,11 @@ For build instructions, please see the [BUILD page](../../BUILD.md#AMD-MIGraphX)
|
|||
|
||||
## Using the MIGraphX execution provider
|
||||
### C/C++
|
||||
The MIGraphX execution provider needs to be registered with ONNX Runtime to enable in the inference session.
|
||||
```
|
||||
string log_id = "Foo";
|
||||
auto logging_manager = std::make_unique<LoggingManager>
|
||||
(std::unique_ptr<ISink>{new CLogSink{}},
|
||||
static_cast<Severity>(lm_info.default_warning_level),
|
||||
false,
|
||||
LoggingManager::InstanceType::Default,
|
||||
&log_id)
|
||||
Environment::Create(std::move(logging_manager), env)
|
||||
InferenceSession session_object{so,env};
|
||||
session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::MIGraphXExecutionProvider>());
|
||||
status = session_object.Load(model_file_name);
|
||||
Ort::Env env = Ort::Env{ORT_LOGGING_LEVEL_ERROR, "Default"};
|
||||
Ort::SessionOptions sf;
|
||||
int device_id = 0;
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_MiGraphX(sf, device_id));
|
||||
```
|
||||
You can check [here](https://github.com/scxiao/ort_test/tree/master/char_rnn) for a specific c/c++ program.
|
||||
|
||||
|
|
|
|||
|
|
@ -12,18 +12,10 @@ For build instructions, please see the [BUILD page](../../BUILD.md#Android-NNAPI
|
|||
|
||||
## Using NNAPI EP in C/C++
|
||||
|
||||
To use NNAPI EP for inferencing, please register it as below.
|
||||
```
|
||||
string log_id = "Foo";
|
||||
auto logging_manager = std::make_unique<LoggingManager>
|
||||
(std::unique_ptr<ISink>{new CLogSink{}},
|
||||
static_cast<Severity>(lm_info.default_warning_level),
|
||||
false,
|
||||
LoggingManager::InstanceType::Default,
|
||||
&log_id)
|
||||
Environment::Create(std::move(logging_manager), env)
|
||||
InferenceSession session_object{so,env};
|
||||
session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::NnapiExecutionProvider>());
|
||||
status = session_object.Load(model_file_name);
|
||||
Ort::Env env = Ort::Env{ORT_LOGGING_LEVEL_ERROR, "Default"};
|
||||
Ort::SessionOptions sf;
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Nnapi(sf));
|
||||
Ort::Session session(env, model_path, sf);
|
||||
```
|
||||
The C API details are [here](../C_API.md#c-api).
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Nuphar Execution Provider (preview)
|
||||
|
||||
NUPHAR stands for Neural-network Unified Preprocessing Heterogeneous ARchitecture. As an execution provider in the ONNX Runtime, it is built on top of [TVM](https://github.com/dmlc/tvm) and [LLVM](https://llvm.org) to accelerate ONNX models by compiling nodes in subgraphs into optimized functions via JIT. It also provides JIT caching to save compilation time at runtime.
|
||||
NUPHAR stands for Neural-network Unified Preprocessing Heterogeneous Architecture. As an execution provider in the ONNX Runtime, it is built on top of [TVM](https://github.com/dmlc/tvm) and [LLVM](https://llvm.org) to accelerate ONNX models by compiling nodes in subgraphs into optimized functions via JIT. It also provides JIT caching to save compilation time at runtime.
|
||||
|
||||
Developers can tap into the power of Nuphar through ONNX Runtime to accelerate inferencing of ONNX models. The Nuphar execution provider comes with a common ONNX to TVM lowering [library](../../onnxruntime/core/codegen) that can potentially be reused by other execution providers to leverage TVM. With the Nuphar execution provider, the ONNX Runtime delivers better inferencing performance on the same hardware compared to generic X64 CPU acceleration, especially for quantized recurrent neural networks. Various products at Microsoft have seen up to a 5x improvement in performance with no loss of accuracy, by running quantized LSTMs via the Nuphar execution provider in the ONNX Runtime.
|
||||
|
||||
|
|
@ -9,7 +9,10 @@ For build instructions, please see the [BUILD page](../../BUILD.md#nuphar).
|
|||
|
||||
## Using the Nuphar execution provider
|
||||
### C/C++
|
||||
The Nuphar execution provider needs to be registered with ONNX Runtime to enable in the inference session. The C API details are [here](../C_API.md#c-api).
|
||||
Ort::Env env = Ort::Env{ORT_LOGGING_LEVEL_ERROR, "Default"};
|
||||
Ort::SessionOptions sf;
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Nuphar(sf, /*allow_unaligned_buffers*/ 1, ""));
|
||||
Ort::Session session(env, model_path, sf);
|
||||
|
||||
### Python
|
||||
You can use the Nuphar execution provider via the python wheel from the ONNX Runtime build. The Nuphar execution provider will be automatically prioritized over the default CPU execution providers, thus no need to separately register the execution provider. Python APIs details are [here](../python/api_summary.rst#api-summary).
|
||||
|
|
|
|||
|
|
@ -15,17 +15,10 @@ For build instructions, please see the [BUILD page](../../BUILD.md#RKNPU).
|
|||
### C/C++
|
||||
To use RKNPU as execution provider for inferencing, please register it as below.
|
||||
```
|
||||
string log_id = "Foo";
|
||||
auto logging_manager = std::make_unique<LoggingManager>
|
||||
(std::unique_ptr<ISink>{new CLogSink{}},
|
||||
static_cast<Severity>(lm_info.default_warning_level),
|
||||
false,
|
||||
LoggingManager::InstanceType::Default,
|
||||
&log_id)
|
||||
Environment::Create(std::move(logging_manager), env)
|
||||
InferenceSession session_object{so,env};
|
||||
session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::RknpuExecutionProvider>());
|
||||
status = session_object.Load(model_file_name);
|
||||
Ort::Env env = Ort::Env{ORT_LOGGING_LEVEL_ERROR, "Default"};
|
||||
Ort::SessionOptions sf;
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_RKNPU(sf));
|
||||
Ort::Session session(env, model_path, sf);
|
||||
```
|
||||
The C API details are [here](../C_API.md#c-api).
|
||||
|
||||
|
|
|
|||
|
|
@ -11,19 +11,13 @@ The TensorRT execution provider for ONNX Runtime is built and tested with Tensor
|
|||
|
||||
## Using the TensorRT execution provider
|
||||
### C/C++
|
||||
The TensorRT execution provider needs to be registered with ONNX Runtime to enable in the inference session.
|
||||
```
|
||||
string log_id = "Foo";
|
||||
auto logging_manager = std::make_unique<LoggingManager>
|
||||
(std::unique_ptr<ISink>{new CLogSink{}},
|
||||
static_cast<Severity>(lm_info.default_warning_level),
|
||||
false,
|
||||
LoggingManager::InstanceType::Default,
|
||||
&log_id)
|
||||
Environment::Create(std::move(logging_manager), env)
|
||||
InferenceSession session_object{so,env};
|
||||
session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::TensorrtExecutionProvider>());
|
||||
status = session_object.Load(model_file_name);
|
||||
Ort::Env env = Ort::Env{ORT_LOGGING_LEVEL_ERROR, "Default"};
|
||||
Ort::SessionOptions sf;
|
||||
int device_id = 0;
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Tensorrt(sf, device_id));
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, device_id));
|
||||
Ort::Session session(env, model_path, sf);
|
||||
```
|
||||
The C API details are [here](../C_API.md#c-api).
|
||||
|
||||
|
|
|
|||
|
|
@ -32,19 +32,11 @@ While the nGraph Compiler stack supports various operating systems and backends
|
|||
|
||||
## Using the nGraph execution provider
|
||||
### C/C++
|
||||
To use nGraph as execution provider for inferencing, please register it as below.
|
||||
```
|
||||
string log_id = "Foo";
|
||||
auto logging_manager = std::make_unique<LoggingManager>
|
||||
(std::unique_ptr<ISink>{new CLogSink{}},
|
||||
static_cast<Severity>(lm_info.default_warning_level),
|
||||
false,
|
||||
LoggingManager::InstanceType::Default,
|
||||
&log_id)
|
||||
Environment::Create(std::move(logging_manager), env)
|
||||
InferenceSession session_object{so,env};
|
||||
session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::NGRAPHExecutionProvider>());
|
||||
status = session_object.Load(model_file_name);
|
||||
Ort::Env env = Ort::Env{ORT_LOGGING_LEVEL_ERROR, "Default"};
|
||||
Ort::SessionOptions sf;
|
||||
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Ngraph(sf, "cpu"));
|
||||
Ort::Session session(env, model_path, sf);
|
||||
```
|
||||
The C API details are [here](../C_API.md#c-api).
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue