From e617f97f4e9b916e317e41ecc79022a652daf852 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Mon, 11 Oct 2021 11:31:25 -0700 Subject: [PATCH] Update mobile EP (NNAPI/CoreML) op coverage (#9308) * Add op supported * Update the build and create session * minor update * Add new NNNAPI flag doc * Address CR comments * Correct strings in the sample code * Update docs/tutorials/mobile/model-execution.md Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com> Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com> --- docs/build/android-ios.md | 4 -- .../CoreML-ExecutionProvider.md | 33 ++++++++-- .../NNAPI-ExecutionProvider.md | 65 +++++++++++++++++-- docs/tutorials/mobile/model-execution.md | 41 ++++++++++-- 4 files changed, 126 insertions(+), 17 deletions(-) diff --git a/docs/build/android-ios.md b/docs/build/android-ios.md index 5ec26070e9..f236acbbd6 100644 --- a/docs/build/android-ios.md +++ b/docs/build/android-ios.md @@ -161,10 +161,6 @@ Android NNAPI Execution Provider can be built using building commands in [Androi If the code signing development team ID or code signing identity is specified, and has a valid code signing certificate, Xcode will code sign the onnxruntime library in the building process. Otherwise, the onnxruntime will be built without code signing. It may be required or desired to code sign the library for iOS devices. For more information, see [Code Signing](https://developer.apple.com/support/code-signing/). -* Bitcode - - Bitcode is an Apple technology that enables you to recompile your app to reduce its size. It is by default enabled for building onnxruntime. Bitcode can be disabled by using the building commands in [iOS Build instructions](#build-instructions-1) with `--apple_disable_bitcode`. For more information about bitcode, please see [Doing Basic Optimization to Reduce Your App’s Size](https://developer.apple.com/documentation/xcode/doing-basic-optimization-to-reduce-your-app-s-size). - ### Build Instructions Run one of the following build scripts from the ONNX Runtime repository root: diff --git a/docs/execution-providers/CoreML-ExecutionProvider.md b/docs/execution-providers/CoreML-ExecutionProvider.md index 78487a7339..3c24f666f1 100644 --- a/docs/execution-providers/CoreML-ExecutionProvider.md +++ b/docs/execution-providers/CoreML-ExecutionProvider.md @@ -17,7 +17,7 @@ nav_order: 2 ## Requirements -The CoreML Execution Provider (EP) requires iOS devices with iOS 13 or higher, or Mac computers with macOS 10.15 or higher. +The CoreML Execution Provider (EP) requires iOS devices with iOS 13 or higher, or Mac computers with macOS 10.15 or higher. It is recommended to use Apple devices equipped with Apple Neural Engine to achieve optimal performance. @@ -33,7 +33,7 @@ For build instructions for iOS devices, please see [How to: Build for Android/iO ## Usage -The ONNX Runtime API details are [here](../api). +The ONNX Runtime API details are [here](../api). The CoreML EP can be used via the C or C++ APIs currently. Additional support via the Objective-C API is in progress. @@ -75,8 +75,33 @@ Enable CoreML EP to run on a subgraph in the body of a control flow operator (i. By default the CoreML EP will be enabled for all compatible Apple devices. -Setting this option will only enable CoreML EP for Apple devices with a compatible Apple Neural Engine (ANE). -Note, enabling this option does not guarantee the entire model to be executed using ANE only. +Setting this option will only enable CoreML EP for Apple devices with a compatible Apple Neural Engine (ANE). +Note, enabling this option does not guarantee the entire model to be executed using ANE only. For more information, see [Which devices have an ANE?](https://github.com/hollance/neural-engine/blob/master/docs/supported-devices.md) +## Supported ops +Following ops are supported by the CoreML Execution Provider, + +|Operator|Note| +|--------|------| +|ai.onnx:Add|| +|ai.onnx:ArgMax|| +|ai.onnx:AveragePool|Only 2D Pool is supported.| +|ai.onnx:BatchNormalization|| +|ai.onnx:Cast|| +|ai.onnx:Clip|| +|ai.onnx:Concat|| +|ai.onnx:Conv|Only 1D/2D Conv is supported.
Weights and bias should be constant.| +|ai.onnx:Gemm|Input B should be constant.| +|ai.onnx:GlobalAveragePool|Only 2D Pool is supported.| +|ai.onnx:GlobalMaxPool|Only 2D Pool is supported.| +|ai.onnx:MatMul|Input B should be constant.| +|ai.onnx:MaxPool|Only 2D Pool is supported.| +|ai.onnx:Relu|| +|ai.onnx:Reshape|| +|ai.onnx:Resize|| +|ai.onnx:Sigmoid|| +|ai.onnx:Squeeze|| +|ai.onnx:Tanh|| +|ai.onnx:Transpose|| diff --git a/docs/execution-providers/NNAPI-ExecutionProvider.md b/docs/execution-providers/NNAPI-ExecutionProvider.md index f15343ff32..6a307c75c1 100644 --- a/docs/execution-providers/NNAPI-ExecutionProvider.md +++ b/docs/execution-providers/NNAPI-ExecutionProvider.md @@ -28,7 +28,7 @@ Please see the [ONNX Runtime Mobile](../tutorials/mobile/) deployment informatio ## Usage -The ONNX Runtime API details are [here](../api). +The ONNX Runtime API details are [here](../api). The NNAPI EP can be used via the C, C++ or Java APIs @@ -56,15 +56,15 @@ nnapi_flags |= NNAPI_FLAG_USE_FP16; ### Available Options ##### NNAPI_FLAG_USE_FP16 -Use fp16 relaxation in NNAPI EP. +Use fp16 relaxation in NNAPI EP. This may improve performance but can also reduce accuracy due to the lower precision. ##### NNAPI_FLAG_USE_NCHW -Use the NCHW layout in NNAPI EP. +Use the NCHW layout in NNAPI EP. -This is only available for Android API level 29 and later. Please note that for now, NNAPI might have worse performance using NCHW compared to using NHWC. +This is only available for Android API level 29 and higher. Please note that for now, NNAPI might have worse performance using NCHW compared to using NHWC. ##### NNAPI_FLAG_CPU_DISABLED @@ -74,11 +74,66 @@ NNAPI is more efficient using GPU or NPU for execution, however NNAPI might fall For some models, if NNAPI would use CPU to execute an operation, and this flag is set, the execution of the model may fall back to ORT kernels. -This option is only available after Android API level 29, and will be ignored for Android API level 28 and lower. +This option is only available for Android API level 29 and higher, and will be ignored for Android API level 28 and lower. For NNAPI device assignments, see [https://developer.android.com/ndk/guides/neuralnetworks#device-assignment](https://developer.android.com/ndk/guides/neuralnetworks#device-assignment) For NNAPI CPU fallback, see [https://developer.android.com/ndk/guides/neuralnetworks#cpu-fallback](https://developer.android.com/ndk/guides/neuralnetworks#cpu-fallback) +##### NNAPI_FLAG_CPU_ONLY + +Using CPU only in NNAPI EP, this may decrease the perf but will provide reference output value without precision loss, which is useful for validation. + +This option is only available for Android API level 29 and higher, and will be ignored for Android API level 28 and lower. + +## Supported ops +Following ops are supported by the NNAPI Execution Provider, + +|Operator|Note| +|--------|------| +|ai.onnx:Abs|| +|ai.onnx:Add|| +|ai.onnx:AveragePool|Only 2D Pool is supported.| +|ai.onnx:Cast|| +|ai.onnx:Clip|| +|ai.onnx:Concat|| +|ai.onnx:Conv|Only 2D Conv is supported.
Weights and bias should be constant.| +|ai.onnx:DequantizeLinear|All quantization scales and zero points should be constant.| +|ai.onnx:Div|| +|ai.onnx:Elu|| +|ai.onnx:Exp|| +|ai.onnx:Flatten|| +|ai.onnx:Floor|| +|ai.onnx:Gemm|If input B is not constant, transB should be 1.| +|ai.onnx:GlobalAveragePool|Only 2D Pool is supported.| +|ai.onnx:GlobalMaxPool|Only 2D Pool is supported.| +|ai.onnx:Identity|| +|ai.onnx:Log|| +|ai.onnx:MatMul|| +|ai.onnx:MaxPool|Only 2D Pool is supported.| +|ai.onnx:Max|| +|ai.onnx:Min|| +|ai.onnx:Mul|| +|ai.onnx:Neg|| +|ai.onnx:Pow|| +|ai.onnx:QLinearConv|Only 2D Conv is supported.
Weights and bias should be constant.
All quantization scales and zero points should be constant.| +|ai.onnx:QLinearMatMul|All quantization scales and zero points should be constant.| +|ai.onnx:QuantizeLinear|All quantization scales and zero points should be constant.| +|ai.onnx:Relu|| +|ai.onnx:Reshape|| +|ai.onnx:Resize|Only 2D Resize is supported.| +|ai.onnx:Sigmoid|| +|ai.onnx:Sin|| +|ai.onnx:Slice|| +|ai.onnx:Softmax|| +|ai.onnx:Sqrt|| +|ai.onnx:Squeeze|| +|ai.onnx:Sub|| +|ai.onnx:Tanh|| +|ai.onnx:Transpose|| +|com.microsoft:QLinearAdd|All quantization scales and zero points should be constant.| +|com.microsoft:QLinearAveragePool|Only 2D Pool is supported.
All quantization scales and zero points should be constant.| +|com.microsoft:QLinearSigmoid|All quantization scales and zero points should be constant.| + diff --git a/docs/tutorials/mobile/model-execution.md b/docs/tutorials/mobile/model-execution.md index 43f0a7f354..cbcdf8cad2 100644 --- a/docs/tutorials/mobile/model-execution.md +++ b/docs/tutorials/mobile/model-execution.md @@ -9,7 +9,7 @@ nav_order: 5 # Executing an ORT format model -The API for executing ORT format models is the same as for ONNX models. +The API for executing ORT format models is the same as for ONNX models. See the [ONNX Runtime API documentation](../../api) for details on individual API usage. @@ -18,7 +18,7 @@ See the [ONNX Runtime API documentation](../../api) for details on individual AP | Platform | Available APIs | |----------|----------------| -| Android | C, C++, Java | +| Android | C, C++, Java, Kotlin | | iOS | C, C++, Objective-C (Swift via bridge) | ## ORT format model loading @@ -29,10 +29,12 @@ If you provide in-memory bytes for the ORT format model, a marker in those bytes If you wish to explicitly say that the InferenceSession input is an ORT format model you can do so via SessionOptions, although this generally should not be necessary. +### Load ORT format model from a file path + C++ API ```c++ Ort::SessionOptions session_options; -session_options.AddConfigEntry('session.load_model_format', 'ORT'); +session_options.AddConfigEntry("session.load_model_format", "ORT"); Ort::Env env; Ort::Session session(env, , session_options); @@ -44,7 +46,38 @@ SessionOptions session_options = new SessionOptions(); session_options.addConfigEntry("session.load_model_format", "ORT"); OrtEnvironment env = OrtEnvironment.getEnvironment(); -OrtSession session = env.createSession(, opsession_optionstions); +OrtSession session = env.createSession(, session_options); +``` + +### Load ORT format model from an in-memory byte array + +If a session is created using an input byte array containing the ORT format model data, by default we will copy the model bytes at the time of session creation to ensure the model bytes buffer is valid. + +You may also enable the option to use the model bytes directly by setting the Session Options config `session.use_ort_model_bytes_directly` to `1`, this may reduce the peak memory usage of ONNX Runtime Mobile, you will need to guarantee that the model bytes are valid throughout the lifespan of the ORT session using the model bytes. + +C++ API +```c++ +Ort::SessionOptions session_options; +session_options.AddConfigEntry("session.load_model_format", "ORT"); +session_options.AddConfigEntry("session.use_ort_model_bytes_directly", "1"); + +std::ifstream stream(, std::ios::in | std::ios::binary); +std::vector model_bytes((std::istreambuf_iterator(stream)), std::istreambuf_iterator()); + +Ort::Env env; +Ort::Session session(env, model_bytes.data(), model_bytes.size(), session_options); +``` + +Java API +```java +SessionOptions session_options = new SessionOptions(); +session_options.addConfigEntry("session.load_model_format", "ORT"); +session_options.addConfigEntry("session.use_ort_model_bytes_directly", "1"); + +byte[] model_bytes = Files.readAllBytes(Paths.get()); + +OrtEnvironment env = OrtEnvironment.getEnvironment(); +OrtSession session = env.createSession(model_bytes, session_options); ``` ------