From 7fd952393473c530b0f07f7af6dd35b2525cde88 Mon Sep 17 00:00:00 2001 From: FFrog <85333600+FFFrog@users.noreply.github.com> Date: Wed, 19 Oct 2022 01:18:09 +0800 Subject: [PATCH] add doc about CANN EP (#13239) #### Description This PR adds docs of Ascend CANN excution provider. #### Changes - Preview Github page: [https://fffrog.github.io/](https://fffrog.github.io/) - Add onnxruntime build with CANN: [https://fffrog.github.io/docs/build/eps.html#cann](https://fffrog.github.io/docs/build/eps.html#cann) - Add CANN ExecutionProvider Page: [https://fffrog.github.io/docs/execution-providers/CANN-ExecutionProvider.html](https://fffrog.github.io/docs/execution-providers/CANN-ExecutionProvider.html) --- docs/build/eps.md | 37 +++++- .../CANN-ExecutionProvider.md | 119 ++++++++++++++++++ .../CUDA-ExecutionProvider.md | 2 +- .../CoreML-ExecutionProvider.md | 2 +- .../DirectML-ExecutionProvider.md | 2 +- .../MIGraphX-ExecutionProvider.md | 4 +- .../NNAPI-ExecutionProvider.md | 2 +- .../OpenVINO-ExecutionProvider.md | 2 +- .../RKNPU-ExecutionProvider.md | 4 +- .../ROCm-ExecutionProvider.md | 2 +- .../SNPE-ExecutionProvider.md | 2 +- .../TVM-ExecutionProvider.md | 4 +- .../TensorRT-ExecutionProvider.md | 2 +- .../Vitis-AI-ExecutionProvider.md | 2 +- .../Xnnpack-ExecutionProvider.md | 2 +- .../add-execution-provider.md | 2 +- docs/execution-providers/index.md | 2 +- .../oneDNN-ExecutionProvider.md | 4 +- index.html | 2 + js/script.js | 18 +++ 20 files changed, 194 insertions(+), 22 deletions(-) create mode 100644 docs/execution-providers/CANN-ExecutionProvider.md diff --git a/docs/build/eps.md b/docs/build/eps.md index 040d24d7dc..ab4d464db0 100644 --- a/docs/build/eps.md +++ b/docs/build/eps.md @@ -17,7 +17,7 @@ redirect_from: /docs/how-to/build/eps ## Execution Provider Shared Libraries -The oneDNN, TensorRT, and OpenVINO™ providers are built as shared libraries vs being statically linked into the main onnxruntime. This enables them to be loaded only when needed, and if the dependent libraries of the provider are not installed onnxruntime will still run fine, it just will not be able to use that provider. For non shared library providers, all dependencies of the provider must exist to load onnxruntime. +The oneDNN, TensorRT, OpenVINO™, and CANN providers are built as shared libraries vs being statically linked into the main onnxruntime. This enables them to be loaded only when needed, and if the dependent libraries of the provider are not installed onnxruntime will still run fine, it just will not be able to use that provider. For non shared library providers, all dependencies of the provider must exist to load onnxruntime. ### Built files {: .no_toc } @@ -768,4 +768,37 @@ Linux example: ### Build for Linux ```bash ./build.sh --config --use_xnnpack -``` \ No newline at end of file +``` + +--- + +## CANN + +See more information on the CANN Execution Provider [here](../execution-providers/CANN-ExecutionProvider.md). + +### Prerequisites +{: .no_toc } + +1. Install the CANN Toolkit for the appropriate OS and target hardware by following [documentation](https://www.hiascend.com/document/detail/en/CANNCommunityEdition/51RC1alphaX/softwareinstall/instg/atlasdeploy_03_0017.html) for detailed instructions, please. + +2. Initialize the CANN environment by running the script as shown below. + + ```bash + # Default path, change it if needed. + source /usr/local/Ascend/ascend-toolkit/set_env.sh + ``` + +### Build Instructions +{: .no_toc } + +#### Linux + +```bash +./build.sh --config RelWithDebInfo --build_shared_lib --parallel --use_cann --build_wheel +``` + +### Notes +{: .no_toc } + +* The CANN execution provider supports building for both x64 and aarch64 architectures. +* CANN excution provider now is only supported on Linux. diff --git a/docs/execution-providers/CANN-ExecutionProvider.md b/docs/execution-providers/CANN-ExecutionProvider.md new file mode 100644 index 0000000000..dc648cd5d1 --- /dev/null +++ b/docs/execution-providers/CANN-ExecutionProvider.md @@ -0,0 +1,119 @@ +--- +title: CANN (Huawei) +description: Instructions to execute ONNX Runtime with the Huawei CANN execution provider +parent: Execution Providers +nav_order: 3 +redirect_from: /docs/reference/execution-providers/CANN-ExecutionProvider +--- + +# CANN Execution Provider +{: .no_toc } + +Huawei Compute Architecture for Neural Networks (CANN) is a heterogeneous computing architecture for AI scenarios and provides multi-layer programming interfaces to help users quickly build AI applications and services based on the Ascend platform. + +Using CANN Excution Provider for ONNX Runtime can help you accelerate ONNX models on Huawei Ascend hardware. + +The CANN Execution Provider (EP) for ONNX Runtime is developed by Huawei. + +## Contents +{: .no_toc } + +* TOC placeholder +{:toc} + +## Requirements + +Please reference table below for official CANN packages dependencies for the ONNX Runtime inferencing package. + +|ONNX Runtime|CANN|OS| +|---|---|---|---| +|v1.12.1|6.0|Ubuntu 18.04
Ubuntu 20.04
CentOS 7.8| +|v1.13.1|6.0|Ubuntu 18.04
Ubuntu 20.04
CentOS 7.8| + +## Build + +For build instructions, please see the [BUILD page](../build/eps.md#cann). + +## Install + +Pre-built binaries of ONNX Runtime with CANN EP are published for most language bindings. Please reference [Install ORT](../install). + +## Samples + +Currently, users can use C/C++ and Python API on CANN EP. + +### C/C++ + +```c +const static OrtApi *g_ort = OrtGetApiBase()->GetApi(ORT_API_VERSION); + +OrtSessionOptions *session_options; +g_ort->CreateSessionOptions(&session_options); + +OrtCANNProviderOptions *cann_options = nullptr; +g_ort->CreateCANNProviderOptions(&cann_options); + +std::vector keys{"device_id", "max_opqueue_num", "npu_mem_limit", "arena_extend_strategy", "do_copy_in_default_stream"}; +std::vector values{"1", "10000", "2147483648", "kSameAsRequested", "1"}; + +g_ort->UpdateCANNProviderOptions(cann_options, keys.data(), values.data(), keys.size()); + +g_ort->SessionOptionsAppendExecutionProvider_CANN(session_options, cann_options); + +// Finally, don't forget to release the provider options and session options +g_ort->ReleaseSessionOptions(session_options); +g_ort->ReleaseCANNProviderOptions(cann_options); +``` + +### Python + +```python +import onnxruntime as ort + +model_path = '' + +options = ort.SessionOptions() + +providers = [ + ('CANNExecutionProvider', { + 'device_id': 0, + 'max_opqueue_num': 10000, + 'arena_extend_strategy': 'kNextPowerOfTwo', + 'npu_mem_limit': 2 * 1024 * 1024 * 1024, + 'do_copy_in_default_stream': True, + }), + 'CPUExecutionProvider', +] + +session = ort.InferenceSession(model_path, sess_options=options, providers=providers) +``` + +## Supported ops + +Following ops are supported by the CANN Execution Provider, + +|Operator|Note| +|--------|------| +|ai.onnx:Add|| +|ai.onnx:AveragePool|Only 2D Pool is supported.| +|ai.onnx:BatchNormalization|| +|ai.onnx:Conv|Only 1D/2D Conv is supported.
Weights and bias should be constant.| +|ai.onnx:Div|| +|ai.onnx:Dropout|| +|ai.onnx:Flatten|| +|ai.onnx:Gemm|Input B should be constant.| +|ai.onnx:GlobalAveragePool|Only 2D Pool is supported.| +|ai.onnx:GlobalMaxPool|Only 2D Pool is supported.| +|ai.onnx:Identity|| +|ai.onnx:MatMul|Input B should be constant.| +|ai.onnx:MaxPool|Only 2D Pool is supported.| +|ai.onnx:Mul|| +|ai.onnx:Relu|| +|ai.onnx:Sub|| + +## Additional Resources + +Additional operator support and performance tuning will be added soon. + +* [Ascend](https://www.hiascend.com/en/) +* [CANN](https://www.hiascend.com/en/software/cann) diff --git a/docs/execution-providers/CUDA-ExecutionProvider.md b/docs/execution-providers/CUDA-ExecutionProvider.md index a20b11b0bf..ed48967297 100644 --- a/docs/execution-providers/CUDA-ExecutionProvider.md +++ b/docs/execution-providers/CUDA-ExecutionProvider.md @@ -2,7 +2,7 @@ title: CUDA (NVIDIA) description: Instructions to execute ONNX Runtime applications with CUDA parent: Execution Providers -nav_order: 4 +nav_order: 5 redirect_from: /docs/reference/execution-providers/CUDA-ExecutionProvider --- diff --git a/docs/execution-providers/CoreML-ExecutionProvider.md b/docs/execution-providers/CoreML-ExecutionProvider.md index 07458cb39b..918a286b95 100644 --- a/docs/execution-providers/CoreML-ExecutionProvider.md +++ b/docs/execution-providers/CoreML-ExecutionProvider.md @@ -2,7 +2,7 @@ title: CoreML (Apple) description: Instructions to execute ONNX Runtime with CoreML parent: Execution Providers -nav_order: 3 +nav_order: 4 redirect_from: /docs/reference/execution-providers/CoreML-ExecutionProvider --- {::options toc_levels="2" /} diff --git a/docs/execution-providers/DirectML-ExecutionProvider.md b/docs/execution-providers/DirectML-ExecutionProvider.md index 1ce8f12471..3f06e59f5a 100644 --- a/docs/execution-providers/DirectML-ExecutionProvider.md +++ b/docs/execution-providers/DirectML-ExecutionProvider.md @@ -2,7 +2,7 @@ title: DirectML (Windows) description: Instructions to execute ONNX Runtime with the DirectML execution provider parent: Execution Providers -nav_order: 5 +nav_order: 6 redirect_from: /docs/reference/execution-providers/DirectML-ExecutionProvider --- diff --git a/docs/execution-providers/MIGraphX-ExecutionProvider.md b/docs/execution-providers/MIGraphX-ExecutionProvider.md index 2689c06ddf..758461ecb2 100644 --- a/docs/execution-providers/MIGraphX-ExecutionProvider.md +++ b/docs/execution-providers/MIGraphX-ExecutionProvider.md @@ -2,7 +2,7 @@ title: MIGraphX (AMD) description: Instructions to execute ONNX Runtime with the AMD MIGraphX execution provider parent: Execution Providers -nav_order: 6 +nav_order: 7 redirect_from: /docs/reference/execution-providers/MIGraphX-ExecutionProvider --- @@ -50,4 +50,4 @@ MIGraphX providers an environment variable ORT_MIGRAPHX_FP16_ENABLE to enable th ## Performance Tuning For performance tuning, please see guidance on this page: [ONNX Runtime Perf Tuning](../performance/tune-performance.md) -When/if using [onnxruntime_perf_test](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/test/perftest#onnxruntime-performance-test), use the flag `-e migraphx` \ No newline at end of file +When/if using [onnxruntime_perf_test](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/test/perftest#onnxruntime-performance-test), use the flag `-e migraphx` diff --git a/docs/execution-providers/NNAPI-ExecutionProvider.md b/docs/execution-providers/NNAPI-ExecutionProvider.md index e669514bce..a846da19a1 100644 --- a/docs/execution-providers/NNAPI-ExecutionProvider.md +++ b/docs/execution-providers/NNAPI-ExecutionProvider.md @@ -2,7 +2,7 @@ title: NNAPI (Android) description: Instructions to execute ONNX Runtime with the NNAPI execution provider parent: Execution Providers -nav_order: 7 +nav_order: 8 redirect_from: /docs/reference/execution-providers/NNAPI-ExecutionProvider --- {::options toc_levels="2" /} diff --git a/docs/execution-providers/OpenVINO-ExecutionProvider.md b/docs/execution-providers/OpenVINO-ExecutionProvider.md index d267a89e9a..4b4ccee938 100644 --- a/docs/execution-providers/OpenVINO-ExecutionProvider.md +++ b/docs/execution-providers/OpenVINO-ExecutionProvider.md @@ -2,7 +2,7 @@ title: OpenVINO™ (Intel) description: Instructions to execute OpenVINO™ Execution Provider for ONNX Runtime. parent: Execution Providers -nav_order: 9 +nav_order: 10 redirect_from: /docs/reference/execution-providers/OpenVINO-ExecutionProvider --- diff --git a/docs/execution-providers/RKNPU-ExecutionProvider.md b/docs/execution-providers/RKNPU-ExecutionProvider.md index 1bdcdd7cc9..049b60f22c 100644 --- a/docs/execution-providers/RKNPU-ExecutionProvider.md +++ b/docs/execution-providers/RKNPU-ExecutionProvider.md @@ -2,7 +2,7 @@ title: RKNPU description: Instructions to execute ONNX Runtime on Rockchip NPUs with the RKNPU execution provider parent: Execution Providers -nav_order: 10 +nav_order: 11 redirect_from: /docs/reference/execution-providers/RKNPU-ExecutionProvider --- @@ -89,4 +89,4 @@ The following models from the ONNX model zoo are supported using the RKNPU Execu **Object Detection** - ssd -- yolov3 \ No newline at end of file +- yolov3 diff --git a/docs/execution-providers/ROCm-ExecutionProvider.md b/docs/execution-providers/ROCm-ExecutionProvider.md index a37d3c8fc8..f1db133da4 100644 --- a/docs/execution-providers/ROCm-ExecutionProvider.md +++ b/docs/execution-providers/ROCm-ExecutionProvider.md @@ -2,7 +2,7 @@ title: ROCm (AMD) description: Instructions to execute ONNX Runtime with the AMD ROCm execution provider parent: Execution Providers -nav_order: 11 +nav_order: 12 redirect_from: /docs/reference/execution-providers/ROCm-ExecutionProvider --- diff --git a/docs/execution-providers/SNPE-ExecutionProvider.md b/docs/execution-providers/SNPE-ExecutionProvider.md index 759a04d881..6d49c42490 100644 --- a/docs/execution-providers/SNPE-ExecutionProvider.md +++ b/docs/execution-providers/SNPE-ExecutionProvider.md @@ -2,7 +2,7 @@ title: SNPE (Qualcomm) description: Execute ONNX models with SNPE Execution Provider parent: Execution Providers -nav_order: 12 +nav_order: 13 redirect_from: /docs/reference/execution-providers/SNPE-ExecutionProvider --- diff --git a/docs/execution-providers/TVM-ExecutionProvider.md b/docs/execution-providers/TVM-ExecutionProvider.md index 7915ff043b..d3b8b5e517 100644 --- a/docs/execution-providers/TVM-ExecutionProvider.md +++ b/docs/execution-providers/TVM-ExecutionProvider.md @@ -2,7 +2,7 @@ title: TVM (Apache) description: Instructions to execute ONNX Runtime with the Apache TVM execution provider parent: Execution Providers -nav_order: 14 +nav_order: 15 --- # TVM Execution Provider @@ -260,4 +260,4 @@ pip3 install protobuf==3.19.1 The following pair of ONNX and protobuf versions have been found to be compatible: - 3.17.3 and 1.8.0 -- 3.19.1 and 1.10.1 \ No newline at end of file +- 3.19.1 and 1.10.1 diff --git a/docs/execution-providers/TensorRT-ExecutionProvider.md b/docs/execution-providers/TensorRT-ExecutionProvider.md index 4b2003dd3d..62a3ffafef 100644 --- a/docs/execution-providers/TensorRT-ExecutionProvider.md +++ b/docs/execution-providers/TensorRT-ExecutionProvider.md @@ -2,7 +2,7 @@ title: TensorRT (NVIDIA) description: Instructions to execute ONNX Runtime on NVIDIA GPUs with the TensorRT execution provider parent: Execution Providers -nav_order: 13 +nav_order: 14 redirect_from: /docs/reference/execution-providers/TensorRT-ExecutionProvider --- diff --git a/docs/execution-providers/Vitis-AI-ExecutionProvider.md b/docs/execution-providers/Vitis-AI-ExecutionProvider.md index ea959d591e..3c9cb10822 100644 --- a/docs/execution-providers/Vitis-AI-ExecutionProvider.md +++ b/docs/execution-providers/Vitis-AI-ExecutionProvider.md @@ -2,7 +2,7 @@ title: Vitis AI description: Instructions to execute ONNX Runtime on Xilinx devices with the Vitis AI execution provider parent: Execution Providers -nav_order: 15 +nav_order: 16 redirect_from: /docs/reference/execution-providers/Vitis-AI-ExecutionProvider --- diff --git a/docs/execution-providers/Xnnpack-ExecutionProvider.md b/docs/execution-providers/Xnnpack-ExecutionProvider.md index f89d042c0a..6beedd267b 100644 --- a/docs/execution-providers/Xnnpack-ExecutionProvider.md +++ b/docs/execution-providers/Xnnpack-ExecutionProvider.md @@ -2,7 +2,7 @@ title: XNNPACK description: Instructions to execute ONNX Runtime with the XNNPACK execution provider parent: Execution Providers -nav_order: 16 +nav_order: 17 --- {::options toc_levels="2" /} diff --git a/docs/execution-providers/add-execution-provider.md b/docs/execution-providers/add-execution-provider.md index e6d69146fb..3844185773 100644 --- a/docs/execution-providers/add-execution-provider.md +++ b/docs/execution-providers/add-execution-provider.md @@ -2,7 +2,7 @@ title: Add a new execution provider description: Instructions to add a new execution provider to ONNX Runtime parent: Execution Providers -nav_order: 17 +nav_order: 18 redirect_from: /docs/how-to/add-execution-provider --- diff --git a/docs/execution-providers/index.md b/docs/execution-providers/index.md index ff3949ff82..4c2e4d102d 100644 --- a/docs/execution-providers/index.md +++ b/docs/execution-providers/index.md @@ -30,7 +30,7 @@ ONNX Runtime supports many different execution providers today. Some of the EPs ---|---|---|--- |Default CPU|[NVIDIA CUDA](../execution-providers/CUDA-ExecutionProvider.md)|[Intel OpenVINO](../execution-providers/OpenVINO-ExecutionProvider.md)|[Rockchip NPU](../execution-providers/RKNPU-ExecutionProvider.md) (*preview*)| |[Intel DNNL](../execution-providers/oneDNN-ExecutionProvider.md)|[NVIDIA TensorRT](../execution-providers/TensorRT-ExecutionProvider.md)|[ARM Compute Library](../execution-providers/ACL-ExecutionProvider.md) (*preview*)|[Xilinx Vitis-AI](../execution-providers/Vitis-AI-ExecutionProvider.md) (*preview*)| -|[TVM](../execution-providers/TVM-ExecutionProvider.md) (*preview*)|[DirectML](../execution-providers/DirectML-ExecutionProvider.md)|[Android Neural Networks API](../execution-providers/NNAPI-ExecutionProvider.md)|| +|[TVM](../execution-providers/TVM-ExecutionProvider.md) (*preview*)|[DirectML](../execution-providers/DirectML-ExecutionProvider.md)|[Android Neural Networks API](../execution-providers/NNAPI-ExecutionProvider.md)|[Huawei CANN](../execution-providers/CANN-ExecutionProvider.md) (*preview*)| |[Intel OpenVINO](../execution-providers/OpenVINO-ExecutionProvider.md)|[AMD MIGraphX](../execution-providers/MIGraphX-ExecutionProvider.md) (*preview*)|[ARM-NN](../execution-providers/ArmNN-ExecutionProvider.md) (*preview*)| ||[AMD ROCm](../execution-providers/ROCm-ExecutionProvider.md) (*preview*)|[CoreML](../execution-providers/CoreML-ExecutionProvider.md) (*preview*)| ||[TVM](../execution-providers/TVM-ExecutionProvider.md) (*preview*)|[TVM](../execution-providers/TVM-ExecutionProvider.md) (*preview*)| diff --git a/docs/execution-providers/oneDNN-ExecutionProvider.md b/docs/execution-providers/oneDNN-ExecutionProvider.md index 0c33dbd52a..353f8550c4 100644 --- a/docs/execution-providers/oneDNN-ExecutionProvider.md +++ b/docs/execution-providers/oneDNN-ExecutionProvider.md @@ -2,7 +2,7 @@ title: oneDNN (Intel) description: Instructions to execute ONNX Runtime with the Intel oneDNN execution provider parent: Execution Providers -nav_order: 8 +nav_order: 9 redirect_from: /docs/reference/execution-providers/oneDNN-ExecutionProvider --- @@ -129,4 +129,4 @@ In SubgraphPrimitve::Compute() method, we iterate thru Dnnl Kernels and bind inp * CPU ## Additional Resources -* [DNNL documentation](https://intel.github.io/mkl-dnn/) \ No newline at end of file +* [DNNL documentation](https://intel.github.io/mkl-dnn/) diff --git a/index.html b/index.html index 14eda35b52..cd70ac6176 100644 --- a/index.html +++ b/index.html @@ -267,6 +267,8 @@ ACL (Preview)
ArmNN (Preview)
+
+ CANN (Preview)
MIGraphX (Preview)
diff --git a/js/script.js b/js/script.js index c465e15c74..b1db07c45e 100644 --- a/js/script.js +++ b/js/script.js @@ -1177,6 +1177,24 @@ var validCombos = { "linux,C++,X86,XNNPACK": "Follow build instructions from here", + + "linux,Python,ARM64,CANN": + "Follow build instructions from here.", + + "linux,C-API,ARM64,CANN": + "Follow build instructions from here.", + + "linux,C++,ARM64,CANN": + "Follow build instructions from here.", + + "linux,Python,X64,CANN": + "Follow build instructions from here.", + + "linux,C-API,X64,CANN": + "Follow build instructions from here.", + + "linux,C++,X64,CANN": + "Follow build instructions from here.", }; function commandMessage(key) {