Initial PR for RKNPU execution provider (#3609)

* Initial RKNPU execution provider

    * Init

    * Support Ops:
        Conv, Relu, Clip, LeakyRelu,
        MaxPool, AveragePool, GlobalAveragePool,
        Concat, Softmax, BatchNormalization, Gemm,
        Add, Mul, Sub,
        Reshape, Squeeze, Unsqueeze,
        Flatten, Transpose,
        QLinearConv, DequantizeLinear

    * Add rknpu unittest

    * Update BUILD.md and Add RKNPU-ExecutionProvider.md

* misc code update

* fix CLIP accuracy issue.

* fix "Error: Duplicate definition of name".

* move rknpu_ddk out of onnxruntime submodule.

* remove temporary code.

* add rknpu namespace.

* update misc of node_attr_helper

* add const & comment for onnx_converter

* add const & comment for shaper

* unify variable name

Co-authored-by: dkm <dkm@rock-chips.com>
Co-authored-by: George Wu <jywu@microsoft.com>

BUILD.md

@@ -104,6 +104,7 @@ The complete list of build options can be found by running `./build.sh (or .\bui
 * [Nuphar Model Compiler](#Nuphar)
 * [DirectML](#DirectML)
 * [ARM Compute Library](#ARM-Compute-Library)
+* [Rockchip RKNPU](#RKNPU)
 
 **Options**
 * [OpenMP](#OpenMP)
@@ -449,6 +450,35 @@ export LD_LIBRARY_PATH=~/ComputeLibrary/build/
 
 ---
 
+### RKNPU
+See more information on the RKNPU Execution Provider [here](./docs/execution_providers/RKNPU-ExecutionProvider.md).
+
+#### Pre-Requisites
+
+* Supported platform: RK1808 Linux
+* See [Build ARM](#ARM) below for information on building for ARM devices
+* Use gcc-linaro-6.3.1-2017.05-x86_64_aarch64-linux-gnu instead of gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf, and modify CMAKE_CXX_COMPILER & CMAKE_C_COMPILER in tool.cmake:
+  ```
+  set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)
+  set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)
+  ```
+
+#### Build Instructions
+##### Linux
+1. Download [rknpu_ddk](#https://github.com/airockchip/rknpu_ddk.git) to any directory.
+
+2. Build ONNX Runtime library and test:
+    ```
+    ./build.sh --arm --use_rknpu --parallel --build_shared_lib --build_dir build_arm --config MinSizeRel --cmake_extra_defines RKNPU_DDK_PATH=<Path To rknpu_ddk> CMAKE_TOOLCHAIN_FILE=<Path To tool.cmake> ONNX_CUSTOM_PROTOC_EXECUTABLE=<Path To protoc>
+    ```
+3. Deploy ONNX runtime and librknpu_ddk.so on the RK1808 board:
+    ```
+    libonnxruntime.so.1.2.0
+    onnxruntime_test_all
+    rknpu_ddk/lib64/librknpu_ddk.so
+    ```
+---
+
 ## Options
 ### OpenMP
 #### Build Instructions

cmake/CMakeLists.txt

@@ -56,6 +56,7 @@ option(onnxruntime_USE_CUDA "Build with CUDA support" OFF)
 option(onnxruntime_USE_OPENVINO "Build with OpenVINO support" OFF)
 option(onnxruntime_USE_EIGEN_FOR_BLAS "Use eign for blas" ON)
 option(onnxruntime_USE_NNAPI "Build with DNNLibrary for Android NNAPI support" OFF)
+option(onnxruntime_USE_RKNPU "Build with RKNPU support" OFF)
 option(onnxruntime_USE_DNNL "Build with DNNL support" OFF)
 option(onnxruntime_USE_MKLML "Build the default cpu provider with MKL-ML binary dependency" OFF)
 option(onnxruntime_USE_FEATURIZERS "Build ML Featurizers support" OFF)

cmake/onnxruntime.cmake

@@ -75,6 +75,7 @@ target_link_libraries(onnxruntime PRIVATE
     ${PROVIDERS_DNNL}
     ${PROVIDERS_NGRAPH}
     ${PROVIDERS_NNAPI}
+    ${PROVIDERS_RKNPU}
     ${PROVIDERS_TENSORRT}
     ${PROVIDERS_OPENVINO}
     ${PROVIDERS_NUPHAR}

cmake/onnxruntime_providers.cmake

@@ -55,6 +55,10 @@ if(onnxruntime_USE_NNAPI)
   set(PROVIDERS_NNAPI onnxruntime_providers_nnapi)
   list(APPEND ONNXRUNTIME_PROVIDER_NAMES nnapi)
 endif()
+if(onnxruntime_USE_RKNPU)
+  set(PROVIDERS_RKNPU onnxruntime_providers_rknpu)
+  list(APPEND ONNXRUNTIME_PROVIDER_NAMES rknpu)
+endif()
 if(onnxruntime_USE_DML)
   set(PROVIDERS_DML onnxruntime_providers_dml)
   list(APPEND ONNXRUNTIME_PROVIDER_NAMES dml)
@@ -473,6 +477,40 @@ if (onnxruntime_USE_NNAPI)
   set_target_properties(onnxruntime_providers_nnapi PROPERTIES LINKER_LANGUAGE CXX)
 endif()
 
+if (onnxruntime_USE_RKNPU)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable -Wno-unused-parameter")
+  add_definitions(-DUSE_RKNPU=1)
+  option(DNN_READ_ONNX "" ON)
+  set(DNN_CUSTOM_PROTOC_EXECUTABLE ${ONNX_CUSTOM_PROTOC_EXECUTABLE})
+  option(DNN_CMAKE_INSTALL "" OFF)
+  option(DNN_BUILD_BIN "" OFF)
+  if (NOT RKNPU_DDK_PATH)
+    message(FATAL_ERROR "RKNPU_DDK_PATH required for onnxruntime_USE_RKNPU")
+  endif()
+  set(RKNPU_DDK_INCLUDE_DIR ${RKNPU_DDK_PATH}/include)
+  if (CMAKE_SIZEOF_VOID_P EQUAL 8)
+    set(RKNPU_DDK_LIB_DIR ${RKNPU_DDK_PATH}/lib64)
+  else()
+    set(RKNPU_DDK_LIB_DIR ${RKNPU_DDK_PATH}/lib)
+  endif()
+  file(GLOB_RECURSE
+    onnxruntime_providers_rknpu_cc_srcs CONFIGURE_DEPENDS
+    "${ONNXRUNTIME_ROOT}/core/providers/rknpu/*.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/rknpu/*.cc"
+  )
+  source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_rknpu_cc_srcs})
+  add_library(onnxruntime_providers_rknpu ${onnxruntime_providers_rknpu_cc_srcs})
+  onnxruntime_add_include_to_target(onnxruntime_providers_rknpu onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf-lite)
+  target_link_libraries(onnxruntime_providers_rknpu PRIVATE -lrknpu_ddk)
+  add_dependencies(onnxruntime_providers_rknpu onnx ${onnxruntime_EXTERNAL_DEPENDENCIES})
+  set_target_properties(onnxruntime_providers_rknpu PROPERTIES CXX_STANDARD 14)
+  set_target_properties(onnxruntime_providers_rknpu PROPERTIES CXX_STANDARD_REQUIRED ON)
+  set_target_properties(onnxruntime_providers_rknpu PROPERTIES FOLDER "ONNXRuntime")
+  target_include_directories(onnxruntime_providers_rknpu PRIVATE ${ONNXRUNTIME_ROOT} ${rknpu_INCLUDE_DIRS} ${RKNPU_DDK_INCLUDE_DIR})
+  link_directories(onnxruntime_providers_rknpu ${RKNPU_DDK_LIB_DIR})
+  set_target_properties(onnxruntime_providers_rknpu PROPERTIES LINKER_LANGUAGE CXX)
+endif()
+
 if (onnxruntime_USE_DML)
   file(GLOB_RECURSE onnxruntime_providers_dml_cc_srcs CONFIGURE_DEPENDS
     "${ONNXRUNTIME_ROOT}/core/providers/dml/*.h"

cmake/onnxruntime_python.cmake

@@ -89,6 +89,7 @@ set(onnxruntime_pybind11_state_libs
     ${PROVIDERS_OPENVINO}
     ${PROVIDERS_NUPHAR}
     ${PROVIDERS_NNAPI}
+    ${PROVIDERS_RKNPU}
     ${PROVIDERS_DML}
     onnxruntime_optimizer
     onnxruntime_providers

cmake/onnxruntime_unittests.cmake

@@ -211,6 +211,13 @@ if (onnxruntime_USE_NNAPI)
   list(APPEND onnxruntime_test_providers_src ${onnxruntime_test_providers_nnapi_src})
 endif()
 
+if (onnxruntime_USE_RKNPU)
+  file(GLOB_RECURSE onnxruntime_test_providers_rknpu_src CONFIGURE_DEPENDS
+    "${TEST_SRC_DIR}/providers/rknpu/*"
+    )
+  list(APPEND onnxruntime_test_providers_src ${onnxruntime_test_providers_rknpu_src})
+endif()
+
 set (ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR "${ONNXRUNTIME_ROOT}/test/shared_lib")
 set (ONNXRUNTIME_GLOBAL_THREAD_POOLS_TEST_SRC_DIR "${ONNXRUNTIME_ROOT}/test/global_thread_pools")
 
@@ -293,6 +300,10 @@ if(onnxruntime_USE_NNAPI)
   list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_nnapi)
 endif()
 
+if(onnxruntime_USE_RKNPU)
+  list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_rknpu)
+endif()
+
 if(onnxruntime_USE_FEATURIZERS)
    list(APPEND onnxruntime_test_providers_dependencies onnxruntime_featurizers)
    list(APPEND onnxruntime_test_providers_libs onnxruntime_featurizers re2)
@@ -334,6 +345,7 @@ set(ONNXRUNTIME_TEST_LIBS
     ${PROVIDERS_OPENVINO}
     ${PROVIDERS_NUPHAR}
     ${PROVIDERS_NNAPI}
+    ${PROVIDERS_RKNPU}
     ${PROVIDERS_DML}
     ${PROVIDERS_ACL}
     onnxruntime_optimizer
@@ -370,6 +382,13 @@ if(onnxruntime_USE_NNAPI)
   list(APPEND onnxruntime_test_providers_libs onnxruntime_providers_nnapi)
 endif()
 
+if(onnxruntime_USE_RKNPU)
+  list(APPEND onnxruntime_test_framework_src_patterns  ${TEST_SRC_DIR}/providers/rknpu/*)
+  list(APPEND onnxruntime_test_framework_libs onnxruntime_providers_rknpu)
+  list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_rknpu)
+  list(APPEND onnxruntime_test_providers_libs onnxruntime_providers_rknpu)
+endif()
+
 if(WIN32)
   if (onnxruntime_USE_TVM)
     list(APPEND disabled_warnings ${DISABLED_WARNINGS_FOR_TVM})

docs/execution_providers/RKNPU-ExecutionProvider.md

@@ -0,0 +1,76 @@
+# RKNPU Execution Provider
+RKNPU DDK is an advanced interface to access Rockchip NPU, currently support platform as follow:
+- RK1808 Linux
+
+  Note: RK3399Pro platform is not supported.
+
+RKNPU Execution Provider enables deep learning inference on Rockchip NPU via RKNPU DDK.
+
+
+## Build 
+For build instructions, please see the [BUILD page](../../BUILD.md#RKNPU).
+
+## Using
+### C/C++
+To use RKNPU as execution provider for inferencing, please register it as below.
+```
+string log_id = "Foo";
+auto logging_manager = std::make_unique<LoggingManager>
+(std::unique_ptr<ISink>{new CLogSink{}},
+                                  static_cast<Severity>(lm_info.default_warning_level),
+                                  false,
+                                  LoggingManager::InstanceType::Default,
+                                  &log_id)
+Environment::Create(std::move(logging_manager), env)
+InferenceSession session_object{so,env};
+session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::RknpuExecutionProvider>());
+status = session_object.Load(model_file_name);
+```
+The C API details are [here](../C_API.md#c-api).
+
+
+## ONNX Op supported using RKNPU
+
+The table below shows the ONNX Ops supported using RKNPU Execution Provider and the mapping between ONNX Ops and RKNPU Ops. 
+
+| **ONNX Ops** | **RKNPU Ops** |
+| --- | --- |
+| Add | ADD |
+| Mul | MULTIPLY |
+| Conv | CONV2D |
+| QLinearConv | CONV2D |
+| Gemm | FULLCONNECT |
+| Softmax | SOFTMAX |
+| AveragePool | POOL |
+| GlobalAveragePool | POOL |
+| MaxPool | POOL |
+| GlobalMaxPool | POOL |
+| LeakyRelu | LEAKY_RELU |
+| Concat | CONCAT |
+| BatchNormalization | BATCH_NORM |
+| Reshape | RESHAPE |
+| Flatten | RESHAPE |
+| Squeeze | RESHAPE |
+| Unsqueeze | RESHAPE |
+| Transpose | PERMUTE |
+| Relu | RELU |
+| Sub | SUBTRACT |
+| Clip(0~6)| RELU6 |
+| DequantizeLinear | DATACONVERT |
+| Clip | CLIP |
+
+
+## Model Supported
+
+Below Models are supported from ONNX open model zoo using RKNPU Execution Provider
+
+### Image Classification
+- squeezenet
+- mobilenetv2-1.0
+- resnet50v1
+- resnet50v2
+- inception_v2
+
+### Object Detection
+- ssd
+- yolov3

include/onnxruntime/core/graph/constants.h

@@ -30,6 +30,7 @@ constexpr const char* kOpenVINOExecutionProvider = "OpenVINOExecutionProvider";
 constexpr const char* kNupharExecutionProvider = "NupharExecutionProvider";
 constexpr const char* kTensorrtExecutionProvider = "TensorrtExecutionProvider";
 constexpr const char* kNnapiExecutionProvider = "NnapiExecutionProvider";
+constexpr const char* kRknpuExecutionProvider = "RknpuExecutionProvider";
 constexpr const char* kDmlExecutionProvider = "DmlExecutionProvider";
 constexpr const char* kAclExecutionProvider = "ACLExecutionProvider";
 }  // namespace onnxruntime

include/onnxruntime/core/providers/rknpu/rknpu_provider_factory.h

@@ -0,0 +1,14 @@
+// Copyright 2020 rock-chips.com Inc.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Rknpu,
+               _In_ OrtSessionOptions* options);
+
+#ifdef __cplusplus
+}
+#endif

onnxruntime/core/framework/utils.cc

@@ -100,7 +100,8 @@ bool ProviderIsCpuBased(const std::string& provider_type) {
          provider_type == onnxruntime::kNGraphExecutionProvider ||
          provider_type == onnxruntime::kNupharExecutionProvider ||
          provider_type == onnxruntime::kOpenVINOExecutionProvider ||
-         provider_type == onnxruntime::kNnapiExecutionProvider;
+         provider_type == onnxruntime::kNnapiExecutionProvider ||
+         provider_type == onnxruntime::kRknpuExecutionProvider;
 }
 
 common::Status AllocateHelper(const IExecutionProvider& execution_provider, const OrtDevice& device,

onnxruntime/core/providers/rknpu/node_attr_helper.cc

@@ -0,0 +1,112 @@
+//
+// Created by daquexian on 8/3/18.
+//
+
+#include "node_attr_helper.h"
+
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace onnxruntime {
+namespace rknpu {
+
+NodeAttrHelper::NodeAttrHelper(ONNX_NAMESPACE::NodeProto proto)
+    : node_(proto) {
+}
+
+float NodeAttrHelper::get(const std::string& key,
+                          const float def_val) const {
+  for (const auto& attr : node_.attribute()) {
+    if (attr.name() == key) {
+      return attr.f();
+    }
+  }
+
+  return def_val;
+}
+
+int NodeAttrHelper::get(const std::string& key,
+                        const int def_val) const {
+  for (const auto& attr : node_.attribute()) {
+    if (attr.name() == key) {
+      return static_cast<int>(attr.i());
+    }
+  }
+
+  return def_val;
+}
+
+std::string NodeAttrHelper::get(const std::string& key,
+                                const std::string& def_val) const {
+  for (const auto& attr : node_.attribute()) {
+    if (attr.name() == key) {
+      return attr.s();
+    }
+  }
+
+  return def_val;
+}
+
+std::vector<int> NodeAttrHelper::get(const std::string& key,
+                                     const std::vector<int>& def_val) const {
+  if (!has_attr(key)) {
+    return def_val;
+  }
+  std::vector<int> v;
+
+  for (const auto& attr : node_.attribute()) {
+    if (attr.name() == key) {
+      v.reserve(static_cast<size_t>(attr.ints_size()));
+      for (int j = 0; j < attr.ints_size(); j++) {
+        v.push_back(static_cast<int>(attr.ints(j)));
+      }
+
+      break;
+    }
+  }
+
+  if (v.empty()) {
+    return def_val;
+  }
+
+  return v;
+}
+
+std::vector<float> NodeAttrHelper::get(const std::string& key,
+                                       const std::vector<float>& def_val) const {
+  if (!has_attr(key)) {
+    return def_val;
+  }
+  std::vector<float> v;
+
+  for (const auto& attr : node_.attribute()) {
+    if (attr.name() == key) {
+      v.reserve(static_cast<size_t>(attr.floats_size()));
+      for (int j = 0; j < attr.floats_size(); j++) {
+        v.push_back(attr.floats(j));
+      }
+
+      break;
+    }
+  }
+
+  if (v.empty()) {
+    return def_val;
+  }
+
+  return v;
+}
+
+bool NodeAttrHelper::has_attr(const std::string& key) const {
+  for (const auto& attr : node_.attribute()) {
+    if (attr.name() == key) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+}  // namespace rknpu
+}  // namespace onnxruntime

onnxruntime/core/providers/rknpu/node_attr_helper.h

@@ -0,0 +1,39 @@
+//
+// Created by daquexian on 8/3/18.
+//
+
+#pragma once
+
+#include <onnx/onnx_pb.h>
+#include <vector>
+#include <string>
+
+namespace onnxruntime {
+namespace rknpu {
+
+/**
+ * Wrapping onnx::NodeProto for retrieving attribute values
+ */
+class NodeAttrHelper {
+ public:
+  explicit NodeAttrHelper(ONNX_NAMESPACE::NodeProto proto);
+
+  float get(const std::string& key,
+            const float def_val) const;
+  int get(const std::string& key,
+          const int def_val) const;
+  std::vector<float> get(const std::string& key,
+                         const std::vector<float>& def_val) const;
+  std::vector<int> get(const std::string& key,
+                       const std::vector<int>& def_val) const;
+  std::string get(const std::string& key,
+                  const std::string& def_val) const;
+
+  bool has_attr(const std::string& key) const;
+
+ private:
+  ONNX_NAMESPACE::NodeProto node_;
+};
+
+}  // namespace rknpu
+}  // namespace onnxruntime

onnxruntime/core/providers/rknpu/onnx_converter.h

@@ -0,0 +1,280 @@
+// Copyright 2020 rock-chips.com Inc.
+
+#pragma once
+
+#include <onnx/onnx_pb.h>
+
+#include <map>
+#include <memory>
+#include <utility>
+#include <vector>
+#include <unordered_map>
+#include <string>
+
+#include "shaper.h"
+
+#include "rknpu/rknpu_pub.h"
+
+namespace onnxruntime {
+namespace rknpu {
+
+/**
+ *  For convert from onnx::ModelProto to rk::nn::Graph.
+ */
+class OnnxConverter {
+ public:
+  OnnxConverter() {}
+  ~OnnxConverter() { Clear(); }
+
+  /** Get the supported subgraph.
+  */
+  std::vector<std::vector<int>> GetSupportedNodes(
+      const ONNX_NAMESPACE::ModelProto& model_proto);
+
+  /** Convert onnx::ModelProto to rk::nn::Graph.
+   *  Because some attributes of rk::nn::Tensor are used as input in onnx and these attributes
+   *  can't be found in onnx::ModelProto, so additional input-related information is required.
+  */
+  void Convert(const ONNX_NAMESPACE::ModelProto& model,
+               rk::nn::Graph* graph,
+               const std::vector<const void*>& input_bufs,
+               const std::unordered_map<std::string, int>& input_maps);
+
+  std::string m(const std::string& str) const;
+
+ private:
+  Shaper shaper_;
+
+  enum class FuseCode { FUSED_NONE,
+                        FUSED_RELU,
+                        FUSED_RELU1,
+                        FUSED_RELU6 };
+
+  std::map<std::string, std::string> name_map_;
+
+  ONNX_NAMESPACE::ModelProto model_proto_;
+  rk::nn::Graph* graph_;
+  std::vector<int> skipped_act_;
+  std::vector<std::string> dequantize_after_;
+
+  std::vector<std::string> operands_;
+  std::map<std::string, std::shared_ptr<rk::nn::Tensor>> rk_tensors_;
+
+  // for GetSupportedNodes
+  std::map<std::string, std::vector<uint32_t>> tensor_dims_;
+
+  std::vector<void*> free_list_;  // remember free
+
+  std::pair<std::pair<int, ONNX_NAMESPACE::NodeProto>, FuseCode>
+  FindActivation(const ONNX_NAMESPACE::ModelProto& model_proto,
+                 const std::string& output);
+
+  std::shared_ptr<rk::nn::Tensor>
+  CreateRknnTensor(const std::string& name,
+                   const std::vector<uint32_t>& dims,
+                   const void* data = NULL,
+                   const rk::nn::TensorRole role = rk::nn::TensorRole::VAR,
+                   const rk::nn::PrecisionType precision = rk::nn::PrecisionType::FLOAT32,
+                   const rk::nn::DataLayoutType layout = rk::nn::DataLayoutType::NCHW,
+                   const rk::nn::QuantizationType qntType = rk::nn::QuantizationType::NONE,
+                   const uint8_t bits = 8,
+                   const float scale = 1.0,
+                   const uint32_t zero_point = 0,
+                   const int8_t fl = 0);
+
+  void HandleInitializer();
+  std::vector<std::shared_ptr<rk::nn::Tensor>> GetInputOfOnnxModel(
+      const std::vector<const void*>& input_bufs,
+      const std::unordered_map<std::string, int>& input_maps);
+  std::vector<std::shared_ptr<rk::nn::Tensor>> GetOutputOfOnnxModel();
+
+  std::pair<bool, std::string> IsNodeSupported(
+      const ONNX_NAMESPACE::ModelProto& model_proto,
+      const ONNX_NAMESPACE::NodeProto& node_proto) const;
+
+  void AddConv(const std::string& input,
+               const std::vector<int>& strides,
+               const std::vector<int>& pads,
+               const std::vector<int>& dilations,
+               const int32_t group,
+               const std::string& ori_weight,
+               const std::string& bias,
+               const std::string& auto_pad,
+               const std::string& output);
+  void AddQLinearConv(const std::string& input,
+                      const std::string& input_scale,
+                      const std::string& input_zp,
+                      const std::vector<int>& strides,
+                      const std::vector<int>& pads,
+                      const std::vector<int>& dilations,
+                      const int group,
+                      const std::string& auto_pad,
+                      const std::string& weight,
+                      const std::string& weight_scale,
+                      const std::string& weight_zp,
+                      const std::string& bias,
+                      const std::string& output,
+                      const std::string& output_scale,
+                      const std::string& output_zp);
+  void AddLayerPool(const std::string& op,
+                    const std::string& input,
+                    const std::vector<int>& kernel_shape,
+                    const std::vector<int>& pads,
+                    const std::vector<int>& strides,
+                    const int32_t ceil_mode,
+                    const std::string& output);
+  void SetIdentity(const std::string& input,
+                   const std::string& output);
+  void AddLayerConvImpl(const std::string& input,
+                        const std::string& weight,
+                        const std::string& bias,
+                        const std::vector<int32_t>& pads,
+                        const std::vector<int32_t>& strides,
+                        const int32_t group,
+                        const std::string& auto_pad,
+                        const std::string& output);
+  void AddLayerQLinearConvImpl(const std::string& input,
+                               const std::string& input_scale,
+                               const std::string& input_zp,
+                               const std::string& weight,
+                               const std::string& weight_scale,
+                               const std::string& weight_zp,
+                               const std::string& bias,
+                               const std::vector<int>& pads,
+                               const std::vector<int>& strides,
+                               const int group,
+                               const std::string& auto_pad,
+                               const std::string& output,
+                               const std::string& output_scale,
+                               const std::string& output_zp);
+  void AddLayerAvePoolImpl(const std::string& input,
+                           const std::vector<int32_t>& kernel_shape,
+                           const std::vector<int32_t>& pads,
+                           const std::vector<int32_t>& strides,
+                           const int32_t ceil_mode,
+                           const std::string& output);
+  void AddLayerMaxPoolImpl(const std::string& input,
+                           const std::vector<int32_t>& kernel_shape,
+                           const std::vector<int32_t>& pads,
+                           const std::vector<int32_t>& strides,
+                           const int32_t ceil_mode,
+                           const std::string& output);
+  void AddLayerReLU(const std::string& input,
+                    const std::string& output);
+  void AddLayerSoftmax(const std::string& input,
+                       const std::string& output);
+  void AddLayerFC(const std::string& input,
+                  const std::string& weight,
+                  const std::string& bias,
+                  const std::string& output);
+  void AddLayerAdd(const std::string& input1,
+                   const std::string& input2,
+                   const std::string& output);
+  void AddLayerSub(const std::string& input1,
+                   const std::string& input2,
+                   const std::string& output);
+  void AddLayerConcat(const std::vector<std::string>& inputs,
+                      const int32_t axis,
+                      const std::string& output);
+  void AddLayerDepthwiseConvImpl(const std::string& input,
+                                 const std::string& weight,
+                                 const std::string& bias,
+                                 const std::vector<int32_t>& pads,
+                                 const std::vector<int32_t>& strides,
+                                 const int32_t depth_multiplier,
+                                 const int32_t group,
+                                 const std::string& output);
+  void AddLayerBatchToSpaceND(const std::string& input,
+                              const std::vector<int32_t>& block_sizes,
+                              const std::string& output);
+  void AddLayerSpaceToBatchND(const std::string& input,
+                              const std::vector<int32_t>& block_sizes,
+                              const std::vector<int32_t>& pads,
+                              const std::string& output);
+  void AddLayerSlice(const std::string& input,
+                     const std::string& starts,
+                     const std::string& ends,
+                     const std::string& axes,
+                     const std::string& steps,
+                     const std::string& output);
+  void AddLayerStridedSlice(const std::string& input,
+                            const std::vector<int32_t>& starts,
+                            const std::vector<int32_t>& ends,
+                            const std::vector<int32_t>& strides,
+                            const int32_t begin_mask,
+                            const int32_t end_mask,
+                            const int32_t shrink_axis_mask,
+                            const std::string& output);
+  void AddLayerMul(const std::string& input1,
+                   const std::string& input2,
+                   const std::string& output);
+  void AddLayerAdd(const std::string& input,
+                   const float scalar,
+                   const std::string& output);
+  void AddLayerMul(const std::string& input,
+                   const float scalar,
+                   const std::string& output);
+  void AddLayerDequantize(const std::string& input,
+                          const std::string& output);
+  void AddLayerLRN(const std::string& input,
+                   const int32_t radius,
+                   const float bias,
+                   const float alpha,
+                   const float beta,
+                   const std::string& output);
+  void AddLayerTanh(const std::string& input,
+                    const std::string& output);
+  void AddLayerFloor(const std::string& input,
+                     const std::string& output);
+  void AddLayerLogistic(const std::string& input,
+                        const std::string& output);
+  void AddLayerBatchNorm(const std::string& input,
+                         const std::string& scale,
+                         const std::string& bias,
+                         const std::string& mean,
+                         const std::string& var,
+                         const float eps,
+                         const std::string& output);
+  void AddLayerReshape(const std::string& input,
+                       const std::string& shape,
+                       const std::string& output);
+  void AddLayerFlatten(const std::string& input,
+                       const int32_t axis,
+                       const std::string& output);
+  void AddLayerTranspose(const std::string& input,
+                         const std::vector<int32_t>& perm,
+                         const std::string& output);
+  void AddLayerSqueeze(const std::string& input,
+                       const std::vector<int32_t>& axes,
+                       const std::string& output);
+  void AddLayerUnsqueeze(const std::string& input,
+                         const std::vector<int32_t>& axes,
+                         const std::string& output);
+  void AddLayerGather(const std::string& input,
+                      const std::string& indices,
+                      const int32_t axis,
+                      const std::string& output);
+  void AddLayerLeakyRelu(const std::string& input,
+                         const float alpha,
+                         const std::string& output);
+  void AddLayerClip(const std::string& input,
+                    const int32_t min,
+                    const int32_t max,
+                    const std::string& output);
+  void AddLayerDequantizeLinear(const std::string& input,
+                                const std::string& input_scale,
+                                const std::string& input_zp,
+                                const std::string& output);
+  void AddLayerQuantizeLinear(const std::string& input,
+                              const std::string& output_scale,
+                              const std::string& output_zp,
+                              const std::string& output);
+
+  void Clear();
+
+  OnnxConverter(const OnnxConverter&);
+  OnnxConverter& operator=(const OnnxConverter&);
+};
+
+}  // namespace rknpu
+}  // namespace onnxruntime

onnxruntime/core/providers/rknpu/rknpu_execution_provider.cc

@@ -0,0 +1,578 @@
+// Copyright 2020 rock-chips.com Inc.
+
+#include <unistd.h>
+#include <limits>
+#include <set>
+#include <unordered_set>
+#include <map>
+#include <utility>
+#include <functional>
+#include "rknpu_execution_provider.h"
+#include "core/common/logging/logging.h"
+#include "core/framework/allocatormgr.h"
+#include "core/framework/compute_capability.h"
+#include "core/session/onnxruntime_cxx_api.h"
+#include "core/session/inference_session.h"
+#include "core/graph/model.h"
+#include "core/framework/memcpy.h"
+#include "node_attr_helper.h"
+#include "rknpu/rknpu_pub.h"
+#include "onnx_converter.h"
+
+using std::string;
+using std::vector;
+
+namespace onnxruntime {
+
+constexpr const char* RKNPU = "Rknpu";
+
+struct RknpuFuncState {
+  std::string uniq_input_shape;
+
+  std::unique_ptr<rk::nn::Exection> exector;
+  ONNX_NAMESPACE::ModelProto model_proto;
+  std::unordered_map<std::string, int> input_map;
+  std::unordered_map<std::string, int> output_map;
+  std::vector<int> input_indexes;
+  std::vector<int> output_indexes;
+};
+
+RknpuExecutionProvider::RknpuExecutionProvider()
+    : IExecutionProvider{onnxruntime::kRknpuExecutionProvider} {
+  auto default_allocator_factory = [](int) {
+    auto memory_info = onnxruntime::make_unique<OrtMemoryInfo>(RKNPU, OrtAllocatorType::OrtDeviceAllocator);
+    return onnxruntime::make_unique<CPUAllocator>(std::move(memory_info));
+  };
+  DeviceAllocatorRegistrationInfo default_memory_info{
+      OrtMemTypeDefault,
+      std::move(default_allocator_factory),
+      std::numeric_limits<size_t>::max()};
+  InsertAllocator(CreateAllocator(default_memory_info));
+
+  auto cpu_allocator_factory = [](int) {
+    auto memory_info = onnxruntime::make_unique<OrtMemoryInfo>(
+        RKNPU, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeCPUOutput);
+    return onnxruntime::make_unique<CPUAllocator>(std::move(memory_info));
+  };
+  DeviceAllocatorRegistrationInfo cpu_memory_info{
+      OrtMemTypeCPUOutput,
+      std::move(cpu_allocator_factory),
+      std::numeric_limits<size_t>::max()};
+  InsertAllocator(CreateAllocator(cpu_memory_info));
+}
+
+RknpuExecutionProvider::~RknpuExecutionProvider() {}
+
+std::vector<std::vector<int>> RknpuExecutionProvider::GetSupportedNodes(
+    const ONNX_NAMESPACE::ModelProto& model_proto) const {
+  rknpu::OnnxConverter converter;
+  return converter.GetSupportedNodes(model_proto);
+}
+
+std::vector<std::unique_ptr<ComputeCapability>>
+RknpuExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer,
+                                      const std::vector<const KernelRegistry*>& /*kernel_registries*/) const {
+  // Find inputs, initializers and outputs for each supported subgraph
+  std::vector<std::unique_ptr<ComputeCapability>> result;
+
+  // Handle If and Loop operators
+  if (graph_viewer.IsSubgraph()) {
+    return result;
+  }
+
+  // Need access to model_path_
+  for (const auto& tensor : graph_viewer.GetAllInitializedTensors()) {
+    if (tensor.second->has_data_location() &&
+        tensor.second->data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) {
+      LOGS_DEFAULT(WARNING) << "Rknpu: Initializers with external data"
+                               " location are not currently supported";
+      return result;
+    }
+  }
+
+  // This method is based on that of TRT EP
+  // Construct modelproto from graph_viewer
+  onnxruntime::Model model(graph_viewer.Name(), true, ModelMetaData(),
+                           PathString(),
+                           IOnnxRuntimeOpSchemaRegistryList(),
+                           graph_viewer.DomainToVersionMap(),
+                           std::vector<ONNX_NAMESPACE::FunctionProto>(),
+                           *GetLogger());
+  onnxruntime::Graph& graph_build = model.MainGraph();
+  const std::vector<NodeIndex>& node_index =
+      graph_viewer.GetNodesInTopologicalOrder();
+  std::set<NodeArg*> all_node_inputs;
+  for (const auto& node : graph_viewer.Nodes()) {
+    std::vector<onnxruntime::NodeArg*> inputs, outputs;
+    for (const auto input : node.InputDefs()) {
+      auto& n_input = graph_build.GetOrCreateNodeArg(
+          input->Name(), input->TypeAsProto());
+      inputs.push_back(&n_input);
+      all_node_inputs.insert(&n_input);
+    }
+    for (const auto output : node.OutputDefs()) {
+      auto& n_output = graph_build.GetOrCreateNodeArg(
+          output->Name(), output->TypeAsProto());
+      outputs.push_back(&n_output);
+    }
+    graph_build.AddNode(node.Name(), node.OpType(), node.Description(),
+                        inputs, outputs, &node.GetAttributes(), node.Domain());
+  }
+  const auto graph_outputs = graph_viewer.GetOutputs();
+  // Add initializer to graph_viewer
+  const auto& init_tensors = graph_viewer.GetAllInitializedTensors();
+  for (const auto& tensor : init_tensors) {
+    graph_build.AddInitializedTensor(*(tensor.second));
+  }
+
+  ORT_ENFORCE(graph_build.Resolve().IsOK());
+  ONNX_NAMESPACE::ModelProto model_proto = model.ToProto();
+  model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
+
+  const auto supported_nodes_vector = GetSupportedNodes(model_proto);
+
+  int counter = 0;
+
+  for (const auto& group : supported_nodes_vector) {
+    if (!group.empty()) {
+      std::unordered_set<size_t> node_set;
+      node_set.reserve(group.size());
+      for (const auto& index : group) {
+        node_set.insert(node_index[index]);
+      }
+      std::unique_ptr<IndexedSubGraph> sub_graph =
+          onnxruntime::make_unique<IndexedSubGraph>();
+      // Find inputs and outputs of the subgraph
+      std::unordered_map<const NodeArg*, int>
+          fused_inputs, fused_outputs, fused_outputs_to_add;
+      std::unordered_set<const NodeArg*> erased;
+      int input_order = 0;
+      int output_order = 0;
+
+      for (const auto& index : group) {
+        sub_graph->nodes.push_back(node_index[index]);
+        const auto& node = graph_viewer.GetNode(node_index[index]);
+
+        for (const auto& input : node->InputDefs()) {
+          const auto& it = fused_outputs.find(input);
+
+          if (it != fused_outputs.end()) {
+            fused_outputs.erase(it);
+            erased.insert(input);
+          }
+          // only when input is neither in output list nor erased list, add the
+          // input to input list
+          else if (erased.find(input) == erased.end()) {
+            fused_inputs[input] = input_order++;
+          }
+        }
+
+        // For output searching, there is a special case:
+        // If node's OutputEdges are more than its outputs, meaning certain
+        // output is used more than once,
+        // if the output is connected to nodes that don't belong to the
+        // subgraph, the output need to be added to the output list
+        if (node->GetOutputEdgesCount() > node->OutputDefs().size()) {
+          for (auto it = node->OutputEdgesBegin(),
+                    end = node->OutputEdgesEnd();
+               it != end; ++it) {
+            const auto& node_idx = it->GetNode().Index();
+            const auto& output = (it->GetNode()).InputDefs()[it->GetDstArgIndex()];
+
+            if (node_set.find(node_idx) != node_set.end()) {
+              const auto& iter = fused_inputs.find(output);
+
+              if (iter != fused_inputs.end()) {
+                fused_inputs.erase(iter);
+                erased.insert(output);
+              } else if (erased.find(output) == erased.end()) {
+                fused_outputs[output] = output_order++;
+              }
+            } else {
+              fused_outputs_to_add[output] = output_order++;
+            }
+          }
+        } else {
+          for (const auto& output : node->OutputDefs()) {
+            const auto& it = fused_inputs.find(output);
+
+            if (it != fused_inputs.end()) {
+              fused_inputs.erase(it);
+              erased.insert(output);
+            }
+            // only when output is neither in input list nor erased list,
+            // add the output to output list
+            else if (erased.find(output) == erased.end()) {
+              fused_outputs[output] = output_order++;
+            }
+          }
+        }
+      }
+
+      fused_outputs.insert(
+          fused_outputs_to_add.begin(), fused_outputs_to_add.end());
+
+      // Sort inputs and outputs by the order they were added
+      std::map<int, const NodeArg*> inputs, outputs;
+
+      for (auto it = fused_inputs.begin(),
+                end = fused_inputs.end();
+           it != end; ++it) {
+        inputs.insert(std::pair<int, const NodeArg*>(it->second, it->first));
+      }
+
+      for (auto it = fused_outputs.begin(),
+                end = fused_outputs.end();
+           it != end; ++it) {
+        for (const auto& x : all_node_inputs) {
+          if (x->Name() == it->first->Name()) {
+            outputs.insert(
+                std::pair<int, const NodeArg*>(it->second, it->first));
+            break;
+          }
+        }
+        if (std::find(graph_outputs.begin(),
+                      graph_outputs.end(), it->first) != graph_outputs.end()) {
+          outputs.insert(std::pair<int, const NodeArg*>(it->second, it->first));
+        }
+      }
+
+      // Assign inputs and outputs to subgraph's meta_def
+      auto meta_def =
+          onnxruntime::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>();
+      meta_def->name = "RKNPU_" + std::to_string(counter++);
+      meta_def->domain = kMSDomain;
+
+      for (const auto& input : inputs) {
+        meta_def->inputs.push_back(input.second->Name());
+      }
+
+      for (const auto& output : outputs) {
+        meta_def->outputs.push_back(output.second->Name());
+      }
+
+      meta_def->since_version = 1;
+      sub_graph->SetMetaDef(meta_def);
+
+      result.push_back(
+          onnxruntime::make_unique<ComputeCapability>(std::move(sub_graph)));
+    }
+  }
+
+  return result;
+}
+
+common::Status RknpuExecutionProvider::Compile(
+    const std::vector<onnxruntime::Node*>& fused_nodes,
+    std::vector<NodeComputeInfo>& node_compute_funcs) {
+  for (const auto* fused_node : fused_nodes) {
+    // Reconstruct graph proto from fused node's function body
+    const auto* func_body = fused_node->GetFunctionBody();
+    if (!func_body) {
+      return common::Status(common::ONNXRUNTIME,
+                            common::INVALID_ARGUMENT, "Function body is empty");
+    }
+    const Graph& graph_body = func_body->Body();
+    onnxruntime::Model model(graph_body.Name(), true, ModelMetaData(),
+                             PathString(),
+                             IOnnxRuntimeOpSchemaRegistryList(),
+                             graph_body.DomainToVersionMap(),
+                             std::vector<ONNX_NAMESPACE::FunctionProto>(),
+                             *GetLogger());
+    ONNX_NAMESPACE::ModelProto model_proto = model.ToProto();
+    *(model_proto.mutable_graph()) = graph_body.ToGraphProto();
+    model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
+
+    // Build map from input name to its index in input definitions
+    std::unordered_map<std::string, int> input_map;
+    const auto& input_defs = fused_node->InputDefs();
+    input_map.reserve(input_defs.size());
+    for (int i = 0, end = input_defs.size(); i < end; ++i) {
+      input_map[input_defs[i]->Name()] = i;
+    }
+
+    // Build map from output name to its index in output definitions
+    std::unordered_map<std::string, int> output_map;
+    const auto& output_defs = fused_node->OutputDefs();
+    output_map.reserve(output_defs.size());
+    for (int i = 0, end = output_defs.size(); i < end; ++i) {
+      output_map[output_defs[i]->Name()] = i;
+    }
+
+    model_proto_[fused_node->Name()] = model_proto;
+    input_info_[fused_node->Name()] = input_map;
+    output_info_[fused_node->Name()] = output_map;
+
+    NodeComputeInfo compute_info;
+    compute_info.create_state_func = [&](ComputeContext* context,
+                                         FunctionState* state) {
+      std::unique_ptr<RknpuFuncState> p =
+          onnxruntime::make_unique<RknpuFuncState>();
+      rk::nn::Graph* graph = new rk::nn::Graph();
+      *p = {"", std::unique_ptr<rk::nn::Exection>(new rk::nn::Exection(graph)),
+            model_proto_[context->node_name], input_info_[context->node_name],
+            output_info_[context->node_name],
+            std::vector<int>{}, std::vector<int>{}};
+      *state = p.release();
+      return 0;
+    };
+
+    compute_info.release_state_func = [](FunctionState state) {
+      if (state) {
+        RknpuFuncState* p = static_cast<RknpuFuncState*>(state);
+        rk::nn::Graph* graph = p->exector->GetGraph();
+        delete graph;
+        delete p;
+      }
+    };
+
+    compute_info.compute_func = [](FunctionState state,
+                                   const OrtCustomOpApi* api,
+                                   OrtKernelContext* context) {
+      Ort::CustomOpApi ort{*api};
+      RknpuFuncState* rk_state = reinterpret_cast<RknpuFuncState*>(state);
+      const size_t n_inputs = ort.KernelContext_GetInputCount(context);
+      const size_t n_outputs = ort.KernelContext_GetOutputCount(context);
+
+      std::string input_shape;
+      input_shape.reserve(4 * sizeof(int64_t) * n_inputs + n_inputs);
+      for (size_t i = 0; i < n_inputs; i++) {
+        const OrtValue* input_tensor = ort.KernelContext_GetInput(context, i);
+        auto tensor_info = ort.GetTensorTypeAndShape(input_tensor);
+        auto tensor_shape = ort.GetTensorShape(tensor_info);
+        ort.ReleaseTensorTypeAndShapeInfo(tensor_info);
+
+        const auto ndim = tensor_shape.size();
+        input_shape.append(reinterpret_cast<const char*>(&ndim), sizeof(ndim));
+        input_shape.append(reinterpret_cast<const char*>(tensor_shape.data()),
+                           ndim * sizeof(int64_t));
+      }
+
+      bool rebuild = false;
+      std::vector<const void*> input_bufs(n_inputs);
+      if (rk_state->uniq_input_shape == "") {
+        auto graph_proto = rk_state->model_proto.mutable_graph();
+        for (size_t i = 0; i < n_inputs; i++) {
+          const OrtValue* input_tensor = ort.KernelContext_GetInput(context, i);
+          input_bufs[i] = const_cast<const void*>(
+              ort.GetTensorData<void>(input_tensor));
+          auto tensor_info = ort.GetTensorTypeAndShape(input_tensor);
+          auto tensor_shape = ort.GetTensorShape(tensor_info);
+          ort.ReleaseTensorTypeAndShapeInfo(tensor_info);
+
+          auto g_in_shape =
+              graph_proto->mutable_input((int)i)->mutable_type()->mutable_tensor_type()->mutable_shape();
+          g_in_shape->clear_dim();
+          for (size_t dim = 0; dim < tensor_shape.size(); dim++) {
+            g_in_shape->add_dim()->set_dim_value(tensor_shape[dim]);
+          }
+        }
+        rk_state->uniq_input_shape = input_shape;
+        rebuild = true;
+      } else if (rk_state->uniq_input_shape != input_shape) {
+        // TODO
+        throw std::invalid_argument(
+            "The input_shape is not match"
+            " the rk_state->uniq_input_shape!");
+      } else {
+        LOGS_DEFAULT(INFO) << "input_shape equal to rk_state->uniq_input_shape,"
+                              " skip rebuild!";
+      }
+
+      rk::nn::Graph* graph = rk_state->exector->GetGraph();
+      if (rebuild) {
+        rknpu::OnnxConverter converter;
+        converter.Convert(rk_state->model_proto, graph, input_bufs, rk_state->input_map);
+
+        rk_state->exector->Build();
+
+        auto input_map = rk_state->input_map;
+        auto output_map = rk_state->output_map;
+        for (auto it = output_map.begin(); it != output_map.end(); it++) {
+          if (converter.m(it->first) != it->first)
+            output_map[converter.m(it->first)] = output_map[it->first];
+        }
+
+        int n_inputs = graph->GetInputs().size();
+        rk_state->input_indexes.resize(n_inputs);
+        for (int i = 0; i < n_inputs; ++i) {
+          const auto input = graph->GetInputs()[i];
+          const std::string& name = input->GetName();
+          auto iter = input_map.find(name);
+          if (iter != input_map.end()) {
+            rk_state->input_indexes[i] = iter->second;
+          }
+        }
+
+        int n_outputs = graph->GetOutputs().size();
+        rk_state->output_indexes.resize(n_outputs);
+        for (int i = 0; i < n_outputs; ++i) {
+          const auto output = graph->GetOutputs()[i];
+          const std::string& name = output->GetName();
+          auto iter = output_map.find(name);
+          if (iter != output_map.end()) {
+            rk_state->output_indexes[i] = iter->second;
+          }
+        }
+      }
+
+      ORT_ENFORCE(graph->GetInputs().size() <= n_inputs,
+                  "Inconsistent input sizes");
+      ORT_ENFORCE(graph->GetOutputs().size() == n_outputs,
+                  "Inconsistent output sizes");
+
+      std::vector<rk::nn::InputInfo> inputs;
+      inputs.resize(graph->GetInputs().size());
+      for (size_t i = 0; i < graph->GetInputs().size(); i++) {
+        const OrtValue* input_tensor =
+            ort.KernelContext_GetInput(context, rk_state->input_indexes[i]);
+        float* input_buf =
+            const_cast<float*>(ort.GetTensorData<float>(input_tensor));
+
+        const auto tensor_info = ort.GetTensorTypeAndShape(input_tensor);
+        const auto& tensor_shape = ort.GetTensorShape(tensor_info);
+
+        auto type = graph->GetInputs()[i]->GetPrecision();
+        size_t type_size = 4;
+        switch (type) {
+          case rk::nn::PrecisionType::FLOAT32:
+            type_size = 4;
+            break;
+          case rk::nn::PrecisionType::UINT8:
+            type_size = 1;
+            break;
+          case rk::nn::PrecisionType::INT32:
+            type_size = 4;
+            break;
+          case rk::nn::PrecisionType::INT64:
+            type_size = 8;
+            break;
+          default:
+            // TODO
+            throw std::invalid_argument(
+                "compute_func: unknow input data type!");
+            break;
+        }
+
+        inputs[i].index = i;
+        inputs[i].buf = (void*)input_buf;
+        inputs[i].size = std::accumulate(tensor_shape.begin(),
+                                         tensor_shape.end(), 1,
+                                         std::multiplies<int64_t>()) *
+                         type_size;
+        inputs[i].pass_through = false;
+        inputs[i].type = type;
+        inputs[i].layout = rk::nn::DataLayoutType::NCHW;
+      }
+
+      std::vector<rk::nn::OutputInfo> outputs;
+      outputs.resize(n_outputs);
+      for (size_t i = 0; i < n_outputs; i++) {
+        const auto output = graph->GetOutputs()[i];
+        const auto output_shape = output->GetDims();
+        std::vector<int64_t>
+            int64_output_shape(output_shape.begin(), output_shape.end());
+        const auto* output_tensor = ort.KernelContext_GetOutput(
+            context, rk_state->output_indexes[i],
+            int64_output_shape.data(),
+            int64_output_shape.size());
+        float* output_buf =
+            const_cast<float*>(ort.GetTensorData<float>(output_tensor));
+
+        const auto type = output->GetPrecision();
+        size_t type_size = 4;
+        switch (type) {
+          case rk::nn::PrecisionType::FLOAT32:
+            type_size = 4;
+            break;
+          case rk::nn::PrecisionType::UINT8:
+            type_size = 1;
+            break;
+          case rk::nn::PrecisionType::INT32:
+            type_size = 4;
+            break;
+          case rk::nn::PrecisionType::INT64:
+            type_size = 8;
+            break;
+          default:
+            // TODO
+            throw std::invalid_argument(
+                "compute_func: unknow output data type!");
+            break;
+        }
+
+        outputs[i].index = i;
+        outputs[i].buf = (void*)output_buf;
+        outputs[i].size = accumulate(output_shape.begin(),
+                                     output_shape.end(), 1,
+                                     std::multiplies<uint32_t>()) *
+                          type_size;
+        outputs[i].type = type;
+        outputs[i].layout = rk::nn::DataLayoutType::NCHW;
+        outputs[i].want_float = false;
+      }
+
+      rk_state->exector->SetInputs(inputs);
+      rk_state->exector->Run();
+      rk_state->exector->GetOutputs(outputs);
+
+      return Status::OK();
+    };
+
+    node_compute_funcs.push_back(compute_info);
+  }
+  return Status::OK();
+}
+
+ONNX_OPERATOR_KERNEL_EX(
+    MemcpyFromHost,
+    kOnnxDomain,
+    1,
+    kRknpuExecutionProvider,
+    KernelDefBuilder()
+        .InputMemoryType<OrtMemTypeCPUInput>(0)
+        .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()),
+    Memcpy);
+
+ONNX_OPERATOR_KERNEL_EX(
+    MemcpyToHost,
+    kOnnxDomain,
+    1,
+    kRknpuExecutionProvider,
+    KernelDefBuilder()
+        .OutputMemoryType<OrtMemTypeCPUOutput>(0)
+        .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()),
+    Memcpy);
+
+class ONNX_OPERATOR_KERNEL_CLASS_NAME(
+    kRknpuExecutionProvider, kOnnxDomain, 1, MemcpyFromHost);
+class ONNX_OPERATOR_KERNEL_CLASS_NAME(
+    kRknpuExecutionProvider, kOnnxDomain, 1, MemcpyToHost);
+
+static void RegisterRknpuKernels(KernelRegistry& kernel_registry) {
+  static const BuildKernelCreateInfoFn function_table[] = {
+      BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kRknpuExecutionProvider, kOnnxDomain, 1, MemcpyFromHost)>,
+      BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kRknpuExecutionProvider, kOnnxDomain, 1, MemcpyToHost)>,
+  };
+
+  for (auto& function_table_entry : function_table) {
+    kernel_registry.Register(function_table_entry());
+  }
+}
+
+std::shared_ptr<KernelRegistry> GetRknpuKernelRegistry() {
+  std::shared_ptr<KernelRegistry> kernel_registry =
+      std::make_shared<KernelRegistry>();
+  RegisterRknpuKernels(*kernel_registry);
+
+  return kernel_registry;
+}
+
+std::shared_ptr<KernelRegistry>
+RknpuExecutionProvider::GetKernelRegistry() const {
+  static std::shared_ptr<KernelRegistry> kernel_registry =
+      onnxruntime::GetRknpuKernelRegistry();
+  return kernel_registry;
+}
+
+}  // namespace onnxruntime

onnxruntime/core/providers/rknpu/rknpu_execution_provider.h

@@ -0,0 +1,34 @@
+// Copyright 2020 rock-chips.com Inc.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "core/framework/execution_provider.h"
+#include "core/graph/onnx_protobuf.h"
+
+namespace onnxruntime {
+
+class RknpuExecutionProvider : public IExecutionProvider {
+ public:
+  RknpuExecutionProvider();
+  virtual ~RknpuExecutionProvider();
+
+  std::vector<std::unique_ptr<ComputeCapability>>
+  GetCapability(const onnxruntime::GraphViewer& graph,
+                const std::vector<const KernelRegistry*>& /*kernel_registries*/) const override;
+  common::Status Compile(const std::vector<onnxruntime::Node*>& fused_nodes,
+                         std::vector<NodeComputeInfo>& node_compute_funcs) override;
+  std::shared_ptr<KernelRegistry> GetKernelRegistry() const override;
+
+ private:
+  std::unordered_map<std::string, ONNX_NAMESPACE::ModelProto> model_proto_;
+  std::unordered_map<std::string, std::unordered_map<std::string, int>> input_info_;
+  std::unordered_map<std::string, std::unordered_map<std::string, int>> output_info_;
+  std::vector<std::vector<int>> GetSupportedNodes(
+      const ONNX_NAMESPACE::ModelProto& model_proto) const;
+};
+}  // namespace onnxruntime

onnxruntime/core/providers/rknpu/rknpu_provider_factory.cc

@@ -0,0 +1,33 @@
+// Copyright 2020 rock-chips.com Inc.
+
+#include "core/providers/rknpu/rknpu_provider_factory.h"
+#include "rknpu_execution_provider.h"
+#include "core/session/abi_session_options_impl.h"
+
+using namespace onnxruntime;
+
+namespace onnxruntime {
+
+struct RknpuProviderFactory : IExecutionProviderFactory {
+  RknpuProviderFactory() {}
+  ~RknpuProviderFactory() override {}
+
+  std::unique_ptr<IExecutionProvider> CreateProvider() override;
+};
+
+std::unique_ptr<IExecutionProvider> RknpuProviderFactory::CreateProvider() {
+  return onnxruntime::make_unique<RknpuExecutionProvider>();
+}
+
+std::shared_ptr<IExecutionProviderFactory>
+CreateExecutionProviderFactory_Rknpu() {
+  return std::make_shared<onnxruntime::RknpuProviderFactory>();
+}
+}  // namespace onnxruntime
+
+ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Rknpu,
+                    _In_ OrtSessionOptions* options) {
+  options->provider_factories.push_back(
+      onnxruntime::CreateExecutionProviderFactory_Rknpu());
+  return nullptr;
+}

onnxruntime/core/providers/rknpu/shaper.cc

@@ -0,0 +1,561 @@
+#include "shaper.h"
+#include <numeric>
+#include <algorithm>
+#include <functional>
+
+namespace onnxruntime {
+namespace rknpu {
+
+template <typename T>
+T Product(const std::vector<T>& v) {
+  return static_cast<T>(
+      accumulate(v.begin(), v.end(), 1, std::multiplies<T>()));
+}
+
+Shaper::len_t Shaper::total(const Shape& shape) {
+  return Product(shape);
+}
+
+/**
+ *  strides: [stride_y, stride_x]
+ *  paddings: [top, left, bottom, right]
+ */
+void Shaper::Conv(const std::string& input,
+                  const std::string& weight,
+                  const std::vector<int32_t>& paddings,
+                  const std::vector<int32_t>& strides,
+                  const std::string& auto_pad,
+                  const std::string& output) {
+  Shaper::Conv(input, strides[1], strides[0], 1, 1, paddings[1],
+               paddings[3], paddings[0], paddings[2], weight, auto_pad,
+               output);
+}
+
+void Shaper::Conv(const std::string& input,
+                  const std::vector<int32_t>& paddings,
+                  const std::vector<int32_t>& strides,
+                  const std::vector<int32_t>& dilations,
+                  const std::string& weight,
+                  const std::string& auto_pad,
+                  const std::string& output) {
+  Shaper::Conv(input, strides[1], strides[0], dilations[1], dilations[0],
+               paddings[1], paddings[3], paddings[0], paddings[2],
+               weight, auto_pad, output);
+}
+void Shaper::Conv(const std::string& input,
+                  const std::string& weight,
+                  const int32_t padding_left,
+                  const int32_t padding_right,
+                  const int32_t padding_top,
+                  const int32_t padding_bottom,
+                  const int32_t stride_x,
+                  const int32_t stride_y,
+                  const std::string& auto_pad,
+                  const std::string& output) {
+  Shaper::Conv(input, stride_x, stride_y, 1, 1, padding_left, padding_right,
+               padding_top, padding_bottom, weight, auto_pad, output);
+}
+
+void Shaper::Conv(const std::string& input,
+                  const int32_t strideX,
+                  const int32_t strideY,
+                  const int32_t dilationX,
+                  const int32_t dilationY,
+                  const int32_t paddingLeft,
+                  const int32_t paddingRight,
+                  const int32_t paddingTop,
+                  const int32_t paddingBottom,
+                  const std::string& weight,
+                  const std::string& auto_pad,
+                  const std::string& output) {
+  Shape weightDimen =
+      shape_map_.at(weight);  // num_output, height, width, num_input
+  // NCHW
+  Shape inputDimen = shape_map_.at(input);
+
+  if (auto_pad == "VALID") {
+    Shape outputDimen{inputDimen[0],
+                      weightDimen[0],
+                      (inputDimen[2] - ((weightDimen[2] - 1) * dilationY + 1)) / strideY + 1,
+                      (inputDimen[3] - ((weightDimen[3] - 1) * dilationX + 1)) / strideX + 1};
+    shape_map_[output] = outputDimen;
+  } else if (auto_pad == "SAME_UPPER" || auto_pad == "SAME_LOWER") {
+    int32_t legacy_target_size_Y = (inputDimen[2] + strideY - 1) / strideY;
+    int32_t legacy_target_size_X = (inputDimen[3] + strideX - 1) / strideX;
+    int32_t pad_needed_Y =
+        (legacy_target_size_Y - 1) * strideY + weightDimen[2] - inputDimen[2];
+    int32_t pad_needed_X =
+        (legacy_target_size_X - 1) * strideX + weightDimen[3] - inputDimen[3];
+    Shape outputDimen{inputDimen[0],
+                      weightDimen[0],
+                      (inputDimen[2] - ((weightDimen[2] - 1) * dilationY + 1) + pad_needed_Y) / strideY + 1,
+                      (inputDimen[3] - ((weightDimen[3] - 1) * dilationX + 1) + pad_needed_X) / strideX + 1};
+    shape_map_[output] = outputDimen;
+  } else {  // default: NOTSET
+    Shape outputDimen{inputDimen[0],
+                      weightDimen[0],
+                      (inputDimen[2] - ((weightDimen[2] - 1) * dilationY + 1) + paddingTop + paddingBottom) / strideY + 1,
+                      (inputDimen[3] - ((weightDimen[3] - 1) * dilationX + 1) + paddingLeft + paddingRight) / strideX + 1};
+    shape_map_[output] = outputDimen;
+  }
+}
+
+void Shaper::DepthwiseConv(const std::string& input,
+                           const std::vector<int32_t>& paddings,
+                           const std::vector<int32_t>& strides,
+                           const std::vector<int32_t>& dilations,
+                           const std::string& weight,
+                           const std::string& output) {
+  Shaper::DepthwiseConv(input, strides[1], strides[0], dilations[1],
+                        dilations[0], paddings[1], paddings[3], paddings[0],
+                        paddings[2], weight, output);
+}
+
+void Shaper::DepthwiseConv(const std::string& input,
+                           const std::string& weight,
+                           const int32_t padding_left,
+                           const int32_t padding_right,
+                           const int32_t padding_top,
+                           const int32_t padding_bottom,
+                           const int32_t stride_x,
+                           const int32_t stride_y,
+                           const std::string& output) {
+  DepthwiseConv(input, stride_x, stride_y, 1, 1, padding_left,
+                padding_right, padding_top, padding_bottom, weight,
+                output);
+}
+
+void Shaper::DepthwiseConv(const std::string& input,
+                           const int32_t strideX,
+                           const int32_t strideY,
+                           const int32_t dilationX,
+                           const int32_t dilationY,
+                           const int32_t paddingLeft,
+                           const int32_t paddingRight,
+                           const int32_t paddingTop,
+                           const int32_t paddingBottom,
+                           const std::string& weight,
+                           const std::string& output) {
+  Shape weightDimen =
+      shape_map_.at(weight);  // 1, height, width, num_output
+  // NCHW
+  Shape inputDimen = shape_map_.at(input);
+  Shape outputDimen{
+      inputDimen[0],
+      weightDimen[0],
+      (inputDimen[2] - ((weightDimen[2] - 1) * dilationY + 1) +
+       paddingTop + paddingBottom) /
+              strideY +
+          1,
+      (inputDimen[3] - ((weightDimen[3] - 1) * dilationX + 1) +
+       paddingLeft + paddingRight) /
+              strideX +
+          1,
+  };
+  shape_map_[output] = outputDimen;
+}
+
+void Shaper::DepthwiseConv(const std::string& input,
+                           const std::string& weight,
+                           const std::vector<int32_t>& paddings,
+                           const std::vector<int32_t>& strides,
+                           const std::string& output) {
+  DepthwiseConv(input, weight, paddings[1], paddings[3],
+                paddings[0], paddings[2], strides[1], strides[0],
+                output);
+}
+
+void Shaper::Slice(const std::string& input,
+                   const std::vector<int32_t>& starts,
+                   const std::vector<int32_t>& ends,
+                   const std::vector<int32_t>& axes,
+                   const std::vector<int32_t>& steps,
+                   const std::string& output) {
+  std::vector<uint32_t> inputDimen = shape_map_.at(input);
+  std::vector<uint32_t> outputDimen = inputDimen;
+  for (size_t i = 0; i < axes.size(); i++) {
+    int32_t axis =
+        (axes[i] < 0) ? (axes[i] + (int32_t)inputDimen.size()) : axes[i];
+    int32_t dim = outputDimen[axis];
+    if (dim > 0) {
+      int32_t start = starts[i] < 0 ? (starts[i] + dim) : starts[i];
+      int32_t end = ends[i] < 0 ? (ends[i] + dim) : ends[i];
+      start = std::max(start, 0);
+      end = std::max(end, 0);
+      end = std::min(end, dim);
+      outputDimen[axis] = end - start;
+    }
+  }
+
+  shape_map_[output] = outputDimen;
+}
+
+void Shaper::StridedSlice(const std::string& input,
+                          const std::vector<int32_t>& starts,
+                          const std::vector<int32_t>& ends,
+                          const std::vector<int32_t>& strides,
+                          const int32_t beginMask,
+                          const int32_t endMask,
+                          const int32_t shrinkAxisMask,
+                          const std::string& output) {
+  // NHWC
+  std::vector<uint32_t> inputDimen = shape_map_.at(input);
+  std::vector<uint32_t> outputDimen;
+  for (size_t i = 0; i < inputDimen.size(); i++) {
+    if (shrinkAxisMask & (1 << i)) {
+      continue;
+    }
+    int32_t start = starts[i], end = ends[i], stride = strides[i];
+    if (beginMask & (1 << i)) {
+      start = 0;
+    }
+    if (endMask & (1 << i)) {
+      end = inputDimen[i];
+    }
+    outputDimen.emplace_back((end - start) / stride);
+  }
+  shape_map_[output] = outputDimen;
+}
+
+void Shaper::Gather(const std::string& input,
+                    const std::string& indices,
+                    const int32_t axis,
+                    const std::string& output) {
+  auto inputDimen = shape_map_.at(input);
+  auto indicesDimen = shape_map_.at(indices);
+  int32_t input_rank = indicesDimen.size();
+  int32_t axis_new = (axis < 0) ? (axis + input_rank) : axis;
+
+  std::vector<uint32_t> outputDimen;
+  outputDimen.reserve(input_rank - 1 + indicesDimen.size());
+
+  // replace the dimension for axis with the shape from the indices
+  for (int32_t i = 0; i < axis_new; ++i)
+    outputDimen.push_back(inputDimen[i]);
+
+  for (const auto dim : indicesDimen)
+    outputDimen.push_back(dim);
+
+  for (int32_t i = axis_new + 1; i < input_rank; ++i)
+    outputDimen.push_back(inputDimen[i]);
+
+  shape_map_[output] = outputDimen;
+}
+
+void Shaper::Pool(const std::string& input,
+                  const int32_t padding_left,
+                  const int32_t padding_right,
+                  const int32_t padding_top,
+                  const int32_t padding_bottom,
+                  const int32_t stride_x,
+                  const int32_t stride_y,
+                  const int32_t width,
+                  const int32_t height,
+                  const std::string& output) {
+  auto inputDimen = shape_map_.at(input);
+
+  // NCHW
+  Shape outputDimen;
+  if (height == -1 && width == -1) {
+    outputDimen = {inputDimen[0], inputDimen[1], 1, 1};
+  } else {
+    outputDimen = {
+        inputDimen[0],
+        inputDimen[1],
+        (inputDimen[2] - height + padding_top + padding_bottom) / stride_y +
+            1,
+        (inputDimen[3] - width + padding_left + padding_right) / stride_x +
+            1};
+  }
+  shape_map_[output] = outputDimen;
+}
+
+/**
+ *  kernel_shape: [height, width]
+ *  strides: [stride_y, stride_x]
+ *  pads: [top, left, bottom, right]
+ */
+void Shaper::Pool(const std::string& input,
+                  const std::vector<int32_t>& kernel_shape,
+                  const std::vector<int32_t>& pads,
+                  const std::vector<int32_t>& strides,
+                  const std::string& output) {
+  Shaper::Pool(input, pads[1], pads[3], pads[0], pads[2], strides[1],
+               strides[0], kernel_shape[1], kernel_shape[0], output);
+}
+
+void Shaper::Softmax(const std::string& input,
+                     const std::string& output) {
+  shape_map_[output] = shape_map_.at(input);
+}
+
+void Shaper::Relu(const std::string& input,
+                  const std::string& output) {
+  shape_map_[output] = shape_map_.at(input);
+}
+
+void Shaper::Concat(const std::vector<std::string>& inputs,
+                    const int32_t axis,
+                    const std::string& output) {
+  std::vector<Shape> dimens;
+  for (const auto& input : inputs) {
+    auto& dimen = shape_map_.at(input);
+    if (!dimens.empty()) {
+      for (size_t i = 0; i < dimens[0].size(); i++) {
+        if ((int32_t)i == axis) continue;
+        if (dimen[i] != dimens[0][i]) {
+          throw std::invalid_argument("Wrong input for concat");
+        }
+      }
+    }
+    dimens.push_back(shape_map_.at(input));
+  }
+
+  auto outputDimen = dimens[0];
+  for (size_t i = 1; i < dimens.size(); i++) {
+    outputDimen[axis] += dimens[i][axis];
+  }
+  shape_map_[output] = outputDimen;
+}
+
+void Shaper::LRN(const std::string& input,
+                 const std::string& output) {
+  shape_map_[output] = shape_map_.at(input);
+}
+
+void Shaper::FC(const std::string& input,
+                const std::string& weight,
+                const std::string& output) {
+  Shape weightDimen = shape_map_.at(weight);  // num_units, input_size
+  auto input_dimen = shape_map_.at(input);
+  Shape outputDimen{input_dimen[0], weightDimen[0]};
+  shape_map_[output] = outputDimen;
+}
+
+void Shaper::Eltwise(const std::string& input1,
+                     const std::string& input2,
+                     const std::string& output) {
+  auto shape1 = shape_map_.at(input1);
+  auto shape2 = shape_map_.at(input2);
+
+  // broadcasting support
+  auto max_shape = shape1.size() >= shape2.size() ? shape1 : shape2;
+  auto min_shape = shape1.size() < shape2.size() ? shape1 : shape2;
+  for (int i = (int)max_shape.size() - 1, j = (int)min_shape.size() - 1;
+       i >= 0 && j >= 0;
+       i--, j--) {
+    if (max_shape[i] < min_shape[j])
+      max_shape[i] = min_shape[j];
+  }
+
+  shape_map_[output] = max_shape;
+}
+
+void Shaper::Eltwise(const std::string& input1,
+                     const std::string& output) {
+  shape_map_[output] = shape_map_.at(input1);
+}
+
+void Shaper::BatchToSpace(const std::string& input,
+                          const std::vector<int32_t>& block_sizes,
+                          const std::string& output) {
+  auto input_dimen = shape_map_.at(input);
+  auto output_dimen = {input_dimen[0] / Product(block_sizes),
+                       input_dimen[1] * block_sizes[0],
+                       input_dimen[2] * block_sizes[1], input_dimen[3]};
+  shape_map_[output] = output_dimen;
+}
+
+void Shaper::SpaceToBatch(const std::string& input,
+                          const std::vector<int32_t>& block_sizes,
+                          const std::vector<int32_t>& pads,
+                          const std::string& output) {
+  auto input_dimen = shape_map_.at(input);
+  auto output_dimen = {input_dimen[0] * Product(block_sizes),
+                       (input_dimen[1] + pads[0] + pads[1]) / block_sizes[0],
+                       (input_dimen[2] + pads[2] + pads[3]) / block_sizes[1],
+                       input_dimen[3]};
+  shape_map_[output] = output_dimen;
+}
+
+void Shaper::BatchNorm(const std::string& input,
+                       const std::string& output) {
+  shape_map_[output] = shape_map_.at(input);
+}
+
+void Shaper::Reshape(const std::string& input,
+                     const std::vector<int32_t>& shape,
+                     const std::string& output) {
+  auto input_dimen = shape_map_.at(input);
+  int64_t input_size = std::accumulate(
+      input_dimen.begin(), input_dimen.end(), 1, std::multiplies<uint32_t>());
+  std::vector<int32_t> output_dimen(shape.size());
+
+  int64_t capacity = 1;
+  int unk_dim_idx = -1;
+  for (size_t i = 0; i < shape.size(); i++) {
+    if (shape[i] == -1) {
+      if (unk_dim_idx != -1)
+        throw std::invalid_argument(
+            "Only one input dimension of Attr(shape) can be unknown!");
+      unk_dim_idx = i;
+    } else if (shape[i] == 0) {
+      if (i >= input_dimen.size())
+        throw std::invalid_argument(
+            "The index of dimension to copy from input"
+            " shape must be less than the size of input shape!");
+    } else {
+      if (shape[i] < 0)
+        throw std::invalid_argument(
+            "Each input dimension of Attr(shape) must"
+            " not be negtive except one unknown dimension!");
+    }
+    int32_t output_dim_i = shape[i] ? shape[i] : input_dimen[i];
+    output_dimen[i] = output_dim_i;
+    capacity *= output_dim_i;
+  }
+
+  if (unk_dim_idx != -1) {
+    output_dimen[unk_dim_idx] = -input_size / capacity;
+    if ((output_dimen[unk_dim_idx] * capacity) != (-input_size))
+      throw std::invalid_argument("Invalid shape is given!");
+  } else {
+    if (capacity != input_size)
+      throw std::invalid_argument("Invalid shape is given!");
+  }
+
+  Shape final_dimen(shape.size());
+  for (size_t i = 0; i < shape.size(); i++) {
+    final_dimen[i] = (uint32_t)output_dimen[i];
+  }
+  shape_map_[output] = final_dimen;
+}
+
+void Shaper::Transpose(const std::string& input,
+                       const std::vector<int32_t>& perm,
+                       const std::string& output) {
+  auto input_dimen = shape_map_.at(input);
+
+  Shape outputDimen(perm.size());
+
+  for (size_t i = 0; i < perm.size(); i++) {
+    outputDimen[i] = input_dimen[perm[i]];
+  }
+
+  shape_map_[output] = outputDimen;
+}
+
+void Shaper::Squeeze(const std::string& input,
+                     const std::vector<int32_t>& axes,
+                     const std::string& output) {
+  std::vector<uint32_t> inputDimen = shape_map_.at(input);
+  size_t n_axes = axes.size();
+  int cnt_squeezed_dims = 0;
+  bool should_squeeze[9] = {false};
+
+  if (n_axes == 0) {
+    for (size_t idx = 0; idx < inputDimen.size(); ++idx) {
+      if (inputDimen[idx] == 1) {
+        should_squeeze[idx] = true;
+        ++cnt_squeezed_dims;
+      }
+    }
+  } else {
+    for (size_t idx = 0; idx < n_axes; ++idx) {
+      int32_t current =
+          axes[idx] < 0 ? axes[idx] + inputDimen.size() : axes[idx];
+      if (!(should_squeeze[current])) {
+        ++cnt_squeezed_dims;
+      }
+      should_squeeze[current] = true;
+    }
+  }
+
+  // Make output dimensions
+  std::vector<uint32_t> outputDimen(inputDimen.size() - cnt_squeezed_dims, 0);
+  for (size_t in_idx = 0, out_idx = 0; in_idx < inputDimen.size(); ++in_idx) {
+    if (!should_squeeze[in_idx]) {
+      outputDimen[out_idx++] = inputDimen[in_idx];
+    }
+  }
+
+  shape_map_[output] = outputDimen;
+}
+
+void Shaper::Unsqueeze(const std::string& input,
+                       const std::vector<int32_t>& axes,
+                       const std::string& output) {
+  std::vector<uint32_t> inputDimen = shape_map_.at(input);
+
+  int output_size = inputDimen.size() + axes.size();
+  int cur_output_size = inputDimen.size();
+  std::vector<uint32_t> outputDimen(output_size, 0);
+
+  for (int axis : axes) {
+    int cur = axis;
+
+    // Move old axis, and insert new axis.
+    for (int i = cur_output_size; i >= cur; --i) {
+      if (outputDimen[i] == 1) {
+        // Move axis
+        outputDimen[i + 1] = 1;
+        outputDimen[i] = 0;
+      }
+    }
+
+    outputDimen[cur] = 1;
+    // Add the output size.
+    cur_output_size++;
+  }
+
+  // Make output shape
+  for (int in_idx = 0, out_idx = 0; out_idx < output_size; ++out_idx) {
+    if (outputDimen[out_idx] == 0) {
+      outputDimen[out_idx] = inputDimen[in_idx++];
+    }
+  }
+
+  shape_map_[output] = outputDimen;
+}
+
+void Shaper::Affine(const std::string& input,
+                    const std::string& output) {
+  shape_map_[output] = shape_map_.at(input);
+}
+void Shaper::Affine(const std::string& input,
+                    const std::string& a,
+                    const std::string& b,
+                    const std::string& output) {
+  (void)a;
+  (void)b;
+  Shaper::Affine(input, output);
+}
+
+void Shaper::Identity(const std::string& input,
+                      const std::string& output) {
+  shape_map_[output] = shape_map_.at(input);
+}
+
+void Shaper::AddShape(const std::string& name,
+                      const Shape& shape) {
+  shape_map_[name] = shape;
+}
+
+size_t Shaper::GetSize(const std::string& name) {
+  return static_cast<size_t>(Product(shape_map_.at(name)));
+}
+
+void Shaper::Clear() {
+  shape_map_.clear();
+}
+
+std::ostream& operator<<(std::ostream& os,
+                         const Shaper& shaper) {
+  for (const auto& p : shaper.shape_map_) {
+    os << (p.first + ": ") /*<< p.second*/ << std::endl;
+  }
+  return os;
+}
+
+}  // namespace rknpu
+}  // namespace onnxruntime

onnxruntime/core/providers/rknpu/shaper.h

@@ -0,0 +1,183 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <map>
+
+#include <iostream>
+
+namespace onnxruntime {
+namespace rknpu {
+
+/**
+ * Help to caculate the output shape of nodes.
+ */
+class Shaper {
+ public:
+  using len_t = uint32_t;
+  using Shape = std::vector<len_t>;
+
+  static len_t total(const Shape& shape);
+
+  void Conv(const std::string& input,
+            const std::string& weight,
+            const std::vector<int32_t>& strides,
+            const std::vector<int32_t>& paddings,
+            const std::string& auto_pad,
+            const std::string& output);
+  void Conv(const std::string& input,
+            const std::vector<int32_t>& strides,
+            const std::vector<int32_t>& dilations,
+            const std::vector<int32_t>& paddings,
+            const std::string& weight,
+            const std::string& auto_pad,
+            const std::string& output);
+  void Conv(const std::string& input,
+            const std::string& weight,
+            const int32_t padding_left,
+            const int32_t padding_right,
+            const int32_t padding_top,
+            const int32_t padding_bottom,
+            const int32_t stride_x,
+            const int32_t stride_y,
+            const std::string& auto_pad,
+            const std::string& output);
+  void Conv(const std::string& input,
+            const int32_t strideX,
+            const int32_t strideY,
+            const int32_t dilationX,
+            const int32_t dilationY,
+            const int32_t paddingLeft,
+            const int32_t paddingRight,
+            const int32_t paddingTop,
+            const int32_t paddingBottom,
+            const std::string& weight,
+            const std::string& auto_pad,
+            const std::string& output);
+  void DepthwiseConv(const std::string& input,
+                     const std::vector<int32_t>& strides,
+                     const std::vector<int32_t>& dilations,
+                     const std::vector<int32_t>& paddings,
+                     const std::string& weight,
+                     const std::string& output);
+  void DepthwiseConv(const std::string& input,
+                     const std::string& weight,
+                     const int32_t padding_left,
+                     const int32_t padding_right,
+                     const int32_t padding_top,
+                     const int32_t padding_bottom,
+                     const int32_t stride_x,
+                     const int32_t stride_y,
+                     const std::string& output);
+  void DepthwiseConv(const std::string& input,
+                     const int32_t strideX,
+                     const int32_t strideY,
+                     const int32_t dilationX,
+                     const int32_t dilationY,
+                     const int32_t paddingLeft,
+                     const int32_t paddingRight,
+                     const int32_t paddingTop,
+                     const int32_t paddingBottom,
+                     const std::string& weight,
+                     const std::string& output);
+  void DepthwiseConv(const std::string& input,
+                     const std::string& weight,
+                     const std::vector<int32_t>& paddings,
+                     const std::vector<int32_t>& strides,
+                     const std::string& output);
+  void Slice(const std::string& input,
+             const std::vector<int32_t>& starts,
+             const std::vector<int32_t>& ends,
+             const std::vector<int32_t>& axes,
+             const std::vector<int32_t>& steps,
+             const std::string& output);
+  void StridedSlice(const std::string& input,
+                    const std::vector<int32_t>& starts,
+                    const std::vector<int32_t>& ends,
+                    const std::vector<int32_t>& strides,
+                    const int32_t beginMask,
+                    const int32_t endMask,
+                    const int32_t shrinkAxisMask,
+                    const std::string& output);
+  void Gather(const std::string& input,
+              const std::string& indices,
+              const int32_t axis,
+              const std::string& output);
+  void Pool(const std::string& input,
+            const int32_t padding_left,
+            const int32_t padding_right,
+            const int32_t padding_top,
+            const int32_t padding_bottom,
+            const int32_t stride_x,
+            const int32_t stride_y,
+            const int32_t width,
+            const int32_t height,
+            const std::string& output);
+  void Pool(const std::string& input,
+            const std::vector<int32_t>& kernel_shape,
+            const std::vector<int32_t>& pads,
+            const std::vector<int32_t>& strides,
+            const std::string& output);
+  void Softmax(const std::string& input,
+               const std::string& output);
+  void Relu(const std::string& input,
+            const std::string& output);
+  void Concat(const std::vector<std::string>& inputs,
+              const int32_t axis,
+              const std::string& output);
+  void LRN(const std::string& input,
+           const std::string& output);
+  void FC(const std::string& input,
+          const std::string& weight,
+          const std::string& output);
+  void Eltwise(const std::string& input1,
+               const std::string& input2,
+               const std::string& output);
+  void Eltwise(const std::string& input1,
+               const std::string& output);
+  void Affine(const std::string& input,
+              const std::string& output);
+  void Affine(const std::string& input,
+              const std::string& a,
+              const std::string& b,
+              const std::string& output);
+  void Identity(const std::string& input,
+                const std::string& output);
+  void BatchToSpace(const std::string& input,
+                    const std::vector<int32_t>& block_sizes,
+                    const std::string& output);
+  void SpaceToBatch(const std::string& input,
+                    const std::vector<int32_t>& block_sizes,
+                    const std::vector<int32_t>& pads,
+                    const std::string& output);
+  void BatchNorm(const std::string& input,
+                 const std::string& output);
+  void Reshape(const std::string& input,
+               const std::vector<int32_t>& shape,
+               const std::string& output);
+  void Transpose(const std::string& input,
+                 const std::vector<int32_t>& perm,
+                 const std::string& output);
+  void Squeeze(const std::string& input,
+               const std::vector<int32_t>& axes,
+               const std::string& output);
+  void Unsqueeze(const std::string& input,
+                 const std::vector<int32_t>& axes,
+                 const std::string& output);
+  void AddShape(const std::string& name,
+                const Shape& shape);
+  size_t GetSize(const std::string& name);
+  void Clear();
+
+  inline const Shape& operator[](const std::string& key) {
+    return shape_map_[key];
+  }
+  friend std::ostream& operator<<(std::ostream& os,
+                                  const Shaper& shaper);
+
+ private:
+  std::map<std::string, Shape> shape_map_;
+};
+
+}  // namespace rknpu
+}  // namespace onnxruntime

onnxruntime/core/providers/rknpu/symbols.txt

@@ -0,0 +1 @@
+OrtSessionOptionsAppendExecutionProvider_Rknpu

onnxruntime/test/framework/test_utils.cc

@@ -43,6 +43,13 @@ IExecutionProvider* TestNnapiExecutionProvider() {
 }
 #endif
 
+#ifdef USE_RKNPU
+IExecutionProvider* TestRknpuExecutionProvider() {
+  static RknpuExecutionProvider rknpu_provider;
+  return &rknpu_provider;
+}
+#endif
+
 static void CountOpsInGraphImpl(
     const Graph& graph, bool recurse_into_subgraphs, std::map<std::string, int>& ops) {
   for (auto& node : graph.Nodes()) {

onnxruntime/test/framework/test_utils.h

@@ -24,6 +24,9 @@
 #ifdef USE_NNAPI
 #include "core/providers/nnapi/nnapi_execution_provider.h"
 #endif
+#ifdef USE_RKNPU
+#include "core/providers/rknpu/rknpu_execution_provider.h"
+#endif
 
 namespace onnxruntime {
 class Graph;
@@ -50,6 +53,10 @@ IExecutionProvider* TestOpenVINOExecutionProvider();
 IExecutionProvider* TestNnapiExecutionProvider();
 #endif
 
+#ifdef USE_RKNPU
+IExecutionProvider* TestRknpuExecutionProvider();
+#endif
+
 template <typename T>
 inline void CopyVectorToTensor(const std::vector<T>& value, Tensor& tensor) {
   gsl::copy(gsl::make_span(value), tensor.MutableDataAsSpan<T>());

onnxruntime/test/providers/provider_test_utils.cc

@@ -774,6 +774,8 @@ void OpTester::Run(
           execution_provider = DefaultTensorrtExecutionProvider();
         else if (provider_type == onnxruntime::kNnapiExecutionProvider)
           execution_provider = DefaultNnapiExecutionProvider();
+        else if (provider_type == onnxruntime::kRknpuExecutionProvider)
+          execution_provider = DefaultRknpuExecutionProvider();
         else if (provider_type == onnxruntime::kAclExecutionProvider)
           execution_provider = DefaultAclExecutionProvider();
         // skip if execution provider is disabled

onnxruntime/test/providers/rknpu/rknpu_basic_test.cc

@@ -0,0 +1,102 @@
+#include "core/session/inference_session.h"
+#include "test/providers/provider_test_utils.h"
+#include "test/framework/test_utils.h"
+#include "gtest/gtest.h"
+#include "core/providers/rknpu/rknpu_execution_provider.h"
+#include "core/common/logging/logging.h"
+
+using namespace std;
+using namespace ONNX_NAMESPACE;
+using namespace ::onnxruntime::logging;
+
+namespace onnxruntime {
+
+namespace test {
+void VerifyOutputs(const std::vector<OrtValue>& fetches, const std::vector<int64_t>& expected_dims,
+                   const std::vector<float>& expected_values) {
+  ASSERT_EQ(1, fetches.size());
+  auto& rtensor = fetches.front().Get<Tensor>();
+  TensorShape expected_shape(expected_dims);
+  ASSERT_EQ(expected_shape, rtensor.Shape());
+  const std::vector<float> found(rtensor.template Data<float>(), rtensor.template Data<float>() + expected_values.size());
+  ASSERT_EQ(expected_values, found);
+}
+
+TEST(RknpuExecutionProviderTest, FunctionTest) {
+  onnxruntime::Model model("graph_1", false, DefaultLoggingManager().DefaultLogger());
+  auto& graph = model.MainGraph();
+  std::vector<onnxruntime::NodeArg*> inputs;
+  std::vector<onnxruntime::NodeArg*> outputs;
+
+  // FLOAT tensor.
+  ONNX_NAMESPACE::TypeProto float_tensor;
+  float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT);
+  float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
+  float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
+  float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3);
+  float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2);
+
+  auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor);
+  auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor);
+  inputs.push_back(&input_arg_1);
+  inputs.push_back(&input_arg_2);
+  auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &float_tensor);
+  outputs.push_back(&output_arg);
+  graph.AddNode("node_1", "Add", "node 1.", inputs, outputs);
+
+  auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &float_tensor);
+  inputs.clear();
+  inputs.push_back(&output_arg);
+  inputs.push_back(&input_arg_3);
+  auto& output_arg_2 = graph.GetOrCreateNodeArg("M", &float_tensor);
+  outputs.clear();
+  outputs.push_back(&output_arg_2);
+  graph.AddNode("node_2", "Add", "node 2.", inputs, outputs);
+
+  auto status = graph.Resolve();
+  ASSERT_TRUE(status.IsOK());
+  std::string model_file_name = "rknpu_execution_provider_test_graph.onnx";
+  status = onnxruntime::Model::Save(model, model_file_name);
+
+  std::vector<int64_t> dims_mul_x = {1, 1, 3, 2};
+  std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
+  OrtValue ml_value_x;
+  CreateMLValue<float>(TestRknpuExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_x);
+  OrtValue ml_value_y;
+  CreateMLValue<float>(TestRknpuExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_y);
+  OrtValue ml_value_z;
+  CreateMLValue<float>(TestRknpuExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_z);
+  NameMLValMap feeds;
+  feeds.insert(std::make_pair("X", ml_value_x));
+  feeds.insert(std::make_pair("Y", ml_value_y));
+  feeds.insert(std::make_pair("Z", ml_value_z));
+
+  // prepare outputs
+  std::vector<std::string> output_names;
+  output_names.push_back("M");
+  std::vector<OrtValue> fetches;
+
+  // prepare expected inputs and outputs
+  std::vector<int64_t> expected_dims_mul_m = {1, 1, 3, 2};
+  std::vector<float> expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f};
+
+  SessionOptions so;
+  so.session_logid = "RknpuExecutionProviderTest.FunctionTest";
+  RunOptions run_options;
+  run_options.run_tag = so.session_logid;
+
+  InferenceSession session_object{so, GetEnvironment()};
+  status = session_object.RegisterExecutionProvider(onnxruntime::make_unique<::onnxruntime::RknpuExecutionProvider>());
+  ASSERT_TRUE(status.IsOK());
+  status = session_object.Load(model_file_name);
+  ASSERT_TRUE(status.IsOK());
+  status = session_object.Initialize();
+  ASSERT_TRUE(status.IsOK());
+
+  // Now run
+  status = session_object.Run(run_options, feeds, output_names, &fetches);
+  ASSERT_TRUE(status.IsOK());
+  VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
+}
+}  // namespace test
+}  // namespace onnxruntime

onnxruntime/test/util/default_providers.cc

@@ -18,6 +18,7 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_NGraph
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_OpenVINO(const char* device_id);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nuphar(bool, const char*);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nnapi();
+std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Rknpu();
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tensorrt(int device_id);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ACL(int use_arena);
 
@@ -86,6 +87,14 @@ std::unique_ptr<IExecutionProvider> DefaultNnapiExecutionProvider() {
 #endif
 }
 
+std::unique_ptr<IExecutionProvider> DefaultRknpuExecutionProvider() {
+#ifdef USE_RKNPU
+  return CreateExecutionProviderFactory_Rknpu()->CreateProvider();
+#else
+  return nullptr;
+#endif
+}
+
 std::unique_ptr<IExecutionProvider> DefaultAclExecutionProvider(bool enable_arena) {
 #ifdef USE_ACL
   return CreateExecutionProviderFactory_ACL(enable_arena)->CreateProvider();

onnxruntime/test/util/include/default_providers.h

@@ -15,6 +15,7 @@ std::unique_ptr<IExecutionProvider> DefaultNupharExecutionProvider(bool allow_un
 std::unique_ptr<IExecutionProvider> DefaultTensorrtExecutionProvider();
 std::unique_ptr<IExecutionProvider> DefaultOpenVINOExecutionProvider();
 std::unique_ptr<IExecutionProvider> DefaultNnapiExecutionProvider();
+std::unique_ptr<IExecutionProvider> DefaultRknpuExecutionProvider();
 std::unique_ptr<IExecutionProvider> DefaultAclExecutionProvider(bool enable_arena = true);
 
 }  // namespace test

tools/ci_build/build.py

@@ -257,6 +257,8 @@ def parse_arguments():
         help="Build with OpenVINO for specific hardware.")
     parser.add_argument(
         "--use_dnnlibrary", action='store_true', help="Build with DNNLibrary.")
+    parser.add_argument(
+        "--use_rknpu", action='store_true', help="Build with RKNPU.")
     parser.add_argument(
         "--use_preinstalled_eigen", action='store_true',
         help="Use pre-installed Eigen.")
@@ -560,10 +562,11 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home,
         "-Donnxruntime_USE_OPENVINO_BINARY=" + (
             "ON" if args.use_openvino else "OFF"),
         "-Donnxruntime_USE_NNAPI=" + ("ON" if args.use_dnnlibrary else "OFF"),
+        "-Donnxruntime_USE_RKNPU=" + ("ON" if args.use_rknpu else "OFF"),
         "-Donnxruntime_USE_OPENMP=" + (
             "ON" if args.use_openmp and not (
                 args.use_dnnlibrary or args.use_mklml or args.use_ngraph or
-                args.android or (args.ios and is_macOS()))
+                args.android or (args.ios and is_macOS()) or args.use_rknpu)
             else "OFF"),
         "-Donnxruntime_USE_TVM=" + ("ON" if args.use_tvm else "OFF"),
         "-Donnxruntime_USE_LLVM=" + ("ON" if args.use_llvm else "OFF"),