From 20bc83400b6a0a5f6a27c5d24f5742eb63335115 Mon Sep 17 00:00:00 2001 From: Andrews548 <32704142+Andrews548@users.noreply.github.com> Date: Thu, 22 Oct 2020 19:29:44 +0300 Subject: [PATCH 1/3] ACL/ArmNN update (#5515) * Build ACL and ArmNN with custom library path * Define import to tensor as a separate function for maintenance and readability * Enabled optimized depthwise convolution for ACL v20.02 * Check operation status for ACL and ArmNN Execution Providers * Enabled fused operation for convolution-activation Co-authored-by: Andrei-Alexandru --- BUILD.md | 14 +++++ cmake/CMakeLists.txt | 50 +++++++++++++++++ cmake/onnxruntime_providers.cmake | 4 +- .../core/optimizer/graph_transformer_utils.cc | 3 +- onnxruntime/core/providers/acl/acl_common.cc | 20 +++++++ onnxruntime/core/providers/acl/acl_common.h | 2 + .../providers/acl/acl_execution_provider.cc | 50 ++++++++++------- .../core/providers/acl/nn/batch_norm.cc | 15 +---- onnxruntime/core/providers/acl/nn/conv.cc | 28 +++++++++- .../core/providers/acl/tensor/concat.cc | 15 +---- .../armnn/armnn_execution_provider.cc | 55 ++++++++++++------- .../core/providers/armnn/nn/fused_conv.cc | 41 ++++++++++++++ tools/ci_build/build.py | 32 ++++++++++- 13 files changed, 252 insertions(+), 77 deletions(-) create mode 100644 onnxruntime/core/providers/armnn/nn/fused_conv.cc diff --git a/BUILD.md b/BUILD.md index 87783f6675..82fa51c071 100644 --- a/BUILD.md +++ b/BUILD.md @@ -510,6 +510,11 @@ cmake ../onnxruntime-arm-upstream/cmake -DONNX_CUSTOM_PROTOC_EXECUTABLE=/usr/bin The ```-Donnxruntime_USE_ACL=ON``` option will use, by default, the 19.05 version of the Arm Compute Library. To set the right version you can use: ```-Donnxruntime_USE_ACL_1902=ON```, ```-Donnxruntime_USE_ACL_1905=ON```, ```-Donnxruntime_USE_ACL_1908=ON``` or ```-Donnxruntime_USE_ACL_2002=ON```; +To use a library outside the normal environment you can set a custom path by using ```-Donnxruntime_ACL_HOME``` and ```-Donnxruntime_ACL_LIBS``` tags that defines the path to the ComputeLibrary directory and the build directory respectively. + +```-Donnxruntime_ACL_HOME=/path/to/ComputeLibrary```, ```-Donnxruntime_ACL_LIBS=/path/to/build``` + + 2. Build ONNX Runtime library, test and performance application: ``` make -j 6 @@ -542,6 +547,10 @@ export LD_LIBRARY_PATH=~/ComputeLibrary/build/ ``` ./build.sh --use_acl ``` +To use a library outside the normal environment you can set a custom path by using --acl_home and --acl_libs tags that defines the path to the ComputeLibrary directory and the build directory respectively. +``` +./build.sh --use_acl --acl_home /path/to/ComputeLibrary --acl_libs /path/to/build +``` --- @@ -572,6 +581,11 @@ The Batch Normalization operator is set by default to use the CPU execution prov ./build.sh --use_armnn --armnn_bn ``` +To use a library outside the normal environment you can set a custom path by using --armnn_home and --armnn_libs tags that defines the path to the ArmNN home directory and the build directory respectively. +``` +./build.sh --use_armnn --armnn_home /path/to/ComputeLibrary --armnn_libs /path/to/build +``` + --- ### RKNPU diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 6d5a0576b4..d9b4ab304c 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -675,7 +675,28 @@ if (onnxruntime_USE_ACL OR onnxruntime_USE_ACL_1902 OR onnxruntime_USE_ACL_1905 endif() endif() + if (NOT ${onnxruntime_ACL_LIBS} STREQUAL "") + + add_library(arm_compute SHARED IMPORTED) + set_target_properties(arm_compute PROPERTIES + IMPORTED_NO_SONAME 1 + IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute.so") + + add_library(arm_compute_core SHARED IMPORTED) + set_target_properties(arm_compute_core PROPERTIES + IMPORTED_NO_SONAME 1 + IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute_core.so") + + add_library(arm_compute_graph SHARED IMPORTED) + set_target_properties(arm_compute_graph PROPERTIES + IMPORTED_NO_SONAME 1 + IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute_graph.so") + + link_libraries(arm_compute arm_compute_core arm_compute_graph) + endif() + list(APPEND onnxruntime_EXTERNAL_LIBRARIES arm_compute arm_compute_graph arm_compute_core) + endif() # ArmNN @@ -687,6 +708,35 @@ if (onnxruntime_USE_ARMNN) add_definitions(-DBN_ARMNN=1) endif() + if (NOT onnxruntime_USE_ACL AND NOT ${onnxruntime_ACL_LIBS} STREQUAL "") + + add_library(arm_compute SHARED IMPORTED) + set_target_properties(arm_compute PROPERTIES + IMPORTED_NO_SONAME 1 + IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute.so") + + add_library(arm_compute_core SHARED IMPORTED) + set_target_properties(arm_compute_core PROPERTIES + IMPORTED_NO_SONAME 1 + IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute_core.so") + + add_library(arm_compute_graph SHARED IMPORTED) + set_target_properties(arm_compute_graph PROPERTIES + IMPORTED_NO_SONAME 1 + IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute_graph.so") + + link_libraries(arm_compute arm_compute_core arm_compute_graph) + endif() + + if (NOT ${onnxruntime_ARMNN_LIBS} STREQUAL "") + + find_library(ARMNN_LIBRARY NAMES armnn + PATHS ${onnxruntime_ARMNN_LIBS} + NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH) + + link_libraries(armnn) + endif() + list(APPEND onnxruntime_EXTERNAL_LIBRARIES armnn pthread arm_compute_core arm_compute arm_compute_graph) endif() diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 4cdf1f57bd..6580dd00cb 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -766,7 +766,7 @@ if (onnxruntime_USE_ACL) target_link_libraries(onnxruntime_providers_acl -L$ENV{LD_LIBRARY_PATH}) add_dependencies(onnxruntime_providers_acl ${onnxruntime_EXTERNAL_DEPENDENCIES}) set_target_properties(onnxruntime_providers_acl PROPERTIES FOLDER "ONNXRuntime") - target_include_directories(onnxruntime_providers_acl PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${ACL_INCLUDE_DIR}) + target_include_directories(onnxruntime_providers_acl PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${onnxruntime_ACL_HOME} ${onnxruntime_ACL_HOME}/include) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/acl DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_acl PROPERTIES LINKER_LANGUAGE CXX) endif() @@ -783,7 +783,7 @@ if (onnxruntime_USE_ARMNN) onnxruntime_add_include_to_target(onnxruntime_providers_armnn onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf flatbuffers) add_dependencies(onnxruntime_providers_armnn ${onnxruntime_EXTERNAL_DEPENDENCIES}) set_target_properties(onnxruntime_providers_armnn PROPERTIES FOLDER "ONNXRuntime") - target_include_directories(onnxruntime_providers_armnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${ARMNN_INCLUDE_DIR}) + target_include_directories(onnxruntime_providers_armnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${onnxruntime_ARMNN_HOME} ${onnxruntime_ARMNN_HOME}/include) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/armnn DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_armnn PROPERTIES LINKER_LANGUAGE CXX) endif() diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 2963caaf51..a2569c4852 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -137,8 +137,9 @@ std::vector> GenerateTransformers(TransformerL transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); std::unordered_set cpu_acl_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider}; + std::unordered_set cpu_acl_armnn_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider, onnxruntime::kArmNNExecutionProvider}; - transformers.emplace_back(onnxruntime::make_unique(cpu_acl_execution_providers)); + transformers.emplace_back(onnxruntime::make_unique(cpu_acl_armnn_execution_providers)); std::unordered_set cpu_cuda_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kCudaExecutionProvider}; transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_execution_providers)); diff --git a/onnxruntime/core/providers/acl/acl_common.cc b/onnxruntime/core/providers/acl/acl_common.cc index 36b494c3f3..e87af75fe3 100644 --- a/onnxruntime/core/providers/acl/acl_common.cc +++ b/onnxruntime/core/providers/acl/acl_common.cc @@ -59,6 +59,26 @@ arm_compute::Status ACLImportMemory(arm_compute::TensorAllocator* allocator, voi #endif } +template +void importDataToTensor(arm_compute::Tensor* tensor, const T* data){ + + arm_compute::Window aclInpuWindow; + aclInpuWindow.use_tensor_dimensions(tensor->info()->tensor_shape()); + + arm_compute::Iterator aclInputIt(tensor, aclInpuWindow); + int index = 0; + + // copy input tensor into the larger buffer + arm_compute::execute_window_loop( + aclInpuWindow, + [&](const arm_compute::Coordinates& co) { + *reinterpret_cast(aclInputIt.ptr()) = data[index]; + index++; + }, + aclInputIt); +} +template void importDataToTensor(arm_compute::Tensor*, const float*); + template void importDataFromTensor(arm_compute::Tensor* tensor, T* data){ diff --git a/onnxruntime/core/providers/acl/acl_common.h b/onnxruntime/core/providers/acl/acl_common.h index 7fcbc800a2..899736c477 100644 --- a/onnxruntime/core/providers/acl/acl_common.h +++ b/onnxruntime/core/providers/acl/acl_common.h @@ -19,6 +19,8 @@ void ACLPrintTensorShape(const char*, arm_compute::Tensor& t); std::shared_ptr ACLCreateMemoryManager(); arm_compute::Status ACLImportMemory(arm_compute::TensorAllocator* allocator, void* memory, size_t size); template +void importDataToTensor(arm_compute::Tensor* tensor, const T* data); +template void importDataFromTensor(arm_compute::Tensor* tensor, T* data); } // namespace acl diff --git a/onnxruntime/core/providers/acl/acl_execution_provider.cc b/onnxruntime/core/providers/acl/acl_execution_provider.cc index af726ee3c9..5cb217c419 100644 --- a/onnxruntime/core/providers/acl/acl_execution_provider.cc +++ b/onnxruntime/core/providers/acl/acl_execution_provider.cc @@ -40,36 +40,48 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 11, Co class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kMSDomain, 1, float, FusedConv); -static void RegisterACLKernels(KernelRegistry& kernel_registry) { - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); +Status RegisterACLKernels(KernelRegistry& kernel_registry) { + static const BuildKernelCreateInfoFn function_table[] = { + //BuildKernelCreateInfo, //default entry to avoid the list become empty after ops-reducing + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, + BuildKernelCreateInfo, // Opset 10 - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, + BuildKernelCreateInfo, - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, + }; + + for (auto& function_table_entry : function_table) { + KernelCreateInfo info = function_table_entry(); + if (info.kernel_def != nullptr) { // filter disabled entries where type is void + ORT_RETURN_IF_ERROR(kernel_registry.Register(std::move(info))); + } + } + + return Status::OK(); } std::shared_ptr GetAclKernelRegistry() { std::shared_ptr kernel_registry = std::make_shared(); - RegisterACLKernels(*kernel_registry); + ORT_THROW_IF_ERROR(RegisterACLKernels(*kernel_registry)); return kernel_registry; } diff --git a/onnxruntime/core/providers/acl/nn/batch_norm.cc b/onnxruntime/core/providers/acl/nn/batch_norm.cc index 933205dd62..7a682a4241 100755 --- a/onnxruntime/core/providers/acl/nn/batch_norm.cc +++ b/onnxruntime/core/providers/acl/nn/batch_norm.cc @@ -109,20 +109,7 @@ Status BatchNorm::Compute(OpKernelContext* context) const { if(X->Shape().Size() != 0 && pBatchNorm->in->info()->has_padding() ){ - arm_compute::Window aclInpuWindow; - aclInpuWindow.use_tensor_dimensions(pBatchNorm->in->info()->tensor_shape()); - - arm_compute::Iterator aclInputIt(pBatchNorm->in.get(), aclInpuWindow); - const unsigned int aclWidth = pBatchNorm->in->info()->dimension(0); - const unsigned int aclHeight = pBatchNorm->in->info()->dimension(1); - - // copy input tensor into the larger buffer - arm_compute::execute_window_loop( - aclInpuWindow, - [&](const arm_compute::Coordinates& co) { - *reinterpret_cast(aclInputIt.ptr()) = x_data[co.z() * (aclWidth * aclHeight) + co.y() * aclHeight + co.x()]; - }, - aclInputIt); + importDataToTensor(pBatchNorm->in.get(), x_data); }else{ ACLImportMemory(pBatchNorm->in->allocator(), (void*)x_data, X->Shape().Size() * 4); } diff --git a/onnxruntime/core/providers/acl/nn/conv.cc b/onnxruntime/core/providers/acl/nn/conv.cc index 664c248008..1aed263847 100644 --- a/onnxruntime/core/providers/acl/nn/conv.cc +++ b/onnxruntime/core/providers/acl/nn/conv.cc @@ -216,7 +216,16 @@ Status Conv::Compute(OpKernelContext* context) const { 1 /* depth multiplier */, arm_compute::Size2D(aclDilation0, dilations[0])); #elif defined(ACL_2002) - bool optimizable = false; + bool optimizable = bool(arm_compute::NEDepthwiseConvolutionLayerOptimized::validate(tconv.in->info(), + tconv.k->info(), + (B != nullptr) ? tconv.b->info() : nullptr, + tconv.out->info(), + aclPadStride, + 1 /* depth multiplier */, + acl_activ_enabled ? + arm_compute::ActivationLayerInfo(acl_activ_func, conv_attrs_.alpha) : + arm_compute::ActivationLayerInfo(), + arm_compute::Size2D(aclDilation0, dilations[0]))); #endif if (optimizable) { @@ -286,7 +295,12 @@ Status Conv::Compute(OpKernelContext* context) const { } const T* x_data = X->template Data(); - ACLImportMemory(pConv->in->allocator(), (void*)x_data, X->Shape().Size() * 4); + if(X->Shape().Size() != 0 && pConv->in->info()->has_padding() ){ + pConv->in->allocator()->allocate(); + importDataToTensor(pConv->in.get(), x_data); + }else{ + ACLImportMemory(pConv->in->allocator(), (void*)x_data, X->Shape().Size() * 4); + } const T* k_data = W->template Data(); ACLImportMemory(pConv->k->allocator(), (void*)k_data, W->Shape().Size() * 4); @@ -297,13 +311,21 @@ Status Conv::Compute(OpKernelContext* context) const { } T* y_data = Y->template MutableData(); - ACLImportMemory(pConv->out->allocator(), (void*)y_data, Y->Shape().Size() * 4); + if(Y->Shape().Size() != 0 && pConv->out->info()->has_padding() ){ + pConv->out->allocator()->allocate(); + } else { + ACLImportMemory(pConv->out->allocator(), (void*)y_data, Y->Shape().Size() * 4); + } arm_compute::Allocator alloc_mm{}; pConv->mm_layer->populate(alloc_mm, 1); pConv->layer->run(); pConv->mm_layer->clear(); + if(Y->Shape().Size() != 0 && pConv->out->info()->has_padding() ){ + importDataFromTensor(pConv->out.get(), y_data); + } + pConv->in->allocator()->free(); pConv->k->allocator()->free(); if (B != nullptr) diff --git a/onnxruntime/core/providers/acl/tensor/concat.cc b/onnxruntime/core/providers/acl/tensor/concat.cc index 9aad25f0b0..a7489970df 100644 --- a/onnxruntime/core/providers/acl/tensor/concat.cc +++ b/onnxruntime/core/providers/acl/tensor/concat.cc @@ -87,20 +87,7 @@ Status Concat::Compute(OpKernelContext* ctx) const { if (X->Shape().Size() != 0 && in->info()->has_padding() ){ in->allocator()->allocate(); - arm_compute::Window aclInpuWindow; - aclInpuWindow.use_tensor_dimensions(in->info()->tensor_shape()); - - arm_compute::Iterator aclInputIt(in, aclInpuWindow); - int index = 0; - - // copy input tensor into the larger buffer - arm_compute::execute_window_loop( - aclInpuWindow, - [&](const arm_compute::Coordinates& co) { - *reinterpret_cast(aclInputIt.ptr()) = x_data[index]; - index++; - }, - aclInputIt); + importDataToTensor(in, x_data); } else { ACLImportMemory(in->allocator(), (void*)x_data, X->Shape().Size() * 4); } diff --git a/onnxruntime/core/providers/armnn/armnn_execution_provider.cc b/onnxruntime/core/providers/armnn/armnn_execution_provider.cc index 67218da12c..274d1f8fb5 100644 --- a/onnxruntime/core/providers/armnn/armnn_execution_provider.cc +++ b/onnxruntime/core/providers/armnn/armnn_execution_provider.cc @@ -30,7 +30,6 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 9, float, AveragePool); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 10, 10, float, AveragePool); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, float, AveragePool); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, AveragePool); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 8, 11, float, MaxPool); @@ -45,38 +44,52 @@ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDo class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 4, 10, Concat); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Concat); -static void RegisterArmNNKernels(KernelRegistry& kernel_registry) { +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kMSDomain, 1, float, FusedConv); + +Status RegisterArmNNKernels(KernelRegistry& kernel_registry) { + static const BuildKernelCreateInfoFn function_table[] = { #ifdef RELU_ARMNN - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, #endif - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, + BuildKernelCreateInfo, - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, + BuildKernelCreateInfo, #ifdef BN_ARMNN - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, #endif - kernel_registry.Register(BuildKernelCreateInfo()); - kernel_registry.Register(BuildKernelCreateInfo()); + BuildKernelCreateInfo, + BuildKernelCreateInfo, + + BuildKernelCreateInfo, + }; + + for (auto& function_table_entry : function_table) { + KernelCreateInfo info = function_table_entry(); + if (info.kernel_def != nullptr) { // filter disabled entries where type is void + ORT_RETURN_IF_ERROR(kernel_registry.Register(std::move(info))); + } + } + + return Status::OK(); } std::shared_ptr GetArmNNKernelRegistry() { std::shared_ptr kernel_registry = std::make_shared(); - RegisterArmNNKernels(*kernel_registry); + ORT_THROW_IF_ERROR(RegisterArmNNKernels(*kernel_registry)); return kernel_registry; } diff --git a/onnxruntime/core/providers/armnn/nn/fused_conv.cc b/onnxruntime/core/providers/armnn/nn/fused_conv.cc new file mode 100644 index 0000000000..53c513f8c7 --- /dev/null +++ b/onnxruntime/core/providers/armnn/nn/fused_conv.cc @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/armnn/armnn_execution_provider.h" +#include "core/providers/armnn/nn/conv.h" +#include "core/providers/armnn/armnn_common.h" +#include "core/providers/armnn/armnn_fwd.h" +#include "contrib_ops/cpu/fused_activation.h" + +#include "armnn/ArmNN.hpp" + +#include +#include + +namespace onnxruntime { +namespace armnn_ep{ + +class FusedConv final : public armnn_ep::Conv { +public: + explicit FusedConv(const OpKernelInfo& info) : armnn_ep::Conv(info) { + ORT_ENFORCE(info.GetAttr("activation", &(this->activation_type)).IsOK()); + ORT_ENFORCE(GetFusedActivationAttr(info, activation_).IsOK()); + } +}; + +ONNX_OPERATOR_TYPED_KERNEL_EX( + FusedConv, + kMSDomain, + 1, + float, + kArmNNExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + FusedConv); + +} // namespace armnn_ep +} // namespace onnxruntime diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index c69e6c5477..12d2a97650 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -347,6 +347,10 @@ def parse_arguments(): "--use_acl", nargs="?", const="ACL_1905", choices=["ACL_1902", "ACL_1905", "ACL_1908", "ACL_2002"], help="Build with ACL for ARM architectures.") + parser.add_argument( + "--acl_home", help="Path to ACL home dir") + parser.add_argument( + "--acl_libs", help="Path to ACL libraries") parser.add_argument( "--use_armnn", action='store_true', help="Enable ArmNN Execution Provider.") @@ -356,6 +360,10 @@ def parse_arguments(): parser.add_argument( "--armnn_bn", action='store_true', help="Use the Batch Normalization operator implementation from the ArmNN EP.") + parser.add_argument( + "--armnn_home", help="Path to ArmNN home dir") + parser.add_argument( + "--armnn_libs", help="Path to ArmNN libraries") parser.add_argument( "--build_micro_benchmarks", action='store_true', help="Build ONNXRuntime micro-benchmarks.") @@ -578,7 +586,7 @@ def use_dev_mode(args): def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, - mpi_home, nccl_home, tensorrt_home, migraphx_home, + mpi_home, nccl_home, tensorrt_home, migraphx_home, acl_home, acl_libs, armnn_home, armnn_libs, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args): log.info("Generating CMake build tree") cmake_dir = os.path.join(source_dir, "cmake") @@ -687,6 +695,18 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "ON" if args.build_micro_benchmarks else "OFF") ] + if acl_home and os.path.exists(acl_home): + cmake_args += ["-Donnxruntime_ACL_HOME=" + acl_home] + + if acl_libs and os.path.exists(acl_libs): + cmake_args += ["-Donnxruntime_ACL_LIBS=" + acl_libs] + + if armnn_home and os.path.exists(armnn_home): + cmake_args += ["-Donnxruntime_ARMNN_HOME=" + armnn_home] + + if armnn_libs and os.path.exists(armnn_libs): + cmake_args += ["-Donnxruntime_ARMNN_LIBS=" + armnn_libs] + if mpi_home and os.path.exists(mpi_home): cmake_args += ["-Donnxruntime_MPI_HOME=" + mpi_home] @@ -1767,6 +1787,12 @@ def main(): mpi_home = args.mpi_home nccl_home = args.nccl_home + acl_home = args.acl_home + acl_libs = args.acl_libs + + armnn_home = args.armnn_home + armnn_libs = args.armnn_libs + # if using tensorrt, setup tensorrt paths tensorrt_home = setup_tensorrt_vars(args) @@ -1863,8 +1889,8 @@ def main(): setup_test_data(build_dir, configs) generate_build_tree( cmake_path, source_dir, build_dir, cuda_home, cudnn_home, mpi_home, nccl_home, - tensorrt_home, migraphx_home, path_to_protoc_exe, configs, cmake_extra_defines, - args, cmake_extra_args) + tensorrt_home, migraphx_home, acl_home, acl_libs, armnn_home, armnn_libs, + path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args) if args.clean: clean_targets(cmake_path, build_dir, configs) From 7da5949279577ee21fa09ed6484833dfe10f127d Mon Sep 17 00:00:00 2001 From: ytaous <4484531+ytaous@users.noreply.github.com> Date: Thu, 22 Oct 2020 10:34:20 -0700 Subject: [PATCH 2/3] NVTX label change (#5562) * label change * more info on label Co-authored-by: Ethan Tao --- onnxruntime/core/framework/sequential_executor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/framework/sequential_executor.cc b/onnxruntime/core/framework/sequential_executor.cc index f8820d6abf..cbcef6a521 100644 --- a/onnxruntime/core/framework/sequential_executor.cc +++ b/onnxruntime/core/framework/sequential_executor.cc @@ -302,7 +302,7 @@ Status SequentialExecutor::Execute(const SessionState& session_state, const std: #endif #ifdef ENABLE_NVTX_PROFILE profile::NvtxRangeCreator node_compute_range( - MakeString(node.OpType(), ".", node.Index()), profile::Color::Blue); + MakeString(node.OpType(), ".", node.Index(), "(", node.Name(), ")"), profile::Color::Blue); node_compute_range.Begin(); #endif ORT_TRY { From 3f06286154df52d5900e306b5a6dd19b53194ed1 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Thu, 22 Oct 2020 18:15:53 -0700 Subject: [PATCH 3/3] Add Flatten support for NNAPI (#5545) * Add flatten support for NNAPI, correct some typo in NNAPI code files * Address review comments * Update CanSkipReshape * Add test for verify NNAPI is actually running for a supported model * Adding test for reshape/flatten test for NNAPI * Add one extra verbose log for skipping reshape * Fix Android CI failure * Correct test file name to fix Android CI failure --- .../nnapi/nnapi_builtin/builders/helper.h | 2 +- .../nnapi_builtin/builders/model_builder.cc | 30 +-- .../nnapi_builtin/builders/model_builder.h | 2 +- .../nnapi_builtin/builders/op_builder.cc | 204 +++++++++++++++--- .../providers/nnapi/nnapi_builtin/model.cc | 2 +- .../providers/nnapi/nnapi_builtin/model.h | 4 +- .../nnapi_builtin/nnapi_execution_provider.cc | 4 +- .../test/providers/nnapi/nnapi_basic_test.cc | 145 ++++++++----- .../testdata/nnapi_reshape_flatten_test.onnx | Bin 0 -> 602 bytes .../testdata/nnapi_reshape_flatten_test.py | 45 ++++ 10 files changed, 338 insertions(+), 100 deletions(-) create mode 100644 onnxruntime/test/testdata/nnapi_reshape_flatten_test.onnx create mode 100644 onnxruntime/test/testdata/nnapi_reshape_flatten_test.py diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index c948cb5c71..9758581a32 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -65,7 +65,7 @@ enum class QLinearOpType : uint8_t { QLinearOpType GetQLinearOpType(const onnxruntime::Node& node); -// This qlinear op is an operator takes 2 input and producce 1 output +// This qlinear op is an operator takes 2 input and produces 1 output // Such as QLinearConv, QLinearMatMul, QLinearAdd, ... bool IsQLinearBinaryOp(QLinearOpType qlinear_op_type); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index c3aed42a69..99684a0264 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -34,23 +34,23 @@ bool ModelBuilder::IsNodeSupported(const Node& node) { } bool IsValidSupportedNodesVec(const std::vector& supported_node_vec, const GraphViewer& graph_viewer) { - if (!supported_node_vec.empty()) { - if (supported_node_vec.size() == 1) { - const auto& node_indices = graph_viewer.GetNodesInTopologicalOrder(); - const auto* node(graph_viewer.GetNode(node_indices[supported_node_vec[0]])); - const auto& op = node->OpType(); - // It is not worth it to perform a single Reshape/Dropout/Identity operator - // which is only copying the data in NNAPI - // If this is the case, let it fall back - if (op == "Reshape" || - op == "Dropout" || - op == "Identity") { - return false; - } + if (supported_node_vec.empty()) + return false; + + if (supported_node_vec.size() == 1) { + const auto& node_indices = graph_viewer.GetNodesInTopologicalOrder(); + const auto* node(graph_viewer.GetNode(node_indices[supported_node_vec[0]])); + const auto& op = node->OpType(); + // It is not worth it to perform a single Reshape/Flatten/Identity operator + // which is only copying the data in NNAPI + // If this is the case, let it fall back + if (op == "Reshape" || + op == "Flatten" || + op == "Identity") { + return false; } - return true; } - return false; + return true; } std::vector> ModelBuilder::GetSupportedNodes() { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index 35369fb560..9f67fafb88 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -132,7 +132,7 @@ class ModelBuilder { std::unordered_map initializers_; std::unordered_set skipped_initializers_; - // All activation nodes (Relu, Relu1, Relu6) as a map + // All activation nodes (Relu, Relu1, Relu6) as a map std::unordered_map activation_nodes_; std::unordered_map> op_builders_; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 8dcdea4f6e..c56a50afa6 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -524,8 +524,8 @@ static Status GetBinaryOpQuantizationScaleAndZeroPoint( return Status::OK(); } -// NNAPI has the qunatization scale and zero point embedded in the ANeuralNetworksOperandType -// ONNX has the qunatization scale and zero point as the inputs of the qlinear operators +// NNAPI has the quantization scale and zero point embedded in the ANeuralNetworksOperandType +// ONNX has the quantization scale and zero point as the inputs of the qlinear operators // We want to verify the scale and zeropoint of the ONNX inputs matches the values embedded in the NNAPI inputs static Status IsValidInputQuantizedType(const ModelBuilder& model_builder, const std::string& input_name, @@ -886,10 +886,10 @@ Status BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const if (input1_is_nhwc == input2_is_nhwc) { output_is_nhwc = input1_is_nhwc; } else if (input1_is_nhwc) { - // need transpsoe input1 back to nchw + // need transpose input1 back to nchw ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, a_idx, input1)); } else { // input2_is_nhwc - // need transpsoe input2 back to nchw + // need transpose input2 back to nchw ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, b_idx, input2)); } @@ -1024,17 +1024,113 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co class ReshapeOpBuilder : public BaseOpBuilder { public: void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; + static Status AddReshapeOperator(ModelBuilder& model_builder, const Node& node, + const std::string& input, const std::vector& shape) ORT_MUST_USE_RESULT; private: bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - Status AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override ORT_MUST_USE_RESULT; + static bool CanSkipReshape(const Node& node, size_t input_rank, size_t output_rank); }; void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); } +// We can skip the Reshape if all the output edges satisfies, +// 1. The output of the reshape/flatten is the input 0 of the GEMM/Matmul, +// and the input rank >= 2 and output_rank == 2 +// This is because Gemm/Matmul will map to ANEURALNETWORKS_FULLY_CONNECTED in NNAPI, +// ANEURALNETWORKS_FULLY_CONNECTED will flatten the 2+ dim input 0 to 2d +// 2. Or the output the reshape/flatten is the output of the graph +// (no op in the graph is using the output except can be used by Gemm/Matmul satisfying condition 1 above) +// The reason we want to skip Reshape is that Reshape is not running on Hardware (NPU,...) in NNAPI for +// some CPU (e.g. Qualcomm SD for now), skipping unnecessary Reshape will prevent context switching +// between NNAPI CPU impl and Hardware Accelerator impl and will speed up the execution +// If we are going to skip the reshape, we will still add correct shape and operand type for the output in +// onnxruntime::nnapi::Model. +// If the Reshape output is also a graph output, since NNAPI output is a void* buffer, we can find the shape +// information in onnxruntime::nnapi::Model and pass the correct shape information back to ORT to be used as output shape +/* static */ bool ReshapeOpBuilder::CanSkipReshape(const Node& node, size_t input_rank, size_t output_rank) { + const auto& output = node.OutputDefs()[0]->Name(); + // We will go through all the output edges + for (auto it = node.OutputEdgesBegin(), end = node.OutputEdgesEnd(); it != end; ++it) { + const auto& op_type = it->GetNode().OpType(); + // TODO add quantized matmul when reshape support quantized input + if (op_type != "Gemm" && op_type != "MatMul") { + LOGS_DEFAULT(VERBOSE) << "Reshape/Flatten can only be skipped when the output is Gemm/Matmul" + << " or no op is using the output (output is graph output)" + << ", output name, " << output + << " is used by " << op_type; + return false; + } + + // NNAPI ANEURALNETWORKS_FULLY_CONNECTED will only flatten the input 0 + if (it->GetDstArgIndex() != 0) { + LOGS_DEFAULT(VERBOSE) << "Reshape/Flatten can only be skipped when the output is input 0 of Gemm/Matmul" + << ", output name, " << output; + return false; + } + + // We only support 2d matmul/gemm here + // And NNAPI ANEURALNETWORKS_FULLY_CONNECTED will only flatten input rank >= 2 + if (input_rank < 2 || output_rank != 2) { + LOGS_DEFAULT(VERBOSE) << "Reshape/Flatten can only be skipped when input_rank >= 2 and output_rank == 2" + << ", output name, " << output + << ", the actual input_rank, " << input_rank + << ", the actual output_rank, " << output_rank; + return false; + } + } + + // If we reach here, we have either, + // all the Reshape outputs are used by gemm/matmul, the output can also be a model output [doesn't really matter here] + // or + // Reshape has no output edge ==> the output is a graph output or a dead end [which we don't care] + // we can skip this Reshape now + LOGS_DEFAULT(VERBOSE) << "Skipping Reshape/Flatten node [" + << node.Name() << "] with output, " << output; + return true; +} + +/* static */ Status ReshapeOpBuilder::AddReshapeOperator(ModelBuilder& model_builder, + const Node& node, + const std::string& input, + const std::vector& shape) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + const auto& output = node.OutputDefs()[0]->Name(); + ORT_RETURN_IF_ERROR(shaper.Reshape(input, shape, output)); + auto input_rank = shaper[input].size(); + auto output_rank = shaper[output].size(); + + // Since Reshape is not running using hardware in NNAPI for some CPU (e.g. Qualcomm SD for now) + // We will try to see if we the skip the Reshape to prevent context switching between + // NNAPI CPU impl and NNAPI hardware accelerator impl + if (CanSkipReshape(node, input_rank, output_rank)) { + // Since reshape can be skipped, only register the dimension and type, with same index and new name + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, false); + } else { + // We still need to perform a reshape here + // Add input + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); + // Add new shape + Shape shape_dimen = {static_cast(shape.size())}; + std::string shape_name = model_builder.GetUniqueName(node.Name() + input + "newshape"); + OperandType shape_operand_type(Type::TENSOR_INT32, shape_dimen); + ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(shape_name, shape.data(), shape_operand_type)); + input_indices.push_back(operand_indices.at(shape_name)); + + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_RESHAPE, input_indices, {output}, {output_operand_type}, {false})); + } + + return Status::OK(); +} + bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { const auto& initializers(model_builder.GetInitializerTensors()); const auto& perm_name = node.InputDefs()[1]->Name(); @@ -1070,8 +1166,6 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); auto input = node.InputDefs()[0]->Name(); @@ -1080,10 +1174,6 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, 0, input)); } - const auto& output = node.OutputDefs()[0]->Name(); - std::vector input_indices; - input_indices.push_back(operand_indices.at(input)); // input - const auto& shape_tensor = initializers.at(node.InputDefs()[1]->Name()); const int64_t* rawShape = GetTensorInt64Data(shape_tensor); const auto size = SafeInt(shape_tensor.dims()[0]); @@ -1096,18 +1186,7 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons shape[i] = dim == 0 ? input_shape[i] : dim; } - Shape shape_dimen = {size}; - std::string shape_name = model_builder.GetUniqueName(node.Name() + input + "newshape"); - OperandType shape_operand_type(Type::TENSOR_INT32, shape_dimen); - ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(shape_name, shape.data(), shape_operand_type)); - input_indices.push_back(operand_indices.at(shape_name)); - - ORT_RETURN_IF_ERROR(shaper.Reshape(input, shape, output)); - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_RESHAPE, input_indices, - {output}, {output_operand_type}, {false})); - - return Status::OK(); + return AddReshapeOperator(model_builder, node, input, shape); } #pragma endregion op_reshape @@ -1287,7 +1366,7 @@ bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const N const auto input_size = input_shape.size(); if (input_size != 4) { LOGS_DEFAULT(VERBOSE) - << op_type << " only supportes rank-4 tensor, input [" + << op_type << " only supports rank-4 tensor, input [" << node.InputDefs()[0]->Name() << "] has actual dim count " << input_size; return false; } @@ -1499,7 +1578,7 @@ bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& n const auto android_sdk_ver = model_builder.GetAndroidSdkVer(); if (android_sdk_ver < 29) { - LOGS_DEFAULT(VERBOSE) << op_type << " dilations is only supported on Android API levle 29+, " + LOGS_DEFAULT(VERBOSE) << op_type << " dilations is only supported on Android API level 29+, " << "actual API level: " << android_sdk_ver; return false; } @@ -2307,7 +2386,7 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const } int rank = shaper[input0].size(); - int32_t axis = SafeInt(HandleNegativeAxis(helper.Get("axis", 1), rank)); + int32_t axis = static_cast(HandleNegativeAxis(helper.Get("axis", 1), rank)); if (output_is_nhwc) { ORT_RETURN_IF_NOT(rank == 4, @@ -2372,7 +2451,7 @@ Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons const auto& input_shape(shaper[input]); auto input_dims = input_shape.size(); for (auto& axis : axes) { - axis = SafeInt(HandleNegativeAxis(axis, input_dims)); + axis = static_cast(HandleNegativeAxis(axis, input_dims)); } if (axes.empty()) { // Squeeze all @@ -2918,6 +2997,76 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const #pragma endregion +#pragma region op_flatten + +class FlattenOpBuilder : public BaseOpBuilder { + private: + bool IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) override; + static void GetFlattenShape(const Node& node, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2); + Status AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override ORT_MUST_USE_RESULT; +}; + +/* static */ void FlattenOpBuilder::GetFlattenShape(const Node& node, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2) { + int32_t rank = static_cast(input_shape.size()); + NodeAttrHelper helper(node); + int32_t axis = helper.Get("axis", 1); + // axis == rank is a valid input, but invalid for HandleNegativeAxis + // Skip non-negative axis here + if (axis < 0) + axis = static_cast(HandleNegativeAxis(axis, rank)); + + dim_1 = std::accumulate(input_shape.cbegin(), input_shape.cbegin() + axis, 1, std::multiplies()); + dim_2 = std::accumulate(input_shape.cbegin() + axis, input_shape.cend(), 1, std::multiplies()); +} + +bool FlattenOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { + Shape input_shape; + if (!GetShape(*node.InputDefs()[0], input_shape)) + return false; + + if (input_shape.size() > 4 || input_shape.empty()) { + LOGS_DEFAULT(VERBOSE) << "Flatten only supports up to 1-4d shape, input is " + << input_shape.size() << "d shape"; + return false; + } + + int32_t dim_1 = 1; + int32_t dim_2 = 1; + GetFlattenShape(node, input_shape, dim_1, dim_2); + + if (dim_1 == 0 && dim_2 == 0) { + LOGS_DEFAULT(VERBOSE) << "The dynamical input shape " << Shape2String(input_shape) + << " is not supported"; + return false; + } + + return true; +} + +Status FlattenOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { + auto input = node.InputDefs()[0]->Name(); + if (model_builder.IsOperandNHWC(input)) { + // We want to transpose nhwc operand back to nchw before reshape + ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, 0, input)); + } + + // Flatten is basically a reshape to 2d tensor + // Get the shape for Reshape here + Shape input_shape; + GetShape(*node.InputDefs()[0], input_shape); + int32_t dim_1 = 1; + int32_t dim_2 = 1; + GetFlattenShape(node, input_shape, dim_1, dim_2); + // If the input is of dynamic shape, replace 0 (dynamic) dimension with -1 + // We cannot have dim_1 and dim_2 both be 0 here, it was checked in IsOpSupportedImpl + dim_1 = dim_1 == 0 ? -1 : dim_1; + dim_2 = dim_2 == 0 ? -1 : dim_2; + std::vector shape{dim_1, dim_2}; + return ReshapeOpBuilder::AddReshapeOperator(model_builder, node, input, shape); +} + +#pragma endregion op_reshape + #pragma region CreateOpBuilders std::unordered_map> @@ -2982,6 +3131,7 @@ CreateOpBuilders() { op_map.emplace("LRN", std::make_shared()); op_map.emplace("Clip", std::make_shared()); op_map.emplace("Resize", std::make_shared()); + op_map.emplace("Flatten", std::make_shared()); return op_map; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc index 110bf7f4f2..1aff71f01a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc @@ -78,7 +78,7 @@ size_t Model::GetMappedOutputIdx(const std::string& name) const { } bool Model::SupportsDynamicOutputShape() const { - // dynamic output shape is only supported on Android API levle 29+ + // dynamic output shape is only supported on Android API level 29+ return GetAndroidSdkVer() >= 29 && dynamic_output_buffer_size_ > 0; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h index 7527e1c4e5..3bfb09e54f 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h @@ -81,11 +81,11 @@ class Model { bool SupportsDynamicOutputShape() const; // Set and Get the number of elements in the buffer for a dynamic output - // If the buffer is not big enough, ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE will be returned by exection + // If the buffer is not big enough, ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE will be returned by execution // Note: this will return number of elements of the buffer not the byte size of the buffer // and each output will have its separated buffer // TODO: - // 1. Consider an adaptive aproach to automatically increase the buffer size if the execution reports + // 1. Consider an adaptive approach to automatically increase the buffer size if the execution reports // insufficient size // 2. Experiment with bigger initial buffer size (currently 1024) size_t GetDynamicOutputBufferSize() const { return dynamic_output_buffer_size_; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index f25839b6a5..939188c2b8 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -312,7 +312,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vectorGetDynamicOutputBufferSize() * model_output_type.GetElementByteSize(); std::unique_ptr buffer_holder(new uint8_t[output_buffer_byte_size]); output_buffer = buffer_holder.get(); diff --git a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc index 877303f921..4432837756 100644 --- a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc +++ b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc @@ -2,8 +2,9 @@ #include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h" #include "core/session/inference_session.h" #include "gtest/gtest.h" -#include "test/providers/provider_test_utils.h" #include "test/framework/test_utils.h" +#include "test/providers/provider_test_utils.h" +#include "test/util/include/inference_session_wrapper.h" using namespace std; using namespace ONNX_NAMESPACE; @@ -22,41 +23,99 @@ void VerifyOutputs(const std::vector& fetches, const std::vector& output_names, + const std::vector& expected_dims, + const std::vector& expected_values) { + SessionOptions so; + so.session_logid = log_id; + RunOptions run_options; + run_options.run_tag = so.session_logid; + + InferenceSessionWrapper session_object{so, GetEnvironment()}; + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(onnxruntime::make_unique<::onnxruntime::NnapiExecutionProvider>())); + ASSERT_STATUS_OK(session_object.Load(model_file_name)); + ASSERT_STATUS_OK(session_object.Initialize()); + + // Since we already know the model is entirely supported by NNAPI, all nodes (2 Add nodes here) will be fused + // Get the graph after session is initialized, and verify the fused node (the only node in the graph) is using NNAPI EP + const auto& graph = session_object.GetGraph(); + ASSERT_EQ(1, graph.NumberOfNodes()); // Make sure the graph has 1 fused node + ASSERT_EQ(onnxruntime::kNnapiExecutionProvider, graph.Nodes().cbegin()->GetExecutionProviderType()); + + // Now run and verify the result + std::vector fetches; + ASSERT_STATUS_OK(session_object.Run(run_options, feeds, output_names, &fetches)); + VerifyOutputs(fetches, expected_dims, expected_values); +} + +// Since NNAPI EP handles Reshape and Flatten differently, +// Please see ReshapeOpBuilder::CanSkipReshape in /onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +// We have a separated test for these skip reshape scenarios +TEST(NnapiExecutionProviderTest, ReshapeFlattenTest) { + std::string model_file_name = "testdata/nnapi_reshape_flatten_test.onnx"; + + std::vector dims_mul_x = {2, 1, 2}; + std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f}; + std::vector dims_mul_y = {3, 2, 2}; + std::vector values_mul_y = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f}; + OrtValue ml_value_x; + CreateMLValue(TestNnapiExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_x); + OrtValue ml_value_y; + CreateMLValue(TestNnapiExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_y, values_mul_y, &ml_value_y); + NameMLValMap feeds; + feeds.insert(std::make_pair("X", ml_value_x)); + feeds.insert(std::make_pair("Y", ml_value_y)); + + // prepare outputs + std::vector output_names; + output_names.push_back("Z"); + + // prepare expected inputs and outputs + std::vector expected_dims_mul_z = {1, 6}; + std::vector expected_values_mul_z = {59.0f, 72.0f, 129.0f, 159.0f, 204.0f, 253.0f}; + + RunAndVerifyOutputs(model_file_name, "NnapiExecutionProviderTest.ReshapeFlattenTest", feeds, output_names, expected_dims_mul_z, expected_values_mul_z); +} + TEST(NnapiExecutionProviderTest, FunctionTest) { - onnxruntime::Model model("graph_1", false, DefaultLoggingManager().DefaultLogger()); - auto& graph = model.MainGraph(); - std::vector inputs; - std::vector outputs; - - // FLOAT tensor. - ONNX_NAMESPACE::TypeProto float_tensor; - float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); - float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); - float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); - float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3); - float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2); - - auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor); - auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor); - inputs.push_back(&input_arg_1); - inputs.push_back(&input_arg_2); - auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &float_tensor); - outputs.push_back(&output_arg); - graph.AddNode("node_1", "Add", "node 1.", inputs, outputs); - - auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &float_tensor); - inputs.clear(); - inputs.push_back(&output_arg); - inputs.push_back(&input_arg_3); - auto& output_arg_2 = graph.GetOrCreateNodeArg("M", &float_tensor); - outputs.clear(); - outputs.push_back(&output_arg_2); - graph.AddNode("node_2", "Add", "node 2.", inputs, outputs); - - auto status = graph.Resolve(); - ASSERT_TRUE(status.IsOK()); std::string model_file_name = "nnapi_execution_provider_test_graph.onnx"; - status = onnxruntime::Model::Save(model, model_file_name); + { // Create the model with 2 add nodes + onnxruntime::Model model("graph_1", false, DefaultLoggingManager().DefaultLogger()); + auto& graph = model.MainGraph(); + std::vector inputs; + std::vector outputs; + + // FLOAT tensor. + ONNX_NAMESPACE::TypeProto float_tensor; + float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2); + + auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor); + auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor); + inputs.push_back(&input_arg_1); + inputs.push_back(&input_arg_2); + auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &float_tensor); + outputs.push_back(&output_arg); + graph.AddNode("node_1", "Add", "node 1.", inputs, outputs); + + auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &float_tensor); + inputs.clear(); + inputs.push_back(&output_arg); + inputs.push_back(&input_arg_3); + auto& output_arg_2 = graph.GetOrCreateNodeArg("M", &float_tensor); + outputs.clear(); + outputs.push_back(&output_arg_2); + graph.AddNode("node_2", "Add", "node 2.", inputs, outputs); + + ASSERT_STATUS_OK(graph.Resolve()); + ASSERT_STATUS_OK(onnxruntime::Model::Save(model, model_file_name)); + } std::vector dims_mul_x = {1, 1, 3, 2}; std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; @@ -80,23 +139,7 @@ TEST(NnapiExecutionProviderTest, FunctionTest) { std::vector expected_dims_mul_m = {1, 1, 3, 2}; std::vector expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f}; - SessionOptions so; - so.session_logid = "NnapiExecutionProviderTest.FunctionTest"; - RunOptions run_options; - run_options.run_tag = so.session_logid; - - InferenceSession session_object{so, GetEnvironment()}; - status = session_object.RegisterExecutionProvider(onnxruntime::make_unique<::onnxruntime::NnapiExecutionProvider>()); - ASSERT_TRUE(status.IsOK()); - status = session_object.Load(model_file_name); - ASSERT_TRUE(status.IsOK()); - status = session_object.Initialize(); - ASSERT_TRUE(status.IsOK()); - - // Now run - status = session_object.Run(run_options, feeds, output_names, &fetches); - ASSERT_TRUE(status.IsOK()); - VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m); + RunAndVerifyOutputs(model_file_name, "NnapiExecutionProviderTest.FunctionTest", feeds, output_names, expected_dims_mul_m, expected_values_mul_m); } } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/testdata/nnapi_reshape_flatten_test.onnx b/onnxruntime/test/testdata/nnapi_reshape_flatten_test.onnx new file mode 100644 index 0000000000000000000000000000000000000000..120e2ff6f5bb0021761c7e17243225aad727a131 GIT binary patch literal 602 zcmZ`%-AaT&6rLGHr=yfi7j0gJ1d&&*P!~&3&PCD1E)iX%H)3c%pb_apm-{IDD7`{& z(4%zRok>{;{5YQTedm00M!7xtHsKKHgK(~@ylu<3Y@8+j){{;GTh(C3l1C172bb&WpjnE2*1 z=7HH&;r#C$1EXluDC$!2uzQ3qr^x zK8OWkiTFbNAV@6LwS^&P+`x{}e;rbPK$-4!e}VvyQ{e*R3_-it5y^ls6v-ZUfu>>` MT/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +# We have a separated test for these skip reshape scenarios +def GenerateModel(model_name): + nodes = [ + helper.make_node("Flatten", ["X"], ["Flatten_1_Y"], "flatten_1"), + helper.make_node("MatMul", ["Flatten_1_Y", "MatMul_B"], ["MatMul_Y"], "matmul"), + helper.make_node("Reshape", ["Y", "Reshape_1_shape"], ["Reshape_1_Y"], "reshape_1"), + helper.make_node("Gemm", ["Reshape_1_Y", "Gemm_B"], ["Gemm_Y"], "gemm"), + helper.make_node("Reshape", ["MatMul_Y", "Reshape_2_shape"], ["Reshape_2_Y"], "reshape_2"), + helper.make_node("Flatten", ["Gemm_Y"], ["Flatten_2_Y"], "flatten_2", axis=0), + helper.make_node("Add", ["Reshape_2_Y", "Flatten_2_Y"], ["Z"], "add"), + ] + + initializers = [ + helper.make_tensor('Reshape_1_shape', TensorProto.INT64, [2], [3, 4]), + helper.make_tensor('Reshape_2_shape', TensorProto.INT64, [2], [1, 6]), + helper.make_tensor('Gemm_B', TensorProto.FLOAT, [4, 2], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]), + helper.make_tensor('MatMul_B', TensorProto.FLOAT, [2, 3], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + ] + + inputs = [ + helper.make_tensor_value_info('X', TensorProto.FLOAT, [2, 1, 2]), + helper.make_tensor_value_info('Y', TensorProto.FLOAT, [3, 2, 2]) + ] + + graph = helper.make_graph( + nodes, + "NNAPI_Reshape_Flatten_Test", + inputs, + [helper.make_tensor_value_info('Z', TensorProto.FLOAT, [1, 6])], + initializers + ) + + model = helper.make_model(graph) + onnx.save(model, model_name) + + +if __name__ == "__main__": + GenerateModel('nnapi_reshape_flatten_test.onnx')