diff --git a/BUILD.md b/BUILD.md index 182339df84..c4c3be315e 100644 --- a/BUILD.md +++ b/BUILD.md @@ -19,6 +19,7 @@ * [Nuphar Model Compiler](#Nuphar) * [DirectML](#DirectML) * [ARM Compute Library](#ARM-Compute-Library) + * [ArmNN](#ArmNN) * [Rockchip RKNPU](#RKNPU) * [Xilinx Vitis-AI](#Vitis-AI) * Options @@ -491,6 +492,31 @@ export LD_LIBRARY_PATH=~/ComputeLibrary/build/ --- +### ArmNN +See more information on the ArmNN Execution Provider [here](./docs/execution_providers/ArmNN-ExecutionProvider.md). + +#### Prerequisites +* Supported backend: i.MX8QM Armv8 CPUs +* Supported BSP: i.MX8QM BSP + * Install i.MX8QM BSP: `source fsl-imx-xwayland-glibc-x86_64-fsl-image-qt5-aarch64-toolchain-4*.sh` +* Set up the build environment +``` +source /opt/fsl-imx-xwayland/4.*/environment-setup-aarch64-poky-linux +alias cmake="/usr/bin/cmake -DCMAKE_TOOLCHAIN_FILE=$OECORE_NATIVE_SYSROOT/usr/share/cmake/OEToolchainConfig.cmake" +``` +* See [Build ARM](#ARM) below for information on building for ARM devices + +#### Build Instructions +``` +./build.sh --use_armnn +``` +The Relu operator is set by default to use the CPU execution provider for better performance. To use the ArmNN implementation build with --armnn_relu flag +``` +./build.sh --use_armnn --armnn_relu +``` + +--- + ### RKNPU See more information on the RKNPU Execution Provider [here](./docs/execution_providers/RKNPU-ExecutionProvider.md). diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index c253eed572..6596ed2f45 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -92,6 +92,8 @@ option(onnxruntime_USE_ACL "Build with ACL support" OFF) option(onnxruntime_USE_ACL_1902 "Build with ACL version 1902 support" OFF) option(onnxruntime_USE_ACL_1905 "Build with ACL version 1905 support" OFF) option(onnxruntime_USE_ACL_1908 "Build with ACL version 1908 support" OFF) +option(onnxruntime_USE_ARMNN "Build with ArmNN support" OFF) +option(onnxruntime_ARMNN_RELU_USE_CPU "Use the CPU implementation for the Relu operator for the ArmNN EP" ON) option(onnxruntime_ENABLE_INSTRUMENT "Enable Instrument with Event Tracing for Windows (ETW)" OFF) option(onnxruntime_USE_TELEMETRY "Build with Telemetry" OFF) #The onnxruntime_PREFER_SYSTEM_LIB is mainly designed for package managers like apt/yum/vcpkg. @@ -513,6 +515,15 @@ if (onnxruntime_USE_ACL OR onnxruntime_USE_ACL_1902 OR onnxruntime_USE_ACL_1905 list(APPEND onnxruntime_EXTERNAL_LIBRARIES arm_compute arm_compute_graph arm_compute_core) endif() +# ArmNN +if (onnxruntime_USE_ARMNN) + if (NOT onnxruntime_ARMNN_RELU_USE_CPU) + add_definitions(-DRELU_ARMNN=1) + endif() + + list(APPEND onnxruntime_EXTERNAL_LIBRARIES armnn pthread arm_compute_core arm_compute arm_compute_graph) +endif() + # MKLML if (onnxruntime_USE_DNNL OR onnxruntime_USE_MKLML) include(dnnl) diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index a234638b63..2c0768d9c5 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -88,6 +88,7 @@ target_link_libraries(onnxruntime PRIVATE ${PROVIDERS_VITISAI} ${PROVIDERS_DML} ${PROVIDERS_ACL} + ${PROVIDERS_ARMNN} ${onnxruntime_winml} onnxruntime_optimizer onnxruntime_providers diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 2f33a36532..780023362d 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -83,6 +83,10 @@ if(onnxruntime_USE_ACL) set(PROVIDERS_ACL onnxruntime_providers_acl) list(APPEND ONNXRUNTIME_PROVIDER_NAMES acl) endif() +if(onnxruntime_USE_ARMNN) + set(PROVIDERS_ARMNN onnxruntime_providers_armnn) + list(APPEND ONNXRUNTIME_PROVIDER_NAMES armnn) +endif() source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs}) set(onnxruntime_providers_src ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs}) @@ -654,3 +658,20 @@ if (onnxruntime_USE_ACL) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/acl DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_acl PROPERTIES LINKER_LANGUAGE CXX) endif() + +if (onnxruntime_USE_ARMNN) + add_definitions(-DUSE_ARMNN=1) + file(GLOB_RECURSE onnxruntime_providers_armnn_cc_srcs + "${ONNXRUNTIME_ROOT}/core/providers/armnn/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/armnn/*.cc" + ) + + source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_armnn_cc_srcs}) + add_library(onnxruntime_providers_armnn ${onnxruntime_providers_armnn_cc_srcs}) + onnxruntime_add_include_to_target(onnxruntime_providers_armnn onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf) + add_dependencies(onnxruntime_providers_armnn ${onnxruntime_EXTERNAL_DEPENDENCIES}) + set_target_properties(onnxruntime_providers_armnn PROPERTIES FOLDER "ONNXRuntime") + target_include_directories(onnxruntime_providers_armnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${ARMNN_INCLUDE_DIR}) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/armnn DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) + set_target_properties(onnxruntime_providers_armnn PROPERTIES LINKER_LANGUAGE CXX) +endif() diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 19a947057f..48ea4866fd 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -336,6 +336,10 @@ if(onnxruntime_USE_ACL) list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_acl) endif() +if(onnxruntime_USE_ARMNN) + list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_armnn) +endif() + if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) set(ONNXRUNTIME_INTEROP_TEST_LIBS PRIVATE onnxruntime_language_interop onnxruntime_pyop) endif() @@ -355,6 +359,7 @@ set(ONNXRUNTIME_TEST_LIBS ${PROVIDERS_RKNPU} ${PROVIDERS_DML} ${PROVIDERS_ACL} + ${PROVIDERS_ARMNN} onnxruntime_optimizer onnxruntime_providers onnxruntime_util diff --git a/docs/execution_providers/ArmNN-ExecutionProvider.md b/docs/execution_providers/ArmNN-ExecutionProvider.md new file mode 100644 index 0000000000..76078bf408 --- /dev/null +++ b/docs/execution_providers/ArmNN-ExecutionProvider.md @@ -0,0 +1,29 @@ +## ArmNN Execution Provider + +[ArmNN](https://github.com/ARM-software/armnn) is an open source inference engine maintained by Arm and Linaro companies. The integration of ArmNN as an execution provider (EP) into ONNX Runtime accelerates performance of ONNX model workloads across Armv8 cores. + +### Build ArmNN execution provider +For build instructions, please see the [BUILD page](../../BUILD.md#ArmNN). + +### Using the ArmNN execution provider +#### C/C++ +To use ArmNN as execution provider for inferencing, please register it as below. +``` +string log_id = "Foo"; +auto logging_manager = std::make_unique +(std::unique_ptr{new CLogSink{}}, + static_cast(lm_info.default_warning_level), + false, + LoggingManager::InstanceType::Default, + &log_id) +Environment::Create(std::move(logging_manager), env) +InferenceSession session_object{so, env}; +session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::ArmNNExecutionProvider>()); +status = session_object.Load(model_file_name); +``` +The C API details are [here](../C_API.md#c-api). + +### Performance Tuning +For performance tuning, please see guidance on this page: [ONNX Runtime Perf Tuning](../ONNX_Runtime_Perf_Tuning.md) + +When/if using [onnxruntime_perf_test](../../onnxruntime/test/perftest), use the flag -e armnn diff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.h index fa69c90113..1cf3677e2e 100644 --- a/include/onnxruntime/core/graph/constants.h +++ b/include/onnxruntime/core/graph/constants.h @@ -37,4 +37,5 @@ constexpr const char* kRknpuExecutionProvider = "RknpuExecutionProvider"; constexpr const char* kDmlExecutionProvider = "DmlExecutionProvider"; constexpr const char* kMIGraphXExecutionProvider = "MIGraphXExecutionProvider"; constexpr const char* kAclExecutionProvider = "ACLExecutionProvider"; +constexpr const char* kArmNNExecutionProvider = "ArmNNExecutionProvider"; } // namespace onnxruntime diff --git a/include/onnxruntime/core/providers/armnn/armnn_provider_factory.h b/include/onnxruntime/core/providers/armnn/armnn_provider_factory.h new file mode 100644 index 0000000000..35276db323 --- /dev/null +++ b/include/onnxruntime/core/providers/armnn/armnn_provider_factory.h @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "onnxruntime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \param use_arena zero: false. non-zero: true. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ArmNN, _In_ OrtSessionOptions* options, int use_arena) +ORT_ALL_ARGS_NONNULL; + +#ifdef __cplusplus +} +#endif diff --git a/onnxruntime/core/optimizer/transformer_memcpy.cc b/onnxruntime/core/optimizer/transformer_memcpy.cc index 560d5cfe67..9100f9bf62 100644 --- a/onnxruntime/core/optimizer/transformer_memcpy.cc +++ b/onnxruntime/core/optimizer/transformer_memcpy.cc @@ -76,7 +76,8 @@ common::Status MemcpyTransformer::ApplyImpl(Graph& graph, bool& modified, int gr provider != onnxruntime::kVitisAIExecutionProvider && provider != onnxruntime::kOpenVINOExecutionProvider && provider != onnxruntime::kNnapiExecutionProvider && - provider != onnxruntime::kAclExecutionProvider) { + provider != onnxruntime::kAclExecutionProvider && + provider != onnxruntime::kArmNNExecutionProvider) { TransformerMemcpyImpl copy_impl(graph, provider); auto current_modified = copy_impl.ModifyGraph(registry_manager_); modified = modified || current_modified; diff --git a/onnxruntime/core/providers/armnn/activation/activations.cc b/onnxruntime/core/providers/armnn/activation/activations.cc new file mode 100644 index 0000000000..b76a2bf3b4 --- /dev/null +++ b/onnxruntime/core/providers/armnn/activation/activations.cc @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License + +#ifdef RELU_ARMNN + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/armnn/armnn_common.h" +#include "core/providers/armnn/activation/activations.h" +#include "core/providers/armnn/armnn_fwd.h" + +namespace onnxruntime { +namespace armnn_ep { + +template +thread_local std::map Relu::reluLayers; + +template +armnn::IRuntimePtr Relu::run = Relu::initRuntime(); + +template +Status Relu::Compute(OpKernelContext* context) const { + + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + + const T* src_data = X->template Data(); + T* dst_data = Y->template MutableData(); + + armnn::NetworkId* pNetworkId; + ReluLayersIterator it = Relu::reluLayers.find((OpKernel*)this); + if (it == Relu::reluLayers.end()) { + + armnn::NetworkId networkId; + armnn::INetworkPtr myNetwork = armnn::INetwork::Create(); + + armnn::TensorShape inputShape = ArmNNTensorShape(X->Shape()); + armnn::TensorShape outputShape = ArmNNTensorShape(Y->Shape()); + + armnn::ActivationDescriptor desc; + desc.m_Function = armnn::ActivationFunction::ReLu; + + armnn::IConnectableLayer* activation = myNetwork->AddActivationLayer(desc, "relu_armnn"); + + armnn::IConnectableLayer *InputLayer = myNetwork->AddInputLayer(0); + armnn::IConnectableLayer *OutputLayer = myNetwork->AddOutputLayer(0); + + InputLayer->GetOutputSlot(0).Connect(activation->GetInputSlot(0)); + activation->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0)); + + //Set the tensors in the network. + armnn::TensorInfo inputTensorInfo(inputShape, armnn::DataType::Float32); + InputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo); + + armnn::TensorInfo outputTensorInfo(outputShape, armnn::DataType::Float32); + activation->GetOutputSlot(0).SetTensorInfo(outputTensorInfo); + + // Optimise ArmNN network + armnn::IOptimizedNetworkPtr optNet = armnn::Optimize(*myNetwork, {armnn::Compute::CpuAcc}, Relu::run->GetDeviceSpec()); + + if (optNet == nullptr) { + ORT_NOT_IMPLEMENTED("Something went wrong when creating the layer"); + } + + // Load graph into runtime + Relu::run->LoadNetwork(networkId, std::move(optNet)); + + std::pair ret; + ret = Relu::reluLayers.insert(std::pair((OpKernel*)this, networkId)); + pNetworkId = &ret.first->second; + + } else { + pNetworkId = &it->second; + } + + armnn::InputTensors inputTensors{{0, armnn::ConstTensor(Relu::run->GetInputTensorInfo(*pNetworkId, 0), + src_data)}}; + armnn::OutputTensors outputTensors{{0, armnn::Tensor(Relu::run->GetOutputTensorInfo(*pNetworkId, 0), + dst_data)}}; + + Relu::run->EnqueueWorkload(*pNetworkId, inputTensors, outputTensors); + + return Status::OK(); +} + +ONNX_OPERATOR_KERNEL_EX( + Relu, + kOnnxDomain, + 6, + kArmNNExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Relu); + +} // namespace armnn_ep +} // namespace onnxruntime + +#endif diff --git a/onnxruntime/core/providers/armnn/activation/activations.h b/onnxruntime/core/providers/armnn/activation/activations.h new file mode 100644 index 0000000000..ff191cd890 --- /dev/null +++ b/onnxruntime/core/providers/armnn/activation/activations.h @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License + +#ifdef RELU_ARMNN + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/activation/activations.h" +#include "core/providers/armnn/armnn_execution_provider.h" + +#include "armnn/ArmNN.hpp" + +#include +#include + +namespace onnxruntime { +namespace armnn_ep { + +typedef std::map::iterator ReluLayersIterator; + +template +class Relu : public OpKernel { + public: + explicit Relu(const OpKernelInfo& info) : OpKernel(info) { + provider_ = (const_cast( + dynamic_cast(info.GetExecutionProvider()))); + } + + ~Relu() { + Relu::reluLayers.erase(this); + } + + Status Compute(OpKernelContext* context) const override; + + static armnn::IRuntimePtr initRuntime(){ + if (Relu::run) + return std::move(Relu::run); + armnn::IRuntime::CreationOptions options; + return std::move(armnn::IRuntime::Create(options)); + } + + private: + static thread_local std::map reluLayers; + ArmNNExecutionProvider* provider_; + static armnn::IRuntimePtr run; +}; + +} // namespace armnn_ep +} // namespace onnxruntime + +#endif diff --git a/onnxruntime/core/providers/armnn/armnn_common.cc b/onnxruntime/core/providers/armnn/armnn_common.cc new file mode 100644 index 0000000000..847260d747 --- /dev/null +++ b/onnxruntime/core/providers/armnn/armnn_common.cc @@ -0,0 +1,25 @@ +// Copyright(C) 2018 Intel Corporation +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/armnn/armnn_common.h" + +namespace onnxruntime { +namespace armnn_ep { + +armnn::TensorShape ArmNNTensorShape(const TensorShape& tensorShape) { + std::vector dims; + unsigned int inDim = tensorShape.NumDimensions(); + + for (unsigned int i = 0; i < inDim; ++i) + dims.push_back(tensorShape.GetDims()[i]); + + return armnn::TensorShape{static_cast(dims.size()), dims.data()}; +} + +} // namespace armnn_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/armnn/armnn_common.h b/onnxruntime/core/providers/armnn/armnn_common.h new file mode 100644 index 0000000000..2195c837dd --- /dev/null +++ b/onnxruntime/core/providers/armnn/armnn_common.h @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +#include "armnn/ArmNN.hpp" + + +namespace onnxruntime { +namespace armnn_ep { + +armnn::TensorShape ArmNNTensorShape(const TensorShape& tensorShape); + +} // namespace armnn_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/armnn/armnn_execution_provider.cc b/onnxruntime/core/providers/armnn/armnn_execution_provider.cc new file mode 100644 index 0000000000..a60f3db5c6 --- /dev/null +++ b/onnxruntime/core/providers/armnn/armnn_execution_provider.cc @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#include "armnn_execution_provider.h" +#include "core/framework/allocator.h" +#include "core/framework/op_kernel.h" +#include "core/framework/kernel_registry.h" +#include "core/framework/compute_capability.h" +#include "contrib_ops/cpu/cpu_contrib_kernels.h" +#include "armnn_fwd.h" + +namespace onnxruntime { + +constexpr const char* ArmNN = "ArmNN"; +constexpr const char* ArmNN_CPU = "ArmNNCpu"; + +namespace armnn_ep { + +// Forward declarations of op kernels +#ifdef RELU_ARMNN +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 6, Relu); +#endif +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 10, Conv); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Conv); + +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 8, Gemm); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 9, 10, Gemm); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Gemm); + +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 9, float, AveragePool); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 10, 10, float, AveragePool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, float, AveragePool); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 8, 11, float, MaxPool); + +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, float, GlobalAveragePool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, float, GlobalMaxPool); + +static void RegisterArmNNKernels(KernelRegistry& kernel_registry) { + +#ifdef RELU_ARMNN + kernel_registry.Register(BuildKernelCreateInfo()); +#endif + kernel_registry.Register(BuildKernelCreateInfo()); + kernel_registry.Register(BuildKernelCreateInfo()); + + kernel_registry.Register(BuildKernelCreateInfo()); + kernel_registry.Register(BuildKernelCreateInfo()); + kernel_registry.Register(BuildKernelCreateInfo()); + + kernel_registry.Register(BuildKernelCreateInfo()); + kernel_registry.Register(BuildKernelCreateInfo()); + kernel_registry.Register(BuildKernelCreateInfo()); + kernel_registry.Register(BuildKernelCreateInfo()); + kernel_registry.Register(BuildKernelCreateInfo()); + + kernel_registry.Register(BuildKernelCreateInfo()); + kernel_registry.Register(BuildKernelCreateInfo()); + +} + +std::shared_ptr GetArmNNKernelRegistry() { + std::shared_ptr kernel_registry = std::make_shared(); + RegisterArmNNKernels(*kernel_registry); + + return kernel_registry; +} + +} // namespace armnn_ep + +ArmNNExecutionProvider::ArmNNExecutionProvider(const ArmNNExecutionProviderInfo& info) + : IExecutionProvider{onnxruntime::kArmNNExecutionProvider} { + ORT_UNUSED_PARAMETER(info); + + auto default_allocator_factory = [](int) { + auto memory_info = onnxruntime::make_unique(ArmNN, OrtAllocatorType::OrtDeviceAllocator); + return onnxruntime::make_unique(std::move(memory_info)); + }; + + DeviceAllocatorRegistrationInfo default_memory_info{ + OrtMemTypeDefault, + std::move(default_allocator_factory), + std::numeric_limits::max()}; + + InsertAllocator(CreateAllocator(default_memory_info)); + + auto cpu_allocator_factory = [](int) { + auto memory_info = onnxruntime::make_unique( + ArmNN_CPU, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeCPUOutput); + return onnxruntime::make_unique(std::move(memory_info)); + }; + + DeviceAllocatorRegistrationInfo cpu_memory_info{ + OrtMemTypeCPUOutput, + std::move(cpu_allocator_factory), + std::numeric_limits::max()}; + + InsertAllocator(CreateAllocator(cpu_memory_info)); +} + +ArmNNExecutionProvider::~ArmNNExecutionProvider() { +} + +std::shared_ptr ArmNNExecutionProvider::GetKernelRegistry() const { + static std::shared_ptr kernel_registry = onnxruntime::armnn_ep::GetArmNNKernelRegistry(); + return kernel_registry; +} + +std::vector> +ArmNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& kernel_registries) const { + std::vector> + result = IExecutionProvider::GetCapability(graph, kernel_registries); + + return result; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/armnn/armnn_execution_provider.h b/onnxruntime/core/providers/armnn/armnn_execution_provider.h new file mode 100755 index 0000000000..d16bacc5c5 --- /dev/null +++ b/onnxruntime/core/providers/armnn/armnn_execution_provider.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/allocatormgr.h" +#include "core/framework/execution_provider.h" +#include "core/graph/constants.h" + +namespace onnxruntime { + +// Information needed to construct ArmNN execution providers. +struct ArmNNExecutionProviderInfo { + bool create_arena{true}; + + explicit ArmNNExecutionProviderInfo(bool use_arena) + : create_arena(use_arena) {} + + ArmNNExecutionProviderInfo() = default; +}; + +// Logical device representation. +class ArmNNExecutionProvider : public IExecutionProvider { + public: + explicit ArmNNExecutionProvider(const ArmNNExecutionProviderInfo& info); + virtual ~ArmNNExecutionProvider(); + + std::vector> GetCapability( + const onnxruntime::GraphViewer& graph, + const std::vector& kernel_registries) const override; + + const void* GetExecutionHandle() const noexcept override { + // The ArmNN interface does not return anything interesting. + return nullptr; + } + + std::shared_ptr GetKernelRegistry() const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/armnn/armnn_fwd.h b/onnxruntime/core/providers/armnn/armnn_fwd.h new file mode 100755 index 0000000000..4e60b14889 --- /dev/null +++ b/onnxruntime/core/providers/armnn/armnn_fwd.h @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace armnn_ep { +template +KernelCreateInfo BuildKernelCreateInfo(); +} +} diff --git a/onnxruntime/core/providers/armnn/armnn_provider_factory.cc b/onnxruntime/core/providers/armnn/armnn_provider_factory.cc new file mode 100755 index 0000000000..e29d5efba0 --- /dev/null +++ b/onnxruntime/core/providers/armnn/armnn_provider_factory.cc @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/armnn/armnn_provider_factory.h" +#include "armnn_execution_provider.h" +#include "core/session/abi_session_options_impl.h" + +namespace onnxruntime { + +struct ArmNNProviderFactory : IExecutionProviderFactory { + ArmNNProviderFactory(bool create_arena) : create_arena_(create_arena) {} + ~ArmNNProviderFactory() override {} + std::unique_ptr CreateProvider() override; + + private: + bool create_arena_; +}; + +std::unique_ptr ArmNNProviderFactory::CreateProvider() { + ArmNNExecutionProviderInfo info; + info.create_arena = create_arena_; + return std::make_unique(info); +} + +std::shared_ptr CreateExecutionProviderFactory_ArmNN(int use_arena) { + return std::make_shared(use_arena != 0); +} + +} // namespace onnxruntime + +ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_ArmNN, _In_ OrtSessionOptions* options, int use_arena) { + options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_ArmNN(use_arena)); + return nullptr; +} diff --git a/onnxruntime/core/providers/armnn/math/gemm.cc b/onnxruntime/core/providers/armnn/math/gemm.cc new file mode 100755 index 0000000000..d0943b08b0 --- /dev/null +++ b/onnxruntime/core/providers/armnn/math/gemm.cc @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/armnn/armnn_common.h" +#include "core/providers/armnn/math/gemm.h" +#include "core/providers/armnn/armnn_fwd.h" + +namespace onnxruntime { +namespace armnn_ep { + +ONNX_OPERATOR_VERSIONED_KERNEL_EX( + Gemm, + kOnnxDomain, + 7, + 8, + kArmNNExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Gemm); + +ONNX_OPERATOR_VERSIONED_KERNEL_EX( + Gemm, + kOnnxDomain, + 9, + 10, + kArmNNExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Gemm); + +ONNX_OPERATOR_KERNEL_EX( + Gemm, + kOnnxDomain, + 11, + kArmNNExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Gemm); + +} // namespace armnn_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/armnn/math/gemm.h b/onnxruntime/core/providers/armnn/math/gemm.h new file mode 100644 index 0000000000..ca31210cb8 --- /dev/null +++ b/onnxruntime/core/providers/armnn/math/gemm.h @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/op_kernel.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" +#include "core/providers/cpu/math/gemm.h" +#include "core/providers/armnn/armnn_execution_provider.h" + +namespace onnxruntime { +namespace armnn_ep { + +typedef std::map::iterator GEMMLayersIterator; + +template +class Gemm : public onnxruntime::Gemm { + public: + Gemm(const OpKernelInfo& info) : onnxruntime::Gemm(info) { + int64_t temp; + + ORT_ENFORCE(info.GetAttr("transA", &temp).IsOK()); + trans_A_ = temp == 0 ? CblasNoTrans : CblasTrans; + ORT_ENFORCE(info.GetAttr("transB", &temp).IsOK()); + trans_B_ = temp == 0 ? CblasNoTrans : CblasTrans; + + ORT_ENFORCE(info.GetAttr("alpha", &alpha_).IsOK()); + ORT_ENFORCE(info.GetAttr("beta", &beta_).IsOK()); + } + + Status Compute(OpKernelContext* context) const override { + const auto X = context->Input(0); + const auto W = context->Input(1); + const auto B = context->Input(2); + + bool useBias = B != nullptr && beta_ != 0; + bool FC = alpha_ == 1 && (beta_ == 1 || beta_ == 0); + if (!FC) { + return onnxruntime::Gemm::Compute(context); + } + + GemmHelper helper(X->Shape(), trans_A_ != CblasNoTrans, W->Shape(), trans_B_ != CblasNoTrans, useBias ? B->Shape() : TensorShape({})); + + if (!helper.State().IsOK()) + return helper.State(); + + int64_t M = helper.M(); + int64_t N = helper.N(); + auto Y = context->Output(0, TensorShape({M, N})); + + if (trans_A_ == CblasTrans) { // transpose input + return onnxruntime::Gemm::Compute(context); + } + + int64_t K = helper.K(); + LOGS_DEFAULT(VERBOSE) << "Gemm ArmNN:" << std::endl; + if (X) LOGS_DEFAULT(VERBOSE) << "X " << X->Shape().ToString().c_str() << std::endl; + if (W) LOGS_DEFAULT(VERBOSE) << "W " << W->Shape().ToString().c_str() << std::endl; + if (B) LOGS_DEFAULT(VERBOSE) << "B " << B->Shape().ToString().c_str() << std::endl; + LOGS_DEFAULT(VERBOSE) << "Y " << Y->Shape().ToString().c_str() << std::endl; + LOGS_DEFAULT(VERBOSE) << "M " << (int)M << ", N " << (int)N << ", K " << (int)K << std::endl; + LOGS_DEFAULT(VERBOSE) << "Alfa " << alpha_ << ", Beta " << beta_ << std::endl; + LOGS_DEFAULT(VERBOSE) << "trans_A_ " << (trans_A_ == CblasTrans) << std::endl; + LOGS_DEFAULT(VERBOSE) << "trans_B_ " << (trans_B_ == CblasTrans) << std::endl; + LOGS_DEFAULT(VERBOSE) << std::endl; + + const T* x_data = X->template Data(); + const T* w_data = W->template Data(); + const T* b_data; + if (useBias) + b_data = B->template Data(); + T* y_data = Y->template MutableData(); + + armnn::NetworkId* pNetworkId; + GEMMLayersIterator it = Gemm::gemmLayers.find((OpKernel*)this); + if (it == Gemm::gemmLayers.end()) { + + armnn::NetworkId networkId; + + armnn::INetworkPtr myNetwork = armnn::INetwork::Create(); + + armnn::TensorShape inputShape = ArmNNTensorShape(X->Shape()); + armnn::TensorShape weightShape = ArmNNTensorShape(W->Shape()); + armnn::TensorShape outputShape = ArmNNTensorShape(Y->Shape()); + + armnn::FullyConnectedDescriptor fcDescriptor; + fcDescriptor.m_BiasEnabled = useBias; + fcDescriptor.m_TransposeWeightMatrix = trans_B_ == CblasTrans; + + armnn::IConnectableLayer* fc_armnn; + + armnn::TensorInfo weightsInfo(weightShape, armnn::DataType::Float32); + armnn::ConstTensor weights(weightsInfo, w_data); + + if (fcDescriptor.m_BiasEnabled) { + armnn::TensorShape biasShape = ArmNNTensorShape(B->Shape()); + if(B->Shape().NumDimensions() == 2){ + if(B->Shape().GetDims()[0] == 1 && B->Shape().GetDims()[1] > 1) + biasShape = {B->Shape().GetDims()[1]}; + } + armnn::TensorInfo biasDesc(biasShape, armnn::DataType::Float32); + armnn::ConstTensor bias(biasDesc, b_data); + fc_armnn = myNetwork->AddFullyConnectedLayer(fcDescriptor, + weights, + armnn::Optional(bias), + "fc_armnn"); + } else { + fc_armnn = myNetwork->AddFullyConnectedLayer(fcDescriptor, + weights, + armnn::EmptyOptional(), + "fc_armnn"); + } + + armnn::IConnectableLayer *InputLayer = myNetwork->AddInputLayer(0); + armnn::IConnectableLayer *OutputLayer = myNetwork->AddOutputLayer(0); + + InputLayer->GetOutputSlot(0).Connect(fc_armnn->GetInputSlot(0)); + fc_armnn->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0)); + + //Set the tensors in the network. + armnn::TensorInfo inputTensorInfo(inputShape, armnn::DataType::Float32); + InputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo); + + armnn::TensorInfo outputTensorInfo(outputShape, armnn::DataType::Float32); + fc_armnn->GetOutputSlot(0).SetTensorInfo(outputTensorInfo); + + // Optimise ArmNN network + armnn::IOptimizedNetworkPtr optNet = armnn::Optimize(*myNetwork, {armnn::Compute::CpuAcc}, Gemm::run->GetDeviceSpec()); + + if (optNet == nullptr) { + return onnxruntime::Gemm::Compute(context); + } + + // Load graph into runtime + Gemm::run->LoadNetwork(networkId, std::move(optNet)); + + std::pair ret; + ret = Gemm::gemmLayers.insert(std::pair((OpKernel*)this, networkId)); + pNetworkId = &ret.first->second; + + } else { + pNetworkId = &it->second; + } + + armnn::InputTensors inputTensors{{0, armnn::ConstTensor(Gemm::run->GetInputTensorInfo(*pNetworkId, 0), + x_data)}}; + armnn::OutputTensors outputTensors{{0, armnn::Tensor(Gemm::run->GetOutputTensorInfo(*pNetworkId, 0), + y_data)}}; + + Gemm::run->EnqueueWorkload(*pNetworkId, inputTensors, outputTensors); + + return Status::OK(); + } + + ~Gemm() { + gemmLayers.erase(this); + } + + static armnn::IRuntimePtr initRuntime(){ + if (Gemm::run) + return std::move(Gemm::run); + armnn::IRuntime::CreationOptions options; + return std::move(armnn::IRuntime::Create(options)); + } + + private: + static thread_local std::map gemmLayers; + ArmNNExecutionProvider* provider_; + static armnn::IRuntimePtr run; + + CBLAS_TRANSPOSE trans_A_; + CBLAS_TRANSPOSE trans_B_; + float alpha_; + float beta_; +}; + +template +thread_local std::map onnxruntime::armnn_ep::Gemm::gemmLayers; + +template +armnn::IRuntimePtr Gemm::run = Gemm::initRuntime(); + +} // namespace armnn_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/armnn/nn/conv.cc b/onnxruntime/core/providers/armnn/nn/conv.cc new file mode 100644 index 0000000000..f793d5acba --- /dev/null +++ b/onnxruntime/core/providers/armnn/nn/conv.cc @@ -0,0 +1,300 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" + +#include "core/providers/armnn/nn/conv.h" +#include "core/providers/armnn/armnn_common.h" +#include "core/providers/armnn/armnn_fwd.h" + +#define PREF_DIM 4 + +namespace onnxruntime { +namespace armnn_ep { + +template +thread_local std::map Conv::convLayers; + +template +armnn::IRuntimePtr Conv::run = Conv::initRuntime(); + +armnn::Convolution2dDescriptor createConvDescriptor(std::vector pads, std::vector dilations, std::vector strides, bool biasEnabled){ + + std::vector armnnStrides(2); + armnnStrides[0] = (strides.size() == 2) ? strides[1] : 1; + armnnStrides[1] = strides[0]; + + std::vector armnnDilations(2); + armnnDilations[0] = (dilations.size() == 2) ? dilations[1] : 1; + armnnDilations[1] = dilations[0]; + + std::vector armnnPads(4); + if (pads.size() == 2) { + if (strides.size() == 1) { + armnnPads[0] = 0; + armnnPads[1] = 0; + armnnPads[2] = pads[1]; + armnnPads[3] = pads[0]; + } else { + armnnPads[0] = pads[1]; + armnnPads[1] = pads[0]; + armnnPads[2] = pads[1]; + armnnPads[3] = pads[0]; + } + } else { + armnnPads[0] = pads[1]; + armnnPads[1] = pads[3]; + armnnPads[2] = pads[0]; + armnnPads[3] = pads[2]; + } + + armnn::Convolution2dDescriptor convolutionDescriptor; + convolutionDescriptor.m_PadLeft = armnnPads[0]; + convolutionDescriptor.m_PadRight = armnnPads[1]; + convolutionDescriptor.m_PadTop = armnnPads[2]; + convolutionDescriptor.m_PadBottom = armnnPads[3]; + convolutionDescriptor.m_StrideX = armnnStrides[0]; + convolutionDescriptor.m_StrideY = armnnStrides[1]; + convolutionDescriptor.m_DilationX = armnnDilations[0]; + convolutionDescriptor.m_DilationY = armnnDilations[1]; + convolutionDescriptor.m_BiasEnabled = biasEnabled; + convolutionDescriptor.m_DataLayout = armnn::DataLayout::NCHW; + + return convolutionDescriptor; +} + +armnn::DepthwiseConvolution2dDescriptor createDepthwiseDescriptor(armnn::Convolution2dDescriptor convolutionDescriptor){ + + armnn::DepthwiseConvolution2dDescriptor depthwiseDescriptor; + depthwiseDescriptor.m_PadLeft = convolutionDescriptor.m_PadLeft; + depthwiseDescriptor.m_PadRight = convolutionDescriptor.m_PadRight; + depthwiseDescriptor.m_PadTop = convolutionDescriptor.m_PadTop; + depthwiseDescriptor.m_PadBottom = convolutionDescriptor.m_PadBottom; + depthwiseDescriptor.m_StrideX = convolutionDescriptor.m_StrideX; + depthwiseDescriptor.m_StrideY = convolutionDescriptor.m_StrideY; + depthwiseDescriptor.m_DilationX = convolutionDescriptor.m_DilationX; + depthwiseDescriptor.m_DilationY = convolutionDescriptor.m_DilationY; + depthwiseDescriptor.m_BiasEnabled = convolutionDescriptor.m_BiasEnabled; + depthwiseDescriptor.m_DataLayout = convolutionDescriptor.m_DataLayout; + + return depthwiseDescriptor; +} + +template +Status Conv::Compute(OpKernelContext* context) const { + size_t num_inputs = OpKernel::Node().InputDefs().size(); + const Tensor* X = context->Input(0); + const Tensor* W = context->Input(1); + const Tensor* B = num_inputs == 3 ? context->Input(2) : nullptr; + + const int64_t N = X->Shape()[0]; + const int64_t M = W->Shape()[0]; + + if (X->Shape().NumDimensions() != PREF_DIM) { + Status s = onnxruntime::Conv::Compute(context); + return s; + } + + ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X, W)); + + std::vector kernel_shape; + ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape)); + + std::vector pads(conv_attrs_.pads); + if (pads.empty()) { + pads.resize(kernel_shape.size() * 2, 0); + } + std::vector dilations(conv_attrs_.dilations); + if (dilations.empty()) { + dilations.resize(kernel_shape.size(), 1); + } + std::vector strides(conv_attrs_.strides); + if (strides.empty()) { + strides.resize(kernel_shape.size(), 1); + } + + std::vector Y_dims; + Y_dims.insert(Y_dims.begin(), {N, M}); + TensorShape input_shape = X->Shape().Slice(2); + ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShape(input_shape, kernel_shape, strides, dilations, &pads, &Y_dims)); + Tensor* Y = context->Output(0, TensorShape(Y_dims)); + + bool biasEnabled = B != nullptr; + + const T* x_data = X->template Data(); + const T* k_data = W->template Data(); + + const T* b_data; + if (biasEnabled) { + b_data = B->template Data(); + } + + T* y_data = Y->template MutableData(); + + armnn::NetworkId* pNetworkId; + ConvLayersIterator it = Conv::convLayers.find((OpKernel*)this); + if (it == Conv::convLayers.end()) { + + armnn::NetworkId networkId; + armnn::INetworkPtr myNetwork = armnn::INetwork::Create(); + + armnn::Convolution2dDescriptor convolutionDescriptor = createConvDescriptor(pads, dilations, strides, biasEnabled); + + armnn::IConnectableLayer *convolution_armnn; + armnn::TensorShape inputShape = ArmNNTensorShape(X->Shape()); + armnn::TensorShape weightShape = ArmNNTensorShape(W->Shape()); + + if (weightShape[2] == 1 && weightShape[3] == 1) { + Status s = onnxruntime::Conv::Compute(context); + return s; + } + + if (conv_attrs_.group > 1) { + + if (conv_attrs_.group == inputShape[1]) { + // depthwise convolution + armnn::DepthwiseConvolution2dDescriptor depthwiseDescriptor = createDepthwiseDescriptor(convolutionDescriptor); + + weightShape[1] = weightShape[0]; + weightShape[0] = 1; + armnn::TensorInfo weightsInfo(weightShape, armnn::DataType::Float32); + armnn::ConstTensor weights(weightsInfo, k_data); + + if (biasEnabled) { + armnn::TensorInfo biasDesc(ArmNNTensorShape(B->Shape()), armnn::DataType::Float32); + armnn::ConstTensor bias(biasDesc, b_data); + convolution_armnn = myNetwork->AddDepthwiseConvolution2dLayer(depthwiseDescriptor, + weights, + armnn::Optional(bias), + "depthwise_convolution_armnn"); + } else { + convolution_armnn = myNetwork->AddDepthwiseConvolution2dLayer(depthwiseDescriptor, + weights, + armnn::EmptyOptional(), + "depthwise_convolution_armnn"); + } + } else { + // NCHWc convolution + Status s = onnxruntime::Conv::Compute(context); + return s; + } + } else { + // normal convolution + armnn::TensorInfo weightsInfo(weightShape, armnn::DataType::Float32); + armnn::ConstTensor weights(weightsInfo, k_data); + + if (biasEnabled) { + armnn::TensorInfo biasDesc(ArmNNTensorShape(B->Shape()), armnn::DataType::Float32); + armnn::ConstTensor bias(biasDesc, b_data); + convolution_armnn = myNetwork->AddConvolution2dLayer(convolutionDescriptor, + weights, + armnn::Optional(bias), + "convolution_armnn"); + } else { + convolution_armnn = myNetwork->AddConvolution2dLayer(convolutionDescriptor, + weights, + armnn::EmptyOptional(), + "convolution_armnn"); + } + } + + bool armnn_activ_enabled = false; + armnn::ActivationDescriptor desc; + desc.m_A = conv_attrs_.alpha; + + if (activation_type == "Relu") { + desc.m_Function = armnn::ActivationFunction::ReLu; + armnn_activ_enabled = true; + } else if (activation_type == "LeakyRelu") { + desc.m_Function = armnn::ActivationFunction::LeakyReLu; + armnn_activ_enabled = true; + } else if (activation_type == "Tanh") { + desc.m_Function = armnn::ActivationFunction::TanH; + armnn_activ_enabled = true; + } else if (activation_type == "Sigmoid") { + desc.m_Function = armnn::ActivationFunction::Sigmoid; + armnn_activ_enabled = true; + } else if (!activation_type.empty()) { + ORT_NOT_IMPLEMENTED("Not implemented fused activation: ", activation_type); + } + + armnn::IConnectableLayer* activation = myNetwork->AddActivationLayer(desc, "activation_armnn"); + + armnn::IConnectableLayer *InputLayer = myNetwork->AddInputLayer(0); + armnn::IConnectableLayer *OutputLayer = myNetwork->AddOutputLayer(0); + + InputLayer->GetOutputSlot(0).Connect(convolution_armnn->GetInputSlot(0)); + if (armnn_activ_enabled) { + convolution_armnn->GetOutputSlot(0).Connect(activation->GetInputSlot(0)); + activation->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0)); + } + else { + convolution_armnn->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0)); + } + + //Set the tensors in the network. + armnn::TensorInfo inputTensorInfo(inputShape, armnn::DataType::Float32); + InputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo); + + armnn::TensorInfo outputTensorInfo(ArmNNTensorShape(Y->Shape()), armnn::DataType::Float32); + convolution_armnn->GetOutputSlot(0).SetTensorInfo(outputTensorInfo); + + if (armnn_activ_enabled) { + activation->GetOutputSlot(0).SetTensorInfo(outputTensorInfo); + } + + // Optimise ArmNN network + armnn::IOptimizedNetworkPtr optNet = armnn::Optimize(*myNetwork, {armnn::Compute::CpuAcc}, Conv::run->GetDeviceSpec()); + + if (optNet == nullptr) { + return onnxruntime::Conv::Compute(context); + } + + // Load graph into runtime + Conv::run->LoadNetwork(networkId, std::move(optNet)); + + std::pair ret; + ret = Conv::convLayers.insert(std::pair((OpKernel*)this, networkId)); + pNetworkId = &ret.first->second; + + } else { + pNetworkId = &it->second; + } + + armnn::InputTensors inputTensors{{0, armnn::ConstTensor(Conv::run->GetInputTensorInfo(*pNetworkId, 0), + x_data)}}; + armnn::OutputTensors outputTensors{{0, armnn::Tensor(Conv::run->GetOutputTensorInfo(*pNetworkId, 0), + y_data)}}; + + // Execute network + Conv::run->EnqueueWorkload(*pNetworkId, inputTensors, outputTensors); + + return Status::OK(); +} + +ONNX_OPERATOR_VERSIONED_KERNEL_EX( + Conv, + kOnnxDomain, + 1, 10, + kArmNNExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Conv); + +ONNX_OPERATOR_KERNEL_EX( + Conv, + kOnnxDomain, + 11, + kArmNNExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Conv); + +} // namespace armnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/armnn/nn/conv.h b/onnxruntime/core/providers/armnn/nn/conv.h new file mode 100644 index 0000000000..c6a463bdc8 --- /dev/null +++ b/onnxruntime/core/providers/armnn/nn/conv.h @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/conv.h" +#include "core/providers/armnn/armnn_execution_provider.h" + +#include "armnn/ArmNN.hpp" + +#include +#include + +namespace onnxruntime { +namespace armnn_ep{ + +typedef std::map::iterator ConvLayersIterator; + +template +class Conv : public onnxruntime::Conv { + public: + explicit Conv(const OpKernelInfo& info) : onnxruntime::Conv(info), conv_attrs_(info) { + provider_ = (const_cast( + dynamic_cast(info.GetExecutionProvider()))); + } + + ~Conv() { + Conv::convLayers.erase(this); + } + + Status Compute(OpKernelContext* context) const override; + + static armnn::IRuntimePtr initRuntime(){ + if (Conv::run) + return std::move(Conv::run); + armnn::IRuntime::CreationOptions options; + return std::move(armnn::IRuntime::Create(options)); + } + + protected: + static thread_local std::map convLayers; + ConvAttributes conv_attrs_; + ArmNNExecutionProvider* provider_; + static armnn::IRuntimePtr run; + std::string activation_type; + +}; + +} // namespace armnn_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/armnn/nn/pool.cc b/onnxruntime/core/providers/armnn/nn/pool.cc new file mode 100644 index 0000000000..f53f6c18eb --- /dev/null +++ b/onnxruntime/core/providers/armnn/nn/pool.cc @@ -0,0 +1,310 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" + +#include "core/providers/armnn/nn/pool.h" +#include "core/providers/armnn/armnn_common.h" +#include "core/providers/armnn/armnn_fwd.h" + +#define PREF_DIM 4 + +namespace onnxruntime { +namespace armnn_ep { + +template +thread_local std::map Pool::poolLayers; + +template +armnn::IRuntimePtr Pool::run = Pool::initRuntime(); + +template +thread_local std::map MaxPoolV8::maxPoolLayers; + +template +armnn::IRuntimePtr MaxPoolV8::run = MaxPoolV8::initRuntime(); + +armnn::Pooling2dDescriptor createDescriptor(std::vector pads, std::vector strides, std::vector kernel_shape, armnn::PoolingAlgorithm pool_type, onnxruntime::PoolAttributes pool_attrs){ + + std::vector armnnStrides(2); + armnnStrides[0] = (strides.size() == 2) ? strides[1] : 1; + armnnStrides[1] = strides[0]; + + std::vector armnnKernelShape(2); + armnnKernelShape[0] = (kernel_shape.size() > 1) ? kernel_shape[1] : 1; + armnnKernelShape[1] = kernel_shape[0]; + + std::vector armnnPads(4); + if (pads.size() == 2) { + if (strides.size() == 1) { + armnnPads[0] = 0; + armnnPads[1] = 0; + armnnPads[2] = pads[1]; + armnnPads[3] = pads[0]; + } else { + armnnPads[0] = pads[1]; + armnnPads[1] = pads[0]; + armnnPads[2] = pads[1]; + armnnPads[3] = pads[0]; + } + } else { + armnnPads[0] = pads[1]; + armnnPads[1] = pads[3]; + armnnPads[2] = pads[0]; + armnnPads[3] = pads[2]; + } + + armnn::Pooling2dDescriptor poolDescriptor; + poolDescriptor.m_PoolType = pool_type; + poolDescriptor.m_PadLeft = armnnPads[0]; + poolDescriptor.m_PadRight = armnnPads[1]; + poolDescriptor.m_PadTop = armnnPads[2]; + poolDescriptor.m_PadBottom = armnnPads[3]; + poolDescriptor.m_PoolWidth = armnnKernelShape[0]; + poolDescriptor.m_PoolHeight = armnnKernelShape[1]; + poolDescriptor.m_StrideX = armnnStrides[0]; + poolDescriptor.m_StrideY = armnnStrides[1]; + poolDescriptor.m_OutputShapeRounding = pool_attrs.ceil_mode ? armnn::OutputShapeRounding::Ceiling : armnn::OutputShapeRounding::Floor; + poolDescriptor.m_PaddingMethod = armnn::PaddingMethod::Exclude; + if (pool_type == armnn::PoolingAlgorithm::Average && pool_attrs.count_include_pad) + poolDescriptor.m_PaddingMethod = armnn::PaddingMethod::IgnoreValue; + poolDescriptor.m_DataLayout = armnn::DataLayout::NCHW; + + return poolDescriptor; +} + +template +Status Pool::Compute(OpKernelContext* context) const { + + const Tensor* X = context->Input(0); + const TensorShape& x_shape = X->Shape(); + + std::vector dilations(PoolBase::pool_attrs_.dilations); + std::vector armnnDilations(2); + armnnDilations[0] = (dilations.size() == 2) ? dilations[1] : 1; + armnnDilations[1] = (!dilations.empty()) ? dilations[0] : 1; + + if ((X->Shape().NumDimensions() != PREF_DIM) || + (armnnDilations[0] * armnnDilations[1] > 1)) { + Status s = onnxruntime::Pool::Compute(context); + return s; + } + + std::vector pads = PoolBase::pool_attrs_.pads; + std::vector strides = PoolBase::pool_attrs_.strides; + std::vector kernel_shape = PoolBase::pool_attrs_.kernel_shape; + + if (PoolBase::pool_attrs_.global_pooling) { + const auto& input_dims = x_shape.GetDims(); + kernel_shape.assign(input_dims.begin() + 2, input_dims.end()); + strides.assign(kernel_shape.size(), 0); + pads.assign(kernel_shape.size(), 0); + } + + std::vector output_dims = PoolBase::pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); + Tensor* Y = context->Output(0, TensorShape(output_dims)); + + const T* x_data = X->template Data(); + T* y_data = Y->template MutableData(); + + armnn::NetworkId* pNetworkId; + PoolLayersIterator it = Pool::poolLayers.find((OpKernel*)this); + if (it == Pool::poolLayers.end()) { + + armnn::PoolingAlgorithm pool_type; + if (PoolBase::op_name_ == "GlobalAveragePool" || PoolBase::op_name_ == "AveragePool"){ + pool_type = armnn::PoolingAlgorithm::Average; + } else if (PoolBase::op_name_ == "GlobalMaxPool" || PoolBase::op_name_ == "MaxPool"){ + pool_type = armnn::PoolingAlgorithm::Max; + } else + return onnxruntime::Pool::Compute(context); + + armnn::NetworkId networkId; + + armnn::INetworkPtr myNetwork = armnn::INetwork::Create(); + + armnn::Pooling2dDescriptor poolDescriptor = createDescriptor(pads, strides, kernel_shape, pool_type, PoolBase::pool_attrs_); + + armnn::IConnectableLayer *pool_armnn = myNetwork->AddPooling2dLayer(poolDescriptor, "pool_armnn"); + armnn::TensorShape inputShape = ArmNNTensorShape(X->Shape()); + armnn::TensorShape outputShape = ArmNNTensorShape(Y->Shape()); + + armnn::IConnectableLayer *InputLayer = myNetwork->AddInputLayer(0); + armnn::IConnectableLayer *OutputLayer = myNetwork->AddOutputLayer(0); + + InputLayer->GetOutputSlot(0).Connect(pool_armnn->GetInputSlot(0)); + pool_armnn->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0)); + + //Set the tensors in the network. + armnn::TensorInfo inputTensorInfo(inputShape, armnn::DataType::Float32); + InputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo); + + armnn::TensorInfo outputTensorInfo(outputShape, armnn::DataType::Float32); + pool_armnn->GetOutputSlot(0).SetTensorInfo(outputTensorInfo); + + // Optimise ArmNN network + armnn::IOptimizedNetworkPtr optNet = armnn::Optimize(*myNetwork, {armnn::Compute::CpuAcc}, Pool::run->GetDeviceSpec()); + + if (optNet == nullptr) { + return onnxruntime::Pool::Compute(context); + } + + // Load graph into runtime + Pool::run->LoadNetwork(networkId, std::move(optNet)); + + std::pair ret; + ret = Pool::poolLayers.insert(std::pair((OpKernel*)this, networkId)); + pNetworkId = &ret.first->second; + + } else { + pNetworkId = &it->second; + } + + armnn::InputTensors inputTensors{{0, armnn::ConstTensor(Pool::run->GetInputTensorInfo(*pNetworkId, 0), + x_data)}}; + armnn::OutputTensors outputTensors{{0, armnn::Tensor(Pool::run->GetOutputTensorInfo(*pNetworkId, 0), + y_data)}}; + + // Execute network + Pool::run->EnqueueWorkload(*pNetworkId, inputTensors, outputTensors); + + return Status::OK(); +} + +template +Status MaxPoolV8::Compute(OpKernelContext* context) const { + + const Tensor* X = context->Input(0); + const TensorShape& x_shape = X->Shape(); + + std::vector dilations(PoolBase::pool_attrs_.dilations); + std::vector armnnDilations(2); + armnnDilations[0] = (dilations.size() == 2) ? dilations[1] : 1; + armnnDilations[1] = (!dilations.empty()) ? dilations[0] : 1; + + if ((X->Shape().NumDimensions() != PREF_DIM) || + (armnnDilations[0] * armnnDilations[1] > 1)) { + Status s = onnxruntime::MaxPoolV8::Compute(context); + return s; + } + + std::vector pads = PoolBase::pool_attrs_.pads; + std::vector strides = PoolBase::pool_attrs_.strides; + std::vector kernel_shape = PoolBase::pool_attrs_.kernel_shape; + + if (PoolBase::pool_attrs_.global_pooling) { + const auto& input_dims = x_shape.GetDims(); + kernel_shape.assign(input_dims.begin() + 2, input_dims.end()); + strides.assign(kernel_shape.size(), 0); + pads.assign(kernel_shape.size(), 0); + } + + std::vector output_dims = PoolBase::pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads); + Tensor* Y = context->Output(0, TensorShape(output_dims)); + + const T* x_data = X->template Data(); + T* y_data = Y->template MutableData(); + + armnn::NetworkId* pNetworkId; + PoolLayersIterator it = MaxPoolV8::maxPoolLayers.find((OpKernel*)this); + if (it == MaxPoolV8::maxPoolLayers.end()) { + + armnn::NetworkId networkId; + + armnn::INetworkPtr myNetwork = armnn::INetwork::Create(); + + armnn::Pooling2dDescriptor poolDescriptor = createDescriptor(pads, strides, kernel_shape, armnn::PoolingAlgorithm::Max, PoolBase::pool_attrs_); + + armnn::IConnectableLayer *pool_armnn = myNetwork->AddPooling2dLayer(poolDescriptor, "pool_armnn"); + armnn::TensorShape inputShape = ArmNNTensorShape(X->Shape()); + armnn::TensorShape outputShape = ArmNNTensorShape(Y->Shape()); + + armnn::IConnectableLayer *InputLayer = myNetwork->AddInputLayer(0); + armnn::IConnectableLayer *OutputLayer = myNetwork->AddOutputLayer(0); + + InputLayer->GetOutputSlot(0).Connect(pool_armnn->GetInputSlot(0)); + pool_armnn->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0)); + + //Set the tensors in the network. + armnn::TensorInfo inputTensorInfo(inputShape, armnn::DataType::Float32); + InputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo); + + armnn::TensorInfo outputTensorInfo(outputShape, armnn::DataType::Float32); + pool_armnn->GetOutputSlot(0).SetTensorInfo(outputTensorInfo); + + // Optimise ArmNN network + armnn::IOptimizedNetworkPtr optNet = armnn::Optimize(*myNetwork, {armnn::Compute::CpuAcc}, MaxPoolV8::run->GetDeviceSpec()); + + if (optNet == nullptr) { + return onnxruntime::MaxPoolV8::Compute(context); + } + + // Load graph into runtime + MaxPoolV8::run->LoadNetwork(networkId, std::move(optNet)); + + std::pair ret; + ret = MaxPoolV8::maxPoolLayers.insert(std::pair((OpKernel*)this, networkId)); + pNetworkId = &ret.first->second; + + } else { + pNetworkId = &it->second; + } + + armnn::InputTensors inputTensors{{0, armnn::ConstTensor(MaxPoolV8::run->GetInputTensorInfo(*pNetworkId, 0), + x_data)}}; + armnn::OutputTensors outputTensors{{0, armnn::Tensor(MaxPoolV8::run->GetOutputTensorInfo(*pNetworkId, 0), + y_data)}}; + + // Execute network + MaxPoolV8::run->EnqueueWorkload(*pNetworkId, inputTensors, outputTensors); + + return Status::OK(); +} + +#define POOLING_KERNEL_VERSIONED(op_name, data_type, pool_type, since_version, end_version) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + op_name, \ + kOnnxDomain, \ + since_version, \ + end_version, \ + data_type, \ + kArmNNExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Pool); + +#define POOLING_KERNEL(op_name, data_type, pool_type, since_version) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + op_name, \ + kOnnxDomain, \ + since_version, \ + data_type, \ + kArmNNExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Pool); + +POOLING_KERNEL_VERSIONED(MaxPool, float, MaxPool<1>, 1, 7) +POOLING_KERNEL_VERSIONED(AveragePool, float, AveragePool, 7, 9) +POOLING_KERNEL_VERSIONED(AveragePool, float, AveragePool, 10, 10) +POOLING_KERNEL(AveragePool, float, AveragePool, 11) +POOLING_KERNEL(GlobalAveragePool, float, AveragePool, 1) +POOLING_KERNEL(GlobalMaxPool, float, MaxPool<1>, 1) + +ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + MaxPool, \ + kOnnxDomain, \ + 8, \ + 11, \ + float, \ + kArmNNExecutionProvider, \ + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + MaxPoolV8); + +} // namespace armnn_ep +} // namespace onnxruntime + diff --git a/onnxruntime/core/providers/armnn/nn/pool.h b/onnxruntime/core/providers/armnn/nn/pool.h new file mode 100644 index 0000000000..264abba894 --- /dev/null +++ b/onnxruntime/core/providers/armnn/nn/pool.h @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/pool.h" +#include "core/providers/armnn/armnn_execution_provider.h" + +#include "armnn/ArmNN.hpp" + +#include +#include + +namespace onnxruntime { +namespace armnn_ep { + +typedef std::map::iterator PoolLayersIterator; + +template +class Pool final : public onnxruntime::Pool { + public: + explicit Pool(const OpKernelInfo& info) : onnxruntime::Pool(info) { + provider_ = (const_cast( + dynamic_cast(info.GetExecutionProvider()))); + } + + ~Pool() { + poolLayers.erase(this); + } + + Status Compute(OpKernelContext* context) const override; + + static armnn::IRuntimePtr initRuntime(){ + if (Pool::run) + return std::move(Pool::run); + armnn::IRuntime::CreationOptions options; + return std::move(armnn::IRuntime::Create(options)); + } + + private: + static thread_local std::map poolLayers; + ArmNNExecutionProvider* provider_; + static armnn::IRuntimePtr run; +}; + +template +class MaxPoolV8 final : public onnxruntime::MaxPoolV8 { + public: + explicit MaxPoolV8(const OpKernelInfo& info) : onnxruntime::MaxPoolV8(info) { + provider_ = (const_cast( + dynamic_cast(info.GetExecutionProvider()))); + } + + ~MaxPoolV8() { + maxPoolLayers.erase(this); + } + + Status Compute(OpKernelContext* context) const override; + + static armnn::IRuntimePtr initRuntime(){ + if (MaxPoolV8::run) + return std::move(MaxPoolV8::run); + armnn::IRuntime::CreationOptions options; + return std::move(armnn::IRuntime::Create(options)); + } + + private: + static thread_local std::map maxPoolLayers; + ArmNNExecutionProvider* provider_; + static armnn::IRuntimePtr run; +}; + +} // namespace armnn_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/armnn/symbols.txt b/onnxruntime/core/providers/armnn/symbols.txt new file mode 100755 index 0000000000..eb1dc56615 --- /dev/null +++ b/onnxruntime/core/providers/armnn/symbols.txt @@ -0,0 +1 @@ +OrtSessionOptionsAppendExecutionProvider_ArmNN diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index f004698107..18368d4511 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -37,7 +37,7 @@ void usage() { "\t-v: verbose\n" "\t-n [test_case_name]: Specifies a single test case to run.\n" "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'dnnl', 'tensorrt', 'ngraph', " - "'openvino', 'nuphar', 'migraphx' or 'acl'. " + "'openvino', 'nuphar', 'migraphx', 'acl' or 'armnn'. " "Default: 'cpu'.\n" "\t-x: Use parallel executor, default (without -x): sequential executor.\n" "\t-d [device_id]: Specifies the device id for multi-device (e.g. GPU). The value should > 0\n" @@ -101,6 +101,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) { bool enable_nnapi = false; bool enable_dml = false; bool enable_acl = false; + bool enable_armnn = false; bool enable_migraphx = false; int device_id = 0; GraphOptimizationLevel graph_optimization_level = ORT_ENABLE_ALL; @@ -168,9 +169,11 @@ int real_main(int argc, char* argv[], Ort::Env& env) { enable_dml = true; } else if (!CompareCString(optarg, ORT_TSTR("acl"))) { enable_acl = true; + } else if (!CompareCString(optarg, ORT_TSTR("armnn"))) { + enable_armnn = true; } else if (!CompareCString(optarg, ORT_TSTR("migraphx"))) { enable_migraphx = true; - }else { + } else { usage(); return -1; } @@ -362,6 +365,14 @@ int real_main(int argc, char* argv[], Ort::Env& env) { #else fprintf(stderr, "ACL is not supported in this build"); return -1; +#endif + } + if (enable_armnn) { +#ifdef USE_ARMNN + Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_ArmNN(sf, enable_cpu_mem_arena ? 1 : 0)); +#else + fprintf(stderr, "ArmNN is not supported in this build\n"); + return -1; #endif } if (enable_migraphx) { diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index faa2055a16..97ae257c7b 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -98,6 +98,8 @@ namespace perftest { test_config.machine_config.provider_type_name = onnxruntime::kDmlExecutionProvider; } else if (!CompareCString(optarg, ORT_TSTR("acl"))) { test_config.machine_config.provider_type_name = onnxruntime::kAclExecutionProvider; + } else if (!CompareCString(optarg, ORT_TSTR("armnn"))) { + test_config.machine_config.provider_type_name = onnxruntime::kArmNNExecutionProvider; } else { return false; } diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 42284995b3..c3270d78f3 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -85,6 +85,13 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device performance_test_config.run_config.enable_cpu_mem_arena ? 1 : 0)); #else ORT_THROW("Acl is not supported in this build\n"); +#endif + } else if (provider_name == onnxruntime::kArmNNExecutionProvider) { +#ifdef USE_ARMNN + Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_ArmNN(session_options, + performance_test_config.run_config.enable_cpu_mem_arena ? 1 : 0)); +#else + ORT_THROW("ArmNN is not supported in this build\n"); #endif } else if (provider_name == onnxruntime::kMIGraphXExecutionProvider) { #ifdef USE_MIGRAPHX diff --git a/onnxruntime/test/providers/cpu/nn/pool_op_test.cc b/onnxruntime/test/providers/cpu/nn/pool_op_test.cc index a06955a2f9..bc46df7037 100644 --- a/onnxruntime/test/providers/cpu/nn/pool_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/pool_op_test.cc @@ -162,7 +162,7 @@ static void MaxPool_8_WithIndexTest(bool has_index, int64_t storage_order = 0) { storage_order == 0 ? test.AddOutput("Indices", expected_dims, expected_indices_row) : test.AddOutput("Indices", expected_dims, expected_indices_col); } - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kDnnlExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider}); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kDnnlExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider, kArmNNExecutionProvider}); } TEST(PoolTest, MaxPool_8_With_Index) { diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 3721dfa3a6..ab826ca0f5 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -712,6 +712,7 @@ void OpTester::Run( kOpenVINOExecutionProvider, kDmlExecutionProvider, kAclExecutionProvider, + kArmNNExecutionProvider, }; bool has_run = false; @@ -778,6 +779,8 @@ void OpTester::Run( execution_provider = DefaultRknpuExecutionProvider(); else if (provider_type == onnxruntime::kAclExecutionProvider) execution_provider = DefaultAclExecutionProvider(); + else if (provider_type == onnxruntime::kArmNNExecutionProvider) + execution_provider = DefaultArmNNExecutionProvider(); // skip if execution provider is disabled if (execution_provider == nullptr) continue; diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 0710ad9249..7e6347b95a 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -22,6 +22,7 @@ std::shared_ptr CreateExecutionProviderFactory_Rknpu( std::shared_ptr CreateExecutionProviderFactory_Tensorrt(int device_id); std::shared_ptr CreateExecutionProviderFactory_MIGraphX(int device_id); std::shared_ptr CreateExecutionProviderFactory_ACL(int use_arena); +std::shared_ptr CreateExecutionProviderFactory_ArmNN(int use_arena); namespace test { @@ -113,5 +114,14 @@ std::unique_ptr DefaultAclExecutionProvider(bool enable_aren #endif } +std::unique_ptr DefaultArmNNExecutionProvider(bool enable_arena){ +#ifdef USE_ARMNN + return CreateExecutionProviderFactory_ArmNN(enable_arena)->CreateProvider(); +#else + ORT_UNUSED_PARAMETER(enable_arena); + return nullptr; +#endif +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/util/include/default_providers.h b/onnxruntime/test/util/include/default_providers.h index 69433675ae..21465d746a 100644 --- a/onnxruntime/test/util/include/default_providers.h +++ b/onnxruntime/test/util/include/default_providers.h @@ -18,6 +18,7 @@ std::unique_ptr DefaultOpenVINOExecutionProvider(); std::unique_ptr DefaultNnapiExecutionProvider(); std::unique_ptr DefaultRknpuExecutionProvider(); std::unique_ptr DefaultAclExecutionProvider(bool enable_arena = true); +std::unique_ptr DefaultArmNNExecutionProvider(bool enable_arena = true); } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/util/include/providers.h b/onnxruntime/test/util/include/providers.h index 14cc74a60f..fc01479581 100644 --- a/onnxruntime/test/util/include/providers.h +++ b/onnxruntime/test/util/include/providers.h @@ -31,6 +31,9 @@ #ifdef USE_ACL #include "core/providers/acl/acl_provider_factory.h" #endif +#ifdef USE_ARMNN +#include "core/providers/armnn/armnn_provider_factory.h" +#endif #ifdef USE_MIGRAPHX #include "core/providers/migraphx/migraphx_provider_factory.h" #endif diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 241d8a6a5c..82d1b8e9dc 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -333,6 +333,12 @@ def parse_arguments(): "--use_acl", nargs="?", const="ACL_1905", choices=["ACL_1902", "ACL_1905", "ACL_1908"], help="Build with ACL for ARM architectures.") + parser.add_argument( + "--use_armnn", action='store_true', + help="Enable ArmNN Execution Provider.") + parser.add_argument( + "--armnn_relu", action='store_true', + help="Use the Relu operator implementation from the ArmNN EP.") return parser.parse_args() @@ -523,7 +529,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, "OFF" if args.skip_winml_tests else "ON"), "-Donnxruntime_GENERATE_TEST_REPORTS=ON", "-Donnxruntime_DEV_MODE=" + ( - "OFF" if args.android or args.use_acl or + "OFF" if args.android or args.use_acl or args.use_armnn or (args.ios and is_macOS()) else "ON"), "-DPYTHON_EXECUTABLE=" + sys.executable, "-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"), @@ -612,6 +618,10 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, "ON" if args.use_acl == "ACL_1905" else "OFF"), "-Donnxruntime_USE_ACL_1908=" + ( "ON" if args.use_acl == "ACL_1908" else "OFF"), + "-Donnxruntime_USE_ARMNN=" + ( + "ON" if args.use_armnn else "OFF"), + "-Donnxruntime_ARMNN_RELU_USE_CPU=" + ( + "OFF" if args.armnn_relu else "ON"), # Training related flags "-Donnxruntime_ENABLE_NVTX_PROFILE=" + ( "ON" if args.enable_nvtx_profile else "OFF"), @@ -1659,7 +1669,7 @@ def main(): "Only Windows ARM(64) cross-compiled builds supported " "currently through this script") install_ubuntu_deps(args) - if not is_docker(): + if not is_docker() and not args.use_armnn: install_python_deps() if args.enable_pybind and is_windows(): install_python_deps(args.numpy_version)