saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add ArmNN Execution Provider (#3714)

Browse source 
* Add ArmNN Execution Provider

Add a new execution provider targeting Arm architecture based on ArmNN.
Validated on NXP i.MX8QM CPU with ResNet50, MobileNetv2 and VGG models.

reviewed-by: mike.caraman@nxp.com

* Minor fixes

- renamed onnxruntime_ARMNN_RELU_USECPU to onnxruntime_ARMNN_RELU_USE_CPU
- fixed acl typo

* remove extra includes. added exception for ArmNN in test

* fix indentation

* Separated the activation implementation from the cpu and fixed the blockage from the endif

Co-authored-by: Andrei-Alexandru <andrei-alexandru.avram@nxp.com>
This commit is contained in:
Andrews548 2020-06-03 20:27:51 +03:00 committed by GitHub
parent 62af8da3f6
commit 62b44527e5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
33 changed files with 1530 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
BUILD.md
View file

@ -19,6 +19,7 @@
* [Nuphar Model Compiler](#Nuphar)
* [DirectML](#DirectML)
* [ARM Compute Library](#ARM-Compute-Library)
* [ArmNN](#ArmNN)
* [Rockchip RKNPU](#RKNPU)
* [Xilinx Vitis-AI](#Vitis-AI)
* Options
@ -491,6 +492,31 @@ export LD_LIBRARY_PATH=~/ComputeLibrary/build/
---
### ArmNN
See more information on the ArmNN Execution Provider [here](./docs/execution_providers/ArmNN-ExecutionProvider.md).
#### Prerequisites
* Supported backend: i.MX8QM Armv8 CPUs
* Supported BSP: i.MX8QM BSP
* Install i.MX8QM BSP: `source fsl-imx-xwayland-glibc-x86_64-fsl-image-qt5-aarch64-toolchain-4*.sh`
* Set up the build environment
```
source /opt/fsl-imx-xwayland/4.*/environment-setup-aarch64-poky-linux
alias cmake="/usr/bin/cmake -DCMAKE_TOOLCHAIN_FILE=$OECORE_NATIVE_SYSROOT/usr/share/cmake/OEToolchainConfig.cmake"
```
* See [Build ARM](#ARM) below for information on building for ARM devices
#### Build Instructions
```
./build.sh --use_armnn
```
The Relu operator is set by default to use the CPU execution provider for better performance. To use the ArmNN implementation build with --armnn_relu flag
```
./build.sh --use_armnn --armnn_relu
```
---
### RKNPU
See more information on the RKNPU Execution Provider [here](./docs/execution_providers/RKNPU-ExecutionProvider.md).

11
cmake/CMakeLists.txt
View file

@ -92,6 +92,8 @@ option(onnxruntime_USE_ACL "Build with ACL support" OFF)
option(onnxruntime_USE_ACL_1902 "Build with ACL version 1902 support" OFF)
option(onnxruntime_USE_ACL_1905 "Build with ACL version 1905 support" OFF)
option(onnxruntime_USE_ACL_1908 "Build with ACL version 1908 support" OFF)
option(onnxruntime_USE_ARMNN "Build with ArmNN support" OFF)
option(onnxruntime_ARMNN_RELU_USE_CPU "Use the CPU implementation for the Relu operator for the ArmNN EP" ON)
option(onnxruntime_ENABLE_INSTRUMENT "Enable Instrument with Event Tracing for Windows (ETW)" OFF)
option(onnxruntime_USE_TELEMETRY "Build with Telemetry" OFF)
#The onnxruntime_PREFER_SYSTEM_LIB is mainly designed for package managers like apt/yum/vcpkg.
@ -513,6 +515,15 @@ if (onnxruntime_USE_ACL OR onnxruntime_USE_ACL_1902 OR onnxruntime_USE_ACL_1905
list(APPEND onnxruntime_EXTERNAL_LIBRARIES arm_compute arm_compute_graph arm_compute_core)
endif()
# ArmNN
if (onnxruntime_USE_ARMNN)
if (NOT onnxruntime_ARMNN_RELU_USE_CPU)
add_definitions(-DRELU_ARMNN=1)
endif()
list(APPEND onnxruntime_EXTERNAL_LIBRARIES armnn pthread arm_compute_core arm_compute arm_compute_graph)
endif()
# MKLML
if (onnxruntime_USE_DNNL OR onnxruntime_USE_MKLML)
include(dnnl)

1
cmake/onnxruntime.cmake
View file

@ -88,6 +88,7 @@ target_link_libraries(onnxruntime PRIVATE
${PROVIDERS_VITISAI}
${PROVIDERS_DML}
${PROVIDERS_ACL}
${PROVIDERS_ARMNN}
${onnxruntime_winml}
onnxruntime_optimizer
onnxruntime_providers

21
cmake/onnxruntime_providers.cmake
View file

@ -83,6 +83,10 @@ if(onnxruntime_USE_ACL)
set(PROVIDERS_ACL onnxruntime_providers_acl)
list(APPEND ONNXRUNTIME_PROVIDER_NAMES acl)
endif()
if(onnxruntime_USE_ARMNN)
set(PROVIDERS_ARMNN onnxruntime_providers_armnn)
list(APPEND ONNXRUNTIME_PROVIDER_NAMES armnn)
endif()
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs})
set(onnxruntime_providers_src ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs})
@ -654,3 +658,20 @@ if (onnxruntime_USE_ACL)
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/acl DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
set_target_properties(onnxruntime_providers_acl PROPERTIES LINKER_LANGUAGE CXX)
endif()
if (onnxruntime_USE_ARMNN)
add_definitions(-DUSE_ARMNN=1)
file(GLOB_RECURSE onnxruntime_providers_armnn_cc_srcs
"${ONNXRUNTIME_ROOT}/core/providers/armnn/*.h"
"${ONNXRUNTIME_ROOT}/core/providers/armnn/*.cc"
)
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_armnn_cc_srcs})
add_library(onnxruntime_providers_armnn ${onnxruntime_providers_armnn_cc_srcs})
onnxruntime_add_include_to_target(onnxruntime_providers_armnn onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf)
add_dependencies(onnxruntime_providers_armnn ${onnxruntime_EXTERNAL_DEPENDENCIES})
set_target_properties(onnxruntime_providers_armnn PROPERTIES FOLDER "ONNXRuntime")
target_include_directories(onnxruntime_providers_armnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${ARMNN_INCLUDE_DIR})
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/armnn DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
set_target_properties(onnxruntime_providers_armnn PROPERTIES LINKER_LANGUAGE CXX)
endif()

5
cmake/onnxruntime_unittests.cmake
View file

@ -336,6 +336,10 @@ if(onnxruntime_USE_ACL)
list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_acl)
endif()
if(onnxruntime_USE_ARMNN)
list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_armnn)
endif()
if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
set(ONNXRUNTIME_INTEROP_TEST_LIBS PRIVATE onnxruntime_language_interop onnxruntime_pyop)
endif()
@ -355,6 +359,7 @@ set(ONNXRUNTIME_TEST_LIBS
${PROVIDERS_RKNPU}
${PROVIDERS_DML}
${PROVIDERS_ACL}
${PROVIDERS_ARMNN}
onnxruntime_optimizer
onnxruntime_providers
onnxruntime_util

29
docs/execution_providers/ArmNN-ExecutionProvider.md Normal file
View file

@ -0,0 +1,29 @@
## ArmNN Execution Provider
[ArmNN](https://github.com/ARM-software/armnn) is an open source inference engine maintained by Arm and Linaro companies. The integration of ArmNN as an execution provider (EP) into ONNX Runtime accelerates performance of ONNX model workloads across Armv8 cores.
### Build ArmNN execution provider
For build instructions, please see the [BUILD page](../../BUILD.md#ArmNN).
### Using the ArmNN execution provider
#### C/C++
To use ArmNN as execution provider for inferencing, please register it as below.
```
string log_id = "Foo";
auto logging_manager = std::make_unique<LoggingManager>
(std::unique_ptr<ISink>{new CLogSink{}},
static_cast<Severity>(lm_info.default_warning_level),
false,
LoggingManager::InstanceType::Default,
&log_id)
Environment::Create(std::move(logging_manager), env)
InferenceSession session_object{so, env};
session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::ArmNNExecutionProvider>());
status = session_object.Load(model_file_name);
```
The C API details are [here](../C_API.md#c-api).
### Performance Tuning
For performance tuning, please see guidance on this page: [ONNX Runtime Perf Tuning](../ONNX_Runtime_Perf_Tuning.md)
When/if using [onnxruntime_perf_test](../../onnxruntime/test/perftest), use the flag -e armnn

1
include/onnxruntime/core/graph/constants.h
View file

@ -37,4 +37,5 @@ constexpr const char* kRknpuExecutionProvider = "RknpuExecutionProvider";
constexpr const char* kDmlExecutionProvider = "DmlExecutionProvider";
constexpr const char* kMIGraphXExecutionProvider = "MIGraphXExecutionProvider";
constexpr const char* kAclExecutionProvider = "ACLExecutionProvider";
constexpr const char* kArmNNExecutionProvider = "ArmNNExecutionProvider";
} // namespace onnxruntime

18
include/onnxruntime/core/providers/armnn/armnn_provider_factory.h Normal file
View file

@ -0,0 +1,18 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "onnxruntime_c_api.h"
#ifdef __cplusplus
extern "C" {
#endif
/**
* \param use_arena zero: false. non-zero: true.
*/
ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ArmNN, _In_ OrtSessionOptions* options, int use_arena)
ORT_ALL_ARGS_NONNULL;
#ifdef __cplusplus
}
#endif

3
onnxruntime/core/optimizer/transformer_memcpy.cc
View file

@ -76,7 +76,8 @@ common::Status MemcpyTransformer::ApplyImpl(Graph& graph, bool& modified, int gr
provider != onnxruntime::kVitisAIExecutionProvider &&
provider != onnxruntime::kOpenVINOExecutionProvider &&
provider != onnxruntime::kNnapiExecutionProvider &&
provider != onnxruntime::kAclExecutionProvider) {
provider != onnxruntime::kAclExecutionProvider &&
provider != onnxruntime::kArmNNExecutionProvider) {
TransformerMemcpyImpl copy_impl(graph, provider);
auto current_modified = copy_impl.ModifyGraph(registry_manager_);
modified = modified || current_modified;

100
onnxruntime/core/providers/armnn/activation/activations.cc Normal file
View file

@ -0,0 +1,100 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License
#ifdef RELU_ARMNN
#ifdef _WIN32
#pragma warning(disable : 4244)
#endif
#include "core/providers/armnn/armnn_common.h"
#include "core/providers/armnn/activation/activations.h"
#include "core/providers/armnn/armnn_fwd.h"
namespace onnxruntime {
namespace armnn_ep {
template <typename T>
thread_local std::map<OpKernel*, armnn::NetworkId> Relu<T>::reluLayers;
template <typename T>
armnn::IRuntimePtr Relu<T>::run = Relu<T>::initRuntime();
template <typename T>
Status Relu<T>::Compute(OpKernelContext* context) const {
const Tensor* X = context->Input<Tensor>(0);
Tensor* Y = context->Output(0, X->Shape());
const T* src_data = X->template Data<T>();
T* dst_data = Y->template MutableData<T>();
armnn::NetworkId* pNetworkId;
ReluLayersIterator it = Relu::reluLayers.find((OpKernel*)this);
if (it == Relu::reluLayers.end()) {
armnn::NetworkId networkId;
armnn::INetworkPtr myNetwork = armnn::INetwork::Create();
armnn::TensorShape inputShape = ArmNNTensorShape(X->Shape());
armnn::TensorShape outputShape = ArmNNTensorShape(Y->Shape());
armnn::ActivationDescriptor desc;
desc.m_Function = armnn::ActivationFunction::ReLu;
armnn::IConnectableLayer* activation = myNetwork->AddActivationLayer(desc, "relu_armnn");
armnn::IConnectableLayer *InputLayer = myNetwork->AddInputLayer(0);
armnn::IConnectableLayer *OutputLayer = myNetwork->AddOutputLayer(0);
InputLayer->GetOutputSlot(0).Connect(activation->GetInputSlot(0));
activation->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0));
//Set the tensors in the network.
armnn::TensorInfo inputTensorInfo(inputShape, armnn::DataType::Float32);
InputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
armnn::TensorInfo outputTensorInfo(outputShape, armnn::DataType::Float32);
activation->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
// Optimise ArmNN network
armnn::IOptimizedNetworkPtr optNet = armnn::Optimize(*myNetwork, {armnn::Compute::CpuAcc}, Relu::run->GetDeviceSpec());
if (optNet == nullptr) {
ORT_NOT_IMPLEMENTED("Something went wrong when creating the layer");
}
// Load graph into runtime
Relu::run->LoadNetwork(networkId, std::move(optNet));
std::pair<ReluLayersIterator, bool> ret;
ret = Relu::reluLayers.insert(std::pair<OpKernel*, armnn::NetworkId>((OpKernel*)this, networkId));
pNetworkId = &ret.first->second;
} else {
pNetworkId = &it->second;
}
armnn::InputTensors inputTensors{{0, armnn::ConstTensor(Relu::run->GetInputTensorInfo(*pNetworkId, 0),
src_data)}};
armnn::OutputTensors outputTensors{{0, armnn::Tensor(Relu::run->GetOutputTensorInfo(*pNetworkId, 0),
dst_data)}};
Relu::run->EnqueueWorkload(*pNetworkId, inputTensors, outputTensors);
return Status::OK();
}
ONNX_OPERATOR_KERNEL_EX(
Relu,
kOnnxDomain,
6,
kArmNNExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Relu<float>);
} // namespace armnn_ep
} // namespace onnxruntime
#endif

52
onnxruntime/core/providers/armnn/activation/activations.h Normal file
View file

@ -0,0 +1,52 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License
#ifdef RELU_ARMNN
#pragma once
#include "core/framework/op_kernel.h"
#include "core/providers/cpu/activation/activations.h"
#include "core/providers/armnn/armnn_execution_provider.h"
#include "armnn/ArmNN.hpp"
#include <thread>
#include <mutex>
namespace onnxruntime {
namespace armnn_ep {
typedef std::map<OpKernel*, armnn::NetworkId>::iterator ReluLayersIterator;
template <typename T>
class Relu : public OpKernel {
public:
explicit Relu(const OpKernelInfo& info) : OpKernel(info) {
provider_ = (const_cast<ArmNNExecutionProvider*>(
dynamic_cast<const ArmNNExecutionProvider*>(info.GetExecutionProvider())));
}
~Relu() {
Relu::reluLayers.erase(this);
}
Status Compute(OpKernelContext* context) const override;
static armnn::IRuntimePtr initRuntime(){
if (Relu::run)
return std::move(Relu::run);
armnn::IRuntime::CreationOptions options;
return std::move(armnn::IRuntime::Create(options));
}
private:
static thread_local std::map<OpKernel*, armnn::NetworkId> reluLayers;
ArmNNExecutionProvider* provider_;
static armnn::IRuntimePtr run;
};
} // namespace armnn_ep
} // namespace onnxruntime
#endif

25
onnxruntime/core/providers/armnn/armnn_common.cc Normal file
View file

@ -0,0 +1,25 @@
// Copyright(C) 2018 Intel Corporation
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License
#ifdef _WIN32
#pragma warning(disable : 4244)
#endif
#include "core/providers/armnn/armnn_common.h"
namespace onnxruntime {
namespace armnn_ep {
armnn::TensorShape ArmNNTensorShape(const TensorShape& tensorShape) {
std::vector<unsigned int> dims;
unsigned int inDim = tensorShape.NumDimensions();
for (unsigned int i = 0; i < inDim; ++i)
dims.push_back(tensorShape.GetDims()[i]);
return armnn::TensorShape{static_cast<unsigned int>(dims.size()), dims.data()};
}
} // namespace armnn_ep
} // namespace onnxruntime

18
onnxruntime/core/providers/armnn/armnn_common.h Normal file
View file

@ -0,0 +1,18 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/common/common.h"
#include "core/framework/op_kernel.h"
#include "armnn/ArmNN.hpp"
namespace onnxruntime {
namespace armnn_ep {
armnn::TensorShape ArmNNTensorShape(const TensorShape& tensorShape);
} // namespace armnn_ep
} // namespace onnxruntime

119
onnxruntime/core/providers/armnn/armnn_execution_provider.cc Normal file
View file

@ -0,0 +1,119 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#include "armnn_execution_provider.h"
#include "core/framework/allocator.h"
#include "core/framework/op_kernel.h"
#include "core/framework/kernel_registry.h"
#include "core/framework/compute_capability.h"
#include "contrib_ops/cpu/cpu_contrib_kernels.h"
#include "armnn_fwd.h"
namespace onnxruntime {
constexpr const char* ArmNN = "ArmNN";
constexpr const char* ArmNN_CPU = "ArmNNCpu";
namespace armnn_ep {
// Forward declarations of op kernels
#ifdef RELU_ARMNN
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 6, Relu);
#endif
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 10, Conv);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Conv);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 8, Gemm);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 9, 10, Gemm);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Gemm);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 9, float, AveragePool);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 10, 10, float, AveragePool);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, float, AveragePool);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 8, 11, float, MaxPool);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, float, GlobalAveragePool);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, float, GlobalMaxPool);
static void RegisterArmNNKernels(KernelRegistry& kernel_registry) {
#ifdef RELU_ARMNN
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 6, Relu)>());
#endif
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 10, Conv)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Conv)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 8, Gemm)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 9, 10, Gemm)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Gemm)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 9, float, AveragePool)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 10, 10, float, AveragePool)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, float, AveragePool)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 8, 11, float, MaxPool)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, float, GlobalAveragePool)>());
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, float, GlobalMaxPool)>());
}
std::shared_ptr<KernelRegistry> GetArmNNKernelRegistry() {
std::shared_ptr<KernelRegistry> kernel_registry = std::make_shared<KernelRegistry>();
RegisterArmNNKernels(*kernel_registry);
return kernel_registry;
}
} // namespace armnn_ep
ArmNNExecutionProvider::ArmNNExecutionProvider(const ArmNNExecutionProviderInfo& info)
: IExecutionProvider{onnxruntime::kArmNNExecutionProvider} {
ORT_UNUSED_PARAMETER(info);
auto default_allocator_factory = [](int) {
auto memory_info = onnxruntime::make_unique<OrtMemoryInfo>(ArmNN, OrtAllocatorType::OrtDeviceAllocator);
return onnxruntime::make_unique<CPUAllocator>(std::move(memory_info));
};
DeviceAllocatorRegistrationInfo default_memory_info{
OrtMemTypeDefault,
std::move(default_allocator_factory),
std::numeric_limits<size_t>::max()};
InsertAllocator(CreateAllocator(default_memory_info));
auto cpu_allocator_factory = [](int) {
auto memory_info = onnxruntime::make_unique<OrtMemoryInfo>(
ArmNN_CPU, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeCPUOutput);
return onnxruntime::make_unique<CPUAllocator>(std::move(memory_info));
};
DeviceAllocatorRegistrationInfo cpu_memory_info{
OrtMemTypeCPUOutput,
std::move(cpu_allocator_factory),
std::numeric_limits<size_t>::max()};
InsertAllocator(CreateAllocator(cpu_memory_info));
}
ArmNNExecutionProvider::~ArmNNExecutionProvider() {
}
std::shared_ptr<KernelRegistry> ArmNNExecutionProvider::GetKernelRegistry() const {
static std::shared_ptr<KernelRegistry> kernel_registry = onnxruntime::armnn_ep::GetArmNNKernelRegistry();
return kernel_registry;
}
std::vector<std::unique_ptr<ComputeCapability>>
ArmNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph,
const std::vector<const KernelRegistry*>& kernel_registries) const {
std::vector<std::unique_ptr<ComputeCapability>>
result = IExecutionProvider::GetCapability(graph, kernel_registries);
return result;
}
} // namespace onnxruntime

41
onnxruntime/core/providers/armnn/armnn_execution_provider.h Executable file
View file

@ -0,0 +1,41 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/framework/allocatormgr.h"
#include "core/framework/execution_provider.h"
#include "core/graph/constants.h"
namespace onnxruntime {
// Information needed to construct ArmNN execution providers.
struct ArmNNExecutionProviderInfo {
bool create_arena{true};
explicit ArmNNExecutionProviderInfo(bool use_arena)
: create_arena(use_arena) {}
ArmNNExecutionProviderInfo() = default;
};
// Logical device representation.
class ArmNNExecutionProvider : public IExecutionProvider {
public:
explicit ArmNNExecutionProvider(const ArmNNExecutionProviderInfo& info);
virtual ~ArmNNExecutionProvider();
std::vector<std::unique_ptr<ComputeCapability>> GetCapability(
const onnxruntime::GraphViewer& graph,
const std::vector<const KernelRegistry*>& kernel_registries) const override;
const void* GetExecutionHandle() const noexcept override {
// The ArmNN interface does not return anything interesting.
return nullptr;
}
std::shared_ptr<KernelRegistry> GetKernelRegistry() const override;
};
} // namespace onnxruntime

12
onnxruntime/core/providers/armnn/armnn_fwd.h Executable file
View file

@ -0,0 +1,12 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#pragma once
namespace onnxruntime {
namespace armnn_ep {
template <typename T>
KernelCreateInfo BuildKernelCreateInfo();
}
}

35
onnxruntime/core/providers/armnn/armnn_provider_factory.cc Executable file
View file

@ -0,0 +1,35 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/armnn/armnn_provider_factory.h"
#include "armnn_execution_provider.h"
#include "core/session/abi_session_options_impl.h"
namespace onnxruntime {
struct ArmNNProviderFactory : IExecutionProviderFactory {
ArmNNProviderFactory(bool create_arena) : create_arena_(create_arena) {}
~ArmNNProviderFactory() override {}
std::unique_ptr<IExecutionProvider> CreateProvider() override;
private:
bool create_arena_;
};
std::unique_ptr<IExecutionProvider> ArmNNProviderFactory::CreateProvider() {
ArmNNExecutionProviderInfo info;
info.create_arena = create_arena_;
return std::make_unique<ArmNNExecutionProvider>(info);
}
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ArmNN(int use_arena) {
return std::make_shared<onnxruntime::ArmNNProviderFactory>(use_arena != 0);
}
} // namespace onnxruntime
ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_ArmNN, _In_ OrtSessionOptions* options, int use_arena) {
options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_ArmNN(use_arena));
return nullptr;
}

39
onnxruntime/core/providers/armnn/math/gemm.cc Executable file
View file

@ -0,0 +1,39 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/armnn/armnn_common.h"
#include "core/providers/armnn/math/gemm.h"
#include "core/providers/armnn/armnn_fwd.h"
namespace onnxruntime {
namespace armnn_ep {
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
Gemm,
kOnnxDomain,
7,
8,
kArmNNExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Gemm<float>);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
Gemm,
kOnnxDomain,
9,
10,
kArmNNExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Gemm<float>);
ONNX_OPERATOR_KERNEL_EX(
Gemm,
kOnnxDomain,
11,
kArmNNExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Gemm<float>);
} // namespace armnn_ep
} // namespace onnxruntime

186
onnxruntime/core/providers/armnn/math/gemm.h Normal file
View file

@ -0,0 +1,186 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/framework/op_kernel.h"
#include "core/util/math.h"
#include "core/util/math_cpuonly.h"
#include "core/providers/cpu/math/gemm.h"
#include "core/providers/armnn/armnn_execution_provider.h"
namespace onnxruntime {
namespace armnn_ep {
typedef std::map<OpKernel*, armnn::NetworkId>::iterator GEMMLayersIterator;
template <typename T>
class Gemm : public onnxruntime::Gemm<T> {
public:
Gemm(const OpKernelInfo& info) : onnxruntime::Gemm<T>(info) {
int64_t temp;
ORT_ENFORCE(info.GetAttr<int64_t>("transA", &temp).IsOK());
trans_A_ = temp == 0 ? CblasNoTrans : CblasTrans;
ORT_ENFORCE(info.GetAttr<int64_t>("transB", &temp).IsOK());
trans_B_ = temp == 0 ? CblasNoTrans : CblasTrans;
ORT_ENFORCE(info.GetAttr<float>("alpha", &alpha_).IsOK());
ORT_ENFORCE(info.GetAttr<float>("beta", &beta_).IsOK());
}
Status Compute(OpKernelContext* context) const override {
const auto X = context->Input<Tensor>(0);
const auto W = context->Input<Tensor>(1);
const auto B = context->Input<Tensor>(2);
bool useBias = B != nullptr && beta_ != 0;
bool FC = alpha_ == 1 && (beta_ == 1 || beta_ == 0);
if (!FC) {
return onnxruntime::Gemm<T>::Compute(context);
}
GemmHelper helper(X->Shape(), trans_A_ != CblasNoTrans, W->Shape(), trans_B_ != CblasNoTrans, useBias ? B->Shape() : TensorShape({}));
if (!helper.State().IsOK())
return helper.State();
int64_t M = helper.M();
int64_t N = helper.N();
auto Y = context->Output(0, TensorShape({M, N}));
if (trans_A_ == CblasTrans) { // transpose input
return onnxruntime::Gemm<T>::Compute(context);
}
int64_t K = helper.K();
LOGS_DEFAULT(VERBOSE) << "Gemm ArmNN:" << std::endl;
if (X) LOGS_DEFAULT(VERBOSE) << "X " << X->Shape().ToString().c_str() << std::endl;
if (W) LOGS_DEFAULT(VERBOSE) << "W " << W->Shape().ToString().c_str() << std::endl;
if (B) LOGS_DEFAULT(VERBOSE) << "B " << B->Shape().ToString().c_str() << std::endl;
LOGS_DEFAULT(VERBOSE) << "Y " << Y->Shape().ToString().c_str() << std::endl;
LOGS_DEFAULT(VERBOSE) << "M " << (int)M << ", N " << (int)N << ", K " << (int)K << std::endl;
LOGS_DEFAULT(VERBOSE) << "Alfa " << alpha_ << ", Beta " << beta_ << std::endl;
LOGS_DEFAULT(VERBOSE) << "trans_A_ " << (trans_A_ == CblasTrans) << std::endl;
LOGS_DEFAULT(VERBOSE) << "trans_B_ " << (trans_B_ == CblasTrans) << std::endl;
LOGS_DEFAULT(VERBOSE) << std::endl;
const T* x_data = X->template Data<T>();
const T* w_data = W->template Data<T>();
const T* b_data;
if (useBias)
b_data = B->template Data<T>();
T* y_data = Y->template MutableData<T>();
armnn::NetworkId* pNetworkId;
GEMMLayersIterator it = Gemm::gemmLayers.find((OpKernel*)this);
if (it == Gemm::gemmLayers.end()) {
armnn::NetworkId networkId;
armnn::INetworkPtr myNetwork = armnn::INetwork::Create();
armnn::TensorShape inputShape = ArmNNTensorShape(X->Shape());
armnn::TensorShape weightShape = ArmNNTensorShape(W->Shape());
armnn::TensorShape outputShape = ArmNNTensorShape(Y->Shape());
armnn::FullyConnectedDescriptor fcDescriptor;
fcDescriptor.m_BiasEnabled = useBias;
fcDescriptor.m_TransposeWeightMatrix = trans_B_ == CblasTrans;
armnn::IConnectableLayer* fc_armnn;
armnn::TensorInfo weightsInfo(weightShape, armnn::DataType::Float32);
armnn::ConstTensor weights(weightsInfo, w_data);
if (fcDescriptor.m_BiasEnabled) {
armnn::TensorShape biasShape = ArmNNTensorShape(B->Shape());
if(B->Shape().NumDimensions() == 2){
if(B->Shape().GetDims()[0] == 1 && B->Shape().GetDims()[1] > 1)
biasShape = {B->Shape().GetDims()[1]};
}
armnn::TensorInfo biasDesc(biasShape, armnn::DataType::Float32);
armnn::ConstTensor bias(biasDesc, b_data);
fc_armnn = myNetwork->AddFullyConnectedLayer(fcDescriptor,
weights,
armnn::Optional<armnn::ConstTensor>(bias),
"fc_armnn");
} else {
fc_armnn = myNetwork->AddFullyConnectedLayer(fcDescriptor,
weights,
armnn::EmptyOptional(),
"fc_armnn");
}
armnn::IConnectableLayer *InputLayer = myNetwork->AddInputLayer(0);
armnn::IConnectableLayer *OutputLayer = myNetwork->AddOutputLayer(0);
InputLayer->GetOutputSlot(0).Connect(fc_armnn->GetInputSlot(0));
fc_armnn->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0));
//Set the tensors in the network.
armnn::TensorInfo inputTensorInfo(inputShape, armnn::DataType::Float32);
InputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
armnn::TensorInfo outputTensorInfo(outputShape, armnn::DataType::Float32);
fc_armnn->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
// Optimise ArmNN network
armnn::IOptimizedNetworkPtr optNet = armnn::Optimize(*myNetwork, {armnn::Compute::CpuAcc}, Gemm::run->GetDeviceSpec());
if (optNet == nullptr) {
return onnxruntime::Gemm<T>::Compute(context);
}
// Load graph into runtime
Gemm::run->LoadNetwork(networkId, std::move(optNet));
std::pair<GEMMLayersIterator, bool> ret;
ret = Gemm::gemmLayers.insert(std::pair<OpKernel*, armnn::NetworkId>((OpKernel*)this, networkId));
pNetworkId = &ret.first->second;
} else {
pNetworkId = &it->second;
}
armnn::InputTensors inputTensors{{0, armnn::ConstTensor(Gemm::run->GetInputTensorInfo(*pNetworkId, 0),
x_data)}};
armnn::OutputTensors outputTensors{{0, armnn::Tensor(Gemm::run->GetOutputTensorInfo(*pNetworkId, 0),
y_data)}};
Gemm::run->EnqueueWorkload(*pNetworkId, inputTensors, outputTensors);
return Status::OK();
}
~Gemm() {
gemmLayers.erase(this);
}
static armnn::IRuntimePtr initRuntime(){
if (Gemm::run)
return std::move(Gemm::run);
armnn::IRuntime::CreationOptions options;
return std::move(armnn::IRuntime::Create(options));
}
private:
static thread_local std::map<OpKernel*, armnn::NetworkId> gemmLayers;
ArmNNExecutionProvider* provider_;
static armnn::IRuntimePtr run;
CBLAS_TRANSPOSE trans_A_;
CBLAS_TRANSPOSE trans_B_;
float alpha_;
float beta_;
};
template <typename T>
thread_local std::map<OpKernel*, armnn::NetworkId> onnxruntime::armnn_ep::Gemm<T>::gemmLayers;
template <typename T>
armnn::IRuntimePtr Gemm<T>::run = Gemm<T>::initRuntime();
} // namespace armnn_ep
} // namespace onnxruntime

300
onnxruntime/core/providers/armnn/nn/conv.cc Normal file
View file

@ -0,0 +1,300 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#ifdef _WIN32
#pragma warning(disable : 4244)
#endif
#include "core/common/common.h"
#include "core/framework/op_kernel.h"
#include "core/util/math.h"
#include "core/util/math_cpuonly.h"
#include "core/providers/armnn/nn/conv.h"
#include "core/providers/armnn/armnn_common.h"
#include "core/providers/armnn/armnn_fwd.h"
#define PREF_DIM 4
namespace onnxruntime {
namespace armnn_ep {
template <typename T>
thread_local std::map<OpKernel*, armnn::NetworkId> Conv<T>::convLayers;
template <typename T>
armnn::IRuntimePtr Conv<T>::run = Conv<T>::initRuntime();
armnn::Convolution2dDescriptor createConvDescriptor(std::vector<int64_t> pads, std::vector<int64_t> dilations, std::vector<int64_t> strides, bool biasEnabled){
std::vector<int64_t> armnnStrides(2);
armnnStrides[0] = (strides.size() == 2) ? strides[1] : 1;
armnnStrides[1] = strides[0];
std::vector<int64_t> armnnDilations(2);
armnnDilations[0] = (dilations.size() == 2) ? dilations[1] : 1;
armnnDilations[1] = dilations[0];
std::vector<int64_t> armnnPads(4);
if (pads.size() == 2) {
if (strides.size() == 1) {
armnnPads[0] = 0;
armnnPads[1] = 0;
armnnPads[2] = pads[1];
armnnPads[3] = pads[0];
} else {
armnnPads[0] = pads[1];
armnnPads[1] = pads[0];
armnnPads[2] = pads[1];
armnnPads[3] = pads[0];
}
} else {
armnnPads[0] = pads[1];
armnnPads[1] = pads[3];
armnnPads[2] = pads[0];
armnnPads[3] = pads[2];
}
armnn::Convolution2dDescriptor convolutionDescriptor;
convolutionDescriptor.m_PadLeft = armnnPads[0];
convolutionDescriptor.m_PadRight = armnnPads[1];
convolutionDescriptor.m_PadTop = armnnPads[2];
convolutionDescriptor.m_PadBottom = armnnPads[3];
convolutionDescriptor.m_StrideX = armnnStrides[0];
convolutionDescriptor.m_StrideY = armnnStrides[1];
convolutionDescriptor.m_DilationX = armnnDilations[0];
convolutionDescriptor.m_DilationY = armnnDilations[1];
convolutionDescriptor.m_BiasEnabled = biasEnabled;
convolutionDescriptor.m_DataLayout = armnn::DataLayout::NCHW;
return convolutionDescriptor;
}
armnn::DepthwiseConvolution2dDescriptor createDepthwiseDescriptor(armnn::Convolution2dDescriptor convolutionDescriptor){
armnn::DepthwiseConvolution2dDescriptor depthwiseDescriptor;
depthwiseDescriptor.m_PadLeft = convolutionDescriptor.m_PadLeft;
depthwiseDescriptor.m_PadRight = convolutionDescriptor.m_PadRight;
depthwiseDescriptor.m_PadTop = convolutionDescriptor.m_PadTop;
depthwiseDescriptor.m_PadBottom = convolutionDescriptor.m_PadBottom;
depthwiseDescriptor.m_StrideX = convolutionDescriptor.m_StrideX;
depthwiseDescriptor.m_StrideY = convolutionDescriptor.m_StrideY;
depthwiseDescriptor.m_DilationX = convolutionDescriptor.m_DilationX;
depthwiseDescriptor.m_DilationY = convolutionDescriptor.m_DilationY;
depthwiseDescriptor.m_BiasEnabled = convolutionDescriptor.m_BiasEnabled;
depthwiseDescriptor.m_DataLayout = convolutionDescriptor.m_DataLayout;
return depthwiseDescriptor;
}
template <typename T>
Status Conv<T>::Compute(OpKernelContext* context) const {
size_t num_inputs = OpKernel::Node().InputDefs().size();
const Tensor* X = context->Input<Tensor>(0);
const Tensor* W = context->Input<Tensor>(1);
const Tensor* B = num_inputs == 3 ? context->Input<Tensor>(2) : nullptr;
const int64_t N = X->Shape()[0];
const int64_t M = W->Shape()[0];
if (X->Shape().NumDimensions() != PREF_DIM) {
Status s = onnxruntime::Conv<T>::Compute(context);
return s;
}
ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(X, W));
std::vector<int64_t> kernel_shape;
ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(W->Shape(), kernel_shape));
std::vector<int64_t> pads(conv_attrs_.pads);
if (pads.empty()) {
pads.resize(kernel_shape.size() * 2, 0);
}
std::vector<int64_t> dilations(conv_attrs_.dilations);
if (dilations.empty()) {
dilations.resize(kernel_shape.size(), 1);
}
std::vector<int64_t> strides(conv_attrs_.strides);
if (strides.empty()) {
strides.resize(kernel_shape.size(), 1);
}
std::vector<int64_t> Y_dims;
Y_dims.insert(Y_dims.begin(), {N, M});
TensorShape input_shape = X->Shape().Slice(2);
ORT_RETURN_IF_ERROR(conv_attrs_.InferOutputShape(input_shape, kernel_shape, strides, dilations, &pads, &Y_dims));
Tensor* Y = context->Output(0, TensorShape(Y_dims));
bool biasEnabled = B != nullptr;
const T* x_data = X->template Data<T>();
const T* k_data = W->template Data<T>();
const T* b_data;
if (biasEnabled) {
b_data = B->template Data<T>();
}
T* y_data = Y->template MutableData<T>();
armnn::NetworkId* pNetworkId;
ConvLayersIterator it = Conv::convLayers.find((OpKernel*)this);
if (it == Conv::convLayers.end()) {
armnn::NetworkId networkId;
armnn::INetworkPtr myNetwork = armnn::INetwork::Create();
armnn::Convolution2dDescriptor convolutionDescriptor = createConvDescriptor(pads, dilations, strides, biasEnabled);
armnn::IConnectableLayer *convolution_armnn;
armnn::TensorShape inputShape = ArmNNTensorShape(X->Shape());
armnn::TensorShape weightShape = ArmNNTensorShape(W->Shape());
if (weightShape[2] == 1 && weightShape[3] == 1) {
Status s = onnxruntime::Conv<T>::Compute(context);
return s;
}
if (conv_attrs_.group > 1) {
if (conv_attrs_.group == inputShape[1]) {
// depthwise convolution
armnn::DepthwiseConvolution2dDescriptor depthwiseDescriptor = createDepthwiseDescriptor(convolutionDescriptor);
weightShape[1] = weightShape[0];
weightShape[0] = 1;
armnn::TensorInfo weightsInfo(weightShape, armnn::DataType::Float32);
armnn::ConstTensor weights(weightsInfo, k_data);
if (biasEnabled) {
armnn::TensorInfo biasDesc(ArmNNTensorShape(B->Shape()), armnn::DataType::Float32);
armnn::ConstTensor bias(biasDesc, b_data);
convolution_armnn = myNetwork->AddDepthwiseConvolution2dLayer(depthwiseDescriptor,
weights,
armnn::Optional<armnn::ConstTensor>(bias),
"depthwise_convolution_armnn");
} else {
convolution_armnn = myNetwork->AddDepthwiseConvolution2dLayer(depthwiseDescriptor,
weights,
armnn::EmptyOptional(),
"depthwise_convolution_armnn");
}
} else {
// NCHWc convolution
Status s = onnxruntime::Conv<T>::Compute(context);
return s;
}
} else {
// normal convolution
armnn::TensorInfo weightsInfo(weightShape, armnn::DataType::Float32);
armnn::ConstTensor weights(weightsInfo, k_data);
if (biasEnabled) {
armnn::TensorInfo biasDesc(ArmNNTensorShape(B->Shape()), armnn::DataType::Float32);
armnn::ConstTensor bias(biasDesc, b_data);
convolution_armnn = myNetwork->AddConvolution2dLayer(convolutionDescriptor,
weights,
armnn::Optional<armnn::ConstTensor>(bias),
"convolution_armnn");
} else {
convolution_armnn = myNetwork->AddConvolution2dLayer(convolutionDescriptor,
weights,
armnn::EmptyOptional(),
"convolution_armnn");
}
}
bool armnn_activ_enabled = false;
armnn::ActivationDescriptor desc;
desc.m_A = conv_attrs_.alpha;
if (activation_type == "Relu") {
desc.m_Function = armnn::ActivationFunction::ReLu;
armnn_activ_enabled = true;
} else if (activation_type == "LeakyRelu") {
desc.m_Function = armnn::ActivationFunction::LeakyReLu;
armnn_activ_enabled = true;
} else if (activation_type == "Tanh") {
desc.m_Function = armnn::ActivationFunction::TanH;
armnn_activ_enabled = true;
} else if (activation_type == "Sigmoid") {
desc.m_Function = armnn::ActivationFunction::Sigmoid;
armnn_activ_enabled = true;
} else if (!activation_type.empty()) {
ORT_NOT_IMPLEMENTED("Not implemented fused activation: ", activation_type);
}
armnn::IConnectableLayer* activation = myNetwork->AddActivationLayer(desc, "activation_armnn");
armnn::IConnectableLayer *InputLayer = myNetwork->AddInputLayer(0);
armnn::IConnectableLayer *OutputLayer = myNetwork->AddOutputLayer(0);
InputLayer->GetOutputSlot(0).Connect(convolution_armnn->GetInputSlot(0));
if (armnn_activ_enabled) {
convolution_armnn->GetOutputSlot(0).Connect(activation->GetInputSlot(0));
activation->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0));
}
else {
convolution_armnn->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0));
}
//Set the tensors in the network.
armnn::TensorInfo inputTensorInfo(inputShape, armnn::DataType::Float32);
InputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
armnn::TensorInfo outputTensorInfo(ArmNNTensorShape(Y->Shape()), armnn::DataType::Float32);
convolution_armnn->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
if (armnn_activ_enabled) {
activation->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
}
// Optimise ArmNN network
armnn::IOptimizedNetworkPtr optNet = armnn::Optimize(*myNetwork, {armnn::Compute::CpuAcc}, Conv::run->GetDeviceSpec());
if (optNet == nullptr) {
return onnxruntime::Conv<T>::Compute(context);
}
// Load graph into runtime
Conv::run->LoadNetwork(networkId, std::move(optNet));
std::pair<ConvLayersIterator, bool> ret;
ret = Conv::convLayers.insert(std::pair<OpKernel*, armnn::NetworkId>((OpKernel*)this, networkId));
pNetworkId = &ret.first->second;
} else {
pNetworkId = &it->second;
}
armnn::InputTensors inputTensors{{0, armnn::ConstTensor(Conv::run->GetInputTensorInfo(*pNetworkId, 0),
x_data)}};
armnn::OutputTensors outputTensors{{0, armnn::Tensor(Conv::run->GetOutputTensorInfo(*pNetworkId, 0),
y_data)}};
// Execute network
Conv::run->EnqueueWorkload(*pNetworkId, inputTensors, outputTensors);
return Status::OK();
}
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
Conv,
kOnnxDomain,
1, 10,
kArmNNExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Conv<float>);
ONNX_OPERATOR_KERNEL_EX(
Conv,
kOnnxDomain,
11,
kArmNNExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Conv<float>);
} // namespace armnn
} // namespace onnxruntime

51
onnxruntime/core/providers/armnn/nn/conv.h Normal file
View file

@ -0,0 +1,51 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/framework/op_kernel.h"
#include "core/providers/cpu/nn/conv.h"
#include "core/providers/armnn/armnn_execution_provider.h"
#include "armnn/ArmNN.hpp"
#include <thread>
#include <mutex>
namespace onnxruntime {
namespace armnn_ep{
typedef std::map<OpKernel*, armnn::NetworkId>::iterator ConvLayersIterator;
template <typename T>
class Conv : public onnxruntime::Conv<T> {
public:
explicit Conv(const OpKernelInfo& info) : onnxruntime::Conv<T>(info), conv_attrs_(info) {
provider_ = (const_cast<ArmNNExecutionProvider*>(
dynamic_cast<const ArmNNExecutionProvider*>(info.GetExecutionProvider())));
}
~Conv() {
Conv::convLayers.erase(this);
}
Status Compute(OpKernelContext* context) const override;
static armnn::IRuntimePtr initRuntime(){
if (Conv::run)
return std::move(Conv::run);
armnn::IRuntime::CreationOptions options;
return std::move(armnn::IRuntime::Create(options));
}
protected:
static thread_local std::map<OpKernel*, armnn::NetworkId> convLayers;
ConvAttributes conv_attrs_;
ArmNNExecutionProvider* provider_;
static armnn::IRuntimePtr run;
std::string activation_type;
};
} // namespace armnn_ep
} // namespace onnxruntime

310
onnxruntime/core/providers/armnn/nn/pool.cc Normal file
View file

@ -0,0 +1,310 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#include <cmath>
#include "core/common/common.h"
#include "core/framework/op_kernel.h"
#include "core/util/math.h"
#include "core/util/math_cpuonly.h"
#include "core/providers/armnn/nn/pool.h"
#include "core/providers/armnn/armnn_common.h"
#include "core/providers/armnn/armnn_fwd.h"
#define PREF_DIM 4
namespace onnxruntime {
namespace armnn_ep {
template <typename T, typename PoolType>
thread_local std::map<OpKernel*, armnn::NetworkId> Pool<T, PoolType>::poolLayers;
template <typename T, typename PoolType>
armnn::IRuntimePtr Pool<T, PoolType>::run = Pool<T, PoolType>::initRuntime();
template <typename T>
thread_local std::map<OpKernel*, armnn::NetworkId> MaxPoolV8<T>::maxPoolLayers;
template <typename T>
armnn::IRuntimePtr MaxPoolV8<T>::run = MaxPoolV8<T>::initRuntime();
armnn::Pooling2dDescriptor createDescriptor(std::vector<int64_t> pads, std::vector<int64_t> strides, std::vector<int64_t> kernel_shape, armnn::PoolingAlgorithm pool_type, onnxruntime::PoolAttributes pool_attrs){
std::vector<int64_t> armnnStrides(2);
armnnStrides[0] = (strides.size() == 2) ? strides[1] : 1;
armnnStrides[1] = strides[0];
std::vector<int64_t> armnnKernelShape(2);
armnnKernelShape[0] = (kernel_shape.size() > 1) ? kernel_shape[1] : 1;
armnnKernelShape[1] = kernel_shape[0];
std::vector<int64_t> armnnPads(4);
if (pads.size() == 2) {
if (strides.size() == 1) {
armnnPads[0] = 0;
armnnPads[1] = 0;
armnnPads[2] = pads[1];
armnnPads[3] = pads[0];
} else {
armnnPads[0] = pads[1];
armnnPads[1] = pads[0];
armnnPads[2] = pads[1];
armnnPads[3] = pads[0];
}
} else {
armnnPads[0] = pads[1];
armnnPads[1] = pads[3];
armnnPads[2] = pads[0];
armnnPads[3] = pads[2];
}
armnn::Pooling2dDescriptor poolDescriptor;
poolDescriptor.m_PoolType = pool_type;
poolDescriptor.m_PadLeft = armnnPads[0];
poolDescriptor.m_PadRight = armnnPads[1];
poolDescriptor.m_PadTop = armnnPads[2];
poolDescriptor.m_PadBottom = armnnPads[3];
poolDescriptor.m_PoolWidth = armnnKernelShape[0];
poolDescriptor.m_PoolHeight = armnnKernelShape[1];
poolDescriptor.m_StrideX = armnnStrides[0];
poolDescriptor.m_StrideY = armnnStrides[1];
poolDescriptor.m_OutputShapeRounding = pool_attrs.ceil_mode ? armnn::OutputShapeRounding::Ceiling : armnn::OutputShapeRounding::Floor;
poolDescriptor.m_PaddingMethod = armnn::PaddingMethod::Exclude;
if (pool_type == armnn::PoolingAlgorithm::Average && pool_attrs.count_include_pad)
poolDescriptor.m_PaddingMethod = armnn::PaddingMethod::IgnoreValue;
poolDescriptor.m_DataLayout = armnn::DataLayout::NCHW;
return poolDescriptor;
}
template <typename T, typename PoolType>
Status Pool<T, PoolType>::Compute(OpKernelContext* context) const {
const Tensor* X = context->Input<Tensor>(0);
const TensorShape& x_shape = X->Shape();
std::vector<int64_t> dilations(PoolBase::pool_attrs_.dilations);
std::vector<int64_t> armnnDilations(2);
armnnDilations[0] = (dilations.size() == 2) ? dilations[1] : 1;
armnnDilations[1] = (!dilations.empty()) ? dilations[0] : 1;
if ((X->Shape().NumDimensions() != PREF_DIM) ||
(armnnDilations[0] * armnnDilations[1] > 1)) {
Status s = onnxruntime::Pool<T, PoolType>::Compute(context);
return s;
}
std::vector<int64_t> pads = PoolBase::pool_attrs_.pads;
std::vector<int64_t> strides = PoolBase::pool_attrs_.strides;
std::vector<int64_t> kernel_shape = PoolBase::pool_attrs_.kernel_shape;
if (PoolBase::pool_attrs_.global_pooling) {
const auto& input_dims = x_shape.GetDims();
kernel_shape.assign(input_dims.begin() + 2, input_dims.end());
strides.assign(kernel_shape.size(), 0);
pads.assign(kernel_shape.size(), 0);
}
std::vector<int64_t> output_dims = PoolBase::pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads);
Tensor* Y = context->Output(0, TensorShape(output_dims));
const T* x_data = X->template Data<T>();
T* y_data = Y->template MutableData<T>();
armnn::NetworkId* pNetworkId;
PoolLayersIterator it = Pool::poolLayers.find((OpKernel*)this);
if (it == Pool::poolLayers.end()) {
armnn::PoolingAlgorithm pool_type;
if (PoolBase::op_name_ == "GlobalAveragePool" || PoolBase::op_name_ == "AveragePool"){
pool_type = armnn::PoolingAlgorithm::Average;
} else if (PoolBase::op_name_ == "GlobalMaxPool" || PoolBase::op_name_ == "MaxPool"){
pool_type = armnn::PoolingAlgorithm::Max;
} else
return onnxruntime::Pool<T, PoolType>::Compute(context);
armnn::NetworkId networkId;
armnn::INetworkPtr myNetwork = armnn::INetwork::Create();
armnn::Pooling2dDescriptor poolDescriptor = createDescriptor(pads, strides, kernel_shape, pool_type, PoolBase::pool_attrs_);
armnn::IConnectableLayer *pool_armnn = myNetwork->AddPooling2dLayer(poolDescriptor, "pool_armnn");
armnn::TensorShape inputShape = ArmNNTensorShape(X->Shape());
armnn::TensorShape outputShape = ArmNNTensorShape(Y->Shape());
armnn::IConnectableLayer *InputLayer = myNetwork->AddInputLayer(0);
armnn::IConnectableLayer *OutputLayer = myNetwork->AddOutputLayer(0);
InputLayer->GetOutputSlot(0).Connect(pool_armnn->GetInputSlot(0));
pool_armnn->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0));
//Set the tensors in the network.
armnn::TensorInfo inputTensorInfo(inputShape, armnn::DataType::Float32);
InputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
armnn::TensorInfo outputTensorInfo(outputShape, armnn::DataType::Float32);
pool_armnn->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
// Optimise ArmNN network
armnn::IOptimizedNetworkPtr optNet = armnn::Optimize(*myNetwork, {armnn::Compute::CpuAcc}, Pool::run->GetDeviceSpec());
if (optNet == nullptr) {
return onnxruntime::Pool<T, PoolType>::Compute(context);
}
// Load graph into runtime
Pool::run->LoadNetwork(networkId, std::move(optNet));
std::pair<PoolLayersIterator, bool> ret;
ret = Pool::poolLayers.insert(std::pair<OpKernel*, armnn::NetworkId>((OpKernel*)this, networkId));
pNetworkId = &ret.first->second;
} else {
pNetworkId = &it->second;
}
armnn::InputTensors inputTensors{{0, armnn::ConstTensor(Pool::run->GetInputTensorInfo(*pNetworkId, 0),
x_data)}};
armnn::OutputTensors outputTensors{{0, armnn::Tensor(Pool::run->GetOutputTensorInfo(*pNetworkId, 0),
y_data)}};
// Execute network
Pool::run->EnqueueWorkload(*pNetworkId, inputTensors, outputTensors);
return Status::OK();
}
template <typename T>
Status MaxPoolV8<T>::Compute(OpKernelContext* context) const {
const Tensor* X = context->Input<Tensor>(0);
const TensorShape& x_shape = X->Shape();
std::vector<int64_t> dilations(PoolBase::pool_attrs_.dilations);
std::vector<int64_t> armnnDilations(2);
armnnDilations[0] = (dilations.size() == 2) ? dilations[1] : 1;
armnnDilations[1] = (!dilations.empty()) ? dilations[0] : 1;
if ((X->Shape().NumDimensions() != PREF_DIM) ||
(armnnDilations[0] * armnnDilations[1] > 1)) {
Status s = onnxruntime::MaxPoolV8::Compute(context);
return s;
}
std::vector<int64_t> pads = PoolBase::pool_attrs_.pads;
std::vector<int64_t> strides = PoolBase::pool_attrs_.strides;
std::vector<int64_t> kernel_shape = PoolBase::pool_attrs_.kernel_shape;
if (PoolBase::pool_attrs_.global_pooling) {
const auto& input_dims = x_shape.GetDims();
kernel_shape.assign(input_dims.begin() + 2, input_dims.end());
strides.assign(kernel_shape.size(), 0);
pads.assign(kernel_shape.size(), 0);
}
std::vector<int64_t> output_dims = PoolBase::pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads);
Tensor* Y = context->Output(0, TensorShape(output_dims));
const T* x_data = X->template Data<T>();
T* y_data = Y->template MutableData<T>();
armnn::NetworkId* pNetworkId;
PoolLayersIterator it = MaxPoolV8::maxPoolLayers.find((OpKernel*)this);
if (it == MaxPoolV8::maxPoolLayers.end()) {
armnn::NetworkId networkId;
armnn::INetworkPtr myNetwork = armnn::INetwork::Create();
armnn::Pooling2dDescriptor poolDescriptor = createDescriptor(pads, strides, kernel_shape, armnn::PoolingAlgorithm::Max, PoolBase::pool_attrs_);
armnn::IConnectableLayer *pool_armnn = myNetwork->AddPooling2dLayer(poolDescriptor, "pool_armnn");
armnn::TensorShape inputShape = ArmNNTensorShape(X->Shape());
armnn::TensorShape outputShape = ArmNNTensorShape(Y->Shape());
armnn::IConnectableLayer *InputLayer = myNetwork->AddInputLayer(0);
armnn::IConnectableLayer *OutputLayer = myNetwork->AddOutputLayer(0);
InputLayer->GetOutputSlot(0).Connect(pool_armnn->GetInputSlot(0));
pool_armnn->GetOutputSlot(0).Connect(OutputLayer->GetInputSlot(0));
//Set the tensors in the network.
armnn::TensorInfo inputTensorInfo(inputShape, armnn::DataType::Float32);
InputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo);
armnn::TensorInfo outputTensorInfo(outputShape, armnn::DataType::Float32);
pool_armnn->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
// Optimise ArmNN network
armnn::IOptimizedNetworkPtr optNet = armnn::Optimize(*myNetwork, {armnn::Compute::CpuAcc}, MaxPoolV8::run->GetDeviceSpec());
if (optNet == nullptr) {
return onnxruntime::MaxPoolV8::Compute(context);
}
// Load graph into runtime
MaxPoolV8::run->LoadNetwork(networkId, std::move(optNet));
std::pair<PoolLayersIterator, bool> ret;
ret = MaxPoolV8::maxPoolLayers.insert(std::pair<OpKernel*, armnn::NetworkId>((OpKernel*)this, networkId));
pNetworkId = &ret.first->second;
} else {
pNetworkId = &it->second;
}
armnn::InputTensors inputTensors{{0, armnn::ConstTensor(MaxPoolV8::run->GetInputTensorInfo(*pNetworkId, 0),
x_data)}};
armnn::OutputTensors outputTensors{{0, armnn::Tensor(MaxPoolV8::run->GetOutputTensorInfo(*pNetworkId, 0),
y_data)}};
// Execute network
MaxPoolV8::run->EnqueueWorkload(*pNetworkId, inputTensors, outputTensors);
return Status::OK();
}
#define POOLING_KERNEL_VERSIONED(op_name, data_type, pool_type, since_version, end_version) \
ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \
op_name, \
kOnnxDomain, \
since_version, \
end_version, \
data_type, \
kArmNNExecutionProvider, \
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<data_type>()), \
Pool<data_type, pool_type>);
#define POOLING_KERNEL(op_name, data_type, pool_type, since_version) \
ONNX_OPERATOR_TYPED_KERNEL_EX( \
op_name, \
kOnnxDomain, \
since_version, \
data_type, \
kArmNNExecutionProvider, \
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<data_type>()), \
Pool<data_type, pool_type>);
POOLING_KERNEL_VERSIONED(MaxPool, float, MaxPool<1>, 1, 7)
POOLING_KERNEL_VERSIONED(AveragePool, float, AveragePool, 7, 9)
POOLING_KERNEL_VERSIONED(AveragePool, float, AveragePool, 10, 10)
POOLING_KERNEL(AveragePool, float, AveragePool, 11)
POOLING_KERNEL(GlobalAveragePool, float, AveragePool, 1)
POOLING_KERNEL(GlobalMaxPool, float, MaxPool<1>, 1)
ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \
MaxPool, \
kOnnxDomain, \
8, \
11, \
float, \
kArmNNExecutionProvider, \
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()), \
MaxPoolV8<float>);
} // namespace armnn_ep
} // namespace onnxruntime

75
onnxruntime/core/providers/armnn/nn/pool.h Normal file
View file

@ -0,0 +1,75 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/framework/op_kernel.h"
#include "core/providers/cpu/nn/pool.h"
#include "core/providers/armnn/armnn_execution_provider.h"
#include "armnn/ArmNN.hpp"
#include <thread>
#include <mutex>
namespace onnxruntime {
namespace armnn_ep {
typedef std::map<OpKernel*, armnn::NetworkId>::iterator PoolLayersIterator;
template <typename T, typename PoolType>
class Pool final : public onnxruntime::Pool<T, PoolType> {
public:
explicit Pool(const OpKernelInfo& info) : onnxruntime::Pool<T, PoolType>(info) {
provider_ = (const_cast<ArmNNExecutionProvider*>(
dynamic_cast<const ArmNNExecutionProvider*>(info.GetExecutionProvider())));
}
~Pool() {
poolLayers.erase(this);
}
Status Compute(OpKernelContext* context) const override;
static armnn::IRuntimePtr initRuntime(){
if (Pool::run)
return std::move(Pool::run);
armnn::IRuntime::CreationOptions options;
return std::move(armnn::IRuntime::Create(options));
}
private:
static thread_local std::map<OpKernel*, armnn::NetworkId> poolLayers;
ArmNNExecutionProvider* provider_;
static armnn::IRuntimePtr run;
};
template <typename T>
class MaxPoolV8 final : public onnxruntime::MaxPoolV8 {
public:
explicit MaxPoolV8(const OpKernelInfo& info) : onnxruntime::MaxPoolV8(info) {
provider_ = (const_cast<ArmNNExecutionProvider*>(
dynamic_cast<const ArmNNExecutionProvider*>(info.GetExecutionProvider())));
}
~MaxPoolV8() {
maxPoolLayers.erase(this);
}
Status Compute(OpKernelContext* context) const override;
static armnn::IRuntimePtr initRuntime(){
if (MaxPoolV8::run)
return std::move(MaxPoolV8::run);
armnn::IRuntime::CreationOptions options;
return std::move(armnn::IRuntime::Create(options));
}
private:
static thread_local std::map<OpKernel*, armnn::NetworkId> maxPoolLayers;
ArmNNExecutionProvider* provider_;
static armnn::IRuntimePtr run;
};
} // namespace armnn_ep
} // namespace onnxruntime

1
onnxruntime/core/providers/armnn/symbols.txt Executable file
View file

@ -0,0 +1 @@
OrtSessionOptionsAppendExecutionProvider_ArmNN

15
onnxruntime/test/onnx/main.cc
View file

@ -37,7 +37,7 @@ void usage() {
"\t-v: verbose\n"
"\t-n [test_case_name]: Specifies a single test case to run.\n"
"\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'dnnl', 'tensorrt', 'ngraph', "
"'openvino', 'nuphar', 'migraphx' or 'acl'. "
"'openvino', 'nuphar', 'migraphx', 'acl' or 'armnn'. "
"Default: 'cpu'.\n"
"\t-x: Use parallel executor, default (without -x): sequential executor.\n"
"\t-d [device_id]: Specifies the device id for multi-device (e.g. GPU). The value should > 0\n"
@ -101,6 +101,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
bool enable_nnapi = false;
bool enable_dml = false;
bool enable_acl = false;
bool enable_armnn = false;
bool enable_migraphx = false;
int device_id = 0;
GraphOptimizationLevel graph_optimization_level = ORT_ENABLE_ALL;
@ -168,9 +169,11 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
enable_dml = true;
} else if (!CompareCString(optarg, ORT_TSTR("acl"))) {
enable_acl = true;
} else if (!CompareCString(optarg, ORT_TSTR("armnn"))) {
enable_armnn = true;
} else if (!CompareCString(optarg, ORT_TSTR("migraphx"))) {
enable_migraphx = true;
}else {
} else {
usage();
return -1;
}
@ -362,6 +365,14 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
#else
fprintf(stderr, "ACL is not supported in this build");
return -1;
#endif
}
if (enable_armnn) {
#ifdef USE_ARMNN
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_ArmNN(sf, enable_cpu_mem_arena ? 1 : 0));
#else
fprintf(stderr, "ArmNN is not supported in this build\n");
return -1;
#endif
}
if (enable_migraphx) {

2
onnxruntime/test/perftest/command_args_parser.cc
View file

@ -98,6 +98,8 @@ namespace perftest {
test_config.machine_config.provider_type_name = onnxruntime::kDmlExecutionProvider;
} else if (!CompareCString(optarg, ORT_TSTR("acl"))) {
test_config.machine_config.provider_type_name = onnxruntime::kAclExecutionProvider;
} else if (!CompareCString(optarg, ORT_TSTR("armnn"))) {
test_config.machine_config.provider_type_name = onnxruntime::kArmNNExecutionProvider;
} else {
return false;
}

7
onnxruntime/test/perftest/ort_test_session.cc
View file

@ -85,6 +85,13 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
performance_test_config.run_config.enable_cpu_mem_arena ? 1 : 0));
#else
ORT_THROW("Acl is not supported in this build\n");
#endif
} else if (provider_name == onnxruntime::kArmNNExecutionProvider) {
#ifdef USE_ARMNN
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_ArmNN(session_options,
performance_test_config.run_config.enable_cpu_mem_arena ? 1 : 0));
#else
ORT_THROW("ArmNN is not supported in this build\n");
#endif
} else if (provider_name == onnxruntime::kMIGraphXExecutionProvider) {
#ifdef USE_MIGRAPHX

2
onnxruntime/test/providers/cpu/nn/pool_op_test.cc
View file

@ -162,7 +162,7 @@ static void MaxPool_8_WithIndexTest(bool has_index, int64_t storage_order = 0) {
storage_order == 0 ? test.AddOutput<int64_t>("Indices", expected_dims, expected_indices_row)
: test.AddOutput<int64_t>("Indices", expected_dims, expected_indices_col);
}
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kDnnlExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kDnnlExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider, kArmNNExecutionProvider});
}
TEST(PoolTest, MaxPool_8_With_Index) {

3
onnxruntime/test/providers/provider_test_utils.cc
View file

@ -712,6 +712,7 @@ void OpTester::Run(
kOpenVINOExecutionProvider,
kDmlExecutionProvider,
kAclExecutionProvider,
kArmNNExecutionProvider,
};
bool has_run = false;
@ -778,6 +779,8 @@ void OpTester::Run(
execution_provider = DefaultRknpuExecutionProvider();
else if (provider_type == onnxruntime::kAclExecutionProvider)
execution_provider = DefaultAclExecutionProvider();
else if (provider_type == onnxruntime::kArmNNExecutionProvider)
execution_provider = DefaultArmNNExecutionProvider();
// skip if execution provider is disabled
if (execution_provider == nullptr)
continue;

10
onnxruntime/test/util/default_providers.cc
View file

@ -22,6 +22,7 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Rknpu(
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tensorrt(int device_id);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_MIGraphX(int device_id);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ACL(int use_arena);
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ArmNN(int use_arena);
namespace test {
@ -113,5 +114,14 @@ std::unique_ptr<IExecutionProvider> DefaultAclExecutionProvider(bool enable_aren
#endif
}
std::unique_ptr<IExecutionProvider> DefaultArmNNExecutionProvider(bool enable_arena){
#ifdef USE_ARMNN
return CreateExecutionProviderFactory_ArmNN(enable_arena)->CreateProvider();
#else
ORT_UNUSED_PARAMETER(enable_arena);
return nullptr;
#endif
}
} // namespace test
} // namespace onnxruntime

1
onnxruntime/test/util/include/default_providers.h
View file

@ -18,6 +18,7 @@ std::unique_ptr<IExecutionProvider> DefaultOpenVINOExecutionProvider();
std::unique_ptr<IExecutionProvider> DefaultNnapiExecutionProvider();
std::unique_ptr<IExecutionProvider> DefaultRknpuExecutionProvider();
std::unique_ptr<IExecutionProvider> DefaultAclExecutionProvider(bool enable_arena = true);
std::unique_ptr<IExecutionProvider> DefaultArmNNExecutionProvider(bool enable_arena = true);
} // namespace test
} // namespace onnxruntime

3
onnxruntime/test/util/include/providers.h
View file

@ -31,6 +31,9 @@
#ifdef USE_ACL
#include "core/providers/acl/acl_provider_factory.h"
#endif
#ifdef USE_ARMNN
#include "core/providers/armnn/armnn_provider_factory.h"
#endif
#ifdef USE_MIGRAPHX
#include "core/providers/migraphx/migraphx_provider_factory.h"
#endif

14
tools/ci_build/build.py
View file

@ -333,6 +333,12 @@ def parse_arguments():
"--use_acl", nargs="?", const="ACL_1905",
choices=["ACL_1902", "ACL_1905", "ACL_1908"],
help="Build with ACL for ARM architectures.")
parser.add_argument(
"--use_armnn", action='store_true',
help="Enable ArmNN Execution Provider.")
parser.add_argument(
"--armnn_relu", action='store_true',
help="Use the Relu operator implementation from the ArmNN EP.")
return parser.parse_args()
@ -523,7 +529,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home,
"OFF" if args.skip_winml_tests else "ON"),
"-Donnxruntime_GENERATE_TEST_REPORTS=ON",
"-Donnxruntime_DEV_MODE=" + (
"OFF" if args.android or args.use_acl or
"OFF" if args.android or args.use_acl or args.use_armnn or
(args.ios and is_macOS()) else "ON"),
"-DPYTHON_EXECUTABLE=" + sys.executable,
"-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"),
@ -612,6 +618,10 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home,
"ON" if args.use_acl == "ACL_1905" else "OFF"),
"-Donnxruntime_USE_ACL_1908=" + (
"ON" if args.use_acl == "ACL_1908" else "OFF"),
"-Donnxruntime_USE_ARMNN=" + (
"ON" if args.use_armnn else "OFF"),
"-Donnxruntime_ARMNN_RELU_USE_CPU=" + (
"OFF" if args.armnn_relu else "ON"),
# Training related flags
"-Donnxruntime_ENABLE_NVTX_PROFILE=" + (
"ON" if args.enable_nvtx_profile else "OFF"),
@ -1659,7 +1669,7 @@ def main():
"Only Windows ARM(64) cross-compiled builds supported "
"currently through this script")
install_ubuntu_deps(args)
if not is_docker():
if not is_docker() and not args.use_armnn:
install_python_deps()
if args.enable_pybind and is_windows():
install_python_deps(args.numpy_version)