mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Merge remote-tracking branch 'upstream/master' into DmlDev
This commit is contained in:
ISS Build Account
commit
49ec73e939
24 changed files with 591 additions and 178 deletions
14
BUILD.md
14
BUILD.md
|
|
@ -510,6 +510,11 @@ cmake ../onnxruntime-arm-upstream/cmake -DONNX_CUSTOM_PROTOC_EXECUTABLE=/usr/bin
|
|||
The ```-Donnxruntime_USE_ACL=ON``` option will use, by default, the 19.05 version of the Arm Compute Library. To set the right version you can use:
|
||||
```-Donnxruntime_USE_ACL_1902=ON```, ```-Donnxruntime_USE_ACL_1905=ON```, ```-Donnxruntime_USE_ACL_1908=ON``` or ```-Donnxruntime_USE_ACL_2002=ON```;
|
||||
|
||||
To use a library outside the normal environment you can set a custom path by using ```-Donnxruntime_ACL_HOME``` and ```-Donnxruntime_ACL_LIBS``` tags that defines the path to the ComputeLibrary directory and the build directory respectively.
|
||||
|
||||
```-Donnxruntime_ACL_HOME=/path/to/ComputeLibrary```, ```-Donnxruntime_ACL_LIBS=/path/to/build```
|
||||
|
||||
|
||||
2. Build ONNX Runtime library, test and performance application:
|
||||
```
|
||||
make -j 6
|
||||
|
|
@ -542,6 +547,10 @@ export LD_LIBRARY_PATH=~/ComputeLibrary/build/
|
|||
```
|
||||
./build.sh --use_acl
|
||||
```
|
||||
To use a library outside the normal environment you can set a custom path by using --acl_home and --acl_libs tags that defines the path to the ComputeLibrary directory and the build directory respectively.
|
||||
```
|
||||
./build.sh --use_acl --acl_home /path/to/ComputeLibrary --acl_libs /path/to/build
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -572,6 +581,11 @@ The Batch Normalization operator is set by default to use the CPU execution prov
|
|||
./build.sh --use_armnn --armnn_bn
|
||||
```
|
||||
|
||||
To use a library outside the normal environment you can set a custom path by using --armnn_home and --armnn_libs tags that defines the path to the ArmNN home directory and the build directory respectively.
|
||||
```
|
||||
./build.sh --use_armnn --armnn_home /path/to/ComputeLibrary --armnn_libs /path/to/build
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### RKNPU
|
||||
|
|
|
|||
|
|
@ -675,7 +675,28 @@ if (onnxruntime_USE_ACL OR onnxruntime_USE_ACL_1902 OR onnxruntime_USE_ACL_1905
|
|||
endif()
|
||||
endif()
|
||||
|
||||
if (NOT ${onnxruntime_ACL_LIBS} STREQUAL "")
|
||||
|
||||
add_library(arm_compute SHARED IMPORTED)
|
||||
set_target_properties(arm_compute PROPERTIES
|
||||
IMPORTED_NO_SONAME 1
|
||||
IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute.so")
|
||||
|
||||
add_library(arm_compute_core SHARED IMPORTED)
|
||||
set_target_properties(arm_compute_core PROPERTIES
|
||||
IMPORTED_NO_SONAME 1
|
||||
IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute_core.so")
|
||||
|
||||
add_library(arm_compute_graph SHARED IMPORTED)
|
||||
set_target_properties(arm_compute_graph PROPERTIES
|
||||
IMPORTED_NO_SONAME 1
|
||||
IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute_graph.so")
|
||||
|
||||
link_libraries(arm_compute arm_compute_core arm_compute_graph)
|
||||
endif()
|
||||
|
||||
list(APPEND onnxruntime_EXTERNAL_LIBRARIES arm_compute arm_compute_graph arm_compute_core)
|
||||
|
||||
endif()
|
||||
|
||||
# ArmNN
|
||||
|
|
@ -687,6 +708,35 @@ if (onnxruntime_USE_ARMNN)
|
|||
add_definitions(-DBN_ARMNN=1)
|
||||
endif()
|
||||
|
||||
if (NOT onnxruntime_USE_ACL AND NOT ${onnxruntime_ACL_LIBS} STREQUAL "")
|
||||
|
||||
add_library(arm_compute SHARED IMPORTED)
|
||||
set_target_properties(arm_compute PROPERTIES
|
||||
IMPORTED_NO_SONAME 1
|
||||
IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute.so")
|
||||
|
||||
add_library(arm_compute_core SHARED IMPORTED)
|
||||
set_target_properties(arm_compute_core PROPERTIES
|
||||
IMPORTED_NO_SONAME 1
|
||||
IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute_core.so")
|
||||
|
||||
add_library(arm_compute_graph SHARED IMPORTED)
|
||||
set_target_properties(arm_compute_graph PROPERTIES
|
||||
IMPORTED_NO_SONAME 1
|
||||
IMPORTED_LOCATION "${onnxruntime_ACL_LIBS}/libarm_compute_graph.so")
|
||||
|
||||
link_libraries(arm_compute arm_compute_core arm_compute_graph)
|
||||
endif()
|
||||
|
||||
if (NOT ${onnxruntime_ARMNN_LIBS} STREQUAL "")
|
||||
|
||||
find_library(ARMNN_LIBRARY NAMES armnn
|
||||
PATHS ${onnxruntime_ARMNN_LIBS}
|
||||
NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH)
|
||||
|
||||
link_libraries(armnn)
|
||||
endif()
|
||||
|
||||
list(APPEND onnxruntime_EXTERNAL_LIBRARIES armnn pthread arm_compute_core arm_compute arm_compute_graph)
|
||||
endif()
|
||||
|
||||
|
|
|
|||
|
|
@ -766,7 +766,7 @@ if (onnxruntime_USE_ACL)
|
|||
target_link_libraries(onnxruntime_providers_acl -L$ENV{LD_LIBRARY_PATH})
|
||||
add_dependencies(onnxruntime_providers_acl ${onnxruntime_EXTERNAL_DEPENDENCIES})
|
||||
set_target_properties(onnxruntime_providers_acl PROPERTIES FOLDER "ONNXRuntime")
|
||||
target_include_directories(onnxruntime_providers_acl PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${ACL_INCLUDE_DIR})
|
||||
target_include_directories(onnxruntime_providers_acl PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${onnxruntime_ACL_HOME} ${onnxruntime_ACL_HOME}/include)
|
||||
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/acl DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
|
||||
set_target_properties(onnxruntime_providers_acl PROPERTIES LINKER_LANGUAGE CXX)
|
||||
endif()
|
||||
|
|
@ -783,7 +783,7 @@ if (onnxruntime_USE_ARMNN)
|
|||
onnxruntime_add_include_to_target(onnxruntime_providers_armnn onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf flatbuffers)
|
||||
add_dependencies(onnxruntime_providers_armnn ${onnxruntime_EXTERNAL_DEPENDENCIES})
|
||||
set_target_properties(onnxruntime_providers_armnn PROPERTIES FOLDER "ONNXRuntime")
|
||||
target_include_directories(onnxruntime_providers_armnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${ARMNN_INCLUDE_DIR})
|
||||
target_include_directories(onnxruntime_providers_armnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${onnxruntime_ARMNN_HOME} ${onnxruntime_ARMNN_HOME}/include)
|
||||
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/armnn DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
|
||||
set_target_properties(onnxruntime_providers_armnn PROPERTIES LINKER_LANGUAGE CXX)
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -302,7 +302,7 @@ Status SequentialExecutor::Execute(const SessionState& session_state, const std:
|
|||
#endif
|
||||
#ifdef ENABLE_NVTX_PROFILE
|
||||
profile::NvtxRangeCreator node_compute_range(
|
||||
MakeString(node.OpType(), ".", node.Index()), profile::Color::Blue);
|
||||
MakeString(node.OpType(), ".", node.Index(), "(", node.Name(), ")"), profile::Color::Blue);
|
||||
node_compute_range.Begin();
|
||||
#endif
|
||||
ORT_TRY {
|
||||
|
|
|
|||
|
|
@ -137,8 +137,9 @@ std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(TransformerL
|
|||
transformers.emplace_back(onnxruntime::make_unique<DynamicQuantizeMatMulFusion>(cpu_execution_providers));
|
||||
|
||||
std::unordered_set<std::string> cpu_acl_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider};
|
||||
std::unordered_set<std::string> cpu_acl_armnn_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider, onnxruntime::kArmNNExecutionProvider};
|
||||
|
||||
transformers.emplace_back(onnxruntime::make_unique<ConvActivationFusion>(cpu_acl_execution_providers));
|
||||
transformers.emplace_back(onnxruntime::make_unique<ConvActivationFusion>(cpu_acl_armnn_execution_providers));
|
||||
|
||||
std::unordered_set<std::string> cpu_cuda_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kCudaExecutionProvider};
|
||||
transformers.emplace_back(onnxruntime::make_unique<GeluFusion>(cpu_cuda_execution_providers));
|
||||
|
|
|
|||
|
|
@ -59,6 +59,26 @@ arm_compute::Status ACLImportMemory(arm_compute::TensorAllocator* allocator, voi
|
|||
#endif
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void importDataToTensor(arm_compute::Tensor* tensor, const T* data){
|
||||
|
||||
arm_compute::Window aclInpuWindow;
|
||||
aclInpuWindow.use_tensor_dimensions(tensor->info()->tensor_shape());
|
||||
|
||||
arm_compute::Iterator aclInputIt(tensor, aclInpuWindow);
|
||||
int index = 0;
|
||||
|
||||
// copy input tensor into the larger buffer
|
||||
arm_compute::execute_window_loop(
|
||||
aclInpuWindow,
|
||||
[&](const arm_compute::Coordinates& co) {
|
||||
*reinterpret_cast<float*>(aclInputIt.ptr()) = data[index];
|
||||
index++;
|
||||
},
|
||||
aclInputIt);
|
||||
}
|
||||
template void importDataToTensor<float>(arm_compute::Tensor*, const float*);
|
||||
|
||||
template <typename T>
|
||||
void importDataFromTensor(arm_compute::Tensor* tensor, T* data){
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ void ACLPrintTensorShape(const char*, arm_compute::Tensor& t);
|
|||
std::shared_ptr<arm_compute::MemoryManagerOnDemand> ACLCreateMemoryManager();
|
||||
arm_compute::Status ACLImportMemory(arm_compute::TensorAllocator* allocator, void* memory, size_t size);
|
||||
template <typename T>
|
||||
void importDataToTensor(arm_compute::Tensor* tensor, const T* data);
|
||||
template <typename T>
|
||||
void importDataFromTensor(arm_compute::Tensor* tensor, T* data);
|
||||
|
||||
} // namespace acl
|
||||
|
|
|
|||
|
|
@ -40,36 +40,48 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 11, Co
|
|||
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kMSDomain, 1, float, FusedConv);
|
||||
|
||||
static void RegisterACLKernels(KernelRegistry& kernel_registry) {
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 6, Relu)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 7, 8, Gemm)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 9, 10, Gemm)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 11, Gemm)>());
|
||||
Status RegisterACLKernels(KernelRegistry& kernel_registry) {
|
||||
static const BuildKernelCreateInfoFn function_table[] = {
|
||||
//BuildKernelCreateInfo<void>, //default entry to avoid the list become empty after ops-reducing
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 6, Relu)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 7, 8, Gemm)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 9, 10, Gemm)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 11, Gemm)>,
|
||||
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 1, Conv)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 1, Conv)>,
|
||||
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 7, 9, float, AveragePool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 8, 11, float, MaxPool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 12, MaxPool)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 7, 9, float, AveragePool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 8, 11, float, MaxPool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 12, MaxPool)>,
|
||||
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 1, 8, float, GlobalAveragePool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 1, 8, float, GlobalMaxPool)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 1, 8, float, GlobalAveragePool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 1, 8, float, GlobalMaxPool)>,
|
||||
|
||||
// Opset 10
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 10, 10, float, AveragePool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 11, AveragePool)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 10, 10, float, AveragePool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 11, AveragePool)>,
|
||||
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 7, 9, BatchNormalization)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 4, 10, Concat)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 11, Concat)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 7, 9, BatchNormalization)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 4, 10, Concat)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kAclExecutionProvider, kOnnxDomain, 11, Concat)>,
|
||||
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kMSDomain, 1, float, FusedConv)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kAclExecutionProvider, kMSDomain, 1, float, FusedConv)>,
|
||||
};
|
||||
|
||||
for (auto& function_table_entry : function_table) {
|
||||
KernelCreateInfo info = function_table_entry();
|
||||
if (info.kernel_def != nullptr) { // filter disabled entries where type is void
|
||||
ORT_RETURN_IF_ERROR(kernel_registry.Register(std::move(info)));
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
std::shared_ptr<KernelRegistry> GetAclKernelRegistry() {
|
||||
std::shared_ptr<KernelRegistry> kernel_registry = std::make_shared<KernelRegistry>();
|
||||
RegisterACLKernels(*kernel_registry);
|
||||
ORT_THROW_IF_ERROR(RegisterACLKernels(*kernel_registry));
|
||||
return kernel_registry;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -109,20 +109,7 @@ Status BatchNorm<T>::Compute(OpKernelContext* context) const {
|
|||
|
||||
|
||||
if(X->Shape().Size() != 0 && pBatchNorm->in->info()->has_padding() ){
|
||||
arm_compute::Window aclInpuWindow;
|
||||
aclInpuWindow.use_tensor_dimensions(pBatchNorm->in->info()->tensor_shape());
|
||||
|
||||
arm_compute::Iterator aclInputIt(pBatchNorm->in.get(), aclInpuWindow);
|
||||
const unsigned int aclWidth = pBatchNorm->in->info()->dimension(0);
|
||||
const unsigned int aclHeight = pBatchNorm->in->info()->dimension(1);
|
||||
|
||||
// copy input tensor into the larger buffer
|
||||
arm_compute::execute_window_loop(
|
||||
aclInpuWindow,
|
||||
[&](const arm_compute::Coordinates& co) {
|
||||
*reinterpret_cast<float*>(aclInputIt.ptr()) = x_data[co.z() * (aclWidth * aclHeight) + co.y() * aclHeight + co.x()];
|
||||
},
|
||||
aclInputIt);
|
||||
importDataToTensor<T>(pBatchNorm->in.get(), x_data);
|
||||
}else{
|
||||
ACLImportMemory(pBatchNorm->in->allocator(), (void*)x_data, X->Shape().Size() * 4);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -216,7 +216,16 @@ Status Conv<T>::Compute(OpKernelContext* context) const {
|
|||
1 /* depth multiplier */,
|
||||
arm_compute::Size2D(aclDilation0, dilations[0]));
|
||||
#elif defined(ACL_2002)
|
||||
bool optimizable = false;
|
||||
bool optimizable = bool(arm_compute::NEDepthwiseConvolutionLayerOptimized::validate(tconv.in->info(),
|
||||
tconv.k->info(),
|
||||
(B != nullptr) ? tconv.b->info() : nullptr,
|
||||
tconv.out->info(),
|
||||
aclPadStride,
|
||||
1 /* depth multiplier */,
|
||||
acl_activ_enabled ?
|
||||
arm_compute::ActivationLayerInfo(acl_activ_func, conv_attrs_.alpha) :
|
||||
arm_compute::ActivationLayerInfo(),
|
||||
arm_compute::Size2D(aclDilation0, dilations[0])));
|
||||
#endif
|
||||
|
||||
if (optimizable) {
|
||||
|
|
@ -286,7 +295,12 @@ Status Conv<T>::Compute(OpKernelContext* context) const {
|
|||
}
|
||||
|
||||
const T* x_data = X->template Data<T>();
|
||||
ACLImportMemory(pConv->in->allocator(), (void*)x_data, X->Shape().Size() * 4);
|
||||
if(X->Shape().Size() != 0 && pConv->in->info()->has_padding() ){
|
||||
pConv->in->allocator()->allocate();
|
||||
importDataToTensor<T>(pConv->in.get(), x_data);
|
||||
}else{
|
||||
ACLImportMemory(pConv->in->allocator(), (void*)x_data, X->Shape().Size() * 4);
|
||||
}
|
||||
|
||||
const T* k_data = W->template Data<T>();
|
||||
ACLImportMemory(pConv->k->allocator(), (void*)k_data, W->Shape().Size() * 4);
|
||||
|
|
@ -297,13 +311,21 @@ Status Conv<T>::Compute(OpKernelContext* context) const {
|
|||
}
|
||||
|
||||
T* y_data = Y->template MutableData<T>();
|
||||
ACLImportMemory(pConv->out->allocator(), (void*)y_data, Y->Shape().Size() * 4);
|
||||
if(Y->Shape().Size() != 0 && pConv->out->info()->has_padding() ){
|
||||
pConv->out->allocator()->allocate();
|
||||
} else {
|
||||
ACLImportMemory(pConv->out->allocator(), (void*)y_data, Y->Shape().Size() * 4);
|
||||
}
|
||||
|
||||
arm_compute::Allocator alloc_mm{};
|
||||
pConv->mm_layer->populate(alloc_mm, 1);
|
||||
pConv->layer->run();
|
||||
pConv->mm_layer->clear();
|
||||
|
||||
if(Y->Shape().Size() != 0 && pConv->out->info()->has_padding() ){
|
||||
importDataFromTensor<T>(pConv->out.get(), y_data);
|
||||
}
|
||||
|
||||
pConv->in->allocator()->free();
|
||||
pConv->k->allocator()->free();
|
||||
if (B != nullptr)
|
||||
|
|
|
|||
|
|
@ -87,20 +87,7 @@ Status Concat<T>::Compute(OpKernelContext* ctx) const {
|
|||
|
||||
if (X->Shape().Size() != 0 && in->info()->has_padding() ){
|
||||
in->allocator()->allocate();
|
||||
arm_compute::Window aclInpuWindow;
|
||||
aclInpuWindow.use_tensor_dimensions(in->info()->tensor_shape());
|
||||
|
||||
arm_compute::Iterator aclInputIt(in, aclInpuWindow);
|
||||
int index = 0;
|
||||
|
||||
// copy input tensor into the larger buffer
|
||||
arm_compute::execute_window_loop(
|
||||
aclInpuWindow,
|
||||
[&](const arm_compute::Coordinates& co) {
|
||||
*reinterpret_cast<float*>(aclInputIt.ptr()) = x_data[index];
|
||||
index++;
|
||||
},
|
||||
aclInputIt);
|
||||
importDataToTensor<T>(in, x_data);
|
||||
} else {
|
||||
ACLImportMemory(in->allocator(), (void*)x_data, X->Shape().Size() * 4);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11,
|
|||
|
||||
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 9, float, AveragePool);
|
||||
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 10, 10, float, AveragePool);
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, float, AveragePool);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, AveragePool);
|
||||
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool);
|
||||
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 8, 11, float, MaxPool);
|
||||
|
|
@ -45,38 +44,52 @@ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDo
|
|||
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 4, 10, Concat);
|
||||
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Concat);
|
||||
|
||||
static void RegisterArmNNKernels(KernelRegistry& kernel_registry) {
|
||||
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kMSDomain, 1, float, FusedConv);
|
||||
|
||||
Status RegisterArmNNKernels(KernelRegistry& kernel_registry) {
|
||||
static const BuildKernelCreateInfoFn function_table[] = {
|
||||
#ifdef RELU_ARMNN
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 6, Relu)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 6, Relu)>,
|
||||
#endif
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 10, Conv)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Conv)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 10, Conv)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Conv)>,
|
||||
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 8, Gemm)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 9, 10, Gemm)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Gemm)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 8, Gemm)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 9, 10, Gemm)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Gemm)>,
|
||||
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 9, float, AveragePool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 10, 10, float, AveragePool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, float, AveragePool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, AveragePool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 8, 11, float, MaxPool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 12, MaxPool)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 9, float, AveragePool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 10, 10, float, AveragePool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, AveragePool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 8, 11, float, MaxPool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 12, MaxPool)>,
|
||||
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, float, GlobalAveragePool)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, float, GlobalMaxPool)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, float, GlobalAveragePool)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 1, float, GlobalMaxPool)>,
|
||||
|
||||
#ifdef BN_ARMNN
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 9, BatchNormalization)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 7, 9, BatchNormalization)>,
|
||||
#endif
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 4, 10, Concat)>());
|
||||
kernel_registry.Register(BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Concat)>());
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 4, 10, Concat)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kOnnxDomain, 11, Concat)>,
|
||||
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kArmNNExecutionProvider, kMSDomain, 1, float, FusedConv)>,
|
||||
};
|
||||
|
||||
for (auto& function_table_entry : function_table) {
|
||||
KernelCreateInfo info = function_table_entry();
|
||||
if (info.kernel_def != nullptr) { // filter disabled entries where type is void
|
||||
ORT_RETURN_IF_ERROR(kernel_registry.Register(std::move(info)));
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
std::shared_ptr<KernelRegistry> GetArmNNKernelRegistry() {
|
||||
std::shared_ptr<KernelRegistry> kernel_registry = std::make_shared<KernelRegistry>();
|
||||
RegisterArmNNKernels(*kernel_registry);
|
||||
ORT_THROW_IF_ERROR(RegisterArmNNKernels(*kernel_registry));
|
||||
|
||||
return kernel_registry;
|
||||
}
|
||||
|
|
|
|||
41
onnxruntime/core/providers/armnn/nn/fused_conv.cc
Normal file
41
onnxruntime/core/providers/armnn/nn/fused_conv.cc
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Copyright (c) 2020, NXP Semiconductor, Inc. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#ifdef _WIN32
|
||||
#pragma warning(disable : 4244)
|
||||
#endif
|
||||
|
||||
#include "core/providers/armnn/armnn_execution_provider.h"
|
||||
#include "core/providers/armnn/nn/conv.h"
|
||||
#include "core/providers/armnn/armnn_common.h"
|
||||
#include "core/providers/armnn/armnn_fwd.h"
|
||||
#include "contrib_ops/cpu/fused_activation.h"
|
||||
|
||||
#include "armnn/ArmNN.hpp"
|
||||
|
||||
#include <thread>
|
||||
#include <mutex>
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace armnn_ep{
|
||||
|
||||
class FusedConv final : public armnn_ep::Conv<float> {
|
||||
public:
|
||||
explicit FusedConv(const OpKernelInfo& info) : armnn_ep::Conv<float>(info) {
|
||||
ORT_ENFORCE(info.GetAttr<std::string>("activation", &(this->activation_type)).IsOK());
|
||||
ORT_ENFORCE(GetFusedActivationAttr(info, activation_).IsOK());
|
||||
}
|
||||
};
|
||||
|
||||
ONNX_OPERATOR_TYPED_KERNEL_EX(
|
||||
FusedConv,
|
||||
kMSDomain,
|
||||
1,
|
||||
float,
|
||||
kArmNNExecutionProvider,
|
||||
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
|
||||
FusedConv);
|
||||
|
||||
} // namespace armnn_ep
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -65,7 +65,7 @@ enum class QLinearOpType : uint8_t {
|
|||
|
||||
QLinearOpType GetQLinearOpType(const onnxruntime::Node& node);
|
||||
|
||||
// This qlinear op is an operator takes 2 input and producce 1 output
|
||||
// This qlinear op is an operator takes 2 input and produces 1 output
|
||||
// Such as QLinearConv, QLinearMatMul, QLinearAdd, ...
|
||||
bool IsQLinearBinaryOp(QLinearOpType qlinear_op_type);
|
||||
|
||||
|
|
|
|||
|
|
@ -34,23 +34,23 @@ bool ModelBuilder::IsNodeSupported(const Node& node) {
|
|||
}
|
||||
|
||||
bool IsValidSupportedNodesVec(const std::vector<int>& supported_node_vec, const GraphViewer& graph_viewer) {
|
||||
if (!supported_node_vec.empty()) {
|
||||
if (supported_node_vec.size() == 1) {
|
||||
const auto& node_indices = graph_viewer.GetNodesInTopologicalOrder();
|
||||
const auto* node(graph_viewer.GetNode(node_indices[supported_node_vec[0]]));
|
||||
const auto& op = node->OpType();
|
||||
// It is not worth it to perform a single Reshape/Dropout/Identity operator
|
||||
// which is only copying the data in NNAPI
|
||||
// If this is the case, let it fall back
|
||||
if (op == "Reshape" ||
|
||||
op == "Dropout" ||
|
||||
op == "Identity") {
|
||||
return false;
|
||||
}
|
||||
if (supported_node_vec.empty())
|
||||
return false;
|
||||
|
||||
if (supported_node_vec.size() == 1) {
|
||||
const auto& node_indices = graph_viewer.GetNodesInTopologicalOrder();
|
||||
const auto* node(graph_viewer.GetNode(node_indices[supported_node_vec[0]]));
|
||||
const auto& op = node->OpType();
|
||||
// It is not worth it to perform a single Reshape/Flatten/Identity operator
|
||||
// which is only copying the data in NNAPI
|
||||
// If this is the case, let it fall back
|
||||
if (op == "Reshape" ||
|
||||
op == "Flatten" ||
|
||||
op == "Identity") {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::vector<std::vector<int>> ModelBuilder::GetSupportedNodes() {
|
||||
|
|
|
|||
|
|
@ -132,7 +132,7 @@ class ModelBuilder {
|
|||
std::unordered_map<std::string, const ONNX_NAMESPACE::TensorProto&> initializers_;
|
||||
std::unordered_set<std::string> skipped_initializers_;
|
||||
|
||||
// All activation nodes (Relu, Relu1, Relu6) as a map <NodeIndex, activeation_code>
|
||||
// All activation nodes (Relu, Relu1, Relu6) as a map <NodeIndex, activation_code>
|
||||
std::unordered_map<NodeIndex, int32_t> activation_nodes_;
|
||||
|
||||
std::unordered_map<std::string, std::shared_ptr<IOpBuilder>> op_builders_;
|
||||
|
|
|
|||
|
|
@ -524,8 +524,8 @@ static Status GetBinaryOpQuantizationScaleAndZeroPoint(
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
// NNAPI has the qunatization scale and zero point embedded in the ANeuralNetworksOperandType
|
||||
// ONNX has the qunatization scale and zero point as the inputs of the qlinear operators
|
||||
// NNAPI has the quantization scale and zero point embedded in the ANeuralNetworksOperandType
|
||||
// ONNX has the quantization scale and zero point as the inputs of the qlinear operators
|
||||
// We want to verify the scale and zeropoint of the ONNX inputs matches the values embedded in the NNAPI inputs
|
||||
static Status IsValidInputQuantizedType(const ModelBuilder& model_builder,
|
||||
const std::string& input_name,
|
||||
|
|
@ -886,10 +886,10 @@ Status BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
|
|||
if (input1_is_nhwc == input2_is_nhwc) {
|
||||
output_is_nhwc = input1_is_nhwc;
|
||||
} else if (input1_is_nhwc) {
|
||||
// need transpsoe input1 back to nchw
|
||||
// need transpose input1 back to nchw
|
||||
ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, a_idx, input1));
|
||||
} else { // input2_is_nhwc
|
||||
// need transpsoe input2 back to nchw
|
||||
// need transpose input2 back to nchw
|
||||
ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, b_idx, input2));
|
||||
}
|
||||
|
||||
|
|
@ -1024,17 +1024,113 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co
|
|||
class ReshapeOpBuilder : public BaseOpBuilder {
|
||||
public:
|
||||
void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override;
|
||||
static Status AddReshapeOperator(ModelBuilder& model_builder, const Node& node,
|
||||
const std::string& input, const std::vector<int32_t>& shape) ORT_MUST_USE_RESULT;
|
||||
|
||||
private:
|
||||
bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override;
|
||||
|
||||
Status AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override ORT_MUST_USE_RESULT;
|
||||
static bool CanSkipReshape(const Node& node, size_t input_rank, size_t output_rank);
|
||||
};
|
||||
|
||||
void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) {
|
||||
model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name());
|
||||
}
|
||||
|
||||
// We can skip the Reshape if all the output edges satisfies,
|
||||
// 1. The output of the reshape/flatten is the input 0 of the GEMM/Matmul,
|
||||
// and the input rank >= 2 and output_rank == 2
|
||||
// This is because Gemm/Matmul will map to ANEURALNETWORKS_FULLY_CONNECTED in NNAPI,
|
||||
// ANEURALNETWORKS_FULLY_CONNECTED will flatten the 2+ dim input 0 to 2d
|
||||
// 2. Or the output the reshape/flatten is the output of the graph
|
||||
// (no op in the graph is using the output except can be used by Gemm/Matmul satisfying condition 1 above)
|
||||
// The reason we want to skip Reshape is that Reshape is not running on Hardware (NPU,...) in NNAPI for
|
||||
// some CPU (e.g. Qualcomm SD for now), skipping unnecessary Reshape will prevent context switching
|
||||
// between NNAPI CPU impl and Hardware Accelerator impl and will speed up the execution
|
||||
// If we are going to skip the reshape, we will still add correct shape and operand type for the output in
|
||||
// onnxruntime::nnapi::Model.
|
||||
// If the Reshape output is also a graph output, since NNAPI output is a void* buffer, we can find the shape
|
||||
// information in onnxruntime::nnapi::Model and pass the correct shape information back to ORT to be used as output shape
|
||||
/* static */ bool ReshapeOpBuilder::CanSkipReshape(const Node& node, size_t input_rank, size_t output_rank) {
|
||||
const auto& output = node.OutputDefs()[0]->Name();
|
||||
// We will go through all the output edges
|
||||
for (auto it = node.OutputEdgesBegin(), end = node.OutputEdgesEnd(); it != end; ++it) {
|
||||
const auto& op_type = it->GetNode().OpType();
|
||||
// TODO add quantized matmul when reshape support quantized input
|
||||
if (op_type != "Gemm" && op_type != "MatMul") {
|
||||
LOGS_DEFAULT(VERBOSE) << "Reshape/Flatten can only be skipped when the output is Gemm/Matmul"
|
||||
<< " or no op is using the output (output is graph output)"
|
||||
<< ", output name, " << output
|
||||
<< " is used by " << op_type;
|
||||
return false;
|
||||
}
|
||||
|
||||
// NNAPI ANEURALNETWORKS_FULLY_CONNECTED will only flatten the input 0
|
||||
if (it->GetDstArgIndex() != 0) {
|
||||
LOGS_DEFAULT(VERBOSE) << "Reshape/Flatten can only be skipped when the output is input 0 of Gemm/Matmul"
|
||||
<< ", output name, " << output;
|
||||
return false;
|
||||
}
|
||||
|
||||
// We only support 2d matmul/gemm here
|
||||
// And NNAPI ANEURALNETWORKS_FULLY_CONNECTED will only flatten input rank >= 2
|
||||
if (input_rank < 2 || output_rank != 2) {
|
||||
LOGS_DEFAULT(VERBOSE) << "Reshape/Flatten can only be skipped when input_rank >= 2 and output_rank == 2"
|
||||
<< ", output name, " << output
|
||||
<< ", the actual input_rank, " << input_rank
|
||||
<< ", the actual output_rank, " << output_rank;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// If we reach here, we have either,
|
||||
// all the Reshape outputs are used by gemm/matmul, the output can also be a model output [doesn't really matter here]
|
||||
// or
|
||||
// Reshape has no output edge ==> the output is a graph output or a dead end [which we don't care]
|
||||
// we can skip this Reshape now
|
||||
LOGS_DEFAULT(VERBOSE) << "Skipping Reshape/Flatten node ["
|
||||
<< node.Name() << "] with output, " << output;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* static */ Status ReshapeOpBuilder::AddReshapeOperator(ModelBuilder& model_builder,
|
||||
const Node& node,
|
||||
const std::string& input,
|
||||
const std::vector<int32_t>& shape) {
|
||||
auto& shaper(model_builder.GetShaper());
|
||||
const auto& operand_indices(model_builder.GetOperandIndices());
|
||||
const auto& operand_types(model_builder.GetOperandTypes());
|
||||
const auto& output = node.OutputDefs()[0]->Name();
|
||||
ORT_RETURN_IF_ERROR(shaper.Reshape(input, shape, output));
|
||||
auto input_rank = shaper[input].size();
|
||||
auto output_rank = shaper[output].size();
|
||||
|
||||
// Since Reshape is not running using hardware in NNAPI for some CPU (e.g. Qualcomm SD for now)
|
||||
// We will try to see if we the skip the Reshape to prevent context switching between
|
||||
// NNAPI CPU impl and NNAPI hardware accelerator impl
|
||||
if (CanSkipReshape(node, input_rank, output_rank)) {
|
||||
// Since reshape can be skipped, only register the dimension and type, with same index and new name
|
||||
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
|
||||
model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, false);
|
||||
} else {
|
||||
// We still need to perform a reshape here
|
||||
// Add input
|
||||
std::vector<uint32_t> input_indices;
|
||||
input_indices.push_back(operand_indices.at(input));
|
||||
// Add new shape
|
||||
Shape shape_dimen = {static_cast<uint32_t>(shape.size())};
|
||||
std::string shape_name = model_builder.GetUniqueName(node.Name() + input + "newshape");
|
||||
OperandType shape_operand_type(Type::TENSOR_INT32, shape_dimen);
|
||||
ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(shape_name, shape.data(), shape_operand_type));
|
||||
input_indices.push_back(operand_indices.at(shape_name));
|
||||
|
||||
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
|
||||
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_RESHAPE, input_indices, {output}, {output_operand_type}, {false}));
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) {
|
||||
const auto& initializers(model_builder.GetInitializerTensors());
|
||||
const auto& perm_name = node.InputDefs()[1]->Name();
|
||||
|
|
@ -1070,8 +1166,6 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node
|
|||
|
||||
Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) {
|
||||
auto& shaper(model_builder.GetShaper());
|
||||
const auto& operand_indices(model_builder.GetOperandIndices());
|
||||
const auto& operand_types(model_builder.GetOperandTypes());
|
||||
const auto& initializers(model_builder.GetInitializerTensors());
|
||||
|
||||
auto input = node.InputDefs()[0]->Name();
|
||||
|
|
@ -1080,10 +1174,6 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons
|
|||
ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, 0, input));
|
||||
}
|
||||
|
||||
const auto& output = node.OutputDefs()[0]->Name();
|
||||
std::vector<uint32_t> input_indices;
|
||||
input_indices.push_back(operand_indices.at(input)); // input
|
||||
|
||||
const auto& shape_tensor = initializers.at(node.InputDefs()[1]->Name());
|
||||
const int64_t* rawShape = GetTensorInt64Data(shape_tensor);
|
||||
const auto size = SafeInt<uint32_t>(shape_tensor.dims()[0]);
|
||||
|
|
@ -1096,18 +1186,7 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons
|
|||
shape[i] = dim == 0 ? input_shape[i] : dim;
|
||||
}
|
||||
|
||||
Shape shape_dimen = {size};
|
||||
std::string shape_name = model_builder.GetUniqueName(node.Name() + input + "newshape");
|
||||
OperandType shape_operand_type(Type::TENSOR_INT32, shape_dimen);
|
||||
ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(shape_name, shape.data(), shape_operand_type));
|
||||
input_indices.push_back(operand_indices.at(shape_name));
|
||||
|
||||
ORT_RETURN_IF_ERROR(shaper.Reshape(input, shape, output));
|
||||
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
|
||||
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_RESHAPE, input_indices,
|
||||
{output}, {output_operand_type}, {false}));
|
||||
|
||||
return Status::OK();
|
||||
return AddReshapeOperator(model_builder, node, input, shape);
|
||||
}
|
||||
|
||||
#pragma endregion op_reshape
|
||||
|
|
@ -1287,7 +1366,7 @@ bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const N
|
|||
const auto input_size = input_shape.size();
|
||||
if (input_size != 4) {
|
||||
LOGS_DEFAULT(VERBOSE)
|
||||
<< op_type << " only supportes rank-4 tensor, input ["
|
||||
<< op_type << " only supports rank-4 tensor, input ["
|
||||
<< node.InputDefs()[0]->Name() << "] has actual dim count " << input_size;
|
||||
return false;
|
||||
}
|
||||
|
|
@ -1499,7 +1578,7 @@ bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& n
|
|||
|
||||
const auto android_sdk_ver = model_builder.GetAndroidSdkVer();
|
||||
if (android_sdk_ver < 29) {
|
||||
LOGS_DEFAULT(VERBOSE) << op_type << " dilations is only supported on Android API levle 29+, "
|
||||
LOGS_DEFAULT(VERBOSE) << op_type << " dilations is only supported on Android API level 29+, "
|
||||
<< "actual API level: " << android_sdk_ver;
|
||||
return false;
|
||||
}
|
||||
|
|
@ -2307,7 +2386,7 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
|
|||
}
|
||||
|
||||
int rank = shaper[input0].size();
|
||||
int32_t axis = SafeInt<int32_t>(HandleNegativeAxis(helper.Get("axis", 1), rank));
|
||||
int32_t axis = static_cast<int32_t>(HandleNegativeAxis(helper.Get("axis", 1), rank));
|
||||
|
||||
if (output_is_nhwc) {
|
||||
ORT_RETURN_IF_NOT(rank == 4,
|
||||
|
|
@ -2372,7 +2451,7 @@ Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons
|
|||
const auto& input_shape(shaper[input]);
|
||||
auto input_dims = input_shape.size();
|
||||
for (auto& axis : axes) {
|
||||
axis = SafeInt<int32_t>(HandleNegativeAxis(axis, input_dims));
|
||||
axis = static_cast<int32_t>(HandleNegativeAxis(axis, input_dims));
|
||||
}
|
||||
|
||||
if (axes.empty()) { // Squeeze all
|
||||
|
|
@ -2918,6 +2997,76 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
|
|||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region op_flatten
|
||||
|
||||
class FlattenOpBuilder : public BaseOpBuilder {
|
||||
private:
|
||||
bool IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) override;
|
||||
static void GetFlattenShape(const Node& node, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2);
|
||||
Status AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override ORT_MUST_USE_RESULT;
|
||||
};
|
||||
|
||||
/* static */ void FlattenOpBuilder::GetFlattenShape(const Node& node, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2) {
|
||||
int32_t rank = static_cast<int>(input_shape.size());
|
||||
NodeAttrHelper helper(node);
|
||||
int32_t axis = helper.Get("axis", 1);
|
||||
// axis == rank is a valid input, but invalid for HandleNegativeAxis
|
||||
// Skip non-negative axis here
|
||||
if (axis < 0)
|
||||
axis = static_cast<int32_t>(HandleNegativeAxis(axis, rank));
|
||||
|
||||
dim_1 = std::accumulate(input_shape.cbegin(), input_shape.cbegin() + axis, 1, std::multiplies<int32_t>());
|
||||
dim_2 = std::accumulate(input_shape.cbegin() + axis, input_shape.cend(), 1, std::multiplies<int32_t>());
|
||||
}
|
||||
|
||||
bool FlattenOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) {
|
||||
Shape input_shape;
|
||||
if (!GetShape(*node.InputDefs()[0], input_shape))
|
||||
return false;
|
||||
|
||||
if (input_shape.size() > 4 || input_shape.empty()) {
|
||||
LOGS_DEFAULT(VERBOSE) << "Flatten only supports up to 1-4d shape, input is "
|
||||
<< input_shape.size() << "d shape";
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t dim_1 = 1;
|
||||
int32_t dim_2 = 1;
|
||||
GetFlattenShape(node, input_shape, dim_1, dim_2);
|
||||
|
||||
if (dim_1 == 0 && dim_2 == 0) {
|
||||
LOGS_DEFAULT(VERBOSE) << "The dynamical input shape " << Shape2String(input_shape)
|
||||
<< " is not supported";
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
Status FlattenOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) {
|
||||
auto input = node.InputDefs()[0]->Name();
|
||||
if (model_builder.IsOperandNHWC(input)) {
|
||||
// We want to transpose nhwc operand back to nchw before reshape
|
||||
ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, 0, input));
|
||||
}
|
||||
|
||||
// Flatten is basically a reshape to 2d tensor
|
||||
// Get the shape for Reshape here
|
||||
Shape input_shape;
|
||||
GetShape(*node.InputDefs()[0], input_shape);
|
||||
int32_t dim_1 = 1;
|
||||
int32_t dim_2 = 1;
|
||||
GetFlattenShape(node, input_shape, dim_1, dim_2);
|
||||
// If the input is of dynamic shape, replace 0 (dynamic) dimension with -1
|
||||
// We cannot have dim_1 and dim_2 both be 0 here, it was checked in IsOpSupportedImpl
|
||||
dim_1 = dim_1 == 0 ? -1 : dim_1;
|
||||
dim_2 = dim_2 == 0 ? -1 : dim_2;
|
||||
std::vector<int32_t> shape{dim_1, dim_2};
|
||||
return ReshapeOpBuilder::AddReshapeOperator(model_builder, node, input, shape);
|
||||
}
|
||||
|
||||
#pragma endregion op_reshape
|
||||
|
||||
#pragma region CreateOpBuilders
|
||||
|
||||
std::unordered_map<std::string, std::shared_ptr<IOpBuilder>>
|
||||
|
|
@ -2982,6 +3131,7 @@ CreateOpBuilders() {
|
|||
op_map.emplace("LRN", std::make_shared<LRNOpBuilder>());
|
||||
op_map.emplace("Clip", std::make_shared<ClipOpBuilder>());
|
||||
op_map.emplace("Resize", std::make_shared<ResizeOpBuilder>());
|
||||
op_map.emplace("Flatten", std::make_shared<FlattenOpBuilder>());
|
||||
|
||||
return op_map;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ size_t Model::GetMappedOutputIdx(const std::string& name) const {
|
|||
}
|
||||
|
||||
bool Model::SupportsDynamicOutputShape() const {
|
||||
// dynamic output shape is only supported on Android API levle 29+
|
||||
// dynamic output shape is only supported on Android API level 29+
|
||||
return GetAndroidSdkVer() >= 29 && dynamic_output_buffer_size_ > 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -81,11 +81,11 @@ class Model {
|
|||
bool SupportsDynamicOutputShape() const;
|
||||
|
||||
// Set and Get the number of elements in the buffer for a dynamic output
|
||||
// If the buffer is not big enough, ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE will be returned by exection
|
||||
// If the buffer is not big enough, ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE will be returned by execution
|
||||
// Note: this will return number of elements of the buffer not the byte size of the buffer
|
||||
// and each output will have its separated buffer
|
||||
// TODO:
|
||||
// 1. Consider an adaptive aproach to automatically increase the buffer size if the execution reports
|
||||
// 1. Consider an adaptive approach to automatically increase the buffer size if the execution reports
|
||||
// insufficient size
|
||||
// 2. Experiment with bigger initial buffer size (currently 1024)
|
||||
size_t GetDynamicOutputBufferSize() const { return dynamic_output_buffer_size_; }
|
||||
|
|
|
|||
|
|
@ -312,7 +312,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<onnxruntime::No
|
|||
|
||||
if (input_type.dimensions != model_input_type.dimensions && model_input_type.GetOperandBlobByteSize() != 0) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
|
||||
"The actual input dimanesions should match the model input "
|
||||
"The actual input dimensions should match the model input "
|
||||
"dimensions, or model input dimension has 0 (dynamic)");
|
||||
}
|
||||
|
||||
|
|
@ -361,7 +361,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<onnxruntime::No
|
|||
output_buffer_byte_size = model_output_type.GetOperandBlobByteSize();
|
||||
} else {
|
||||
// This output is dynamic (size unknown), will need allocate a buffer for the result
|
||||
// and copy the content to ORT output tensors afte the execution (will know output shape after the execution)
|
||||
// and copy the content to ORT output tensors after the execution (will know output shape after the execution)
|
||||
output_buffer_byte_size = model->GetDynamicOutputBufferSize() * model_output_type.GetElementByteSize();
|
||||
std::unique_ptr<uint8_t[]> buffer_holder(new uint8_t[output_buffer_byte_size]);
|
||||
output_buffer = buffer_holder.get();
|
||||
|
|
|
|||
|
|
@ -2,8 +2,9 @@
|
|||
#include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h"
|
||||
#include "core/session/inference_session.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "test/providers/provider_test_utils.h"
|
||||
#include "test/framework/test_utils.h"
|
||||
#include "test/providers/provider_test_utils.h"
|
||||
#include "test/util/include/inference_session_wrapper.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace ONNX_NAMESPACE;
|
||||
|
|
@ -22,41 +23,99 @@ void VerifyOutputs(const std::vector<OrtValue>& fetches, const std::vector<int64
|
|||
ASSERT_EQ(expected_values, found);
|
||||
}
|
||||
|
||||
void RunAndVerifyOutputs(const std::string& model_file_name,
|
||||
const char* log_id,
|
||||
const NameMLValMap& feeds,
|
||||
const std::vector<std::string>& output_names,
|
||||
const std::vector<int64_t>& expected_dims,
|
||||
const std::vector<float>& expected_values) {
|
||||
SessionOptions so;
|
||||
so.session_logid = log_id;
|
||||
RunOptions run_options;
|
||||
run_options.run_tag = so.session_logid;
|
||||
|
||||
InferenceSessionWrapper session_object{so, GetEnvironment()};
|
||||
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(onnxruntime::make_unique<::onnxruntime::NnapiExecutionProvider>()));
|
||||
ASSERT_STATUS_OK(session_object.Load(model_file_name));
|
||||
ASSERT_STATUS_OK(session_object.Initialize());
|
||||
|
||||
// Since we already know the model is entirely supported by NNAPI, all nodes (2 Add nodes here) will be fused
|
||||
// Get the graph after session is initialized, and verify the fused node (the only node in the graph) is using NNAPI EP
|
||||
const auto& graph = session_object.GetGraph();
|
||||
ASSERT_EQ(1, graph.NumberOfNodes()); // Make sure the graph has 1 fused node
|
||||
ASSERT_EQ(onnxruntime::kNnapiExecutionProvider, graph.Nodes().cbegin()->GetExecutionProviderType());
|
||||
|
||||
// Now run and verify the result
|
||||
std::vector<OrtValue> fetches;
|
||||
ASSERT_STATUS_OK(session_object.Run(run_options, feeds, output_names, &fetches));
|
||||
VerifyOutputs(fetches, expected_dims, expected_values);
|
||||
}
|
||||
|
||||
// Since NNAPI EP handles Reshape and Flatten differently,
|
||||
// Please see ReshapeOpBuilder::CanSkipReshape in <repo_root>/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc
|
||||
// We have a separated test for these skip reshape scenarios
|
||||
TEST(NnapiExecutionProviderTest, ReshapeFlattenTest) {
|
||||
std::string model_file_name = "testdata/nnapi_reshape_flatten_test.onnx";
|
||||
|
||||
std::vector<int64_t> dims_mul_x = {2, 1, 2};
|
||||
std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f};
|
||||
std::vector<int64_t> dims_mul_y = {3, 2, 2};
|
||||
std::vector<float> values_mul_y = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f};
|
||||
OrtValue ml_value_x;
|
||||
CreateMLValue<float>(TestNnapiExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_x);
|
||||
OrtValue ml_value_y;
|
||||
CreateMLValue<float>(TestNnapiExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_y, values_mul_y, &ml_value_y);
|
||||
NameMLValMap feeds;
|
||||
feeds.insert(std::make_pair("X", ml_value_x));
|
||||
feeds.insert(std::make_pair("Y", ml_value_y));
|
||||
|
||||
// prepare outputs
|
||||
std::vector<std::string> output_names;
|
||||
output_names.push_back("Z");
|
||||
|
||||
// prepare expected inputs and outputs
|
||||
std::vector<int64_t> expected_dims_mul_z = {1, 6};
|
||||
std::vector<float> expected_values_mul_z = {59.0f, 72.0f, 129.0f, 159.0f, 204.0f, 253.0f};
|
||||
|
||||
RunAndVerifyOutputs(model_file_name, "NnapiExecutionProviderTest.ReshapeFlattenTest", feeds, output_names, expected_dims_mul_z, expected_values_mul_z);
|
||||
}
|
||||
|
||||
TEST(NnapiExecutionProviderTest, FunctionTest) {
|
||||
onnxruntime::Model model("graph_1", false, DefaultLoggingManager().DefaultLogger());
|
||||
auto& graph = model.MainGraph();
|
||||
std::vector<onnxruntime::NodeArg*> inputs;
|
||||
std::vector<onnxruntime::NodeArg*> outputs;
|
||||
|
||||
// FLOAT tensor.
|
||||
ONNX_NAMESPACE::TypeProto float_tensor;
|
||||
float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2);
|
||||
|
||||
auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor);
|
||||
auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor);
|
||||
inputs.push_back(&input_arg_1);
|
||||
inputs.push_back(&input_arg_2);
|
||||
auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &float_tensor);
|
||||
outputs.push_back(&output_arg);
|
||||
graph.AddNode("node_1", "Add", "node 1.", inputs, outputs);
|
||||
|
||||
auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &float_tensor);
|
||||
inputs.clear();
|
||||
inputs.push_back(&output_arg);
|
||||
inputs.push_back(&input_arg_3);
|
||||
auto& output_arg_2 = graph.GetOrCreateNodeArg("M", &float_tensor);
|
||||
outputs.clear();
|
||||
outputs.push_back(&output_arg_2);
|
||||
graph.AddNode("node_2", "Add", "node 2.", inputs, outputs);
|
||||
|
||||
auto status = graph.Resolve();
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
std::string model_file_name = "nnapi_execution_provider_test_graph.onnx";
|
||||
status = onnxruntime::Model::Save(model, model_file_name);
|
||||
{ // Create the model with 2 add nodes
|
||||
onnxruntime::Model model("graph_1", false, DefaultLoggingManager().DefaultLogger());
|
||||
auto& graph = model.MainGraph();
|
||||
std::vector<onnxruntime::NodeArg*> inputs;
|
||||
std::vector<onnxruntime::NodeArg*> outputs;
|
||||
|
||||
// FLOAT tensor.
|
||||
ONNX_NAMESPACE::TypeProto float_tensor;
|
||||
float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3);
|
||||
float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2);
|
||||
|
||||
auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor);
|
||||
auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor);
|
||||
inputs.push_back(&input_arg_1);
|
||||
inputs.push_back(&input_arg_2);
|
||||
auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &float_tensor);
|
||||
outputs.push_back(&output_arg);
|
||||
graph.AddNode("node_1", "Add", "node 1.", inputs, outputs);
|
||||
|
||||
auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &float_tensor);
|
||||
inputs.clear();
|
||||
inputs.push_back(&output_arg);
|
||||
inputs.push_back(&input_arg_3);
|
||||
auto& output_arg_2 = graph.GetOrCreateNodeArg("M", &float_tensor);
|
||||
outputs.clear();
|
||||
outputs.push_back(&output_arg_2);
|
||||
graph.AddNode("node_2", "Add", "node 2.", inputs, outputs);
|
||||
|
||||
ASSERT_STATUS_OK(graph.Resolve());
|
||||
ASSERT_STATUS_OK(onnxruntime::Model::Save(model, model_file_name));
|
||||
}
|
||||
|
||||
std::vector<int64_t> dims_mul_x = {1, 1, 3, 2};
|
||||
std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
|
||||
|
|
@ -80,23 +139,7 @@ TEST(NnapiExecutionProviderTest, FunctionTest) {
|
|||
std::vector<int64_t> expected_dims_mul_m = {1, 1, 3, 2};
|
||||
std::vector<float> expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f};
|
||||
|
||||
SessionOptions so;
|
||||
so.session_logid = "NnapiExecutionProviderTest.FunctionTest";
|
||||
RunOptions run_options;
|
||||
run_options.run_tag = so.session_logid;
|
||||
|
||||
InferenceSession session_object{so, GetEnvironment()};
|
||||
status = session_object.RegisterExecutionProvider(onnxruntime::make_unique<::onnxruntime::NnapiExecutionProvider>());
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
status = session_object.Load(model_file_name);
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
status = session_object.Initialize();
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
|
||||
// Now run
|
||||
status = session_object.Run(run_options, feeds, output_names, &fetches);
|
||||
ASSERT_TRUE(status.IsOK());
|
||||
VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m);
|
||||
RunAndVerifyOutputs(model_file_name, "NnapiExecutionProviderTest.FunctionTest", feeds, output_names, expected_dims_mul_m, expected_values_mul_m);
|
||||
}
|
||||
} // namespace test
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
BIN
onnxruntime/test/testdata/nnapi_reshape_flatten_test.onnx
vendored
Normal file
BIN
onnxruntime/test/testdata/nnapi_reshape_flatten_test.onnx
vendored
Normal file
Binary file not shown.
45
onnxruntime/test/testdata/nnapi_reshape_flatten_test.py
vendored
Normal file
45
onnxruntime/test/testdata/nnapi_reshape_flatten_test.py
vendored
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import onnx
|
||||
from onnx import helper
|
||||
from onnx import TensorProto
|
||||
|
||||
|
||||
# Since NNAPI EP handles Reshape and Flatten differently,
|
||||
# Please see ReshapeOpBuilder::CanSkipReshape in <repo_root>/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc
|
||||
# We have a separated test for these skip reshape scenarios
|
||||
def GenerateModel(model_name):
|
||||
nodes = [
|
||||
helper.make_node("Flatten", ["X"], ["Flatten_1_Y"], "flatten_1"),
|
||||
helper.make_node("MatMul", ["Flatten_1_Y", "MatMul_B"], ["MatMul_Y"], "matmul"),
|
||||
helper.make_node("Reshape", ["Y", "Reshape_1_shape"], ["Reshape_1_Y"], "reshape_1"),
|
||||
helper.make_node("Gemm", ["Reshape_1_Y", "Gemm_B"], ["Gemm_Y"], "gemm"),
|
||||
helper.make_node("Reshape", ["MatMul_Y", "Reshape_2_shape"], ["Reshape_2_Y"], "reshape_2"),
|
||||
helper.make_node("Flatten", ["Gemm_Y"], ["Flatten_2_Y"], "flatten_2", axis=0),
|
||||
helper.make_node("Add", ["Reshape_2_Y", "Flatten_2_Y"], ["Z"], "add"),
|
||||
]
|
||||
|
||||
initializers = [
|
||||
helper.make_tensor('Reshape_1_shape', TensorProto.INT64, [2], [3, 4]),
|
||||
helper.make_tensor('Reshape_2_shape', TensorProto.INT64, [2], [1, 6]),
|
||||
helper.make_tensor('Gemm_B', TensorProto.FLOAT, [4, 2], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]),
|
||||
helper.make_tensor('MatMul_B', TensorProto.FLOAT, [2, 3], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]),
|
||||
]
|
||||
|
||||
inputs = [
|
||||
helper.make_tensor_value_info('X', TensorProto.FLOAT, [2, 1, 2]),
|
||||
helper.make_tensor_value_info('Y', TensorProto.FLOAT, [3, 2, 2])
|
||||
]
|
||||
|
||||
graph = helper.make_graph(
|
||||
nodes,
|
||||
"NNAPI_Reshape_Flatten_Test",
|
||||
inputs,
|
||||
[helper.make_tensor_value_info('Z', TensorProto.FLOAT, [1, 6])],
|
||||
initializers
|
||||
)
|
||||
|
||||
model = helper.make_model(graph)
|
||||
onnx.save(model, model_name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
GenerateModel('nnapi_reshape_flatten_test.onnx')
|
||||
|
|
@ -347,6 +347,10 @@ def parse_arguments():
|
|||
"--use_acl", nargs="?", const="ACL_1905",
|
||||
choices=["ACL_1902", "ACL_1905", "ACL_1908", "ACL_2002"],
|
||||
help="Build with ACL for ARM architectures.")
|
||||
parser.add_argument(
|
||||
"--acl_home", help="Path to ACL home dir")
|
||||
parser.add_argument(
|
||||
"--acl_libs", help="Path to ACL libraries")
|
||||
parser.add_argument(
|
||||
"--use_armnn", action='store_true',
|
||||
help="Enable ArmNN Execution Provider.")
|
||||
|
|
@ -356,6 +360,10 @@ def parse_arguments():
|
|||
parser.add_argument(
|
||||
"--armnn_bn", action='store_true',
|
||||
help="Use the Batch Normalization operator implementation from the ArmNN EP.")
|
||||
parser.add_argument(
|
||||
"--armnn_home", help="Path to ArmNN home dir")
|
||||
parser.add_argument(
|
||||
"--armnn_libs", help="Path to ArmNN libraries")
|
||||
parser.add_argument(
|
||||
"--build_micro_benchmarks", action='store_true',
|
||||
help="Build ONNXRuntime micro-benchmarks.")
|
||||
|
|
@ -578,7 +586,7 @@ def use_dev_mode(args):
|
|||
|
||||
|
||||
def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home,
|
||||
mpi_home, nccl_home, tensorrt_home, migraphx_home,
|
||||
mpi_home, nccl_home, tensorrt_home, migraphx_home, acl_home, acl_libs, armnn_home, armnn_libs,
|
||||
path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args):
|
||||
log.info("Generating CMake build tree")
|
||||
cmake_dir = os.path.join(source_dir, "cmake")
|
||||
|
|
@ -687,6 +695,18 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
|
|||
"ON" if args.build_micro_benchmarks else "OFF")
|
||||
]
|
||||
|
||||
if acl_home and os.path.exists(acl_home):
|
||||
cmake_args += ["-Donnxruntime_ACL_HOME=" + acl_home]
|
||||
|
||||
if acl_libs and os.path.exists(acl_libs):
|
||||
cmake_args += ["-Donnxruntime_ACL_LIBS=" + acl_libs]
|
||||
|
||||
if armnn_home and os.path.exists(armnn_home):
|
||||
cmake_args += ["-Donnxruntime_ARMNN_HOME=" + armnn_home]
|
||||
|
||||
if armnn_libs and os.path.exists(armnn_libs):
|
||||
cmake_args += ["-Donnxruntime_ARMNN_LIBS=" + armnn_libs]
|
||||
|
||||
if mpi_home and os.path.exists(mpi_home):
|
||||
cmake_args += ["-Donnxruntime_MPI_HOME=" + mpi_home]
|
||||
|
||||
|
|
@ -1767,6 +1787,12 @@ def main():
|
|||
mpi_home = args.mpi_home
|
||||
nccl_home = args.nccl_home
|
||||
|
||||
acl_home = args.acl_home
|
||||
acl_libs = args.acl_libs
|
||||
|
||||
armnn_home = args.armnn_home
|
||||
armnn_libs = args.armnn_libs
|
||||
|
||||
# if using tensorrt, setup tensorrt paths
|
||||
tensorrt_home = setup_tensorrt_vars(args)
|
||||
|
||||
|
|
@ -1863,8 +1889,8 @@ def main():
|
|||
setup_test_data(build_dir, configs)
|
||||
generate_build_tree(
|
||||
cmake_path, source_dir, build_dir, cuda_home, cudnn_home, mpi_home, nccl_home,
|
||||
tensorrt_home, migraphx_home, path_to_protoc_exe, configs, cmake_extra_defines,
|
||||
args, cmake_extra_args)
|
||||
tensorrt_home, migraphx_home, acl_home, acl_libs, armnn_home, armnn_libs,
|
||||
path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args)
|
||||
|
||||
if args.clean:
|
||||
clean_targets(cmake_path, build_dir, configs)
|
||||
|
|
|
|||
Loading…
Reference in a new issue