From de0b04b971be68677dcfa686eeb32289530c1c72 Mon Sep 17 00:00:00 2001 From: gwang-msft <62914304+gwang-msft@users.noreply.github.com> Date: Thu, 30 Jul 2020 16:42:17 -0700 Subject: [PATCH] [Android NNAPI EP] Add support for dynamic output (#4650) * add dynamic output shape support * fix bugs associates with scalar inputs * addressed comments, fixed issue the output buffer size is not correctly set, refactor shaper class * split the execution logic from nnapi::Model into nnapi::Execution * update comments for certain scenarios, 1. dynamic output buffer size, 2. ONNX scalar input * move ctor of nnapi::Execution to public --- .../nnapi/nnapi_builtin/builders/helper.h | 43 +-- .../nnapi_builtin/builders/model_builder.cc | 6 +- .../nnapi/nnapi_builtin/builders/shaper.cc | 256 +++++++++--------- .../nnapi/nnapi_builtin/builders/shaper.h | 39 ++- .../providers/nnapi/nnapi_builtin/model.cc | 162 ++++++----- .../providers/nnapi/nnapi_builtin/model.h | 88 ++++-- .../nnapi_builtin/nnapi_execution_provider.cc | 169 ++++++++---- .../nnapi_builtin/nnapi_execution_provider.h | 1 - .../nnapi_lib/NeuralNetworksWrapper.cc | 29 -- .../cpu/math/element_wise_ops_test.cc | 6 +- 10 files changed, 471 insertions(+), 328 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index d1543ccdbd..94062d969a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -11,25 +11,34 @@ namespace onnxruntime { namespace nnapi { -#define THROW_ON_ERROR(val) \ - { \ - const auto ret = (val); \ - ORT_ENFORCE( \ - ret == ANEURALNETWORKS_NO_ERROR, \ - std::string("Error in ") + __FILE__ + std::string(":") + \ - std::to_string(__LINE__) + std::string(", function name: ") + \ - std::string(__func__) + "error, ret: " + GetErrorCause(ret)); \ +#define THROW_ON_ERROR(val) \ + { \ + const auto ret = (val); \ + ORT_ENFORCE( \ + ret == ANEURALNETWORKS_NO_ERROR, "ResultCode: " + GetErrorCause(ret)); \ } -#define THROW_ON_ERROR_WITH_NOTE(val, note) \ - { \ - const auto ret = (val); \ - ORT_ENFORCE( \ - ret == ANEURALNETWORKS_NO_ERROR, \ - std::string("Error in ") + __FILE__ + std::string(":") + \ - std::to_string(__LINE__) + std::string(", function name: ") + \ - std::string(__func__) + "error, ret: " + GetErrorCause(ret) + \ - std::string(", ") + (note)); \ +#define THROW_ON_ERROR_WITH_NOTE(val, note) \ + { \ + const auto ret = (val); \ + ORT_ENFORCE( \ + ret == ANEURALNETWORKS_NO_ERROR, "ResultCode: " + GetErrorCause(ret) + \ + ", " + (note)); \ + } + +#define RETURN_STATUS_ON_ERROR(val) \ + { \ + const auto ret = (val); \ + ORT_RETURN_IF_NOT( \ + ret == ANEURALNETWORKS_NO_ERROR, "ResultCode: " + GetErrorCause(ret)); \ + } + +#define RETURN_STATUS_ON_ERROR_WITH_NOTE(val, note) \ + { \ + const auto ret = (val); \ + ORT_RETURN_IF_NOT( \ + ret == ANEURALNETWORKS_NO_ERROR, "ResultCode: " + GetErrorCause(ret) + \ + ", " + (note)); \ } template diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index dfb4340476..434ea8df41 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -231,6 +231,8 @@ void ModelBuilder::RegisterInitializers() { shape.push_back(SafeInt(dim)); } + ORT_ENFORCE(!shape.empty(), "NNAPI does not support scalar initializer"); + Type type = Type::TENSOR_FLOAT32; switch (tensor.data_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: @@ -304,6 +306,9 @@ void ModelBuilder::RegisterModelInputs() { shape.push_back(SafeInt(dim.dim_value())); } + ORT_ENFORCE(GetAndroidSdkVer() >= 29 || !shape.empty(), + "0-rank input is only supported on Android API level 29+"); + Type type = Type::TENSOR_FLOAT32; float scale = 0.0f; int32_t zero_point = 0; @@ -370,7 +375,6 @@ void ModelBuilder::RegisterModelOutputs() { } void ModelBuilder::RegisterModelShaper() { - shaper_.Finalize(); nnapi_model_->SetShaper(shaper_); } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc index b716fa93b0..5621d74527 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc @@ -41,6 +41,12 @@ std::pair ComputeConvOutputShape(const uint32_t input_size_y return std::make_pair(static_cast(output_size_y), static_cast(output_size_x)); } +#define SHAPER_FUNC(FUNC, ...) \ + FUNC##Impl(__VA_ARGS__); \ + shape_ops_.push_back([__VA_ARGS__](Shaper& shaper) { \ + shaper.FUNC##Impl(__VA_ARGS__); \ + }); + void Shaper::Conv(const std::string& input_name, const std::string& weight_name, const vector& onnx_pads, @@ -48,6 +54,89 @@ void Shaper::Conv(const std::string& input_name, const vector& onnx_dilations, bool nchw, const std::string& output_name) { + SHAPER_FUNC(Conv, + input_name, weight_name, + onnx_pads, onnx_strides, onnx_dilations, + nchw, + output_name); +} + +void Shaper::DepthwiseConv(const std::string& input_name, + const std::string& weight_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, + bool nchw, + const std::string& output_name) { + SHAPER_FUNC(DepthwiseConv, + input_name, weight_name, + onnx_pads, onnx_strides, onnx_dilations, + nchw, + output_name); +} + +void Shaper::Pool(const std::string& input_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& kernel_shape, + bool nchw, + const std::string& output_name) { + SHAPER_FUNC(Pool, + input_name, + onnx_pads, onnx_strides, kernel_shape, + nchw, + output_name); +} + +void Shaper::Reshape(const std::string& input_name, + const std::vector& shape, + const std::string& output_name) { + SHAPER_FUNC(Reshape, input_name, shape, output_name); +} + +void Shaper::Transpose(const std::string& input_name, + const std::vector& perm, + const std::string& output_name) { + SHAPER_FUNC(Transpose, input_name, perm, output_name); +} + +void Shaper::Eltwise(const std::string& input1_name, + const std::string& input2_name, + const std::string& output_name) { + SHAPER_FUNC(Eltwise, input1_name, input2_name, output_name); +} + +void Shaper::Identity(const std::string& input_name, + const std::string& output_name) { + SHAPER_FUNC(Identity, input_name, output_name); +} + +void Shaper::FC(const std::string& input1_name, const std::string& input2_name, + const std::string& output_name) { + SHAPER_FUNC(FC, input1_name, input2_name, output_name); +} + +void Shaper::Concat(const std::vector& input_names, + const int32_t axis, + const std::string& output_name) { + SHAPER_FUNC(Concat, input_names, axis, output_name); +} + +void Shaper::Squeeze(const std::string& input_name, + const std::vector& axes, + const std::string& output_name) { + SHAPER_FUNC(Squeeze, input_name, axes, output_name); +} + +#undef SHAPER_FUNC + +void Shaper::ConvImpl(const std::string& input_name, + const std::string& weight_name, + const vector& onnx_pads, + const vector& onnx_strides, + const vector& onnx_dilations, + bool nchw, + const std::string& output_name) { const Shape& input_dimen = shape_map_.at(input_name); const Shape& weight_dimen = shape_map_.at(weight_name); // num_output, height, width, num_input @@ -69,28 +158,15 @@ void Shaper::Conv(const std::string& input_name, } shape_map_[output_name] = output_dimen; - - if (!shaper_finalized_) { - shape_ops_.push_back( - [input_name, weight_name, - onnx_pads, onnx_strides, onnx_dilations, - nchw, - output_name](Shaper& shaper) { - shaper.Conv(input_name, weight_name, - onnx_pads, onnx_strides, onnx_dilations, - nchw, - output_name); - }); - } } -void Shaper::DepthwiseConv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) { +void Shaper::DepthwiseConvImpl(const std::string& input_name, + const std::string& weight_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, + bool nchw, + const std::string& output_name) { const Shape& input_dimen = shape_map_.at(input_name); const Shape& weight_dimen = shape_map_.at(weight_name); // 1, height, width, num_output @@ -112,27 +188,14 @@ void Shaper::DepthwiseConv(const std::string& input_name, output_dimen = {input_dimen[0], output_size_y, output_size_x, weight_dimen[3]}; } shape_map_[output_name] = output_dimen; - - if (!shaper_finalized_) { - shape_ops_.push_back( - [input_name, weight_name, - onnx_pads, onnx_strides, onnx_dilations, - nchw, - output_name](Shaper& shaper) { - shaper.DepthwiseConv(input_name, weight_name, - onnx_pads, onnx_strides, onnx_dilations, - nchw, - output_name); - }); - } } -void Shaper::Pool(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name) { +void Shaper::PoolImpl(const std::string& input_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& kernel_shape, + bool nchw, + const std::string& output_name) { const Shape& input_dimen = shape_map_.at(input_name); const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1]; const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2]; @@ -152,24 +215,11 @@ void Shaper::Pool(const std::string& input_name, } shape_map_[output_name] = output_dimen; - - if (!shaper_finalized_) { - shape_ops_.push_back( - [input_name, - onnx_pads, onnx_strides, kernel_shape, - nchw, - output_name](Shaper& shaper) { - shaper.Pool(input_name, - onnx_pads, onnx_strides, kernel_shape, - nchw, - output_name); - }); - } } -void Shaper::Reshape(const std::string& input_name, - const std::vector& shape, - const std::string& output_name) { +void Shaper::ReshapeImpl(const std::string& input_name, + const std::vector& shape, + const std::string& output_name) { const Shape& input_dimen = shape_map_.at(input_name); int64_t input_size = Product(input_dimen); std::vector output_dimen(shape.size()); @@ -200,18 +250,11 @@ void Shaper::Reshape(const std::string& input_name, ORT_ENFORCE(capacity == input_size, "Invalid shape is given!"); shape_map_[output_name] = output_dimen; - - if (!shaper_finalized_) { - shape_ops_.push_back( - [input_name, shape, output_name](Shaper& shaper) { - shaper.Reshape(input_name, shape, output_name); - }); - } } -void Shaper::Transpose(const std::string& input_name, - const std::vector& perm, - const std::string& output_name) { +void Shaper::TransposeImpl(const std::string& input_name, + const std::vector& perm, + const std::string& output_name) { const Shape& input_dimen = shape_map_.at(input_name); ORT_ENFORCE(perm.size() == input_dimen.size(), "Invalid perm is given!"); @@ -222,18 +265,11 @@ void Shaper::Transpose(const std::string& input_name, output_dimen[i] = input_dimen[perm[i]]; shape_map_[output_name] = output_dimen; - - if (!shaper_finalized_) { - shape_ops_.push_back( - [input_name, perm, output_name](Shaper& shaper) { - shaper.Transpose(input_name, perm, output_name); - }); - } } -void Shaper::Eltwise(const std::string& input1_name, - const std::string& input2_name, - const std::string& output_name) { +void Shaper::EltwiseImpl(const std::string& input1_name, + const std::string& input2_name, + const std::string& output_name) { const Shape& shape1 = shape_map_.at(input1_name); const Shape& shape2 = shape_map_.at(input2_name); @@ -262,46 +298,25 @@ void Shaper::Eltwise(const std::string& input1_name, } shape_map_[output_name] = max_shape; - - if (!shaper_finalized_) { - shape_ops_.push_back( - [input1_name, input2_name, output_name](Shaper& shaper) { - shaper.Eltwise(input1_name, input2_name, output_name); - }); - } } -void Shaper::Identity(const std::string& input_name, - const std::string& output_name) { +void Shaper::IdentityImpl(const std::string& input_name, + const std::string& output_name) { shape_map_[output_name] = shape_map_.at(input_name); - - if (!shaper_finalized_) { - shape_ops_.push_back( - [input_name, output_name](Shaper& shaper) { - shaper.Identity(input_name, output_name); - }); - } } -void Shaper::FC(const std::string& input1_name, const std::string& input2_name, - const std::string& output_name) { +void Shaper::FCImpl(const std::string& input1_name, const std::string& input2_name, + const std::string& output_name) { // Currently we only support A*B'+C const Shape& input1_dimen = shape_map_.at(input1_name); const Shape& input2_dimen = shape_map_.at(input2_name); // num_units, input_size Shape output_dimen{input1_dimen[0], input2_dimen[0]}; shape_map_[output_name] = output_dimen; - - if (!shaper_finalized_) { - shape_ops_.push_back( - [input1_name, input2_name, output_name](Shaper& shaper) { - shaper.FC(input1_name, input2_name, output_name); - }); - } } -void Shaper::Concat(const std::vector& input_names, - const int32_t axis, - const std::string& output_name) { +void Shaper::ConcatImpl(const std::vector& input_names, + const int32_t axis, + const std::string& output_name) { std::vector dimens; for (const auto& input_name : input_names) { const Shape& dimen = shape_map_.at(input_name); @@ -323,18 +338,11 @@ void Shaper::Concat(const std::vector& input_names, } shape_map_[output_name] = output_dimen; - - if (!shaper_finalized_) { - shape_ops_.push_back( - [input_names, axis, output_name](Shaper& shaper) { - shaper.Concat(input_names, axis, output_name); - }); - } } -void Shaper::Squeeze(const std::string& input_name, - const std::vector& axes, - const std::string& output_name) { +void Shaper::SqueezeImpl(const std::string& input_name, + const std::vector& axes, + const std::string& output_name) { const Shape& input_dimen = shape_map_.at(input_name); int32_t input_size = input_dimen.size(); size_t axes_size = axes.size(); @@ -358,42 +366,30 @@ void Shaper::Squeeze(const std::string& input_name, } shape_map_[output_name] = output_dimen; - - if (!shaper_finalized_) { - shape_ops_.push_back( - [input_name, axes, output_name](Shaper& shaper) { - shaper.Squeeze(input_name, axes, output_name); - }); - } } void Shaper::AddShape(const std::string& name, const Shape& shape) { shape_map_[name] = shape; } -void Shaper::UpdateShape(const std::string& name, const Shape& new_shape) { - ORT_ENFORCE(shaper_finalized_, - "Cannot UpdateShape while shaper is not finalized"); - +Status Shaper::UpdateShape(const std::string& name, const Shape& new_shape) { const Shape& old_shape = shape_map_.at(name); if (old_shape != new_shape) { - if (Product(old_shape) != 0) - ORT_THROW("The shape should be same size or old shape has size 0 (dynamic shape)"); + ORT_RETURN_IF_NOT(Product(old_shape) == 0 || !old_shape.empty(), + "The shape should be same size or old shape has size 0 (dynamic shape)"); shape_map_[name] = new_shape; } + + return Status::OK(); } void Shaper::UpdateDynamicDimensions() { - ORT_ENFORCE(shaper_finalized_, - "Cannot UpdateDynamicDimensions while shaper is not finalized"); - for (auto& shape_op : shape_ops_) shape_op(*this); } void Shaper::Clear() { - shaper_finalized_ = false; shape_map_.clear(); shape_ops_.clear(); } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h index b18d1f4318..ec4a19d075 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h @@ -56,17 +56,44 @@ class Shaper { // If the shape of certain input is dynamic // Use the following 2 functions to update the particular shape // and calculate the new output shape - void UpdateShape(const std::string& name, const Shape& new_shape); + // Only perform this when the NNAPI model is finalized! + Status UpdateShape(const std::string& name, const Shape& new_shape); void UpdateDynamicDimensions(); - // Need to call Finalize() after the entire graph - // is converted to NNAPI - void Finalize() { shaper_finalized_ = true; } - void Clear(); private: - bool shaper_finalized_{false}; + void ConvImpl(const std::string& input_name, + const std::string& weight_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, + bool nchw, + const std::string& output_name); + + void DepthwiseConvImpl(const std::string& input_name, + const std::string& weight_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& onnx_dilations, + bool nchw, + const std::string& output_name); + + void PoolImpl(const std::string& input_name, + const std::vector& onnx_pads, + const std::vector& onnx_strides, + const std::vector& kernel_shape, + bool nchw, + const std::string& output_name); + + void ReshapeImpl(const std::string& input_name, const std::vector& shape, const std::string& output_name); + void TransposeImpl(const std::string& input_name, const std::vector& perm, const std::string& output_name); + void EltwiseImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); + void IdentityImpl(const std::string& input_name, const std::string& output_name); + void FCImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); + void ConcatImpl(const std::vector& input_names, const int32_t axis, const std::string& output_name); + void SqueezeImpl(const std::string& input, const std::vector& axes, const std::string& output); + std::unordered_map shape_map_; std::vector> shape_ops_; }; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc index a22dcf22cc..fa37538a70 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc @@ -15,12 +15,11 @@ namespace onnxruntime { namespace nnapi { +#pragma region Model + Model::Model() : nnapi_(NnApiImplementation()) {} Model::~Model() { - if (execution_) - nnapi_->ANeuralNetworksExecution_free(execution_); - nnapi_->ANeuralNetworksCompilation_free(compilation_); nnapi_->ANeuralNetworksModel_free(model_); } @@ -53,11 +52,11 @@ const android::nn::wrapper::OperandType& Model::GetInputType(const std::string& return operand_types_.at(name); } -const android::nn::wrapper::OperandType Model::GetOutputType(const std::string& name) const { +android::nn::wrapper::OperandType Model::GetOutputType(const std::string& name, const Execution& execution) const { const auto& nnapi_output_name = onnx_to_nnapi_output_map_.at(name); const auto& output_type = operand_types_.at(nnapi_output_name); android::nn::wrapper::OperandType type( - output_type.type, shaper_for_exeuction_[nnapi_output_name], output_type.operandType.scale, output_type.operandType.zeroPoint); + output_type.type, execution.GetShaper()[nnapi_output_name], output_type.operandType.scale, output_type.operandType.zeroPoint); return type; } @@ -78,77 +77,28 @@ size_t Model::GetMappedOutputIdx(const std::string& name) const { return output_map_.at(name); } -void Model::SetInputBuffer(const int32_t index, const InputBuffer& input) { - PrepareForExecution(); - - THROW_ON_ERROR(nnapi_->ANeuralNetworksExecution_setInput( - execution_, index, &input.type.operandType, input.buffer, input.type.GetOperandBlobByteSize())); +bool Model::SupportsDynamicOutputShape() const { + // dynamic output shape is only supported on Android API levle 29+ + return GetAndroidSdkVer() >= 29 && dynamic_output_buffer_size_ > 0; } -void Model::SetOutputBuffer(const int32_t index, const OutputBuffer& output) { - PrepareForExecution(); +Status Model::PrepareForExecution(std::unique_ptr& execution) { + ORT_RETURN_IF_NOT(nullptr != compilation_, + "Error in PrepareForExecution, compilation_ is null"); - LOGS_DEFAULT(VERBOSE) << "Model::SetOutputBuffer, output shape " - << Shape2String(output.type.dimensions); + ANeuralNetworksExecution* nnapi_execution; + RETURN_STATUS_ON_ERROR( + nnapi_->ANeuralNetworksExecution_create(compilation_, &nnapi_execution)); - THROW_ON_ERROR(nnapi_->ANeuralNetworksExecution_setOutput( - execution_, index, &output.type.operandType, output.buffer, output.type.GetOperandBlobByteSize())); + execution.reset(new Execution(*nnapi_execution, shaper_)); + return Status::OK(); } -void Model::PrepareForExecution() { - if (prepared_for_exe_) - return; - - ORT_ENFORCE(nullptr != compilation_, - "Error in PrepareForExecution, compilation_ is null"); - - // Copy the shaper for calculate the dynamic output shape - // based on the input shape - shaper_for_exeuction_ = shaper_; - - THROW_ON_ERROR( - nnapi_->ANeuralNetworksExecution_create(compilation_, &execution_)); - prepared_for_exe_ = true; +int32_t Model::GetAndroidSdkVer() const { + return nnapi_ ? nnapi_->android_sdk_version : 0; } -void Model::ResetExecution() { - nnapi_->ANeuralNetworksExecution_free(execution_); - execution_ = nullptr; - shaper_for_exeuction_.Clear(); - prepared_for_exe_ = false; -} - -void Model::Predict() { - PrepareForExecution(); - - ANeuralNetworksEvent* event = nullptr; - THROW_ON_ERROR(nnapi_->ANeuralNetworksExecution_startCompute(execution_, &event)); - - THROW_ON_ERROR(nnapi_->ANeuralNetworksEvent_wait(event)); - - nnapi_->ANeuralNetworksEvent_free(event); - - ResetExecution(); -} - -void Model::SetInputBuffers(const std::vector& inputs) { - PrepareForExecution(); - - for (size_t i = 0; i < inputs.size(); i++) { - SetInputBuffer(i, inputs[i]); - shaper_for_exeuction_.UpdateShape(input_names_[i], inputs[i].type.dimensions); - } - - shaper_for_exeuction_.UpdateDynamicDimensions(); -} - -void Model::SetOutputBuffers(const std::vector& outputs) { - PrepareForExecution(); - - for (size_t i = 0; i < outputs.size(); i++) { - SetOutputBuffer(i, outputs[i]); - } -} +#pragma region Model::NNMemory #ifdef USENNAPISHAREDMEM Model::NNMemory::NNMemory(const NnApi* nnapi, const char* name, size_t size) { @@ -181,5 +131,81 @@ Model::NNMemory::NNMemory(const NnApi* /*nnapi*/, const char* name, size_t size) } #endif +#pragma endregion + +#pragma endregion + +#pragma region Execution + +Execution::Execution(ANeuralNetworksExecution& execution, const Shaper& shaper) + : nnapi_(NnApiImplementation()), + execution_(&execution), + shaper_(shaper) { +} + +Execution::~Execution() { + nnapi_->ANeuralNetworksExecution_free(execution_); +} + +Status Execution::SetInputBuffers(const std::vector& inputs) { + for (size_t i = 0; i < inputs.size(); i++) { + const auto& input(inputs[i]); + ORT_RETURN_IF_ERROR(SetInputBuffer(i, input)); + ORT_RETURN_IF_ERROR(shaper_.UpdateShape(input.name, input.type.dimensions)); + } + + shaper_.UpdateDynamicDimensions(); + + return Status::OK(); +} + +Status Execution::SetOutputBuffers(const std::vector& outputs) { + for (size_t i = 0; i < outputs.size(); i++) { + ORT_RETURN_IF_ERROR(SetOutputBuffer(i, outputs[i])); + } + + return Status::OK(); +} + +Status Execution::SetInputBuffer(const int32_t index, const InputBuffer& input) { + RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_setInput( + execution_, index, &input.type.operandType, input.buffer, input.type.GetOperandBlobByteSize())); + + return Status::OK(); +} + +Status Execution::SetOutputBuffer(const int32_t index, const OutputBuffer& output) { + LOGS_DEFAULT(VERBOSE) << "Model::SetOutputBuffer, output shape " + << Shape2String(output.type.dimensions); + + RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_setOutput( + execution_, index, &output.type.operandType, output.buffer, output.buffer_byte_size)); + + return Status::OK(); +} + +Status Execution::Predict(const std::vector& dynamic_outputs, std::vector& dynamic_output_shapes) { + ANeuralNetworksEvent* event = nullptr; + RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_startCompute(execution_, &event)); + RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksEvent_wait(event)); + nnapi_->ANeuralNetworksEvent_free(event); + + dynamic_output_shapes.clear(); + dynamic_output_shapes.reserve(dynamic_outputs.size()); + for (const int32_t i : dynamic_outputs) { + uint32_t output_rank = 0; + RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_getOutputOperandRank(execution_, i, &output_rank)); + + std::vector output_shape(output_rank); + RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_getOutputOperandDimensions(execution_, i, output_shape.data())); + + dynamic_output_shapes.push_back(output_shape); + } + + return Status::OK(); +} + +#pragma endregion + } // namespace nnapi } // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h index 75c3fbb0f9..a83e8e6cb4 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h @@ -12,20 +12,12 @@ namespace nnapi { #define USENNAPISHAREDMEM 1 +class Execution; + class Model { friend class ModelBuilder; public: - struct InputBuffer { - const void* buffer{nullptr}; - android::nn::wrapper::OperandType type; - }; - - struct OutputBuffer { - void* buffer{nullptr}; - android::nn::wrapper::OperandType type; - }; - // Memory for persist data such as initializers and intermediate result #ifdef USENNAPISHAREDMEM // Use NNAPI shared memory @@ -72,7 +64,7 @@ class Model { // Returns the data type and dimension of the given input/output // Please note the output type will have updated dimensions const android::nn::wrapper::OperandType& GetInputType(const std::string& name) const; - const android::nn::wrapper::OperandType GetOutputType(const std::string& name) const; + android::nn::wrapper::OperandType GetOutputType(const std::string& name, const Execution& execution) const; // Set the mapping between input/output name and ORT kernel context // input/output index, at execution time @@ -83,35 +75,43 @@ class Model { size_t GetMappedInputIdx(const std::string& name) const; size_t GetMappedOutputIdx(const std::string& name) const; - // Set the input/output data buffers - // These need to be called before calling Predict() - void SetInputBuffers(const std::vector& inputs); - void SetOutputBuffers(const std::vector& outputs); + // If we support the dynamic output shape, + // This is only for the case where output size cannot be determined at model execution time + // Do not use this for case a determined output shape can be returned from GetOutputType() + bool SupportsDynamicOutputShape() const; - // Execute the NNAPI model - void Predict(); + // Set and Get the number of elements in the buffer for a dynamic output + // If the buffer is not big enough, ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE will be returned by exection + // Note: this will return number of elements of the buffer not the byte size of the buffer + // and each output will have its separated buffer + // TODO: + // 1. Consider an adaptive aproach to automatically increase the buffer size if the execution reports + // insufficient size + // 2. Experiment with bigger initial buffer size (currently 1024) + size_t GetDynamicOutputBufferSize() const { return dynamic_output_buffer_size_; } + void SetDynamicOutputBufferSize(size_t size) { dynamic_output_buffer_size_ = size; } // Mutex for exclusive lock to this model object OrtMutex& GetMutex() { return mutex_; } + Status PrepareForExecution(std::unique_ptr& execution); + private: const NnApi* nnapi_{nullptr}; - bool prepared_for_exe_ = false; ANeuralNetworksModel* model_{nullptr}; ANeuralNetworksCompilation* compilation_{nullptr}; - ANeuralNetworksExecution* execution_{nullptr}; + + size_t dynamic_output_buffer_size_{1024}; std::unique_ptr mem_initializers_; std::vector> mem_persist_buffers_; std::vector input_names_; std::vector output_names_; - std::unordered_map - operand_types_; + std::unordered_map operand_types_; Shaper shaper_; - Shaper shaper_for_exeuction_; std::unordered_map input_map_; std::unordered_map output_map_; @@ -133,11 +133,47 @@ class Model { void SetShaper(const Shaper shaper) { shaper_ = shaper; } - void SetInputBuffer(const int32_t index, const InputBuffer& input); - void SetOutputBuffer(const int32_t index, const OutputBuffer& output); + int32_t GetAndroidSdkVer() const; +}; - void PrepareForExecution(); - void ResetExecution(); +class Execution { + public: + struct InputBuffer { + const std::string& name; + const void* buffer{nullptr}; + android::nn::wrapper::OperandType type; + }; + + struct OutputBuffer { + void* buffer{nullptr}; + android::nn::wrapper::OperandType type; + size_t buffer_byte_size; + }; + + public: + explicit Execution(ANeuralNetworksExecution& execution, const Shaper& shaper); + ~Execution(); + Execution(const Execution&) = delete; + Execution& operator=(const Execution&) = delete; + + const Shaper& GetShaper() const { return shaper_; } + + // Set the input/output data buffers + // These need to be called before calling Predict() + Status SetInputBuffers(const std::vector& inputs); + Status SetOutputBuffers(const std::vector& outputs); + + // Execute the NNAPI model + // if there is dynamic output shape, will output the actual output shapes + Status Predict(const std::vector& dynamic_outputs, std::vector& dynamic_output_shapes); + + private: + Status SetInputBuffer(const int32_t index, const InputBuffer& input); + Status SetOutputBuffer(const int32_t index, const OutputBuffer& output); + + const NnApi* nnapi_{nullptr}; + ANeuralNetworksExecution* execution_; + Shaper shaper_; }; } // namespace nnapi diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 3699d73652..f533eeac57 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -8,7 +8,6 @@ #include "core/framework/compute_capability.h" #include "core/graph/model.h" #include "core/session/onnxruntime_cxx_api.h" -#include "core/session/inference_session.h" namespace onnxruntime { @@ -175,6 +174,40 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view return result; } +static Status GetOutputBuffer(Ort::CustomOpApi& ort, + OrtKernelContext* context, + const nnapi::Model& model, + const std::string& output_name, + const std::vector& output_shape, + const android::nn::wrapper::Type output_type, + void** output_buffer) { + using namespace android::nn::wrapper; + std::vector int64_output_shape(output_shape.begin(), + output_shape.end()); + auto output_idx = model.GetMappedOutputIdx(output_name); + auto* output_tensor = ort.KernelContext_GetOutput(context, output_idx, + int64_output_shape.data(), + int64_output_shape.size()); + + switch (output_type) { + case Type::TENSOR_FLOAT32: + *output_buffer = ort.GetTensorMutableData(output_tensor); + break; + case Type::TENSOR_INT32: + *output_buffer = ort.GetTensorMutableData(output_tensor); + break; + case Type::TENSOR_QUANT8_ASYMM: + *output_buffer = ort.GetTensorMutableData(output_tensor); + break; + default: + return Status(common::ONNXRUNTIME, common::FAIL, + "Unsupported output type: " + TypeToStr(output_type)); + break; + } + + return Status::OK(); +} + common::Status NnapiExecutionProvider::Compile(const std::vector& fused_nodes, std::vector& node_compute_funcs) { using namespace android::nn::wrapper; @@ -225,7 +258,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vector(state); const size_t num_inputs = ort.KernelContext_GetInputCount(context); const size_t num_outputs = ort.KernelContext_GetOutputCount(context); - ORT_ENFORCE(model->GetInputs().size() <= num_inputs, "Inconsistent input sizes"); - ORT_ENFORCE(model->GetOutputs().size() == num_outputs, "Inconsistent output sizes"); + const auto& model_inputs = model->GetInputs(); + const auto& model_outputs = model->GetOutputs(); - std::vector inputs; - inputs.reserve(model->GetInputs().size()); - for (size_t i = 0; i < model->GetInputs().size(); i++) { - const auto& input_name = model->GetInputs()[i]; + ORT_ENFORCE(model_inputs.size() <= num_inputs, "Inconsistent input sizes"); + ORT_ENFORCE(model_outputs.size() == num_outputs, "Inconsistent output sizes"); + + std::vector inputs; + inputs.reserve(model_inputs.size()); + for (size_t i = 0; i < model_inputs.size(); i++) { + const auto& input_name = model_inputs[i]; const auto& model_input_type = model->GetInputType(input_name); auto input_idx = model->GetMappedInputIdx(input_name); @@ -250,73 +286,112 @@ common::Status NnapiExecutionProvider::Compile(const std::vector(dim)); + // NNAPI has strict input type requirements which separates tensor inputs and scalar inputs + // For ONNX the we do not have clear line between scalar inputs and tensor inputs + // Also NNAPI treats a tensor input with empty shape as dynamic shape input + // Disable support of the scalar input (tensor input with an empty shape) for now + // TODO, add support for ONNX scalar input (tensor input with an empty shape) + if (dimensions.empty()) + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "NNAPI does not support scalar input"); + // it is possible that the input has the detailed size while // the model has an operand with unknown size, use the size // of the actual input - OperandType type(model_input_type.type, dimensions, - model_input_type.operandType.scale, - model_input_type.operandType.zeroPoint); + OperandType input_type = model_input_type; + input_type.SetDimensions(dimensions); - if (type.dimensions != model_input_type.dimensions && model_input_type.GetOperandBlobByteSize() != 0) { + if (input_type.GetOperandBlobByteSize() == 0) + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "The actual input cannot have 0 dim (dynamic)"); + + if (input_type.dimensions != model_input_type.dimensions && model_input_type.GetOperandBlobByteSize() != 0) { return Status(common::ONNXRUNTIME, common::FAIL, "The actual input dimanesions should match the model input " "dimensions, or model input dimension has 0 (dynamic)"); } const void* inputBuffer = ort.GetTensorData(input_tensor); - inputs.push_back({inputBuffer, std::move(type)}); + inputs.push_back({input_name, inputBuffer, std::move(input_type)}); ort.ReleaseTensorTypeAndShapeInfo(tensor_info); } // From this point we will need to take the exclusive lock on the model until the Predict is - // performed, to block other threads (if any) to modify this particular model + // performed, to block other threads to perform Predict on the same model + // TODO, investigate concurrent runs for different executions from the same model { + std::unique_ptr execution; std::unique_lock lock(model->GetMutex()); - model->SetInputBuffers(inputs); - std::vector outputs; + model->PrepareForExecution(execution); + + ORT_RETURN_IF_ERROR(execution->SetInputBuffers(inputs)); + std::vector outputs; outputs.reserve(num_outputs); + std::vector dynamic_shape_output_indices; + std::vector dynamic_shape_output_types; + std::vector> dynamic_shape_output_buffers; for (size_t i = 0; i < num_outputs; i++) { - const auto output_name = model->GetOutputs()[i]; - const auto model_output_type = model->GetOutputType(output_name); + const auto output_name = model_outputs[i]; + const auto model_output_type = model->GetOutputType(output_name, *execution); const auto output_shape = model_output_type.dimensions; - std::vector int64_output_shape(output_shape.begin(), - output_shape.end()); - auto output_idx = model->GetMappedOutputIdx(output_name); - auto* output_tensor = ort.KernelContext_GetOutput(context, output_idx, - int64_output_shape.data(), - int64_output_shape.size()); + bool is_dynamic_shape_output = false; + if (model_output_type.GetOperandBlobByteSize() == 0) { + if (!model->SupportsDynamicOutputShape()) { + return Status(common::ONNXRUNTIME, common::FAIL, + "We do not support dynamic output shape or empty output for now"); + } + + is_dynamic_shape_output = true; + } void* output_buffer = nullptr; - switch (model_output_type.type) { - case Type::TENSOR_FLOAT32: - output_buffer = ort.GetTensorMutableData(output_tensor); - break; - case Type::TENSOR_INT32: - output_buffer = ort.GetTensorMutableData(output_tensor); - break; - case Type::TENSOR_QUANT8_ASYMM: - output_buffer = ort.GetTensorMutableData(output_tensor); - break; - default: - return Status(common::ONNXRUNTIME, common::FAIL, - "Unsupported output type: " + TypeToStr(model_output_type.type)); - break; + size_t output_buffer_byte_size; + if (!is_dynamic_shape_output) { + ORT_RETURN_IF_ERROR(GetOutputBuffer(ort, context, + *model, + output_name, output_shape, model_output_type.type, + &output_buffer)); + output_buffer_byte_size = model_output_type.GetOperandBlobByteSize(); + } else { + // This output is dynamic (size unknown), will need allocate a buffer for the result + // and copy the content to ORT output tensors afte the execution (will know output shape after the execution) + output_buffer_byte_size = model->GetDynamicOutputBufferSize() * model_output_type.GetElementByteSize(); + std::unique_ptr buffer_holder(new uint8_t[output_buffer_byte_size]); + output_buffer = buffer_holder.get(); + dynamic_shape_output_types.push_back(model_output_type); + dynamic_shape_output_indices.push_back(static_cast(i)); + dynamic_shape_output_buffers.push_back(std::move(buffer_holder)); } - if (model_output_type.GetOperandBlobByteSize() == 0) { - return Status(common::ONNXRUNTIME, common::FAIL, - "We do not support dynamic output shape or empty output for now"); - } - - outputs.push_back({output_buffer, std::move(model_output_type)}); + outputs.push_back({output_buffer, std::move(model_output_type), output_buffer_byte_size}); } - model->SetOutputBuffers(outputs); - model->Predict(); - } + ORT_RETURN_IF_ERROR(execution->SetOutputBuffers(outputs)); + std::vector> dynamic_output_shapes; + ORT_RETURN_IF_ERROR( + execution->Predict(dynamic_shape_output_indices, dynamic_output_shapes)); + // We have dynamic output buffers, need to copy the content from temp buffers to ORT output tensors + for (size_t i = 0; i < dynamic_shape_output_indices.size(); i++) { + const int32_t model_output_idx = dynamic_shape_output_indices[i]; + const auto output_name = model_outputs[model_output_idx]; + + const auto& output_shape = dynamic_output_shapes[i]; + auto model_output_type = dynamic_shape_output_types[i]; + model_output_type.SetDimensions(output_shape); + ORT_RETURN_IF_NOT(model_output_type.GetOperandBlobByteSize() != 0, "We do not support 0 size output for now"); + + void* model_output_buffer = dynamic_shape_output_buffers[i].get(); + void* onnx_output_buffer = nullptr; + ORT_RETURN_IF_ERROR(GetOutputBuffer(ort, context, + *model, + output_name, output_shape, model_output_type.type, + &onnx_output_buffer)); + + size_t output_buffer_byte_size = model_output_type.GetOperandBlobByteSize(); + memcpy(onnx_output_buffer, model_output_buffer, output_buffer_byte_size); + } + } return Status::OK(); }; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h index 6fab8bbcc3..1eded5e6d0 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h @@ -4,7 +4,6 @@ #pragma once #include "core/framework/execution_provider.h" -#include "core/graph/onnx_protobuf.h" #include "core/providers/nnapi/nnapi_builtin/model.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc index e16e460d71..e44da9635a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc @@ -21,18 +21,8 @@ namespace android { namespace nn { namespace wrapper { -bool IsScalarType(const Type& type) { - return type == Type::FLOAT16 || type == Type::FLOAT32 || type == Type::INT32 || type == Type::BOOL || type == Type::UINT32; -} - OperandType::OperandType(Type type, const std::vector& d, float scale, int32_t zeroPoint) : type(type), dimensions(d) { - if (dimensions.empty()) { - if (!IsScalarType(type)) { - dimensions = {1}; - } - } - operandType = { .type = static_cast(type), .dimensionCount = static_cast(dimensions.size()), @@ -45,13 +35,6 @@ OperandType::OperandType(Type type, const std::vector& d, float scale, OperandType::OperandType(const OperandType& other) { type = other.type; dimensions = other.dimensions; - - if (dimensions.empty()) { - if (!IsScalarType(type)) { - dimensions = {1}; - } - } - operandType = other.operandType; operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr; } @@ -60,13 +43,6 @@ OperandType& OperandType::operator=(const OperandType& other) { if (this != &other) { type = other.type; dimensions = other.dimensions; - - if (dimensions.empty()) { - if (!IsScalarType(type)) { - dimensions = {1}; - } - } - operandType = other.operandType; operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr; } @@ -115,11 +91,6 @@ size_t OperandType::GetOperandBlobByteSize() const { void OperandType::SetDimensions(const std::vector& d) { dimensions = d; - if (dimensions.empty()) { - if (!IsScalarType(type)) { - dimensions = {1}; - } - } operandType.dimensionCount = dimensions.size(); operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr; } diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index 8ad723980f..9341f0d724 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -193,7 +193,7 @@ TEST(MathOpTest, Add_Broadcast_0x1) { test.AddInput("A", {}, {10.0f}); test.AddInput("B", {1}, {2.0f}); test.AddOutput("C", {1}, {12.0f}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider}); // NNAPI: Add does not support scalar input } TEST(MathOpTest, Add_Broadcast_1x0) { @@ -202,7 +202,7 @@ TEST(MathOpTest, Add_Broadcast_1x0) { test.AddInput("A", {1}, {10.0f}); test.AddInput("B", {}, {2.0f}); test.AddOutput("C", {1}, {12.0f}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider}); // NNAPI: Add does not support scalar input } TEST(MathOpTest, Add_Broadcast_1x1) { @@ -369,7 +369,7 @@ TEST(MathOpTest, Sub_Broadcast_Scalar) { {-4.0f, -3.0f, -6.0f, -5.0f, -3.5f, -105.0f, -10.4f, 4.3f, -10005.0f}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider}); // NNAPI: Sub does not support scalar input } TEST(MathOpTest, Mul_int32) {