mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-07 17:15:29 +00:00
[Android NNAPI EP] Add support for dynamic output (#4650)
* add dynamic output shape support * fix bugs associates with scalar inputs * addressed comments, fixed issue the output buffer size is not correctly set, refactor shaper class * split the execution logic from nnapi::Model into nnapi::Execution * update comments for certain scenarios, 1. dynamic output buffer size, 2. ONNX scalar input * move ctor of nnapi::Execution to public
This commit is contained in:
gwang-msft
GitHub
parent
282975aefb
commit
de0b04b971
10 changed files with 471 additions and 328 deletions
|
|
@ -11,25 +11,34 @@
|
|||
namespace onnxruntime {
|
||||
namespace nnapi {
|
||||
|
||||
#define THROW_ON_ERROR(val) \
|
||||
{ \
|
||||
const auto ret = (val); \
|
||||
ORT_ENFORCE( \
|
||||
ret == ANEURALNETWORKS_NO_ERROR, \
|
||||
std::string("Error in ") + __FILE__ + std::string(":") + \
|
||||
std::to_string(__LINE__) + std::string(", function name: ") + \
|
||||
std::string(__func__) + "error, ret: " + GetErrorCause(ret)); \
|
||||
#define THROW_ON_ERROR(val) \
|
||||
{ \
|
||||
const auto ret = (val); \
|
||||
ORT_ENFORCE( \
|
||||
ret == ANEURALNETWORKS_NO_ERROR, "ResultCode: " + GetErrorCause(ret)); \
|
||||
}
|
||||
|
||||
#define THROW_ON_ERROR_WITH_NOTE(val, note) \
|
||||
{ \
|
||||
const auto ret = (val); \
|
||||
ORT_ENFORCE( \
|
||||
ret == ANEURALNETWORKS_NO_ERROR, \
|
||||
std::string("Error in ") + __FILE__ + std::string(":") + \
|
||||
std::to_string(__LINE__) + std::string(", function name: ") + \
|
||||
std::string(__func__) + "error, ret: " + GetErrorCause(ret) + \
|
||||
std::string(", ") + (note)); \
|
||||
#define THROW_ON_ERROR_WITH_NOTE(val, note) \
|
||||
{ \
|
||||
const auto ret = (val); \
|
||||
ORT_ENFORCE( \
|
||||
ret == ANEURALNETWORKS_NO_ERROR, "ResultCode: " + GetErrorCause(ret) + \
|
||||
", " + (note)); \
|
||||
}
|
||||
|
||||
#define RETURN_STATUS_ON_ERROR(val) \
|
||||
{ \
|
||||
const auto ret = (val); \
|
||||
ORT_RETURN_IF_NOT( \
|
||||
ret == ANEURALNETWORKS_NO_ERROR, "ResultCode: " + GetErrorCause(ret)); \
|
||||
}
|
||||
|
||||
#define RETURN_STATUS_ON_ERROR_WITH_NOTE(val, note) \
|
||||
{ \
|
||||
const auto ret = (val); \
|
||||
ORT_RETURN_IF_NOT( \
|
||||
ret == ANEURALNETWORKS_NO_ERROR, "ResultCode: " + GetErrorCause(ret) + \
|
||||
", " + (note)); \
|
||||
}
|
||||
|
||||
template <class Map, class Key>
|
||||
|
|
|
|||
|
|
@ -231,6 +231,8 @@ void ModelBuilder::RegisterInitializers() {
|
|||
shape.push_back(SafeInt<uint32_t>(dim));
|
||||
}
|
||||
|
||||
ORT_ENFORCE(!shape.empty(), "NNAPI does not support scalar initializer");
|
||||
|
||||
Type type = Type::TENSOR_FLOAT32;
|
||||
switch (tensor.data_type()) {
|
||||
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
|
||||
|
|
@ -304,6 +306,9 @@ void ModelBuilder::RegisterModelInputs() {
|
|||
shape.push_back(SafeInt<uint32_t>(dim.dim_value()));
|
||||
}
|
||||
|
||||
ORT_ENFORCE(GetAndroidSdkVer() >= 29 || !shape.empty(),
|
||||
"0-rank input is only supported on Android API level 29+");
|
||||
|
||||
Type type = Type::TENSOR_FLOAT32;
|
||||
float scale = 0.0f;
|
||||
int32_t zero_point = 0;
|
||||
|
|
@ -370,7 +375,6 @@ void ModelBuilder::RegisterModelOutputs() {
|
|||
}
|
||||
|
||||
void ModelBuilder::RegisterModelShaper() {
|
||||
shaper_.Finalize();
|
||||
nnapi_model_->SetShaper(shaper_);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,12 @@ std::pair<uint32_t, uint32_t> ComputeConvOutputShape(const uint32_t input_size_y
|
|||
return std::make_pair(static_cast<uint32_t>(output_size_y), static_cast<uint32_t>(output_size_x));
|
||||
}
|
||||
|
||||
#define SHAPER_FUNC(FUNC, ...) \
|
||||
FUNC##Impl(__VA_ARGS__); \
|
||||
shape_ops_.push_back([__VA_ARGS__](Shaper& shaper) { \
|
||||
shaper.FUNC##Impl(__VA_ARGS__); \
|
||||
});
|
||||
|
||||
void Shaper::Conv(const std::string& input_name,
|
||||
const std::string& weight_name,
|
||||
const vector<int32_t>& onnx_pads,
|
||||
|
|
@ -48,6 +54,89 @@ void Shaper::Conv(const std::string& input_name,
|
|||
const vector<int32_t>& onnx_dilations,
|
||||
bool nchw,
|
||||
const std::string& output_name) {
|
||||
SHAPER_FUNC(Conv,
|
||||
input_name, weight_name,
|
||||
onnx_pads, onnx_strides, onnx_dilations,
|
||||
nchw,
|
||||
output_name);
|
||||
}
|
||||
|
||||
void Shaper::DepthwiseConv(const std::string& input_name,
|
||||
const std::string& weight_name,
|
||||
const std::vector<int32_t>& onnx_pads,
|
||||
const std::vector<int32_t>& onnx_strides,
|
||||
const std::vector<int32_t>& onnx_dilations,
|
||||
bool nchw,
|
||||
const std::string& output_name) {
|
||||
SHAPER_FUNC(DepthwiseConv,
|
||||
input_name, weight_name,
|
||||
onnx_pads, onnx_strides, onnx_dilations,
|
||||
nchw,
|
||||
output_name);
|
||||
}
|
||||
|
||||
void Shaper::Pool(const std::string& input_name,
|
||||
const std::vector<int32_t>& onnx_pads,
|
||||
const std::vector<int32_t>& onnx_strides,
|
||||
const std::vector<int32_t>& kernel_shape,
|
||||
bool nchw,
|
||||
const std::string& output_name) {
|
||||
SHAPER_FUNC(Pool,
|
||||
input_name,
|
||||
onnx_pads, onnx_strides, kernel_shape,
|
||||
nchw,
|
||||
output_name);
|
||||
}
|
||||
|
||||
void Shaper::Reshape(const std::string& input_name,
|
||||
const std::vector<int32_t>& shape,
|
||||
const std::string& output_name) {
|
||||
SHAPER_FUNC(Reshape, input_name, shape, output_name);
|
||||
}
|
||||
|
||||
void Shaper::Transpose(const std::string& input_name,
|
||||
const std::vector<int32_t>& perm,
|
||||
const std::string& output_name) {
|
||||
SHAPER_FUNC(Transpose, input_name, perm, output_name);
|
||||
}
|
||||
|
||||
void Shaper::Eltwise(const std::string& input1_name,
|
||||
const std::string& input2_name,
|
||||
const std::string& output_name) {
|
||||
SHAPER_FUNC(Eltwise, input1_name, input2_name, output_name);
|
||||
}
|
||||
|
||||
void Shaper::Identity(const std::string& input_name,
|
||||
const std::string& output_name) {
|
||||
SHAPER_FUNC(Identity, input_name, output_name);
|
||||
}
|
||||
|
||||
void Shaper::FC(const std::string& input1_name, const std::string& input2_name,
|
||||
const std::string& output_name) {
|
||||
SHAPER_FUNC(FC, input1_name, input2_name, output_name);
|
||||
}
|
||||
|
||||
void Shaper::Concat(const std::vector<std::string>& input_names,
|
||||
const int32_t axis,
|
||||
const std::string& output_name) {
|
||||
SHAPER_FUNC(Concat, input_names, axis, output_name);
|
||||
}
|
||||
|
||||
void Shaper::Squeeze(const std::string& input_name,
|
||||
const std::vector<int32_t>& axes,
|
||||
const std::string& output_name) {
|
||||
SHAPER_FUNC(Squeeze, input_name, axes, output_name);
|
||||
}
|
||||
|
||||
#undef SHAPER_FUNC
|
||||
|
||||
void Shaper::ConvImpl(const std::string& input_name,
|
||||
const std::string& weight_name,
|
||||
const vector<int32_t>& onnx_pads,
|
||||
const vector<int32_t>& onnx_strides,
|
||||
const vector<int32_t>& onnx_dilations,
|
||||
bool nchw,
|
||||
const std::string& output_name) {
|
||||
const Shape& input_dimen = shape_map_.at(input_name);
|
||||
const Shape& weight_dimen = shape_map_.at(weight_name); // num_output, height, width, num_input
|
||||
|
||||
|
|
@ -69,28 +158,15 @@ void Shaper::Conv(const std::string& input_name,
|
|||
}
|
||||
|
||||
shape_map_[output_name] = output_dimen;
|
||||
|
||||
if (!shaper_finalized_) {
|
||||
shape_ops_.push_back(
|
||||
[input_name, weight_name,
|
||||
onnx_pads, onnx_strides, onnx_dilations,
|
||||
nchw,
|
||||
output_name](Shaper& shaper) {
|
||||
shaper.Conv(input_name, weight_name,
|
||||
onnx_pads, onnx_strides, onnx_dilations,
|
||||
nchw,
|
||||
output_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void Shaper::DepthwiseConv(const std::string& input_name,
|
||||
const std::string& weight_name,
|
||||
const std::vector<int32_t>& onnx_pads,
|
||||
const std::vector<int32_t>& onnx_strides,
|
||||
const std::vector<int32_t>& onnx_dilations,
|
||||
bool nchw,
|
||||
const std::string& output_name) {
|
||||
void Shaper::DepthwiseConvImpl(const std::string& input_name,
|
||||
const std::string& weight_name,
|
||||
const std::vector<int32_t>& onnx_pads,
|
||||
const std::vector<int32_t>& onnx_strides,
|
||||
const std::vector<int32_t>& onnx_dilations,
|
||||
bool nchw,
|
||||
const std::string& output_name) {
|
||||
const Shape& input_dimen = shape_map_.at(input_name);
|
||||
const Shape& weight_dimen = shape_map_.at(weight_name); // 1, height, width, num_output
|
||||
|
||||
|
|
@ -112,27 +188,14 @@ void Shaper::DepthwiseConv(const std::string& input_name,
|
|||
output_dimen = {input_dimen[0], output_size_y, output_size_x, weight_dimen[3]};
|
||||
}
|
||||
shape_map_[output_name] = output_dimen;
|
||||
|
||||
if (!shaper_finalized_) {
|
||||
shape_ops_.push_back(
|
||||
[input_name, weight_name,
|
||||
onnx_pads, onnx_strides, onnx_dilations,
|
||||
nchw,
|
||||
output_name](Shaper& shaper) {
|
||||
shaper.DepthwiseConv(input_name, weight_name,
|
||||
onnx_pads, onnx_strides, onnx_dilations,
|
||||
nchw,
|
||||
output_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void Shaper::Pool(const std::string& input_name,
|
||||
const std::vector<int32_t>& onnx_pads,
|
||||
const std::vector<int32_t>& onnx_strides,
|
||||
const std::vector<int32_t>& kernel_shape,
|
||||
bool nchw,
|
||||
const std::string& output_name) {
|
||||
void Shaper::PoolImpl(const std::string& input_name,
|
||||
const std::vector<int32_t>& onnx_pads,
|
||||
const std::vector<int32_t>& onnx_strides,
|
||||
const std::vector<int32_t>& kernel_shape,
|
||||
bool nchw,
|
||||
const std::string& output_name) {
|
||||
const Shape& input_dimen = shape_map_.at(input_name);
|
||||
const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1];
|
||||
const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2];
|
||||
|
|
@ -152,24 +215,11 @@ void Shaper::Pool(const std::string& input_name,
|
|||
}
|
||||
|
||||
shape_map_[output_name] = output_dimen;
|
||||
|
||||
if (!shaper_finalized_) {
|
||||
shape_ops_.push_back(
|
||||
[input_name,
|
||||
onnx_pads, onnx_strides, kernel_shape,
|
||||
nchw,
|
||||
output_name](Shaper& shaper) {
|
||||
shaper.Pool(input_name,
|
||||
onnx_pads, onnx_strides, kernel_shape,
|
||||
nchw,
|
||||
output_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void Shaper::Reshape(const std::string& input_name,
|
||||
const std::vector<int32_t>& shape,
|
||||
const std::string& output_name) {
|
||||
void Shaper::ReshapeImpl(const std::string& input_name,
|
||||
const std::vector<int32_t>& shape,
|
||||
const std::string& output_name) {
|
||||
const Shape& input_dimen = shape_map_.at(input_name);
|
||||
int64_t input_size = Product(input_dimen);
|
||||
std::vector<uint32_t> output_dimen(shape.size());
|
||||
|
|
@ -200,18 +250,11 @@ void Shaper::Reshape(const std::string& input_name,
|
|||
ORT_ENFORCE(capacity == input_size, "Invalid shape is given!");
|
||||
|
||||
shape_map_[output_name] = output_dimen;
|
||||
|
||||
if (!shaper_finalized_) {
|
||||
shape_ops_.push_back(
|
||||
[input_name, shape, output_name](Shaper& shaper) {
|
||||
shaper.Reshape(input_name, shape, output_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void Shaper::Transpose(const std::string& input_name,
|
||||
const std::vector<int32_t>& perm,
|
||||
const std::string& output_name) {
|
||||
void Shaper::TransposeImpl(const std::string& input_name,
|
||||
const std::vector<int32_t>& perm,
|
||||
const std::string& output_name) {
|
||||
const Shape& input_dimen = shape_map_.at(input_name);
|
||||
|
||||
ORT_ENFORCE(perm.size() == input_dimen.size(), "Invalid perm is given!");
|
||||
|
|
@ -222,18 +265,11 @@ void Shaper::Transpose(const std::string& input_name,
|
|||
output_dimen[i] = input_dimen[perm[i]];
|
||||
|
||||
shape_map_[output_name] = output_dimen;
|
||||
|
||||
if (!shaper_finalized_) {
|
||||
shape_ops_.push_back(
|
||||
[input_name, perm, output_name](Shaper& shaper) {
|
||||
shaper.Transpose(input_name, perm, output_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void Shaper::Eltwise(const std::string& input1_name,
|
||||
const std::string& input2_name,
|
||||
const std::string& output_name) {
|
||||
void Shaper::EltwiseImpl(const std::string& input1_name,
|
||||
const std::string& input2_name,
|
||||
const std::string& output_name) {
|
||||
const Shape& shape1 = shape_map_.at(input1_name);
|
||||
const Shape& shape2 = shape_map_.at(input2_name);
|
||||
|
||||
|
|
@ -262,46 +298,25 @@ void Shaper::Eltwise(const std::string& input1_name,
|
|||
}
|
||||
|
||||
shape_map_[output_name] = max_shape;
|
||||
|
||||
if (!shaper_finalized_) {
|
||||
shape_ops_.push_back(
|
||||
[input1_name, input2_name, output_name](Shaper& shaper) {
|
||||
shaper.Eltwise(input1_name, input2_name, output_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void Shaper::Identity(const std::string& input_name,
|
||||
const std::string& output_name) {
|
||||
void Shaper::IdentityImpl(const std::string& input_name,
|
||||
const std::string& output_name) {
|
||||
shape_map_[output_name] = shape_map_.at(input_name);
|
||||
|
||||
if (!shaper_finalized_) {
|
||||
shape_ops_.push_back(
|
||||
[input_name, output_name](Shaper& shaper) {
|
||||
shaper.Identity(input_name, output_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void Shaper::FC(const std::string& input1_name, const std::string& input2_name,
|
||||
const std::string& output_name) {
|
||||
void Shaper::FCImpl(const std::string& input1_name, const std::string& input2_name,
|
||||
const std::string& output_name) {
|
||||
// Currently we only support A*B'+C
|
||||
const Shape& input1_dimen = shape_map_.at(input1_name);
|
||||
const Shape& input2_dimen = shape_map_.at(input2_name); // num_units, input_size
|
||||
Shape output_dimen{input1_dimen[0], input2_dimen[0]};
|
||||
shape_map_[output_name] = output_dimen;
|
||||
|
||||
if (!shaper_finalized_) {
|
||||
shape_ops_.push_back(
|
||||
[input1_name, input2_name, output_name](Shaper& shaper) {
|
||||
shaper.FC(input1_name, input2_name, output_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void Shaper::Concat(const std::vector<std::string>& input_names,
|
||||
const int32_t axis,
|
||||
const std::string& output_name) {
|
||||
void Shaper::ConcatImpl(const std::vector<std::string>& input_names,
|
||||
const int32_t axis,
|
||||
const std::string& output_name) {
|
||||
std::vector<Shape> dimens;
|
||||
for (const auto& input_name : input_names) {
|
||||
const Shape& dimen = shape_map_.at(input_name);
|
||||
|
|
@ -323,18 +338,11 @@ void Shaper::Concat(const std::vector<std::string>& input_names,
|
|||
}
|
||||
|
||||
shape_map_[output_name] = output_dimen;
|
||||
|
||||
if (!shaper_finalized_) {
|
||||
shape_ops_.push_back(
|
||||
[input_names, axis, output_name](Shaper& shaper) {
|
||||
shaper.Concat(input_names, axis, output_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void Shaper::Squeeze(const std::string& input_name,
|
||||
const std::vector<int32_t>& axes,
|
||||
const std::string& output_name) {
|
||||
void Shaper::SqueezeImpl(const std::string& input_name,
|
||||
const std::vector<int32_t>& axes,
|
||||
const std::string& output_name) {
|
||||
const Shape& input_dimen = shape_map_.at(input_name);
|
||||
int32_t input_size = input_dimen.size();
|
||||
size_t axes_size = axes.size();
|
||||
|
|
@ -358,42 +366,30 @@ void Shaper::Squeeze(const std::string& input_name,
|
|||
}
|
||||
|
||||
shape_map_[output_name] = output_dimen;
|
||||
|
||||
if (!shaper_finalized_) {
|
||||
shape_ops_.push_back(
|
||||
[input_name, axes, output_name](Shaper& shaper) {
|
||||
shaper.Squeeze(input_name, axes, output_name);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void Shaper::AddShape(const std::string& name, const Shape& shape) {
|
||||
shape_map_[name] = shape;
|
||||
}
|
||||
|
||||
void Shaper::UpdateShape(const std::string& name, const Shape& new_shape) {
|
||||
ORT_ENFORCE(shaper_finalized_,
|
||||
"Cannot UpdateShape while shaper is not finalized");
|
||||
|
||||
Status Shaper::UpdateShape(const std::string& name, const Shape& new_shape) {
|
||||
const Shape& old_shape = shape_map_.at(name);
|
||||
if (old_shape != new_shape) {
|
||||
if (Product(old_shape) != 0)
|
||||
ORT_THROW("The shape should be same size or old shape has size 0 (dynamic shape)");
|
||||
ORT_RETURN_IF_NOT(Product(old_shape) == 0 || !old_shape.empty(),
|
||||
"The shape should be same size or old shape has size 0 (dynamic shape)");
|
||||
|
||||
shape_map_[name] = new_shape;
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void Shaper::UpdateDynamicDimensions() {
|
||||
ORT_ENFORCE(shaper_finalized_,
|
||||
"Cannot UpdateDynamicDimensions while shaper is not finalized");
|
||||
|
||||
for (auto& shape_op : shape_ops_)
|
||||
shape_op(*this);
|
||||
}
|
||||
|
||||
void Shaper::Clear() {
|
||||
shaper_finalized_ = false;
|
||||
shape_map_.clear();
|
||||
shape_ops_.clear();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,17 +56,44 @@ class Shaper {
|
|||
// If the shape of certain input is dynamic
|
||||
// Use the following 2 functions to update the particular shape
|
||||
// and calculate the new output shape
|
||||
void UpdateShape(const std::string& name, const Shape& new_shape);
|
||||
// Only perform this when the NNAPI model is finalized!
|
||||
Status UpdateShape(const std::string& name, const Shape& new_shape);
|
||||
void UpdateDynamicDimensions();
|
||||
|
||||
// Need to call Finalize() after the entire graph
|
||||
// is converted to NNAPI
|
||||
void Finalize() { shaper_finalized_ = true; }
|
||||
|
||||
void Clear();
|
||||
|
||||
private:
|
||||
bool shaper_finalized_{false};
|
||||
void ConvImpl(const std::string& input_name,
|
||||
const std::string& weight_name,
|
||||
const std::vector<int32_t>& onnx_pads,
|
||||
const std::vector<int32_t>& onnx_strides,
|
||||
const std::vector<int32_t>& onnx_dilations,
|
||||
bool nchw,
|
||||
const std::string& output_name);
|
||||
|
||||
void DepthwiseConvImpl(const std::string& input_name,
|
||||
const std::string& weight_name,
|
||||
const std::vector<int32_t>& onnx_pads,
|
||||
const std::vector<int32_t>& onnx_strides,
|
||||
const std::vector<int32_t>& onnx_dilations,
|
||||
bool nchw,
|
||||
const std::string& output_name);
|
||||
|
||||
void PoolImpl(const std::string& input_name,
|
||||
const std::vector<int32_t>& onnx_pads,
|
||||
const std::vector<int32_t>& onnx_strides,
|
||||
const std::vector<int32_t>& kernel_shape,
|
||||
bool nchw,
|
||||
const std::string& output_name);
|
||||
|
||||
void ReshapeImpl(const std::string& input_name, const std::vector<int32_t>& shape, const std::string& output_name);
|
||||
void TransposeImpl(const std::string& input_name, const std::vector<int32_t>& perm, const std::string& output_name);
|
||||
void EltwiseImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name);
|
||||
void IdentityImpl(const std::string& input_name, const std::string& output_name);
|
||||
void FCImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name);
|
||||
void ConcatImpl(const std::vector<std::string>& input_names, const int32_t axis, const std::string& output_name);
|
||||
void SqueezeImpl(const std::string& input, const std::vector<int32_t>& axes, const std::string& output);
|
||||
|
||||
std::unordered_map<std::string, Shape> shape_map_;
|
||||
std::vector<std::function<void(Shaper&)>> shape_ops_;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -15,12 +15,11 @@
|
|||
namespace onnxruntime {
|
||||
namespace nnapi {
|
||||
|
||||
#pragma region Model
|
||||
|
||||
Model::Model() : nnapi_(NnApiImplementation()) {}
|
||||
|
||||
Model::~Model() {
|
||||
if (execution_)
|
||||
nnapi_->ANeuralNetworksExecution_free(execution_);
|
||||
|
||||
nnapi_->ANeuralNetworksCompilation_free(compilation_);
|
||||
nnapi_->ANeuralNetworksModel_free(model_);
|
||||
}
|
||||
|
|
@ -53,11 +52,11 @@ const android::nn::wrapper::OperandType& Model::GetInputType(const std::string&
|
|||
return operand_types_.at(name);
|
||||
}
|
||||
|
||||
const android::nn::wrapper::OperandType Model::GetOutputType(const std::string& name) const {
|
||||
android::nn::wrapper::OperandType Model::GetOutputType(const std::string& name, const Execution& execution) const {
|
||||
const auto& nnapi_output_name = onnx_to_nnapi_output_map_.at(name);
|
||||
const auto& output_type = operand_types_.at(nnapi_output_name);
|
||||
android::nn::wrapper::OperandType type(
|
||||
output_type.type, shaper_for_exeuction_[nnapi_output_name], output_type.operandType.scale, output_type.operandType.zeroPoint);
|
||||
output_type.type, execution.GetShaper()[nnapi_output_name], output_type.operandType.scale, output_type.operandType.zeroPoint);
|
||||
|
||||
return type;
|
||||
}
|
||||
|
|
@ -78,77 +77,28 @@ size_t Model::GetMappedOutputIdx(const std::string& name) const {
|
|||
return output_map_.at(name);
|
||||
}
|
||||
|
||||
void Model::SetInputBuffer(const int32_t index, const InputBuffer& input) {
|
||||
PrepareForExecution();
|
||||
|
||||
THROW_ON_ERROR(nnapi_->ANeuralNetworksExecution_setInput(
|
||||
execution_, index, &input.type.operandType, input.buffer, input.type.GetOperandBlobByteSize()));
|
||||
bool Model::SupportsDynamicOutputShape() const {
|
||||
// dynamic output shape is only supported on Android API levle 29+
|
||||
return GetAndroidSdkVer() >= 29 && dynamic_output_buffer_size_ > 0;
|
||||
}
|
||||
|
||||
void Model::SetOutputBuffer(const int32_t index, const OutputBuffer& output) {
|
||||
PrepareForExecution();
|
||||
Status Model::PrepareForExecution(std::unique_ptr<Execution>& execution) {
|
||||
ORT_RETURN_IF_NOT(nullptr != compilation_,
|
||||
"Error in PrepareForExecution, compilation_ is null");
|
||||
|
||||
LOGS_DEFAULT(VERBOSE) << "Model::SetOutputBuffer, output shape "
|
||||
<< Shape2String(output.type.dimensions);
|
||||
ANeuralNetworksExecution* nnapi_execution;
|
||||
RETURN_STATUS_ON_ERROR(
|
||||
nnapi_->ANeuralNetworksExecution_create(compilation_, &nnapi_execution));
|
||||
|
||||
THROW_ON_ERROR(nnapi_->ANeuralNetworksExecution_setOutput(
|
||||
execution_, index, &output.type.operandType, output.buffer, output.type.GetOperandBlobByteSize()));
|
||||
execution.reset(new Execution(*nnapi_execution, shaper_));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void Model::PrepareForExecution() {
|
||||
if (prepared_for_exe_)
|
||||
return;
|
||||
|
||||
ORT_ENFORCE(nullptr != compilation_,
|
||||
"Error in PrepareForExecution, compilation_ is null");
|
||||
|
||||
// Copy the shaper for calculate the dynamic output shape
|
||||
// based on the input shape
|
||||
shaper_for_exeuction_ = shaper_;
|
||||
|
||||
THROW_ON_ERROR(
|
||||
nnapi_->ANeuralNetworksExecution_create(compilation_, &execution_));
|
||||
prepared_for_exe_ = true;
|
||||
int32_t Model::GetAndroidSdkVer() const {
|
||||
return nnapi_ ? nnapi_->android_sdk_version : 0;
|
||||
}
|
||||
|
||||
void Model::ResetExecution() {
|
||||
nnapi_->ANeuralNetworksExecution_free(execution_);
|
||||
execution_ = nullptr;
|
||||
shaper_for_exeuction_.Clear();
|
||||
prepared_for_exe_ = false;
|
||||
}
|
||||
|
||||
void Model::Predict() {
|
||||
PrepareForExecution();
|
||||
|
||||
ANeuralNetworksEvent* event = nullptr;
|
||||
THROW_ON_ERROR(nnapi_->ANeuralNetworksExecution_startCompute(execution_, &event));
|
||||
|
||||
THROW_ON_ERROR(nnapi_->ANeuralNetworksEvent_wait(event));
|
||||
|
||||
nnapi_->ANeuralNetworksEvent_free(event);
|
||||
|
||||
ResetExecution();
|
||||
}
|
||||
|
||||
void Model::SetInputBuffers(const std::vector<InputBuffer>& inputs) {
|
||||
PrepareForExecution();
|
||||
|
||||
for (size_t i = 0; i < inputs.size(); i++) {
|
||||
SetInputBuffer(i, inputs[i]);
|
||||
shaper_for_exeuction_.UpdateShape(input_names_[i], inputs[i].type.dimensions);
|
||||
}
|
||||
|
||||
shaper_for_exeuction_.UpdateDynamicDimensions();
|
||||
}
|
||||
|
||||
void Model::SetOutputBuffers(const std::vector<OutputBuffer>& outputs) {
|
||||
PrepareForExecution();
|
||||
|
||||
for (size_t i = 0; i < outputs.size(); i++) {
|
||||
SetOutputBuffer(i, outputs[i]);
|
||||
}
|
||||
}
|
||||
#pragma region Model::NNMemory
|
||||
|
||||
#ifdef USENNAPISHAREDMEM
|
||||
Model::NNMemory::NNMemory(const NnApi* nnapi, const char* name, size_t size) {
|
||||
|
|
@ -181,5 +131,81 @@ Model::NNMemory::NNMemory(const NnApi* /*nnapi*/, const char* name, size_t size)
|
|||
}
|
||||
#endif
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region Execution
|
||||
|
||||
Execution::Execution(ANeuralNetworksExecution& execution, const Shaper& shaper)
|
||||
: nnapi_(NnApiImplementation()),
|
||||
execution_(&execution),
|
||||
shaper_(shaper) {
|
||||
}
|
||||
|
||||
Execution::~Execution() {
|
||||
nnapi_->ANeuralNetworksExecution_free(execution_);
|
||||
}
|
||||
|
||||
Status Execution::SetInputBuffers(const std::vector<InputBuffer>& inputs) {
|
||||
for (size_t i = 0; i < inputs.size(); i++) {
|
||||
const auto& input(inputs[i]);
|
||||
ORT_RETURN_IF_ERROR(SetInputBuffer(i, input));
|
||||
ORT_RETURN_IF_ERROR(shaper_.UpdateShape(input.name, input.type.dimensions));
|
||||
}
|
||||
|
||||
shaper_.UpdateDynamicDimensions();
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Execution::SetOutputBuffers(const std::vector<OutputBuffer>& outputs) {
|
||||
for (size_t i = 0; i < outputs.size(); i++) {
|
||||
ORT_RETURN_IF_ERROR(SetOutputBuffer(i, outputs[i]));
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Execution::SetInputBuffer(const int32_t index, const InputBuffer& input) {
|
||||
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_setInput(
|
||||
execution_, index, &input.type.operandType, input.buffer, input.type.GetOperandBlobByteSize()));
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Execution::SetOutputBuffer(const int32_t index, const OutputBuffer& output) {
|
||||
LOGS_DEFAULT(VERBOSE) << "Model::SetOutputBuffer, output shape "
|
||||
<< Shape2String(output.type.dimensions);
|
||||
|
||||
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_setOutput(
|
||||
execution_, index, &output.type.operandType, output.buffer, output.buffer_byte_size));
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Execution::Predict(const std::vector<int32_t>& dynamic_outputs, std::vector<Shaper::Shape>& dynamic_output_shapes) {
|
||||
ANeuralNetworksEvent* event = nullptr;
|
||||
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_startCompute(execution_, &event));
|
||||
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksEvent_wait(event));
|
||||
nnapi_->ANeuralNetworksEvent_free(event);
|
||||
|
||||
dynamic_output_shapes.clear();
|
||||
dynamic_output_shapes.reserve(dynamic_outputs.size());
|
||||
for (const int32_t i : dynamic_outputs) {
|
||||
uint32_t output_rank = 0;
|
||||
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_getOutputOperandRank(execution_, i, &output_rank));
|
||||
|
||||
std::vector<uint32_t> output_shape(output_rank);
|
||||
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_getOutputOperandDimensions(execution_, i, output_shape.data()));
|
||||
|
||||
dynamic_output_shapes.push_back(output_shape);
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
#pragma endregion
|
||||
|
||||
} // namespace nnapi
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -12,20 +12,12 @@ namespace nnapi {
|
|||
|
||||
#define USENNAPISHAREDMEM 1
|
||||
|
||||
class Execution;
|
||||
|
||||
class Model {
|
||||
friend class ModelBuilder;
|
||||
|
||||
public:
|
||||
struct InputBuffer {
|
||||
const void* buffer{nullptr};
|
||||
android::nn::wrapper::OperandType type;
|
||||
};
|
||||
|
||||
struct OutputBuffer {
|
||||
void* buffer{nullptr};
|
||||
android::nn::wrapper::OperandType type;
|
||||
};
|
||||
|
||||
// Memory for persist data such as initializers and intermediate result
|
||||
#ifdef USENNAPISHAREDMEM
|
||||
// Use NNAPI shared memory
|
||||
|
|
@ -72,7 +64,7 @@ class Model {
|
|||
// Returns the data type and dimension of the given input/output
|
||||
// Please note the output type will have updated dimensions
|
||||
const android::nn::wrapper::OperandType& GetInputType(const std::string& name) const;
|
||||
const android::nn::wrapper::OperandType GetOutputType(const std::string& name) const;
|
||||
android::nn::wrapper::OperandType GetOutputType(const std::string& name, const Execution& execution) const;
|
||||
|
||||
// Set the mapping between input/output name and ORT kernel context
|
||||
// input/output index, at execution time
|
||||
|
|
@ -83,35 +75,43 @@ class Model {
|
|||
size_t GetMappedInputIdx(const std::string& name) const;
|
||||
size_t GetMappedOutputIdx(const std::string& name) const;
|
||||
|
||||
// Set the input/output data buffers
|
||||
// These need to be called before calling Predict()
|
||||
void SetInputBuffers(const std::vector<InputBuffer>& inputs);
|
||||
void SetOutputBuffers(const std::vector<OutputBuffer>& outputs);
|
||||
// If we support the dynamic output shape,
|
||||
// This is only for the case where output size cannot be determined at model execution time
|
||||
// Do not use this for case a determined output shape can be returned from GetOutputType()
|
||||
bool SupportsDynamicOutputShape() const;
|
||||
|
||||
// Execute the NNAPI model
|
||||
void Predict();
|
||||
// Set and Get the number of elements in the buffer for a dynamic output
|
||||
// If the buffer is not big enough, ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE will be returned by exection
|
||||
// Note: this will return number of elements of the buffer not the byte size of the buffer
|
||||
// and each output will have its separated buffer
|
||||
// TODO:
|
||||
// 1. Consider an adaptive aproach to automatically increase the buffer size if the execution reports
|
||||
// insufficient size
|
||||
// 2. Experiment with bigger initial buffer size (currently 1024)
|
||||
size_t GetDynamicOutputBufferSize() const { return dynamic_output_buffer_size_; }
|
||||
void SetDynamicOutputBufferSize(size_t size) { dynamic_output_buffer_size_ = size; }
|
||||
|
||||
// Mutex for exclusive lock to this model object
|
||||
OrtMutex& GetMutex() { return mutex_; }
|
||||
|
||||
Status PrepareForExecution(std::unique_ptr<Execution>& execution);
|
||||
|
||||
private:
|
||||
const NnApi* nnapi_{nullptr};
|
||||
bool prepared_for_exe_ = false;
|
||||
|
||||
ANeuralNetworksModel* model_{nullptr};
|
||||
ANeuralNetworksCompilation* compilation_{nullptr};
|
||||
ANeuralNetworksExecution* execution_{nullptr};
|
||||
|
||||
size_t dynamic_output_buffer_size_{1024};
|
||||
|
||||
std::unique_ptr<NNMemory> mem_initializers_;
|
||||
std::vector<std::unique_ptr<NNMemory>> mem_persist_buffers_;
|
||||
|
||||
std::vector<std::string> input_names_;
|
||||
std::vector<std::string> output_names_;
|
||||
std::unordered_map<std::string, android::nn::wrapper::OperandType>
|
||||
operand_types_;
|
||||
std::unordered_map<std::string, android::nn::wrapper::OperandType> operand_types_;
|
||||
|
||||
Shaper shaper_;
|
||||
Shaper shaper_for_exeuction_;
|
||||
|
||||
std::unordered_map<std::string, size_t> input_map_;
|
||||
std::unordered_map<std::string, size_t> output_map_;
|
||||
|
|
@ -133,11 +133,47 @@ class Model {
|
|||
|
||||
void SetShaper(const Shaper shaper) { shaper_ = shaper; }
|
||||
|
||||
void SetInputBuffer(const int32_t index, const InputBuffer& input);
|
||||
void SetOutputBuffer(const int32_t index, const OutputBuffer& output);
|
||||
int32_t GetAndroidSdkVer() const;
|
||||
};
|
||||
|
||||
void PrepareForExecution();
|
||||
void ResetExecution();
|
||||
class Execution {
|
||||
public:
|
||||
struct InputBuffer {
|
||||
const std::string& name;
|
||||
const void* buffer{nullptr};
|
||||
android::nn::wrapper::OperandType type;
|
||||
};
|
||||
|
||||
struct OutputBuffer {
|
||||
void* buffer{nullptr};
|
||||
android::nn::wrapper::OperandType type;
|
||||
size_t buffer_byte_size;
|
||||
};
|
||||
|
||||
public:
|
||||
explicit Execution(ANeuralNetworksExecution& execution, const Shaper& shaper);
|
||||
~Execution();
|
||||
Execution(const Execution&) = delete;
|
||||
Execution& operator=(const Execution&) = delete;
|
||||
|
||||
const Shaper& GetShaper() const { return shaper_; }
|
||||
|
||||
// Set the input/output data buffers
|
||||
// These need to be called before calling Predict()
|
||||
Status SetInputBuffers(const std::vector<InputBuffer>& inputs);
|
||||
Status SetOutputBuffers(const std::vector<OutputBuffer>& outputs);
|
||||
|
||||
// Execute the NNAPI model
|
||||
// if there is dynamic output shape, will output the actual output shapes
|
||||
Status Predict(const std::vector<int32_t>& dynamic_outputs, std::vector<Shaper::Shape>& dynamic_output_shapes);
|
||||
|
||||
private:
|
||||
Status SetInputBuffer(const int32_t index, const InputBuffer& input);
|
||||
Status SetOutputBuffer(const int32_t index, const OutputBuffer& output);
|
||||
|
||||
const NnApi* nnapi_{nullptr};
|
||||
ANeuralNetworksExecution* execution_;
|
||||
Shaper shaper_;
|
||||
};
|
||||
|
||||
} // namespace nnapi
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
#include "core/framework/compute_capability.h"
|
||||
#include "core/graph/model.h"
|
||||
#include "core/session/onnxruntime_cxx_api.h"
|
||||
#include "core/session/inference_session.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
@ -175,6 +174,40 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view
|
|||
return result;
|
||||
}
|
||||
|
||||
static Status GetOutputBuffer(Ort::CustomOpApi& ort,
|
||||
OrtKernelContext* context,
|
||||
const nnapi::Model& model,
|
||||
const std::string& output_name,
|
||||
const std::vector<uint32_t>& output_shape,
|
||||
const android::nn::wrapper::Type output_type,
|
||||
void** output_buffer) {
|
||||
using namespace android::nn::wrapper;
|
||||
std::vector<int64_t> int64_output_shape(output_shape.begin(),
|
||||
output_shape.end());
|
||||
auto output_idx = model.GetMappedOutputIdx(output_name);
|
||||
auto* output_tensor = ort.KernelContext_GetOutput(context, output_idx,
|
||||
int64_output_shape.data(),
|
||||
int64_output_shape.size());
|
||||
|
||||
switch (output_type) {
|
||||
case Type::TENSOR_FLOAT32:
|
||||
*output_buffer = ort.GetTensorMutableData<float>(output_tensor);
|
||||
break;
|
||||
case Type::TENSOR_INT32:
|
||||
*output_buffer = ort.GetTensorMutableData<int32_t>(output_tensor);
|
||||
break;
|
||||
case Type::TENSOR_QUANT8_ASYMM:
|
||||
*output_buffer = ort.GetTensorMutableData<uint8_t>(output_tensor);
|
||||
break;
|
||||
default:
|
||||
return Status(common::ONNXRUNTIME, common::FAIL,
|
||||
"Unsupported output type: " + TypeToStr(output_type));
|
||||
break;
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
common::Status NnapiExecutionProvider::Compile(const std::vector<onnxruntime::Node*>& fused_nodes,
|
||||
std::vector<NodeComputeInfo>& node_compute_funcs) {
|
||||
using namespace android::nn::wrapper;
|
||||
|
|
@ -225,7 +258,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<onnxruntime::No
|
|||
};
|
||||
|
||||
compute_info.release_state_func = [](FunctionState state) {
|
||||
// the `state` is a dnn::model managed by unique_ptr
|
||||
// the `state` is a nnapi::model managed by unique_ptr
|
||||
ORT_UNUSED_PARAMETER(state);
|
||||
};
|
||||
|
||||
|
|
@ -234,13 +267,16 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<onnxruntime::No
|
|||
nnapi::Model* model = reinterpret_cast<nnapi::Model*>(state);
|
||||
const size_t num_inputs = ort.KernelContext_GetInputCount(context);
|
||||
const size_t num_outputs = ort.KernelContext_GetOutputCount(context);
|
||||
ORT_ENFORCE(model->GetInputs().size() <= num_inputs, "Inconsistent input sizes");
|
||||
ORT_ENFORCE(model->GetOutputs().size() == num_outputs, "Inconsistent output sizes");
|
||||
const auto& model_inputs = model->GetInputs();
|
||||
const auto& model_outputs = model->GetOutputs();
|
||||
|
||||
std::vector<nnapi::Model::InputBuffer> inputs;
|
||||
inputs.reserve(model->GetInputs().size());
|
||||
for (size_t i = 0; i < model->GetInputs().size(); i++) {
|
||||
const auto& input_name = model->GetInputs()[i];
|
||||
ORT_ENFORCE(model_inputs.size() <= num_inputs, "Inconsistent input sizes");
|
||||
ORT_ENFORCE(model_outputs.size() == num_outputs, "Inconsistent output sizes");
|
||||
|
||||
std::vector<nnapi::Execution::InputBuffer> inputs;
|
||||
inputs.reserve(model_inputs.size());
|
||||
for (size_t i = 0; i < model_inputs.size(); i++) {
|
||||
const auto& input_name = model_inputs[i];
|
||||
const auto& model_input_type = model->GetInputType(input_name);
|
||||
|
||||
auto input_idx = model->GetMappedInputIdx(input_name);
|
||||
|
|
@ -250,73 +286,112 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<onnxruntime::No
|
|||
for (const auto& dim : ort.GetTensorShape(tensor_info))
|
||||
dimensions.push_back(static_cast<uint32_t>(dim));
|
||||
|
||||
// NNAPI has strict input type requirements which separates tensor inputs and scalar inputs
|
||||
// For ONNX the we do not have clear line between scalar inputs and tensor inputs
|
||||
// Also NNAPI treats a tensor input with empty shape as dynamic shape input
|
||||
// Disable support of the scalar input (tensor input with an empty shape) for now
|
||||
// TODO, add support for ONNX scalar input (tensor input with an empty shape)
|
||||
if (dimensions.empty())
|
||||
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "NNAPI does not support scalar input");
|
||||
|
||||
// it is possible that the input has the detailed size while
|
||||
// the model has an operand with unknown size, use the size
|
||||
// of the actual input
|
||||
OperandType type(model_input_type.type, dimensions,
|
||||
model_input_type.operandType.scale,
|
||||
model_input_type.operandType.zeroPoint);
|
||||
OperandType input_type = model_input_type;
|
||||
input_type.SetDimensions(dimensions);
|
||||
|
||||
if (type.dimensions != model_input_type.dimensions && model_input_type.GetOperandBlobByteSize() != 0) {
|
||||
if (input_type.GetOperandBlobByteSize() == 0)
|
||||
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "The actual input cannot have 0 dim (dynamic)");
|
||||
|
||||
if (input_type.dimensions != model_input_type.dimensions && model_input_type.GetOperandBlobByteSize() != 0) {
|
||||
return Status(common::ONNXRUNTIME, common::FAIL,
|
||||
"The actual input dimanesions should match the model input "
|
||||
"dimensions, or model input dimension has 0 (dynamic)");
|
||||
}
|
||||
|
||||
const void* inputBuffer = ort.GetTensorData<void>(input_tensor);
|
||||
inputs.push_back({inputBuffer, std::move(type)});
|
||||
inputs.push_back({input_name, inputBuffer, std::move(input_type)});
|
||||
|
||||
ort.ReleaseTensorTypeAndShapeInfo(tensor_info);
|
||||
}
|
||||
|
||||
// From this point we will need to take the exclusive lock on the model until the Predict is
|
||||
// performed, to block other threads (if any) to modify this particular model
|
||||
// performed, to block other threads to perform Predict on the same model
|
||||
// TODO, investigate concurrent runs for different executions from the same model
|
||||
{
|
||||
std::unique_ptr<nnapi::Execution> execution;
|
||||
std::unique_lock<OrtMutex> lock(model->GetMutex());
|
||||
model->SetInputBuffers(inputs);
|
||||
std::vector<nnapi::Model::OutputBuffer> outputs;
|
||||
model->PrepareForExecution(execution);
|
||||
|
||||
ORT_RETURN_IF_ERROR(execution->SetInputBuffers(inputs));
|
||||
std::vector<nnapi::Execution::OutputBuffer> outputs;
|
||||
outputs.reserve(num_outputs);
|
||||
std::vector<int32_t> dynamic_shape_output_indices;
|
||||
std::vector<OperandType> dynamic_shape_output_types;
|
||||
std::vector<std::unique_ptr<uint8_t[]>> dynamic_shape_output_buffers;
|
||||
for (size_t i = 0; i < num_outputs; i++) {
|
||||
const auto output_name = model->GetOutputs()[i];
|
||||
const auto model_output_type = model->GetOutputType(output_name);
|
||||
const auto output_name = model_outputs[i];
|
||||
const auto model_output_type = model->GetOutputType(output_name, *execution);
|
||||
const auto output_shape = model_output_type.dimensions;
|
||||
|
||||
std::vector<int64_t> int64_output_shape(output_shape.begin(),
|
||||
output_shape.end());
|
||||
auto output_idx = model->GetMappedOutputIdx(output_name);
|
||||
auto* output_tensor = ort.KernelContext_GetOutput(context, output_idx,
|
||||
int64_output_shape.data(),
|
||||
int64_output_shape.size());
|
||||
bool is_dynamic_shape_output = false;
|
||||
if (model_output_type.GetOperandBlobByteSize() == 0) {
|
||||
if (!model->SupportsDynamicOutputShape()) {
|
||||
return Status(common::ONNXRUNTIME, common::FAIL,
|
||||
"We do not support dynamic output shape or empty output for now");
|
||||
}
|
||||
|
||||
is_dynamic_shape_output = true;
|
||||
}
|
||||
|
||||
void* output_buffer = nullptr;
|
||||
switch (model_output_type.type) {
|
||||
case Type::TENSOR_FLOAT32:
|
||||
output_buffer = ort.GetTensorMutableData<float>(output_tensor);
|
||||
break;
|
||||
case Type::TENSOR_INT32:
|
||||
output_buffer = ort.GetTensorMutableData<int32_t>(output_tensor);
|
||||
break;
|
||||
case Type::TENSOR_QUANT8_ASYMM:
|
||||
output_buffer = ort.GetTensorMutableData<uint8_t>(output_tensor);
|
||||
break;
|
||||
default:
|
||||
return Status(common::ONNXRUNTIME, common::FAIL,
|
||||
"Unsupported output type: " + TypeToStr(model_output_type.type));
|
||||
break;
|
||||
size_t output_buffer_byte_size;
|
||||
if (!is_dynamic_shape_output) {
|
||||
ORT_RETURN_IF_ERROR(GetOutputBuffer(ort, context,
|
||||
*model,
|
||||
output_name, output_shape, model_output_type.type,
|
||||
&output_buffer));
|
||||
output_buffer_byte_size = model_output_type.GetOperandBlobByteSize();
|
||||
} else {
|
||||
// This output is dynamic (size unknown), will need allocate a buffer for the result
|
||||
// and copy the content to ORT output tensors afte the execution (will know output shape after the execution)
|
||||
output_buffer_byte_size = model->GetDynamicOutputBufferSize() * model_output_type.GetElementByteSize();
|
||||
std::unique_ptr<uint8_t[]> buffer_holder(new uint8_t[output_buffer_byte_size]);
|
||||
output_buffer = buffer_holder.get();
|
||||
dynamic_shape_output_types.push_back(model_output_type);
|
||||
dynamic_shape_output_indices.push_back(static_cast<int32_t>(i));
|
||||
dynamic_shape_output_buffers.push_back(std::move(buffer_holder));
|
||||
}
|
||||
|
||||
if (model_output_type.GetOperandBlobByteSize() == 0) {
|
||||
return Status(common::ONNXRUNTIME, common::FAIL,
|
||||
"We do not support dynamic output shape or empty output for now");
|
||||
}
|
||||
|
||||
outputs.push_back({output_buffer, std::move(model_output_type)});
|
||||
outputs.push_back({output_buffer, std::move(model_output_type), output_buffer_byte_size});
|
||||
}
|
||||
|
||||
model->SetOutputBuffers(outputs);
|
||||
model->Predict();
|
||||
}
|
||||
ORT_RETURN_IF_ERROR(execution->SetOutputBuffers(outputs));
|
||||
std::vector<std::vector<uint32_t>> dynamic_output_shapes;
|
||||
ORT_RETURN_IF_ERROR(
|
||||
execution->Predict(dynamic_shape_output_indices, dynamic_output_shapes));
|
||||
|
||||
// We have dynamic output buffers, need to copy the content from temp buffers to ORT output tensors
|
||||
for (size_t i = 0; i < dynamic_shape_output_indices.size(); i++) {
|
||||
const int32_t model_output_idx = dynamic_shape_output_indices[i];
|
||||
const auto output_name = model_outputs[model_output_idx];
|
||||
|
||||
const auto& output_shape = dynamic_output_shapes[i];
|
||||
auto model_output_type = dynamic_shape_output_types[i];
|
||||
model_output_type.SetDimensions(output_shape);
|
||||
ORT_RETURN_IF_NOT(model_output_type.GetOperandBlobByteSize() != 0, "We do not support 0 size output for now");
|
||||
|
||||
void* model_output_buffer = dynamic_shape_output_buffers[i].get();
|
||||
void* onnx_output_buffer = nullptr;
|
||||
ORT_RETURN_IF_ERROR(GetOutputBuffer(ort, context,
|
||||
*model,
|
||||
output_name, output_shape, model_output_type.type,
|
||||
&onnx_output_buffer));
|
||||
|
||||
size_t output_buffer_byte_size = model_output_type.GetOperandBlobByteSize();
|
||||
memcpy(onnx_output_buffer, model_output_buffer, output_buffer_byte_size);
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
#pragma once
|
||||
|
||||
#include "core/framework/execution_provider.h"
|
||||
#include "core/graph/onnx_protobuf.h"
|
||||
#include "core/providers/nnapi/nnapi_builtin/model.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
|
|
|||
|
|
@ -21,18 +21,8 @@ namespace android {
|
|||
namespace nn {
|
||||
namespace wrapper {
|
||||
|
||||
bool IsScalarType(const Type& type) {
|
||||
return type == Type::FLOAT16 || type == Type::FLOAT32 || type == Type::INT32 || type == Type::BOOL || type == Type::UINT32;
|
||||
}
|
||||
|
||||
OperandType::OperandType(Type type, const std::vector<uint32_t>& d, float scale, int32_t zeroPoint)
|
||||
: type(type), dimensions(d) {
|
||||
if (dimensions.empty()) {
|
||||
if (!IsScalarType(type)) {
|
||||
dimensions = {1};
|
||||
}
|
||||
}
|
||||
|
||||
operandType = {
|
||||
.type = static_cast<int32_t>(type),
|
||||
.dimensionCount = static_cast<uint32_t>(dimensions.size()),
|
||||
|
|
@ -45,13 +35,6 @@ OperandType::OperandType(Type type, const std::vector<uint32_t>& d, float scale,
|
|||
OperandType::OperandType(const OperandType& other) {
|
||||
type = other.type;
|
||||
dimensions = other.dimensions;
|
||||
|
||||
if (dimensions.empty()) {
|
||||
if (!IsScalarType(type)) {
|
||||
dimensions = {1};
|
||||
}
|
||||
}
|
||||
|
||||
operandType = other.operandType;
|
||||
operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr;
|
||||
}
|
||||
|
|
@ -60,13 +43,6 @@ OperandType& OperandType::operator=(const OperandType& other) {
|
|||
if (this != &other) {
|
||||
type = other.type;
|
||||
dimensions = other.dimensions;
|
||||
|
||||
if (dimensions.empty()) {
|
||||
if (!IsScalarType(type)) {
|
||||
dimensions = {1};
|
||||
}
|
||||
}
|
||||
|
||||
operandType = other.operandType;
|
||||
operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr;
|
||||
}
|
||||
|
|
@ -115,11 +91,6 @@ size_t OperandType::GetOperandBlobByteSize() const {
|
|||
|
||||
void OperandType::SetDimensions(const std::vector<uint32_t>& d) {
|
||||
dimensions = d;
|
||||
if (dimensions.empty()) {
|
||||
if (!IsScalarType(type)) {
|
||||
dimensions = {1};
|
||||
}
|
||||
}
|
||||
operandType.dimensionCount = dimensions.size();
|
||||
operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -193,7 +193,7 @@ TEST(MathOpTest, Add_Broadcast_0x1) {
|
|||
test.AddInput<float>("A", {}, {10.0f});
|
||||
test.AddInput<float>("B", {1}, {2.0f});
|
||||
test.AddOutput<float>("C", {1}, {12.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider}); // NNAPI: Add does not support scalar input
|
||||
}
|
||||
|
||||
TEST(MathOpTest, Add_Broadcast_1x0) {
|
||||
|
|
@ -202,7 +202,7 @@ TEST(MathOpTest, Add_Broadcast_1x0) {
|
|||
test.AddInput<float>("A", {1}, {10.0f});
|
||||
test.AddInput<float>("B", {}, {2.0f});
|
||||
test.AddOutput<float>("C", {1}, {12.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider}); // NNAPI: Add does not support scalar input
|
||||
}
|
||||
|
||||
TEST(MathOpTest, Add_Broadcast_1x1) {
|
||||
|
|
@ -369,7 +369,7 @@ TEST(MathOpTest, Sub_Broadcast_Scalar) {
|
|||
{-4.0f, -3.0f, -6.0f,
|
||||
-5.0f, -3.5f, -105.0f,
|
||||
-10.4f, 4.3f, -10005.0f});
|
||||
test.Run();
|
||||
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider}); // NNAPI: Sub does not support scalar input
|
||||
}
|
||||
|
||||
TEST(MathOpTest, Mul_int32) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue