mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Fix MaxUnpool behaviour when output_shape is provided and doesn't match the inferred shape (#2193)
* MaxUnpool should reconstitute what was pooled by MaxPool. The kernel_shape, pads and strides inputs are purely to infer an output shape, if output_shape is not explicitly provided. The unpool should not be adding new padding, so output_shape is not about auto generating pad values and inserting pads. The current ORT implementation misinterprets the usage of output_shape and inserts pads instead of just allocating an output of the specified shape, and directly unpooling into it. Update to simply find the correct output shape to use, and simply unpool into that. Update unit tests to reflect this. * Exclude maxunpool_export_with_output_shape which has invalid data in the output. * Fix test name in backend test series exclusion
This commit is contained in:
Scott McKay
Pranav Sharma
parent
6699c19010
commit
b6b44c90ac
5 changed files with 83 additions and 180 deletions
|
|
@ -53,151 +53,49 @@ Status MaxUnpool::Compute(OpKernelContext* context) const {
|
|||
ORT_RETURN_IF_NOT(I_shape == X_shape, "Index tensor shape should be same as that of the input data tensor to unpool.");
|
||||
|
||||
// Calculate output tensor shape from attributes
|
||||
std::vector<int64_t> inferredOutputShape(X_shape.NumDimensions());
|
||||
std::vector<int64_t> inferred_output_dims(X_shape.NumDimensions());
|
||||
|
||||
// Copy batch and channel dims
|
||||
inferredOutputShape[0] = X_shape[0];
|
||||
inferredOutputShape[1] = X_shape[1];
|
||||
inferred_output_dims[0] = X_shape[0];
|
||||
inferred_output_dims[1] = X_shape[1];
|
||||
|
||||
// For feature dims calculate reversing the formula used for Maxpool
|
||||
// For feature dims calculate reversing the formula used for MaxPool
|
||||
for (size_t dim = 0; dim < kernel_shape_.size(); ++dim) {
|
||||
inferredOutputShape[dim + 2] = (X_shape[dim + 2] - 1) * strides_[dim] - (pads_[dim + 2] + pads_[kernel_shape_.size() + dim + 4]) + kernel_shape_[dim];
|
||||
inferred_output_dims[dim + 2] =
|
||||
(X_shape[dim + 2] - 1) * strides_[dim] - (pads_[dim] + pads_[kernel_shape_.size() + dim]) + kernel_shape_[dim];
|
||||
}
|
||||
|
||||
// If outputshape is provided use that to infer additional padding.
|
||||
std::vector<int64_t> inferredPads;
|
||||
std::vector<int64_t> givenOutputShape;
|
||||
bool padsInferred = false;
|
||||
TensorShape shape(inferred_output_dims);
|
||||
|
||||
if (num_inputs_ == 3) {
|
||||
auto tensor_shape = context->Input<Tensor>(2);
|
||||
if (tensor_shape == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch");
|
||||
ORT_RETURN_IF_NOT(tensor_shape->Shape().GetDims().size() == 1, "Shape must be 1 dimensional as it's tensor data is a shape");
|
||||
if (tensor_shape == nullptr)
|
||||
return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch");
|
||||
ORT_RETURN_IF_NOT(tensor_shape->Shape().GetDims().size() == 1,
|
||||
"Shape must be 1 dimensional as it's tensor data of a shape");
|
||||
|
||||
// Turn the shape tensor data into an actual shape
|
||||
const auto* p_shape = tensor_shape->template Data<int64_t>();
|
||||
std::vector<int64_t> shape{p_shape, p_shape + tensor_shape->Shape().Size()};
|
||||
givenOutputShape = shape;
|
||||
std::vector<int64_t> given_output_dims(p_shape, p_shape + tensor_shape->Shape().Size());
|
||||
TensorShape given_shape(given_output_dims);
|
||||
|
||||
inferredPads.resize(inferredOutputShape.size() * 2, 0);
|
||||
ORT_RETURN_IF_NOT(given_shape.Size() >= shape.Size(),
|
||||
"output_shape is smaller than minimum required. output_shape:", given_shape,
|
||||
" inferred output shape:", shape);
|
||||
|
||||
// calculate if output shape has any padding over the inferred shape for feature dims.
|
||||
for (size_t dim = 2; dim < shape.size(); dim++) {
|
||||
ORT_RETURN_IF_NOT(inferredOutputShape[dim] <= shape[dim], "Incorrect output shape");
|
||||
|
||||
int64_t inferredPad = shape[dim] - inferredOutputShape[dim];
|
||||
ORT_RETURN_IF_NOT(inferredPad <= kernel_shape_[dim - 2], "Incorrect output shape");
|
||||
|
||||
if (inferredPad > 0) {
|
||||
padsInferred = true;
|
||||
if (inferredPad == kernel_shape_[dim - 2]) {
|
||||
inferredPads[dim] = 1;
|
||||
inferredPads[dim + inferredOutputShape.size()] = inferredPad - 1;
|
||||
} else {
|
||||
inferredPads[dim + inferredOutputShape.size()] = inferredPad;
|
||||
}
|
||||
}
|
||||
}
|
||||
shape = std::move(given_shape);
|
||||
}
|
||||
|
||||
// unpool
|
||||
int64_t totalPooledElem = 1;
|
||||
int64_t totalOutputElem = 1;
|
||||
int64_t total_elements = X_shape.Size();
|
||||
|
||||
for (size_t dim = 0; dim < X_shape.NumDimensions(); dim++) {
|
||||
totalPooledElem *= X_shape[dim];
|
||||
totalOutputElem *= inferredOutputShape[dim];
|
||||
}
|
||||
Tensor* Y = context->Output(0, shape);
|
||||
auto* Y_data = Y->template MutableData<float>();
|
||||
auto out = gsl::make_span(Y_data, Y->Shape().Size());
|
||||
std::fill_n(out.data(), out.size(), 0.f);
|
||||
|
||||
// if there are no pads inferred from outputshape simply create the new unpooled tensor
|
||||
if (!padsInferred) {
|
||||
TensorShape shape(inferredOutputShape);
|
||||
|
||||
Tensor* Y = context->Output(0, shape);
|
||||
auto Y_data = Y->template MutableData<float>();
|
||||
auto out = gsl::make_span(Y_data, Y->Shape().Size());
|
||||
std::fill_n(out.data(), out.size(), 0.f);
|
||||
|
||||
for (auto curElem = 0; curElem < totalPooledElem; ++curElem) {
|
||||
out[I_data[curElem]] = X_data[curElem];
|
||||
}
|
||||
} else {
|
||||
// If the output shape has pads over the inferred dims , first
|
||||
// create the tensor with the inferred dims and add the padding.
|
||||
|
||||
// Generate tensor with inferred dims.
|
||||
TensorShape shape(inferredOutputShape);
|
||||
|
||||
AllocatorPtr alloc;
|
||||
ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc));
|
||||
auto element_type = DataTypeImpl::GetType<float>();
|
||||
|
||||
std::unique_ptr<Tensor> p_tensor = onnxruntime::make_unique<Tensor>(element_type,
|
||||
shape,
|
||||
alloc);
|
||||
|
||||
auto* p = p_tensor->template MutableData<float>();
|
||||
|
||||
auto out = gsl::make_span(p, p_tensor->Shape().Size());
|
||||
std::fill_n(out.data(), out.size(), 0.f);
|
||||
|
||||
for (auto curElem = 0; curElem < totalPooledElem; ++curElem) {
|
||||
out[I_data[curElem]] = X_data[curElem];
|
||||
}
|
||||
|
||||
std::vector<int64_t> output_dims(inferredOutputShape);
|
||||
size_t dimension_count = output_dims.size();
|
||||
|
||||
std::vector<int64_t> input_starts;
|
||||
std::vector<int64_t> input_extents;
|
||||
|
||||
// Calculate output dimensions
|
||||
for (size_t i = 0; i < dimension_count; i++) {
|
||||
input_starts.push_back(slices_[i]);
|
||||
input_extents.push_back(output_dims[i] + slices_[i] + slices_[i + dimension_count]);
|
||||
output_dims[i] += inferredPads[i] + inferredPads[i + dimension_count] + slices_[i] + slices_[i + dimension_count];
|
||||
}
|
||||
|
||||
// setup output object
|
||||
TensorShape output_shape(givenOutputShape);
|
||||
Tensor* Y = context->Output(0, output_shape);
|
||||
auto Y_data = Y->template MutableData<float>();
|
||||
|
||||
auto outData = gsl::make_span(Y_data, Y->Shape().Size());
|
||||
|
||||
std::fill_n(outData.data(), outData.size(), 0.f);
|
||||
|
||||
// add padding
|
||||
TensorPitches output_pitches(*Y);
|
||||
size_t alignSkip = 0; // Amount to skip to align to where the next input tensor data needs to be written
|
||||
|
||||
// Initial skip, sum up the begin padding on each axis
|
||||
for (size_t i = 0; i < dimension_count; i++)
|
||||
alignSkip += inferredPads[i] * output_pitches[i];
|
||||
|
||||
size_t inner_axis = dimension_count - 1;
|
||||
|
||||
TensorAxisCounters input_counters(*p_tensor);
|
||||
SliceIterator<float> input(*p_tensor, input_starts, input_extents, {});
|
||||
|
||||
while (input_counters) {
|
||||
Y_data += alignSkip;
|
||||
{
|
||||
Y_data = input.CopyInnermostAxisSolitaryInnerStep(Y_data);
|
||||
int64_t prePad = inferredPads[inner_axis];
|
||||
int64_t postPad = inferredPads[inner_axis + dimension_count];
|
||||
Y_data += postPad;
|
||||
alignSkip = prePad;
|
||||
}
|
||||
// Calculate the size of the next block of padding (skipping over the innermost axis since that's already done)
|
||||
while (input_counters.Increment()) {
|
||||
ptrdiff_t inner_pitch = output_pitches[input_counters.Axis()];
|
||||
int64_t prePad = inferredPads[input_counters.Axis()];
|
||||
int64_t postPad = inferredPads[input_counters.Axis() + dimension_count];
|
||||
Y_data += inner_pitch * postPad;
|
||||
alignSkip += inner_pitch * prePad;
|
||||
}
|
||||
}
|
||||
for (auto cur_elem = 0; cur_elem < total_elements; ++cur_elem) {
|
||||
out[I_data[cur_elem]] = X_data[cur_elem];
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
|
|
|
|||
|
|
@ -38,19 +38,6 @@ class MaxUnpool : public OpKernel {
|
|||
}
|
||||
|
||||
ORT_ENFORCE(strides_.size() == kernel_shape_.size());
|
||||
|
||||
// Add 4 pad values (0) for batch and channel dimensions
|
||||
pads_.insert(pads_.begin(), {0, 0});
|
||||
pads_.insert(pads_.begin() + 2 + kernel_shape_.size(), {0, 0});
|
||||
|
||||
// Separate out any negative pads_ into the slices_ array
|
||||
slices_.resize(pads_.size(), 0);
|
||||
for (size_t index = 0; index < pads_.size(); index++) {
|
||||
if (pads_[index] < 0) {
|
||||
slices_[index] = pads_[index];
|
||||
pads_[index] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
~MaxUnpool() override = default;
|
||||
|
|
@ -61,7 +48,6 @@ class MaxUnpool : public OpKernel {
|
|||
std::vector<int64_t> kernel_shape_;
|
||||
std::vector<int64_t> pads_;
|
||||
std::vector<int64_t> strides_;
|
||||
std::vector<int64_t> slices_; // All of the negative padding values are separated out into slices_
|
||||
int64_t num_inputs_;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -450,6 +450,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
{"bitshift_left_uint16", "BitShift(11) uint16 support not enabled currently"},
|
||||
{"reflect_pad", "test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests"},
|
||||
{"edge_pad", "test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests"},
|
||||
{"maxunpool_export_with_output_shape", "Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398" },
|
||||
};
|
||||
|
||||
#ifdef USE_NGRAPH
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ TEST(UnpoolTest, MaxUnPool3D) {
|
|||
test.AddAttribute("strides", std::vector<int64_t>{2, 2, 2});
|
||||
test.AddAttribute("kernel_shape", vector<int64_t>{2, 2, 2});
|
||||
|
||||
// NOTE: This input doesn't make sense as MaxPool output, but strictly speaking it doesn't need to be
|
||||
std::vector<float> t_vals = {1, 2, 3, 4, 5, 6, 7, 8};
|
||||
std::vector<int64_t> t_dims = {1, 1, 2, 2, 2};
|
||||
|
||||
|
|
@ -274,7 +275,7 @@ TEST(UnpoolTest, MaxUnPool3D_Padding) {
|
|||
test.Run();
|
||||
}
|
||||
|
||||
TEST(UnpoolTest, MaxUnPool1D_WithPaddedOutput) {
|
||||
TEST(UnpoolTest, MaxUnPool1D_WithOutputShape) {
|
||||
OpTester test("MaxUnpool", 9);
|
||||
|
||||
test.AddAttribute("strides", std::vector<int64_t>{2});
|
||||
|
|
@ -286,20 +287,20 @@ TEST(UnpoolTest, MaxUnPool1D_WithPaddedOutput) {
|
|||
std::vector<int64_t> i_vals = {1, 3, 4, 6};
|
||||
std::vector<int64_t> i_dims = {1, 1, 4};
|
||||
|
||||
std::vector<int64_t> expected_dims = {1, 1, 10};
|
||||
std::vector<float> expected_vals = {0, 0, 1, 0, 2, 3, 0, 4, 0, 0};
|
||||
std::vector<int64_t> expected_dims = {1, 1, 9};
|
||||
std::vector<float> expected_vals = {0, 1, 0, 2, 3, 0, 4, 0, 0};
|
||||
|
||||
std::vector<int64_t> inputDims = {3};
|
||||
std::vector<int64_t> expected_dim_size = {3};
|
||||
|
||||
test.AddInput<float>("xT", t_dims, t_vals);
|
||||
test.AddInput<int64_t>("xI", i_dims, i_vals);
|
||||
test.AddInput<int64_t>("output_shape", inputDims, expected_dims);
|
||||
test.AddInput<int64_t>("output_shape", expected_dim_size, expected_dims);
|
||||
|
||||
test.AddOutput<float>("Y", expected_dims, expected_vals);
|
||||
test.Run();
|
||||
}
|
||||
|
||||
TEST(UnpoolTest, MaxUnPool2D_WithPaddedOutput) {
|
||||
TEST(UnpoolTest, MaxUnPool2D_WithOutputShape) {
|
||||
OpTester test("MaxUnpool", 9);
|
||||
|
||||
test.AddAttribute("strides", std::vector<int64_t>{2, 2});
|
||||
|
|
@ -308,7 +309,7 @@ TEST(UnpoolTest, MaxUnPool2D_WithPaddedOutput) {
|
|||
std::vector<float> t_vals = {1, 2, 3, 4};
|
||||
std::vector<int64_t> t_dims = {1, 1, 2, 2};
|
||||
|
||||
std::vector<int64_t> i_vals = {1, 3, 8, 10};
|
||||
std::vector<int64_t> i_vals = {1, 3, 10, 12};
|
||||
std::vector<int64_t> i_dims = {1, 1, 2, 2};
|
||||
|
||||
std::vector<int64_t> expected_dims = {1, 1, 5, 5};
|
||||
|
|
@ -319,60 +320,76 @@ TEST(UnpoolTest, MaxUnPool2D_WithPaddedOutput) {
|
|||
0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0};
|
||||
|
||||
std::vector<int64_t> inputDims = {4};
|
||||
std::vector<int64_t> expected_dims_size = {4};
|
||||
|
||||
test.AddInput<float>("xT", t_dims, t_vals);
|
||||
test.AddInput<int64_t>("xI", i_dims, i_vals);
|
||||
test.AddInput<int64_t>("output_shape", inputDims, expected_dims);
|
||||
test.AddInput<int64_t>("output_shape", expected_dims_size, expected_dims);
|
||||
|
||||
test.AddOutput<float>("Y", expected_dims, expected_vals);
|
||||
test.Run();
|
||||
}
|
||||
|
||||
TEST(UnpoolTest, MaxUnPool3D_WithPaddedOutput) {
|
||||
TEST(UnpoolTest, MaxUnPool3D_WithOutputShape) {
|
||||
OpTester test("MaxUnpool", 9);
|
||||
// original input 1, 1, 3, 3, 3
|
||||
// with these strides and kernel shape there should only be one value
|
||||
/* Python to check the MaxPool output that is the theoretical input to the MaxUnpool in this test.
|
||||
import onnx
|
||||
from onnx import helper, numpy_helper
|
||||
from onnx import AttributeProto, TensorProto, GraphProto
|
||||
import numpy as np
|
||||
import onnxruntime
|
||||
|
||||
graph = helper.make_graph(
|
||||
[helper.make_node("MaxPool", inputs=["X"], outputs=["Y", "indices"], name="MaxPool", kernel_shape=(2,2,2), strides=(2,2,2))],
|
||||
"the graph",
|
||||
[helper.make_tensor_value_info("X", TensorProto.FLOAT, (1, 1, 3, 3, 3))],
|
||||
[helper.make_tensor_value_info("Y", TensorProto.FLOAT, None),
|
||||
helper.make_tensor_value_info("indices", TensorProto.INT64, None)]
|
||||
)
|
||||
|
||||
# Create the model (ModelProto)
|
||||
model_def = helper.make_model(graph, producer_name='me')
|
||||
onnx.save(model_def, "maxpool_model.onnx")
|
||||
sess = onnxruntime.InferenceSession("maxpool_model.onnx")
|
||||
X = np.arange(0, 27, dtype=np.float32).reshape((1, 1, 3, 3, 3))
|
||||
print(X)
|
||||
(Y, indices) = sess.run(None, {"X" : X})
|
||||
print(Y)
|
||||
print(indices)
|
||||
|
||||
*/
|
||||
|
||||
test.AddAttribute("strides", std::vector<int64_t>{2, 2, 2});
|
||||
test.AddAttribute("kernel_shape", vector<int64_t>{2, 2, 2});
|
||||
|
||||
std::vector<float> t_vals = {1, 2, 3, 4, 5, 6, 7, 8};
|
||||
std::vector<int64_t> t_dims = {1, 1, 2, 2, 2};
|
||||
std::vector<float> t_vals = {3};
|
||||
std::vector<int64_t> t_dims = {1, 1, 1, 1, 1};
|
||||
|
||||
std::vector<int64_t> i_vals = {1, 3, 24, 30, 32, 38, 60, 62};
|
||||
std::vector<int64_t> i_dims = {1, 1, 2, 2, 2};
|
||||
std::vector<int64_t> i_vals = {13};
|
||||
std::vector<int64_t> i_dims = {1, 1, 1, 1, 1};
|
||||
|
||||
std::vector<int64_t> expected_dims = {1, 1, 4, 4, 5};
|
||||
std::vector<int64_t> expectedDims_Size = {5};
|
||||
std::vector<int64_t> expected_dims = {1, 1, 3, 3, 3};
|
||||
|
||||
std::vector<float> expected_vals =
|
||||
{
|
||||
//slice 1
|
||||
0, 1, 0, 2, 0,
|
||||
0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0,
|
||||
std::vector<float> expected_vals = {
|
||||
0, 0, 0,
|
||||
0, 0, 0,
|
||||
0, 0, 0,
|
||||
|
||||
// slice 2
|
||||
0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0,
|
||||
3, 0, 0, 0, 0,
|
||||
0, 0, 4, 0, 0,
|
||||
0, 0, 0,
|
||||
0, 3, 0,
|
||||
0, 0, 0,
|
||||
|
||||
//slice 3
|
||||
5, 0, 0, 0, 0,
|
||||
0, 0, 6, 0, 0,
|
||||
0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0,
|
||||
0, 0, 0,
|
||||
0, 0, 0,
|
||||
0, 0, 0};
|
||||
|
||||
// slice 4
|
||||
0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0,
|
||||
7, 0, 8, 0, 0};
|
||||
std::vector<int64_t> expected_dims_size = {5};
|
||||
|
||||
test.AddInput<float>("xT", t_dims, t_vals);
|
||||
test.AddInput<int64_t>("xI", i_dims, i_vals);
|
||||
test.AddInput<int64_t>("output_shape", expectedDims_Size, expected_dims);
|
||||
test.AddInput<int64_t>("output_shape", expected_dims_size, expected_dims);
|
||||
|
||||
test.AddOutput<float>("Y", expected_dims, expected_vals);
|
||||
test.Run();
|
||||
|
|
|
|||
|
|
@ -152,7 +152,8 @@ def create_backend_test(testname=None):
|
|||
'^test_sequence_*',
|
||||
'^test_scatter_.*',
|
||||
'^test_edge_pad_cpu.*', # test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests
|
||||
'^test_reflect_pad_cpu.*' # test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests
|
||||
'^test_reflect_pad_cpu.*', # test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests
|
||||
'^test_maxunpool_export_with_output_shape_cpu', # Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398'
|
||||
]
|
||||
|
||||
# Example of how to disable tests for a specific provider.
|
||||
|
|
|
|||
Loading…
Reference in a new issue