From 664e548e318944be2d1bd46ecc4caf7139b237b6 Mon Sep 17 00:00:00 2001 From: Evgenii Indenbom Date: Tue, 22 Jun 2021 20:44:49 +0300 Subject: [PATCH] Col2im optimization by eliminating integer multiplications: 1. No padding branch performance is improved 8 times 2. Symmetric padding branch is generalized for asymmetric padding case (padding symmetry was not actually used) and further optimized by eliminating integer multiplications. --- onnxruntime/core/util/math_cpu.cc | 135 ++++++++---------- .../cpu/nn/conv_transpose_op_test.cc | 119 +++++++++++++++ 2 files changed, 180 insertions(+), 74 deletions(-) diff --git a/onnxruntime/core/util/math_cpu.cc b/onnxruntime/core/util/math_cpu.cc index ed8e90d664..9062f435b5 100644 --- a/onnxruntime/core/util/math_cpu.cc +++ b/onnxruntime/core/util/math_cpu.cc @@ -652,72 +652,46 @@ void Col2im(const float* data_col, int64 int64_t pad_l, int64_t pad_b, int64_t pad_r, int64_t stride_h, int64_t stride_w, float* data_im, CPUMathUtil* context) { const int64_t output_h = - (height + pad_b + pad_t - (dilation_h * (kernel_h - 1) + 1)) / stride_h + - 1; + (height + pad_b + pad_t - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; const int64_t output_w = - (width + pad_l + pad_r - (dilation_w * (kernel_w - 1) + 1)) / stride_w + - 1; - const int64_t hwc = height * width * channels; + (width + pad_l + pad_r - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + const int64_t output_hw = output_h * output_w; + const int64_t hw = height * width; + const int64_t hwc = hw * channels; Set(gsl::narrow(hwc), 0, data_im, context); // Fast path for zero padding and no dilation // From Torch, modified THNN_(unfolded_acc) - if (dilation_h == 1 && dilation_w == 1 && pad_l == 0 && pad_r == 0 && - pad_t == 0 && pad_b == 0) { - for (auto k = 0; k < channels * kernel_h * kernel_w; k++) { - const auto nip = k / (kernel_h * kernel_w); - const auto rest = k % (kernel_h * kernel_w); - const auto kh = rest / kernel_w; - const auto kw = rest % kernel_w; - const auto* dst = data_col + - nip * (kernel_h * kernel_w * output_h * output_w) + - kh * (kernel_w * output_h * output_w) + kw * (output_h * output_w); - auto* src = data_im + nip * (height * width); - for (auto y = 0; y < output_h; y++) { - const auto iy = y * stride_h + kh; - const auto ix = kw; - if (stride_w == 1) { - auto offsrc = src + (iy * width + ix); - const auto offdst = dst + (y * output_w); - for (auto i = 0; i < output_w; ++i) { - offsrc[i] += offdst[i]; - } - } else { - for (auto x = 0; x < output_w; x++) { - auto offsrc = src + (iy * width + ix + x * stride_w); - const auto offdst = dst + (y * output_w + x); - *offsrc += *offdst; - } - } - } - } - return; - } - - // Fast path for equal padding - if (pad_l == pad_r && pad_t == pad_b) { - // From Intel, https://github.com/BVLC/caffe/pull/3536 - const int64_t pad_h = pad_t; - const int64_t pad_w = pad_l; - const int64_t channel_size = height * width; - for (int64_t channel = channels; channel--; data_im += channel_size) { - for (int64_t kernel_row = 0; kernel_row < kernel_h; kernel_row++) { - for (int64_t kernel_col = 0; kernel_col < kernel_w; kernel_col++) { - int64_t input_row = -pad_h + kernel_row * dilation_h; - for (int64_t output_rows = output_h; output_rows; output_rows--) { - if (!is_a_ge_zero_and_a_lt_b(input_row, height)) { - data_col += output_w; + if (dilation_h == 1 && dilation_w == 1 && pad_l == 0 && pad_r == 0 && pad_t == 0 && pad_b == 0) { + // Src (column) data cursor + auto* src = data_col; + // End of dst (image) data + auto* dst_end = data_im + hwc; + // Dst cursor step at end of row + auto dst_row_step = stride_h * width - stride_w * output_w; + // Dst channel data + for (auto* dst_cb = data_im; dst_cb < dst_end; dst_cb += hw) { + // First dst row for current kernel row + auto* dst_hb = dst_cb; + for (auto kh = 0; kh < kernel_h; ++kh, dst_hb += width) { + // First dst element for current kernel element + auto* dst_wb = dst_hb; + for (auto kw = 0; kw < kernel_w; ++kw, ++dst_wb) { + // Dst cursor + auto* dst = dst_wb; + // End of source data for kernel element + for (auto* src_he = src + output_hw; src < src_he; dst += dst_row_step) { + // End of source row + auto* src_we = src + output_w; + if (stride_w == 1) { + for (; src < src_we; ++src, ++dst) { + *dst += *src; + } } else { - int64_t input_col = -pad_w + kernel_col * dilation_w; - for (int64_t output_col = output_w; output_col; output_col--) { - if (is_a_ge_zero_and_a_lt_b(input_col, width)) { - data_im[input_row * width + input_col] += *data_col; - } - data_col++; - input_col += stride_w; + for (; src < src_we; ++src, dst += stride_w) { + *dst += *src; } } - input_row += stride_h; } } } @@ -726,23 +700,36 @@ void Col2im(const float* data_col, int64 } // Fallback - const int64_t dkernel_h = dilation_h * (kernel_h - 1) + 1; - const int64_t dkernel_w = dilation_w * (kernel_w - 1) + 1; - int64_t height_col = (height + pad_t + pad_b - dkernel_h) / stride_h + 1; - int64_t width_col = (width + pad_l + pad_r - dkernel_w) / stride_w + 1; - int64_t channels_col = channels * kernel_h * kernel_w; - for (int64_t c = 0; c < channels_col; ++c) { - int64_t w_offset = c % kernel_w; - int64_t h_offset = (c / kernel_w) % kernel_h; - int64_t c_im = c / kernel_h / kernel_w; - for (int64_t h = 0; h < height_col; ++h) { - for (int64_t w = 0; w < width_col; ++w) { - int64_t h_pad = h * stride_h - pad_t + h_offset * dilation_h; - int64_t w_pad = w * stride_w - pad_l + w_offset * dilation_w; - if (h_pad >= 0 && h_pad < height && w_pad >= 0 && w_pad < width) { - data_im[(c_im * height + h_pad) * width + w_pad] += - data_col[(c * height_col + h) * width_col + w]; + // Src (col data) cursor + auto* src = data_col; + // End of dst (image) data + auto* dst_end = data_im + hwc; + // Begin of src channel data + for (auto* dst = data_im; dst < dst_end; dst += hw) { + // Current kernel element starting vertical offset in dst data + int64_t h_offset = -pad_t * width; + int64_t h_offset_end = h_offset + kernel_h * dilation_h * width; + for (; h_offset < h_offset_end; h_offset += dilation_h * width) { + // Current kernel element starting horizontal offset in dst data + int64_t w_offset = -pad_l; + int64_t w_offset_end = w_offset + kernel_w * dilation_w; + for (; w_offset < w_offset_end; w_offset += dilation_w) { + // End of src channel data + auto* src_ce = src + output_hw; + // Dst row offset + for (int64_t h = h_offset; src < src_ce; h += stride_h * width) { + // End of src row data + auto* src_we = src + output_w; + if (is_a_ge_zero_and_a_lt_b(h, hw)) { + for (int64_t w = w_offset; src < src_we; src++, w += stride_w) { + if (is_a_ge_zero_and_a_lt_b(w, width)) { + dst[h + w] += *src; + } + } + } else { + src = src_we; + } } } } diff --git a/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc b/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc index 7b2618efe2..4bca7ebb43 100644 --- a/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc @@ -639,6 +639,104 @@ TEST(ConvTransposeTest, ConvTranspose_2D_Dilation_4) { TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); } + +TEST(ConvTransposeTest, ConvTranspose_2D_Dilation_AsymmetricPads_1) { + ConvTransposeOpAttributes attrs = { + vector{2, 2}, + {}, + {}, + vector{2, 2, 1, 1}, + vector{1, 1}, + {3, 3}, + 1, + "NOTSET"}; + + vector X = {3.0f, 8.0f, 1.0f, 9.0f, 5.0f, 7.0f, 3.0f, 2.0f, 6.0f}; + vector X_shape = {1, 1, 3, 3}; + vector W = {7.0f, 2.0f, 1.0f, 9.0f}; + vector W_shape = {1, 1, 2, 2}; + vector Y_shape = {1, 1, 3, 3}; + auto expected_vals = {42.0f, 6.0f, 4.0f, + 1.0f, 27.0f, 72.0f, + 7.0f, 81.0f, 45.0f}; + + TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + + +TEST(ConvTransposeTest, ConvTranspose_2D_Dilation_AsymmetricPads_2) { + ConvTransposeOpAttributes attrs = { + vector{2, 2}, + {}, + {}, + vector{1, 1, 2, 2}, + vector{1, 1}, + {3, 3}, + 1, + "NOTSET"}; + + vector X = {3.0f, 8.0f, 1.0f, 9.0f, 5.0f, 7.0f, 3.0f, 2.0f, 6.0f}; + vector X_shape = {1, 1, 3, 3}; + vector W = {7.0f, 2.0f, 1.0f, 9.0f}; + vector W_shape = {1, 1, 2, 2}; + vector Y_shape = {1, 1, 3, 3}; + auto expected_vals = {35.0f, 49.0f, 18.0f, + 14.0f, 42.0f, 6.0f, + 8.0f, 1.0f, 27.0f}; + + TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + + +TEST(ConvTransposeTest, ConvTranspose_2D_Dilation_AsymmetricPads_3) { + ConvTransposeOpAttributes attrs = { + vector{2, 2}, + {}, + {}, + vector{2, 2, 0, 0}, + vector{1, 1}, + {3, 3}, + 1, + "NOTSET"}; + + vector X = {3.0f, 8.0f, 1.0f, 9.0f, 5.0f, 7.0f, 3.0f, 2.0f, 6.0f}; + vector X_shape = {1, 1, 3, 3}; + vector W = {7.0f, 2.0f, 1.0f, 9.0f}; + vector W_shape = {1, 1, 2, 2}; + vector Y_shape = {1, 1, 4, 4}; + auto expected_vals = {42.0f, 6.0f, 4.0f, 12.0f, + 1.0f, 27.0f, 72.0f, 9.0f, + 7.0f, 81.0f, 45.0f, 63.0f, + 6.0f, 27.0f, 18.0f, 54.0f}; + + TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + + +TEST(ConvTransposeTest, ConvTranspose_2D_Dilation_AsymmetricPads_4) { + ConvTransposeOpAttributes attrs = { + vector{2, 2}, + {}, + {}, + vector{0, 0, 2, 2}, + vector{1, 1}, + {3, 3}, + 1, + "NOTSET"}; + + vector X = {3.0f, 8.0f, 1.0f, 9.0f, 5.0f, 7.0f, 3.0f, 2.0f, 6.0f}; + vector X_shape = {1, 1, 3, 3}; + vector W = {7.0f, 2.0f, 1.0f, 9.0f}; + vector W_shape = {1, 1, 2, 2}; + vector Y_shape = {1, 1, 4, 4}; + auto expected_vals = {21.0f, 56.0f, 7.0f, 6.0f, + 63.0f, 35.0f, 49.0f, 18.0f, + 21.0f, 14.0f, 42.0f, 6.0f, + 3.0f, 8.0f, 1.0f, 27.0f}; + + TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + TEST(ConvTransposeTest, ConvTranspose_2D_Dilation_Group_1) { ConvTransposeOpAttributes attrs = { vector{2, 2}, @@ -723,6 +821,27 @@ TEST(ConvTransposeTest, ConvTranspose_2D_NonDefaultStridesAndDilations) { TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); } +TEST(ConvTransposeTest, ConvTranspose_2D_NonDefaultStridesAndDilations_T) { + ConvTransposeOpAttributes attrs = { + vector{4, 1}, // kernel_shape + {}, // output_padding + {}, // output_shape + vector{0, 0, 0, 0}, // pads + vector{2, 1}, // strides + vector{3, 1}, // dilations + 1, // group + "NOTSET" // auto_pad + }; + vector X = {1., 2.}; + vector X_shape = {1, 1, 2, 1}; + vector W = {1., 1., 1., 1.}; + vector W_shape = {1, 1, 4, 1}; + vector Y_shape = {1, 1, 12, 1}; + auto expected_vals = {1.f, 0.f, 2.f, 1.f, 0.f, 2.f, 1.f, 0.f, 2.f, 1.f, 0.f, 2.f}; + + TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); +} + TEST(ConvTransposeTest, DimWithZero) { ConvTransposeOpAttributes attrs = { vector{3, 3}, // kernel_shape