diff --git a/onnxruntime/core/mlas/lib/convolve.cpp b/onnxruntime/core/mlas/lib/convolve.cpp index 5722f21d5a..5d2c35fbfb 100644 --- a/onnxruntime/core/mlas/lib/convolve.cpp +++ b/onnxruntime/core/mlas/lib/convolve.cpp @@ -676,6 +676,7 @@ Return Value: const size_t GroupCount = Parameters->GroupCount; const size_t BatchGroupCount = Parameters->BatchCount * GroupCount; + const float Beta = Parameters->Beta; size_t BatchGroupStart; size_t BatchGroupRemaining; @@ -709,9 +710,9 @@ Return Value: // Invoke the non-threaded GEMM directly with the input tensor. // - MlasSgemmOperation(CblasNoTrans, Parameters->u.GemmDirect.TransB, FilterCount, - OutputSize, K, 1.0f, filter, K, input, Parameters->u.GemmDirect.ldb, 0.0f, - output, OutputSize); + MlasSgemmOperation(CblasNoTrans, Parameters->u.GemmDirect.TransB, FilterCount, OutputSize, + K, 1.0f, filter, K, input, Parameters->u.GemmDirect.ldb, Beta, output, + OutputSize); // // Apply the activation with optional bias. diff --git a/onnxruntime/core/mlas/lib/scalar/SconvDepthwiseKernelScalar.cpp b/onnxruntime/core/mlas/lib/scalar/SconvDepthwiseKernelScalar.cpp index a805f98fef..da1cdb9606 100644 --- a/onnxruntime/core/mlas/lib/scalar/SconvDepthwiseKernelScalar.cpp +++ b/onnxruntime/core/mlas/lib/scalar/SconvDepthwiseKernelScalar.cpp @@ -47,6 +47,7 @@ Arguments: --*/ { const size_t W = Parameters->InputShape[1]; + const float beta = Parameters->Beta; if (W > 1) { @@ -79,9 +80,8 @@ Arguments: auto out_col = Parameters->OutputShape[1]; if (pad_left == 1) { - float dotsum = w01 * row0[1] + w02 * row0[2] + - w11 * row1[1] + w12 * row1[2] + - w21 * row2[1] + w22 * row2[2]; + float dotsum = w01 * row0[1] + w02 * row0[2] + w11 * row1[1] + w12 * row1[2] + + w21 * row2[1] + w22 * row2[2] + (beta == 0.f ? 0.f : *Output * beta); *Output++ = dotsum; out_col--; row0 += stride_w; @@ -90,10 +90,9 @@ Arguments: } for (; out_col > pad_right; out_col--) { - float dotsum = - w00 * row0[0] + w01 * row0[1] + w02 * row0[2] + - w10 * row1[0] + w11 * row1[1] + w12 * row1[2] + - w20 * row2[0] + w21 * row2[1] + w22 * row2[2]; + float dotsum = w00 * row0[0] + w01 * row0[1] + w02 * row0[2] + w10 * row1[0] + + w11 * row1[1] + w12 * row1[2] + w20 * row2[0] + w21 * row2[1] + + w22 * row2[2] + (beta == 0.f ? 0.f : *Output * beta); *Output++ = dotsum; row0 += stride_w; row1 += stride_w; @@ -101,10 +100,8 @@ Arguments: } if (out_col == 1) { // pad_right == 1 - float dotsum = - w00 * row0[0] + w01 * row0[1] + - w10 * row1[0] + w11 * row1[1] + - w20 * row2[0] + w21 * row2[1]; + float dotsum = w00 * row0[0] + w01 * row0[1] + w10 * row1[0] + w11 * row1[1] + + w20 * row2[0] + w21 * row2[1] + (beta == 0.f ? 0.f : *Output * beta); *Output++ = dotsum; } @@ -129,15 +126,17 @@ Arguments: const float w0 = Filter[pad_left ? 1 : 0]; const float w1 = Filter[pad_left ? 4 : 3]; const float w2 = Filter[pad_left ? 7 : 6]; + auto init_v = (beta == 0.f ? 0.f : *Output * beta); if (pad_top == 1) { - *Output++ = w1 * Input[0] + w2 * ((H + pad_top <= 2) ? 0.0f : Input[1]); + *Output++ = w1 * Input[0] + w2 * ((H + pad_top <= 2) ? 0.0f : Input[1]) + init_v; out_row--; } for (const float* row = Input + pad_top * stride_h - pad_top; out_row > pad_bottom; --out_row) { // All pixels are in the input col - *Output++ = w0 * row[0] + w1 * row[1] + w2 * row[2]; + auto init = (beta == 0.f ? 0.f : *Output * beta); + *Output++ = w0 * row[0] + w1 * row[1] + w2 * row[2] + init; row += stride_h; } @@ -146,9 +145,9 @@ Arguments: // out_row == 1 when arrive here if (pad_bottom == 1) { const float* row = Input + H - 2; - *Output++ = w0 * row[0] + w1 * row[1]; + *Output++ = w0 * row[0] + w1 * row[1] + init_v; } else { // pad_bottom == 2 and H == 1 and padding_top == 0 - *Output++ = w0 * Input[0]; + *Output++ = w0 * Input[0] + init_v; } } } diff --git a/onnxruntime/test/optimizer/conv_add_act_test.cc b/onnxruntime/test/optimizer/conv_add_act_test.cc new file mode 100644 index 0000000000..e97c738082 --- /dev/null +++ b/onnxruntime/test/optimizer/conv_add_act_test.cc @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "gtest/gtest.h" +#include "graph_transform_test_builder.h" + +#include "core/graph/graph.h" + +namespace onnxruntime { +namespace test { + +#ifndef DISABLE_CONTRIB_OPS + +void TestConvPath(const std::vector& input_shape, const std::vector& weights_shape, + const std::vector& output_shape, int64_t group) { + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput(input_shape, -31, 31); + auto* output_arg = builder.MakeOutput(); + auto* bias_arg = builder.MakeInitializer({weights_shape[0]}, -20.f, 20.f); + auto* add_arg = builder.MakeInput(output_shape, -20.f, 20.f); + auto* weight_arg = builder.MakeInitializer(weights_shape, -2.f, 2.f); + auto* conv_out_arg = builder.MakeIntermediate(); + + auto& conv_node = builder.AddNode("Conv", {input_arg, weight_arg, bias_arg}, {conv_out_arg}); + conv_node.AddAttribute("group", group); + builder.AddNode("Add", {conv_out_arg, add_arg}, {output_arg}); + }; + + auto check_graph = [&](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["com.microsoft.FusedConv"], 1); + }; + InlinedHashSet disabled_optimizers = {"NchwcTransformer"}; + TransformerTester(build_test_case, + check_graph, + TransformerLevel::Default, + TransformerLevel::Level3, 12, 0.0001, 0.000001, + 0, {}, disabled_optimizers); +} + +TEST(ConvAddActivationFusionTests, ConvExpandThenGemm) { + // hit MlasConvAlgorithmExpandThenGemm + TestConvPath({1, 16, 5, 5}, {16, 16, 3, 3}, {1, 16, 3, 3}, 1); +} + +TEST(ConvAddActivationFusionTests, ConvDepthwise) { + // MlasConvAlgorithmDepthwise or MlasConvAlgorithmExpandThenGemmSegmented + TestConvPath({1, 16, 5, 5}, {16, 1, 3, 3}, {1, 16, 3, 3}, 16); +} + +TEST(ConvAddActivationFusionTests, ConvGemmDirect) { + // MlasConvAlgorithmGemmDirect + TestConvPath({1, 16, 5, 5}, {16, 16, 1, 1}, {1, 16, 5, 5}, 1); + // MlasConvAlgorithmGemmDirect + TestConvPath({1, 16, 5, 5}, {16, 1, 5, 5}, {1, 16, 1, 1}, 16); +} + +#endif // DISABLE_CONTRIB_OPS + +} // namespace test +} // namespace onnxruntime