From e9d5ed270fadd9225bdaedba715f6c5dd3e1ebb5 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Fri, 13 Mar 2020 22:15:44 +1000 Subject: [PATCH] Normalizer performance improvements (#3201) * Simplify Normalizer as the spec only requires support for 2D input. Tried using eigen (LpNorm<1>(), and norm()) on each row but that was much slower. * Remove unused variable --- .../core/providers/cpu/ml/normalizer.cc | 191 +++++++++--------- .../core/providers/cpu/ml/normalizer.h | 4 +- .../test/providers/cpu/ml/normalizer_test.cc | 63 +++--- 3 files changed, 126 insertions(+), 132 deletions(-) diff --git a/onnxruntime/core/providers/cpu/ml/normalizer.cc b/onnxruntime/core/providers/cpu/ml/normalizer.cc index 73eb46f6a5..5652291962 100644 --- a/onnxruntime/core/providers/cpu/ml/normalizer.cc +++ b/onnxruntime/core/providers/cpu/ml/normalizer.cc @@ -43,146 +43,137 @@ ONNX_CPU_OPERATOR_ML_KERNEL( Normalizer); template -void NormalizeMax(const gsl::span& in, gsl::span& out, - int64_t offset, int64_t stride, int64_t increment_by) { - float max = std::numeric_limits::lowest(); +void NormalizeMax(const T* in, float* out, int64_t num_batches, int64_t batch_size) { + for (int b = 0; b < num_batches; ++b) { + float max = std::numeric_limits::lowest(); - for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { - max = std::max(max, static_cast(in[i])); - } - - if (max != 0.f) { - for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { - out[i] = static_cast(in[i]) / max; + for (int i = 0; i < batch_size; ++i) { + max = std::max(max, static_cast(*in++)); } - } else { - for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { - out[i] = static_cast(in[i]); + + in -= batch_size; + + if (max != 0.f) { + for (int i = 0; i < batch_size; ++i) { + *out++ = static_cast(*in++) / max; + } + } else { + for (int i = 0; i < batch_size; ++i) { + *out++ = static_cast(*in++); + } } } } template -void NormalizeL1(const gsl::span& in, gsl::span& out, - int64_t offset, int64_t stride, int64_t increment_by) { - float sum = 0.f; +static void NormalizeL1(const T* in, float* out, int64_t num_batches, int64_t batch_size) { + for (int b = 0; b < num_batches; ++b) { + float sum = 0.f; - for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { - sum += static_cast(std::abs(in[i])); - } - - if (sum != 0.f) { - for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { - out[i] = static_cast(in[i]) / sum; + for (int i = 0; i < batch_size; ++i) { + sum += static_cast(std::abs(*in++)); } - } else { - for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { - out[i] = static_cast(in[i]); + + in -= batch_size; + + if (sum != 0.f) { + for (int i = 0; i < batch_size; ++i) { + *out++ = static_cast(*in++) / sum; + } + } else { + for (int i = 0; i < batch_size; ++i) { + *out++ = static_cast(*in++); + } } } } template -void NormalizeL2(const gsl::span& in, gsl::span& out, - int64_t offset, int64_t stride, int64_t increment_by) { - float sum = 0.f; - for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { - auto x = in[i]; - auto x_sq = static_cast(x * x); - out[i] = x_sq; - sum += x_sq; - } +void NormalizeL2(const T* in, float* out, int64_t num_batches, int64_t batch_size) { + for (int b = 0; b < num_batches; ++b) { + float sum = 0.f; - if (sum != 0.f) { - for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { - auto x = in[i]; - auto x_sq = out[i]; - - if (x < 0) - out[i] = std::sqrt(x_sq / sum) * -1; - else - out[i] = std::sqrt(x_sq / sum); + for (int i = 0; i < batch_size; ++i) { + auto x = *in++; + auto x_sq = static_cast(x * x); + *out++ = x_sq; + sum += x_sq; } - } else { - for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) { - out[i] = static_cast(in[i]); + + in -= batch_size; + out -= batch_size; + + if (sum != 0.f) { + for (int i = 0; i < batch_size; ++i) { + auto x = *in++; + auto x_sq = *out; + + *out++ = (x < 0) ? std::sqrt(x_sq / sum) * -1 : std::sqrt(x_sq / sum); + } + } else { + for (int i = 0; i < batch_size; ++i) { + *out++ = static_cast(*in++); + } } } } template -void Normalizer::Normalize(OpKernelContext* context) const { +Status Normalizer::Normalize(OpKernelContext* context) const { const Tensor& X = *context->Input(0); const TensorShape& x_shape = X.Shape(); - const auto data_size = x_shape.Size(); + + if (x_shape.NumDimensions() > 2) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Rank of input to Normalized must be less than 2. Got ", + x_shape.NumDimensions()); + } + const auto& x_dims = x_shape.GetDims(); + int64_t num_batches = x_dims.size() == 1 ? 1 : x_dims[0]; + int64_t batch_size = x_dims.size() == 1 ? x_dims[0] : x_dims[1]; Tensor* Y = context->Output(0, x_shape); - auto input = gsl::make_span(X.template Data(), data_size); - auto output = gsl::make_span(Y->template MutableData(), data_size); + const T* input = X.template Data(); + float* output = Y->MutableData(); - int64_t stride = x_dims.size() == 1 ? x_dims[0] : x_dims[1]; - int64_t loops = data_size / stride; - - // we normalize on axis 1 so if there are more than 2 dimensions we need to increment the index - // by more than 1 as we process the stride - // for 1 and 2 dimension tensors we're normalizing across the row/s, so increment_by is 1 - // - // e.g. if you have a tensor of shape {2, 2, 3} - // [[[ 1, 2, 3], - // [ 4, 5, 6]], - // [[ 7, 8, 9], - // [10, 11, 12]]] - // - // we want to normalize (1, 4), (2, 5), (3, 6), - // (7, 10), (8, 11), (9, 12) - // so the stride would be 2, and the increment_by would be 3. - // - // we process a block of stride * increment_by entries before we need to skip to the next row of the 2nd dimension. - // the offset starts at 0 and increases by 1 each loop, for increment_by loops. - // the offset then jumps by stride * increment - - int64_t increment_by = x_dims.size() > 1 ? x_shape.SizeFromDimension(2) : 1; - - for (int64_t n = 0; n < loops; ++n) { - int64_t offset = (n % increment_by) + ((n / increment_by) * (stride * increment_by)); - - switch (normalization_) { - case NORMALIZE::NMAX: { - NormalizeMax(input, output, offset, stride, increment_by); - break; - } - case NORMALIZE::L1: { - NormalizeL1(input, output, offset, stride, increment_by); - break; - } - case NORMALIZE::L2: { - NormalizeL2(input, output, offset, stride, increment_by); - break; - } - default: { - ORT_THROW("Unexpected NORMALIZE value of ", normalization_); - } + switch (normalization_) { + case NORMALIZE::NMAX: { + NormalizeMax(input, output, num_batches, batch_size); + break; + } + case NORMALIZE::L1: { + NormalizeL1(input, output, num_batches, batch_size); + break; + } + case NORMALIZE::L2: { + NormalizeL2(input, output, num_batches, batch_size); + break; + } + default: { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unexpected NORMALIZE value of ", normalization_); } } + + return Status::OK(); } // MLTypeCallDispather implementation wrapper template struct Normalizer::CallNormalizerImpl { - void operator()(const Normalizer* norm, OpKernelContext* ctx) const { - norm->Normalize(ctx); + Status operator()(const Normalizer* norm, OpKernelContext* ctx) const { + return norm->Normalize(ctx); } }; Status Normalizer::Compute(OpKernelContext* context) const { - const auto* input_tensor_ptr = context->Input(0); - ORT_ENFORCE(input_tensor_ptr != nullptr); + const auto& input_tensor_ptr = *context->Input(0); - utils::MLTypeCallDispatcher t_disp(input_tensor_ptr->GetElementType()); - t_disp.Invoke(this, context); - return Status::OK(); + utils::MLTypeCallDispatcherRet + t_disp(input_tensor_ptr.GetElementType()); + + auto status = t_disp.Invoke(this, context); + return status; } } // namespace ml diff --git a/onnxruntime/core/providers/cpu/ml/normalizer.h b/onnxruntime/core/providers/cpu/ml/normalizer.h index 648873a2fa..5172d1e6c6 100644 --- a/onnxruntime/core/providers/cpu/ml/normalizer.h +++ b/onnxruntime/core/providers/cpu/ml/normalizer.h @@ -25,9 +25,9 @@ class Normalizer final : public OpKernel { private: template - void Normalize(OpKernelContext* context) const; + Status Normalize(OpKernelContext* context) const; - template + template struct CallNormalizerImpl; NORMALIZE normalization_; diff --git a/onnxruntime/test/providers/cpu/ml/normalizer_test.cc b/onnxruntime/test/providers/cpu/ml/normalizer_test.cc index 9b9737151a..5befda57af 100644 --- a/onnxruntime/test/providers/cpu/ml/normalizer_test.cc +++ b/onnxruntime/test/providers/cpu/ml/normalizer_test.cc @@ -21,6 +21,8 @@ static void RunTest(const vector& input, test.AddInput("X", dims, input); test.AddOutput("Y", dims, output); + // output 'norm' so if a test fails we know which one + std::cout << "norm=" << norm << "\n"; test.Run(expect_result, expect_error_message); } @@ -120,6 +122,23 @@ TEST(Normalizer, SingleDimensionFloat) { RunTests(input, dims, max_output, l1_output, l2_output); } +TEST(Normalizer, TwoDimensionFloat) { + std::vector dims = {2, 3}; + std::vector input = {-1.0856306f, 0.99734545f, 0.2829785f, + -1.50629471f, -0.57860025f, 1.65143654f}; + + std::vector max_output{-1.0885202f, 1.f, 0.2837317f, + -0.91211176f, -0.35036176f, 1.f}; + + std::vector l1_output{-0.45885524f, 0.42154038f, 0.11960436f, + -0.40314806f, -0.15485784f, 0.44199413f}; + + std::vector l2_output{-0.7232126f, 0.6643998f, 0.18851127f, + -0.65239084f, -0.25059736f, 0.7152532f}; + + RunTests(input, dims, max_output, l1_output, l2_output); +} + TEST(Normalizer, TwoDimensionDouble) { std::vector dims = {2, 3}; std::vector input = {-1.0856306, 0.99734545, 0.2829785, @@ -137,39 +156,23 @@ TEST(Normalizer, TwoDimensionDouble) { RunTests(input, dims, max_output, l1_output, l2_output); } -TEST(Normalizer, ThreeDimensionInt32) { - std::vector dims = {2, 3, 4}; - std::vector input = {-242, -42, 126, -86, - -67, -9, 149, -63, - -44, -43, 220, 218, +TEST(Normalizer, TwoDimensionInt) { + std::vector dims = {3, 2}; + std::vector input = {-242, -42, + 126, -86, + -67, -9}; - 100, 38, 73, 149, - -93, 117, -125, -63, - 90, -142, -14, -86}; + std::vector max_output{5.7619047f, 1.f, + 1.f, -0.6825397f, + 7.4444447f, 1.f}; - std::vector max_output{5.5f, 4.6666665f, 0.57272726f, -0.39449543f, - 1.5227273f, 1.f, 0.67727274f, -0.28899083f, - 1.f, 4.7777777f, 1.f, 1.f, + std::vector l1_output{-0.85211265f, -0.14788732f, + 0.5943396f, -0.4056604f, + -0.8815789f, -0.11842106f}; - 1.f, 0.32478634f, 1.f, 1.f, - -0.93f, 1.f, -1.7123288f, -0.42281878f, - 0.9f, -1.2136753f, -0.19178082f, -0.5771812f}; - - std::vector l1_output{-0.6855524f, -0.44680852f, 0.25454545f, -0.23433243f, - -0.1898017f, -0.09574468f, 0.3010101f, -0.17166212f, - -0.12464589f, -0.4574468f, 0.44444445f, 0.59400547f, - - 0.3533569f, 0.12794612f, 0.3443396f, 0.5f, - -0.3286219f, 0.3939394f, -0.5896226f, -0.21140939f, - 0.3180212f, -0.4781145f, -0.06603774f, -0.2885906f}; - - std::vector l2_output{-0.94928247f, -0.6910363f, 0.4284698f, -0.3543899f, - -0.26281786f, -0.1480792f, 0.5066825f, -0.25961122f, - -0.17259681f, -0.7074895f, 0.74812186f, 0.89833724f, - - 0.6114293f, 0.2022622f, 0.5019584f, 0.8132732f, - -0.56862926f, 0.62275463f, -0.85951775f, -0.34386718f, - 0.55028635f, -0.7558219f, -0.09626599f, -0.469406f}; + std::vector l2_output{-0.98527145f, -0.17099753f, + 0.82594985f, -0.56374353f, + -0.9910982f, -0.13313259f}; RunTests(input, dims, max_output, l1_output, l2_output); }