mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-10 17:37:14 +00:00
Normalizer performance improvements (#3201)
* Simplify Normalizer as the spec only requires support for 2D input. Tried using eigen (LpNorm<1>(), and norm()) on each row but that was much slower. * Remove unused variable
This commit is contained in:
Scott McKay
GitHub
parent
890cb78b20
commit
e9d5ed270f
3 changed files with 126 additions and 132 deletions
|
|
@ -43,146 +43,137 @@ ONNX_CPU_OPERATOR_ML_KERNEL(
|
|||
Normalizer);
|
||||
|
||||
template <typename T>
|
||||
void NormalizeMax(const gsl::span<const T>& in, gsl::span<float>& out,
|
||||
int64_t offset, int64_t stride, int64_t increment_by) {
|
||||
float max = std::numeric_limits<float>::lowest();
|
||||
void NormalizeMax(const T* in, float* out, int64_t num_batches, int64_t batch_size) {
|
||||
for (int b = 0; b < num_batches; ++b) {
|
||||
float max = std::numeric_limits<float>::lowest();
|
||||
|
||||
for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) {
|
||||
max = std::max(max, static_cast<float>(in[i]));
|
||||
}
|
||||
|
||||
if (max != 0.f) {
|
||||
for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) {
|
||||
out[i] = static_cast<float>(in[i]) / max;
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
max = std::max(max, static_cast<float>(*in++));
|
||||
}
|
||||
} else {
|
||||
for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) {
|
||||
out[i] = static_cast<float>(in[i]);
|
||||
|
||||
in -= batch_size;
|
||||
|
||||
if (max != 0.f) {
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
*out++ = static_cast<float>(*in++) / max;
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
*out++ = static_cast<float>(*in++);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void NormalizeL1(const gsl::span<const T>& in, gsl::span<float>& out,
|
||||
int64_t offset, int64_t stride, int64_t increment_by) {
|
||||
float sum = 0.f;
|
||||
static void NormalizeL1(const T* in, float* out, int64_t num_batches, int64_t batch_size) {
|
||||
for (int b = 0; b < num_batches; ++b) {
|
||||
float sum = 0.f;
|
||||
|
||||
for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) {
|
||||
sum += static_cast<float>(std::abs(in[i]));
|
||||
}
|
||||
|
||||
if (sum != 0.f) {
|
||||
for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) {
|
||||
out[i] = static_cast<float>(in[i]) / sum;
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
sum += static_cast<float>(std::abs(*in++));
|
||||
}
|
||||
} else {
|
||||
for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) {
|
||||
out[i] = static_cast<float>(in[i]);
|
||||
|
||||
in -= batch_size;
|
||||
|
||||
if (sum != 0.f) {
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
*out++ = static_cast<float>(*in++) / sum;
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
*out++ = static_cast<float>(*in++);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void NormalizeL2(const gsl::span<const T>& in, gsl::span<float>& out,
|
||||
int64_t offset, int64_t stride, int64_t increment_by) {
|
||||
float sum = 0.f;
|
||||
for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) {
|
||||
auto x = in[i];
|
||||
auto x_sq = static_cast<float>(x * x);
|
||||
out[i] = x_sq;
|
||||
sum += x_sq;
|
||||
}
|
||||
void NormalizeL2(const T* in, float* out, int64_t num_batches, int64_t batch_size) {
|
||||
for (int b = 0; b < num_batches; ++b) {
|
||||
float sum = 0.f;
|
||||
|
||||
if (sum != 0.f) {
|
||||
for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) {
|
||||
auto x = in[i];
|
||||
auto x_sq = out[i];
|
||||
|
||||
if (x < 0)
|
||||
out[i] = std::sqrt(x_sq / sum) * -1;
|
||||
else
|
||||
out[i] = std::sqrt(x_sq / sum);
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
auto x = *in++;
|
||||
auto x_sq = static_cast<float>(x * x);
|
||||
*out++ = x_sq;
|
||||
sum += x_sq;
|
||||
}
|
||||
} else {
|
||||
for (int64_t i = offset, s = 0; s < stride; ++s, i += increment_by) {
|
||||
out[i] = static_cast<float>(in[i]);
|
||||
|
||||
in -= batch_size;
|
||||
out -= batch_size;
|
||||
|
||||
if (sum != 0.f) {
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
auto x = *in++;
|
||||
auto x_sq = *out;
|
||||
|
||||
*out++ = (x < 0) ? std::sqrt(x_sq / sum) * -1 : std::sqrt(x_sq / sum);
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < batch_size; ++i) {
|
||||
*out++ = static_cast<float>(*in++);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void Normalizer::Normalize(OpKernelContext* context) const {
|
||||
Status Normalizer::Normalize(OpKernelContext* context) const {
|
||||
const Tensor& X = *context->Input<Tensor>(0);
|
||||
const TensorShape& x_shape = X.Shape();
|
||||
const auto data_size = x_shape.Size();
|
||||
|
||||
if (x_shape.NumDimensions() > 2) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Rank of input to Normalized must be less than 2. Got ",
|
||||
x_shape.NumDimensions());
|
||||
}
|
||||
|
||||
const auto& x_dims = x_shape.GetDims();
|
||||
int64_t num_batches = x_dims.size() == 1 ? 1 : x_dims[0];
|
||||
int64_t batch_size = x_dims.size() == 1 ? x_dims[0] : x_dims[1];
|
||||
|
||||
Tensor* Y = context->Output(0, x_shape);
|
||||
|
||||
auto input = gsl::make_span(X.template Data<T>(), data_size);
|
||||
auto output = gsl::make_span(Y->template MutableData<float>(), data_size);
|
||||
const T* input = X.template Data<T>();
|
||||
float* output = Y->MutableData<float>();
|
||||
|
||||
int64_t stride = x_dims.size() == 1 ? x_dims[0] : x_dims[1];
|
||||
int64_t loops = data_size / stride;
|
||||
|
||||
// we normalize on axis 1 so if there are more than 2 dimensions we need to increment the index
|
||||
// by more than 1 as we process the stride
|
||||
// for 1 and 2 dimension tensors we're normalizing across the row/s, so increment_by is 1
|
||||
//
|
||||
// e.g. if you have a tensor of shape {2, 2, 3}
|
||||
// [[[ 1, 2, 3],
|
||||
// [ 4, 5, 6]],
|
||||
// [[ 7, 8, 9],
|
||||
// [10, 11, 12]]]
|
||||
//
|
||||
// we want to normalize (1, 4), (2, 5), (3, 6),
|
||||
// (7, 10), (8, 11), (9, 12)
|
||||
// so the stride would be 2, and the increment_by would be 3.
|
||||
//
|
||||
// we process a block of stride * increment_by entries before we need to skip to the next row of the 2nd dimension.
|
||||
// the offset starts at 0 and increases by 1 each loop, for increment_by loops.
|
||||
// the offset then jumps by stride * increment
|
||||
|
||||
int64_t increment_by = x_dims.size() > 1 ? x_shape.SizeFromDimension(2) : 1;
|
||||
|
||||
for (int64_t n = 0; n < loops; ++n) {
|
||||
int64_t offset = (n % increment_by) + ((n / increment_by) * (stride * increment_by));
|
||||
|
||||
switch (normalization_) {
|
||||
case NORMALIZE::NMAX: {
|
||||
NormalizeMax(input, output, offset, stride, increment_by);
|
||||
break;
|
||||
}
|
||||
case NORMALIZE::L1: {
|
||||
NormalizeL1(input, output, offset, stride, increment_by);
|
||||
break;
|
||||
}
|
||||
case NORMALIZE::L2: {
|
||||
NormalizeL2(input, output, offset, stride, increment_by);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
ORT_THROW("Unexpected NORMALIZE value of ", normalization_);
|
||||
}
|
||||
switch (normalization_) {
|
||||
case NORMALIZE::NMAX: {
|
||||
NormalizeMax(input, output, num_batches, batch_size);
|
||||
break;
|
||||
}
|
||||
case NORMALIZE::L1: {
|
||||
NormalizeL1(input, output, num_batches, batch_size);
|
||||
break;
|
||||
}
|
||||
case NORMALIZE::L2: {
|
||||
NormalizeL2(input, output, num_batches, batch_size);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unexpected NORMALIZE value of ", normalization_);
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// MLTypeCallDispather implementation wrapper
|
||||
template <class T>
|
||||
struct Normalizer::CallNormalizerImpl {
|
||||
void operator()(const Normalizer* norm, OpKernelContext* ctx) const {
|
||||
norm->Normalize<T>(ctx);
|
||||
Status operator()(const Normalizer* norm, OpKernelContext* ctx) const {
|
||||
return norm->Normalize<T>(ctx);
|
||||
}
|
||||
};
|
||||
|
||||
Status Normalizer::Compute(OpKernelContext* context) const {
|
||||
const auto* input_tensor_ptr = context->Input<Tensor>(0);
|
||||
ORT_ENFORCE(input_tensor_ptr != nullptr);
|
||||
const auto& input_tensor_ptr = *context->Input<Tensor>(0);
|
||||
|
||||
utils::MLTypeCallDispatcher<CallNormalizerImpl, float, double, int64_t, int32_t> t_disp(input_tensor_ptr->GetElementType());
|
||||
t_disp.Invoke(this, context);
|
||||
return Status::OK();
|
||||
utils::MLTypeCallDispatcherRet<Status, CallNormalizerImpl, float, double, int64_t, int32_t>
|
||||
t_disp(input_tensor_ptr.GetElementType());
|
||||
|
||||
auto status = t_disp.Invoke(this, context);
|
||||
return status;
|
||||
}
|
||||
|
||||
} // namespace ml
|
||||
|
|
|
|||
|
|
@ -25,9 +25,9 @@ class Normalizer final : public OpKernel {
|
|||
|
||||
private:
|
||||
template <typename T>
|
||||
void Normalize(OpKernelContext* context) const;
|
||||
Status Normalize(OpKernelContext* context) const;
|
||||
|
||||
template<class>
|
||||
template <class>
|
||||
struct CallNormalizerImpl;
|
||||
|
||||
NORMALIZE normalization_;
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ static void RunTest(const vector<T>& input,
|
|||
test.AddInput("X", dims, input);
|
||||
test.AddOutput("Y", dims, output);
|
||||
|
||||
// output 'norm' so if a test fails we know which one
|
||||
std::cout << "norm=" << norm << "\n";
|
||||
test.Run(expect_result, expect_error_message);
|
||||
}
|
||||
|
||||
|
|
@ -120,6 +122,23 @@ TEST(Normalizer, SingleDimensionFloat) {
|
|||
RunTests(input, dims, max_output, l1_output, l2_output);
|
||||
}
|
||||
|
||||
TEST(Normalizer, TwoDimensionFloat) {
|
||||
std::vector<int64_t> dims = {2, 3};
|
||||
std::vector<float> input = {-1.0856306f, 0.99734545f, 0.2829785f,
|
||||
-1.50629471f, -0.57860025f, 1.65143654f};
|
||||
|
||||
std::vector<float> max_output{-1.0885202f, 1.f, 0.2837317f,
|
||||
-0.91211176f, -0.35036176f, 1.f};
|
||||
|
||||
std::vector<float> l1_output{-0.45885524f, 0.42154038f, 0.11960436f,
|
||||
-0.40314806f, -0.15485784f, 0.44199413f};
|
||||
|
||||
std::vector<float> l2_output{-0.7232126f, 0.6643998f, 0.18851127f,
|
||||
-0.65239084f, -0.25059736f, 0.7152532f};
|
||||
|
||||
RunTests(input, dims, max_output, l1_output, l2_output);
|
||||
}
|
||||
|
||||
TEST(Normalizer, TwoDimensionDouble) {
|
||||
std::vector<int64_t> dims = {2, 3};
|
||||
std::vector<double> input = {-1.0856306, 0.99734545, 0.2829785,
|
||||
|
|
@ -137,39 +156,23 @@ TEST(Normalizer, TwoDimensionDouble) {
|
|||
RunTests(input, dims, max_output, l1_output, l2_output);
|
||||
}
|
||||
|
||||
TEST(Normalizer, ThreeDimensionInt32) {
|
||||
std::vector<int64_t> dims = {2, 3, 4};
|
||||
std::vector<int32_t> input = {-242, -42, 126, -86,
|
||||
-67, -9, 149, -63,
|
||||
-44, -43, 220, 218,
|
||||
TEST(Normalizer, TwoDimensionInt) {
|
||||
std::vector<int64_t> dims = {3, 2};
|
||||
std::vector<int32_t> input = {-242, -42,
|
||||
126, -86,
|
||||
-67, -9};
|
||||
|
||||
100, 38, 73, 149,
|
||||
-93, 117, -125, -63,
|
||||
90, -142, -14, -86};
|
||||
std::vector<float> max_output{5.7619047f, 1.f,
|
||||
1.f, -0.6825397f,
|
||||
7.4444447f, 1.f};
|
||||
|
||||
std::vector<float> max_output{5.5f, 4.6666665f, 0.57272726f, -0.39449543f,
|
||||
1.5227273f, 1.f, 0.67727274f, -0.28899083f,
|
||||
1.f, 4.7777777f, 1.f, 1.f,
|
||||
std::vector<float> l1_output{-0.85211265f, -0.14788732f,
|
||||
0.5943396f, -0.4056604f,
|
||||
-0.8815789f, -0.11842106f};
|
||||
|
||||
1.f, 0.32478634f, 1.f, 1.f,
|
||||
-0.93f, 1.f, -1.7123288f, -0.42281878f,
|
||||
0.9f, -1.2136753f, -0.19178082f, -0.5771812f};
|
||||
|
||||
std::vector<float> l1_output{-0.6855524f, -0.44680852f, 0.25454545f, -0.23433243f,
|
||||
-0.1898017f, -0.09574468f, 0.3010101f, -0.17166212f,
|
||||
-0.12464589f, -0.4574468f, 0.44444445f, 0.59400547f,
|
||||
|
||||
0.3533569f, 0.12794612f, 0.3443396f, 0.5f,
|
||||
-0.3286219f, 0.3939394f, -0.5896226f, -0.21140939f,
|
||||
0.3180212f, -0.4781145f, -0.06603774f, -0.2885906f};
|
||||
|
||||
std::vector<float> l2_output{-0.94928247f, -0.6910363f, 0.4284698f, -0.3543899f,
|
||||
-0.26281786f, -0.1480792f, 0.5066825f, -0.25961122f,
|
||||
-0.17259681f, -0.7074895f, 0.74812186f, 0.89833724f,
|
||||
|
||||
0.6114293f, 0.2022622f, 0.5019584f, 0.8132732f,
|
||||
-0.56862926f, 0.62275463f, -0.85951775f, -0.34386718f,
|
||||
0.55028635f, -0.7558219f, -0.09626599f, -0.469406f};
|
||||
std::vector<float> l2_output{-0.98527145f, -0.17099753f,
|
||||
0.82594985f, -0.56374353f,
|
||||
-0.9910982f, -0.13313259f};
|
||||
|
||||
RunTests(input, dims, max_output, l1_output, l2_output);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue