saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-20 19:12:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support opset-13 specs of softmax family ops (Softmax, LogSoftmax, Hardmax) (#5707)

Browse source
This commit is contained in:
Hariharan Seshadri 2020-11-11 15:45:03 -08:00 committed by GitHub
parent e5c8040c52
commit a14cd6267b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 687 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
onnxruntime/core/providers/cpu/cpu_execution_provider.cc
View file

@ -345,11 +345,11 @@ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOn
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float, ReduceSumSquare);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, ReduceSumSquare);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, int32_t, ReduceSumSquare);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Hardmax);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, float, LogSoftmax);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, double, LogSoftmax);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, float, Softmax);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, double, Softmax);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, Hardmax);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float, LogSoftmax);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, LogSoftmax);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float, Softmax);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, Softmax);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Loop);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, DepthToSpace);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Scan);
@ -594,6 +594,12 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain,
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Resize);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, int32_t, Resize);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, uint8_t, Resize);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Hardmax);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, LogSoftmax);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double, LogSoftmax);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Softmax);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double, Softmax);
// !!PLEASE READ BELOW!! Following that, add new entries above this comment
/* *** IMPORTANT! ***
@ -1109,15 +1115,15 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
float, ArgMin)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12,
int32_t, ArgMin)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Hardmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, float,
LogSoftmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, double,
LogSoftmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, double,
Softmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, float,
Softmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, Hardmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float,
LogSoftmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double,
LogSoftmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double,
Softmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float,
Softmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Loop)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, DepthToSpace)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Scan)>,
@ -1552,6 +1558,15 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
int32_t, Resize)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13,
uint8_t, Resize)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Hardmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float,
LogSoftmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double,
LogSoftmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double,
Softmax)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float,
Softmax)>,
};
for (auto& function_table_entry : function_table) {

115
onnxruntime/core/providers/cpu/math/hardmax.cc
View file

@ -5,48 +5,123 @@
#include "core/providers/common.h"
#include "core/util/math_cpuonly.h"
#include "core/util/math.h"
#include "core/providers/cpu/tensor/transpose.h"
namespace onnxruntime {
template <>
Status Hardmax<float>::Compute(OpKernelContext* ctx) const {
const auto* X = ctx->Input<Tensor>(0);
const TensorShape& input_shape = X->Shape();
const auto* Xdata = X->template Data<float>();
const TensorShape& X_shape = X->Shape();
size_t rank = X_shape.NumDimensions();
Tensor* Y = ctx->Output(0, X_shape);
auto axis = HandleNegativeAxis(axis_, input_shape.NumDimensions()); // handle negative and enforce axis is valid
size_t tmpN = input_shape.SizeToDimension(axis);
size_t tmpD = input_shape.SizeFromDimension(axis);
// special case when there is a dim value of 0 in the shape.
if (X_shape.Size() == 0)
return Status::OK();
// handle negative and enforce axis is valid
const size_t axis = static_cast<size_t>(HandleNegativeAxis(axis_, rank));
bool is_transpose_required = false;
Tensor transposed_input;
std::vector<int64_t> transposed_input_dims;
Tensor intermediate_output; // output that the hardmax implementation will write into while using transposed input
std::vector<size_t> permutation(rank);
// The "semantic" meaning of axis has changed in opset-13.
// Please compare: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Hardmax
// with https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Hardmax-11 for detailed explanations
// To account for the opset-13 behavior, our plan will be to transpose the "axis" dim to the innermost dim
// and perform softmax and then reverse the transpose. We can skip the transposing aspect if the axis is already
// the innermost dim
if (opset_ >= 13 && axis != (rank - 1)) {
is_transpose_required = true;
}
if (is_transpose_required) {
AllocatorPtr alloc;
auto status = ctx->GetTempSpaceAllocator(&alloc);
if (!status.IsOK())
return status;
std::iota(std::begin(permutation), std::end(permutation), 0);
// swap the innermost dim with the dim corresponding to axis
permutation[axis] = rank - 1;
permutation[rank - 1] = axis;
transposed_input_dims.reserve(rank);
for (auto e : permutation) {
transposed_input_dims.push_back(X_shape[e]);
}
// Allocate a temporary tensor to hold transposed input
Tensor temp_input(X->DataType(), TensorShape(transposed_input_dims), alloc);
// Perform the transpose
ORT_RETURN_IF_ERROR(TransposeBase::DoTranspose(permutation, *X, temp_input));
transposed_input = std::move(temp_input);
// Allocate memory for the intermediate output
Tensor temp_output(Y->DataType(), TensorShape(transposed_input_dims), alloc);
intermediate_output = std::move(temp_output);
}
size_t tmp_N = is_transpose_required ? TensorShape(transposed_input_dims).SizeToDimension(rank - 1) : X_shape.SizeToDimension(axis);
size_t tmp_D = is_transpose_required ? TensorShape(transposed_input_dims).SizeFromDimension(rank - 1) : X_shape.SizeFromDimension(axis);
// Math::RowwiseMax expects int N and D.
if (tmpN * tmpD > INT32_MAX || tmpN > INT32_MAX || tmpD > INT32_MAX) {
if (tmp_N * tmp_D > INT32_MAX || tmp_N > INT32_MAX || tmp_D > INT32_MAX) {
std::ostringstream ss;
ss << "Hardmax inputs N, D and N * D must be < " << INT32_MAX << ". N=" << tmpN << ", D=" << tmpD;
ss << "Hardmax inputs N, D and N * D must be < " << INT32_MAX << ". N=" << tmp_N << ", D=" << tmp_D;
std::string msg = ss.str();
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, msg);
}
const int N = gsl::narrow_cast<int>(tmpN);
const int D = gsl::narrow_cast<int>(tmpD);
const int N = gsl::narrow_cast<int>(tmp_N);
const int D = gsl::narrow_cast<int>(tmp_D);
std::vector<float> rowmax_(N);
float* rowmax_data = rowmax_.data();
math::RowwiseMax<float, CPUMathUtil>(N, D, Xdata, rowmax_data, nullptr);
Tensor* Y = ctx->Output(0, input_shape);
auto* Ydata = Y->template MutableData<float>();
math::Set<float, CPUMathUtil>(input_shape.Size(), 0.f, Ydata, &CPUMathUtil::Instance());
const float* X_data = nullptr;
float* Y_data = nullptr;
if (is_transpose_required) { // use intermediate buffers to compute the hardmax values
X_data = transposed_input.template Data<float>();
Y_data = intermediate_output.template MutableData<float>();
} else { // use the node input/output directly
X_data = X->template Data<float>();
Y_data = Y->template MutableData<float>();
}
math::RowwiseMax<float, CPUMathUtil>(N, D, X_data, rowmax_data, nullptr);
// Even if we had to transpose the input, it is safe to go with X_shape.Size() which computes
// the size of the buffer from the original input's shape as even if we do transpose, the size
// of the transposed buffer will be the same as the original input's buffer
math::Set<float, CPUMathUtil>(X_shape.Size(), 0.f, Y_data, &CPUMathUtil::Instance());
for (int i = 0; i < N; ++i) {
for (int j = 0; j < D; ++j) {
if (Xdata[i * D + j] == rowmax_data[i]) {
Ydata[i * D + j] = 1;
if (X_data[i * D + j] == rowmax_data[i]) {
Y_data[i * D + j] = 1;
break;
}
}
}
if (is_transpose_required) {
std::vector<size_t> reverse_permutation(rank);
for (size_t i = 0, end = permutation.size(); i < end; ++i) {
reverse_permutation[permutation[i]] = i;
}
// Perform the transpose to get the axes back to the original ordering
ORT_RETURN_IF_ERROR(TransposeBase::DoTranspose(reverse_permutation, intermediate_output, *Y));
}
return Status::OK();
}
@ -58,9 +133,17 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL(
Hardmax<float>);
// Opset 11 starts to support Neg Axis.
ONNX_CPU_OPERATOR_KERNEL(
ONNX_CPU_OPERATOR_VERSIONED_KERNEL(
Hardmax,
11,
12,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Hardmax<float>);
// Opset 13 changed the semantic meaning of the axis attribute.
ONNX_CPU_OPERATOR_KERNEL(
Hardmax,
13,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Hardmax<float>);

12
onnxruntime/core/providers/cpu/math/hardmax.h
View file

@ -12,12 +12,21 @@ namespace onnxruntime {
template <typename T>
class Hardmax final : public OpKernel {
public:
Hardmax(const OpKernelInfo& info) : OpKernel{info}, axis_{1} {
Hardmax(const OpKernelInfo& info) : OpKernel{info} {
const auto& node = info.node();
opset_ = node.SinceVersion();
int64_t axis;
Status status = info.GetAttr<int64_t>("axis", &axis);
if (status.IsOK()) {
axis_ = gsl::narrow_cast<int>(axis);
} else {
if (opset_ < 13) {
axis_ = 1; // opset-12 and below, the default axis value is 1
} else {
axis_ = -1; // opset-13, the default axis value is -1
}
}
}
@ -25,5 +34,6 @@ class Hardmax final : public OpKernel {
private:
int axis_;
int opset_;
};
} // namespace onnxruntime

153
onnxruntime/core/providers/cpu/math/softmax.cc
View file

@ -2,6 +2,9 @@
// Licensed under the MIT License.
#include "core/providers/cpu/math/softmax.h"
#include "core/providers/cpu/tensor/transpose.h"
#include <vector>
#include <numeric>
namespace onnxruntime {
@ -14,9 +17,18 @@ ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(
Softmax<float>);
// Opset 11 starts to support Neg Axis.
ONNX_CPU_OPERATOR_TYPED_KERNEL(
ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(
Softmax,
11,
12,
float,
KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Softmax<float>);
// Opset 13 changed the semantic meaning of the axis attribute.
ONNX_CPU_OPERATOR_TYPED_KERNEL(
Softmax,
13,
float,
KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Softmax<float>);
@ -30,9 +42,18 @@ ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(
Softmax<double>);
// Opset 11 starts to support Neg Axis.
ONNX_CPU_OPERATOR_TYPED_KERNEL(
ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(
Softmax,
11,
12,
double,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<double>()),
Softmax<double>);
// Opset 13 changed the semantic meaning of the axis attribute.
ONNX_CPU_OPERATOR_TYPED_KERNEL(
Softmax,
13,
double,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<double>()),
Softmax<double>);
@ -46,9 +67,18 @@ ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(
Softmax<float>);
// Opset 11 starts to support Neg Axis.
ONNX_CPU_OPERATOR_TYPED_KERNEL(
ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(
LogSoftmax,
11,
12,
float,
KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Softmax<float>);
// Opset 13 changed the semantic meaning of the axis attribute.
ONNX_CPU_OPERATOR_TYPED_KERNEL(
LogSoftmax,
13,
float,
KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Softmax<float>);
@ -62,11 +92,126 @@ ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(
Softmax<double>);
// Opset 11 starts to support Neg Axis.
ONNX_CPU_OPERATOR_TYPED_KERNEL(
ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(
LogSoftmax,
11,
12,
double,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<double>()),
Softmax<double>);
// Opset 13 changed the semantic meaning of the axis attribute.
ONNX_CPU_OPERATOR_TYPED_KERNEL(
LogSoftmax,
13,
double,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<double>()),
Softmax<double>);
// opset-12 and below
template <typename T>
Status Softmax<T>::ComputeImpl(const Tensor& input, Tensor& output, size_t axis,
concurrency::ThreadPool* thread_pool) const {
const auto& X_shape = input.Shape();
const size_t N = X_shape.SizeToDimension(axis);
const size_t D = X_shape.SizeFromDimension(axis);
return SoftmaxCPU<T>(N, D, input.template Data<T>(), output.template MutableData<T>(), log_softmax_, thread_pool);
}
// opset-13 and above
template <typename T>
Status Softmax<T>::ComputeImplOpset13(const Tensor& input, Tensor& output, size_t axis,
concurrency::ThreadPool* thread_pool, OpKernelContext* ctx) const {
const auto& X_shape = input.Shape();
size_t rank = X_shape.NumDimensions();
bool is_transpose_required = false;
Tensor transposed_input;
std::vector<int64_t> transposed_input_dims;
Tensor intermediate_output; // output that the softmax implementation will write into while using transposed input
std::vector<size_t> permutation(rank);
// The "semantic" meaning of axis has changed in opset-13.
// Please compare: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softmax
// with https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-11 for detailed explanations
// To account for the opset-13 behavior, our plan will be to transpose the "axis" dim to the innermost dim
// and perform softmax and then reverse the transpose. We can skip the transposing aspect if the axis is already
// the innermost dim
if (axis != (rank - 1)) {
is_transpose_required = true;
}
if (is_transpose_required) {
AllocatorPtr alloc;
auto status = ctx->GetTempSpaceAllocator(&alloc);
if (!status.IsOK())
return status;
std::iota(std::begin(permutation), std::end(permutation), 0);
// swap the innermost dim with the dim corresponding to axis
permutation[axis] = rank - 1;
permutation[rank - 1] = axis;
transposed_input_dims.reserve(rank);
for (auto e : permutation) {
transposed_input_dims.push_back(X_shape[e]);
}
// Allocate a temporary tensor to hold transposed input
Tensor temp_input(input.DataType(), TensorShape(transposed_input_dims), alloc);
// Perform the transpose
ORT_RETURN_IF_ERROR(TransposeBase::DoTranspose(permutation, input, temp_input));
transposed_input = std::move(temp_input);
// Allocate memory for the intermediate output
Tensor temp_output(output.DataType(), TensorShape(transposed_input_dims), alloc);
intermediate_output = std::move(temp_output);
}
const size_t N = is_transpose_required ? TensorShape(transposed_input_dims).SizeToDimension(rank - 1) : X_shape.SizeToDimension(rank - 1);
const size_t D = is_transpose_required ? TensorShape(transposed_input_dims).SizeFromDimension(rank - 1) : X_shape.SizeFromDimension(rank - 1);
ORT_RETURN_IF_ERROR(SoftmaxCPU<T>(N, D,
is_transpose_required ? transposed_input.template Data<T>() : input.template Data<T>(),
is_transpose_required ? intermediate_output.template MutableData<T>() : output.template MutableData<T>(),
log_softmax_, thread_pool));
if (is_transpose_required) {
std::vector<size_t> reverse_permutation(rank);
for (size_t i = 0, end = permutation.size(); i < end; ++i) {
reverse_permutation[permutation[i]] = i;
}
// Perform the transpose to get the axes back to the original ordering
ORT_RETURN_IF_ERROR(TransposeBase::DoTranspose(reverse_permutation, intermediate_output, output));
}
return Status::OK();
}
// compute method of Softmax
template <typename T>
Status Softmax<T>::Compute(OpKernelContext* ctx) const {
const auto* X = ctx->Input<Tensor>(0);
const auto& X_shape = X->Shape();
size_t rank = X_shape.NumDimensions();
auto* Y = ctx->Output(0, X_shape);
// edge case. one or more dims with value of 0. nothing to do
if (X_shape.Size() == 0) {
return Status::OK();
}
const size_t axis = static_cast<size_t>(HandleNegativeAxis(axis_, rank));
concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool();
if (opset_ < 13) {
return ComputeImpl(*X, *Y, axis, thread_pool);
} else {
return ComputeImplOpset13(*X, *Y, axis, thread_pool, ctx);
}
}
} // namespace onnxruntime

37
onnxruntime/core/providers/cpu/math/softmax.h
View file

@ -14,38 +14,37 @@ namespace onnxruntime {
template <typename T>
class Softmax final : public OpKernel {
public:
Softmax(const OpKernelInfo& info) : OpKernel{info}, axis_{1} {
Softmax(const OpKernelInfo& info) : OpKernel{info} {
const auto& node = info.node();
opset_ = node.SinceVersion();
int64_t axis;
Status status = info.GetAttr<int64_t>("axis", &axis);
if (status.IsOK()) {
axis_ = gsl::narrow_cast<int>(axis);
} else {
if (opset_ < 13) {
axis_ = 1; // opset-12 and below, the default axis value is 1
} else {
axis_ = -1; // opset-13, the default axis value is -1
}
}
log_softmax_ = info.GetKernelDef().OpName() == "LogSoftmax";
}
Status Compute(OpKernelContext* ctx) const override {
const auto* X = ctx->Input<Tensor>(0);
const auto& X_shape = X->Shape();
auto* Y = ctx->Output(0, X_shape);
// edge case. one or more dims with value of 0. nothing to do
if (X_shape.Size() == 0) {
return Status::OK();
}
const int64_t axis = HandleNegativeAxis(axis_, X_shape.NumDimensions());
const size_t N = X_shape.SizeToDimension(axis);
const size_t D = X_shape.SizeFromDimension(axis);
concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool();
return SoftmaxCPU(N, D, X->template Data<T>(), Y->template MutableData<T>(), log_softmax_, thread_pool);
}
Status Compute(OpKernelContext* ctx) const override;
private:
Status ComputeImpl(const Tensor& input, Tensor& output, size_t axis,
concurrency::ThreadPool* thread_pool) const;
Status ComputeImplOpset13(const Tensor& input, Tensor& output, size_t axis,
concurrency::ThreadPool* thread_pool, OpKernelContext* ctx) const;
int axis_;
int opset_;
bool log_softmax_;
};

105
onnxruntime/core/providers/cuda/math/softmax.cc
View file

@ -6,6 +6,7 @@
#include "core/providers/common.h"
#include "core/providers/cuda/cudnn_common.h"
#include "core/providers/cuda/shared_inc/accumulation_type.h"
#include "core/providers/cuda/tensor/transpose.h"
namespace onnxruntime {
namespace cuda {
@ -19,10 +20,8 @@ Status SoftMaxComputeHelper(
int64_t axis) {
typedef typename ToCudaType<T>::MappedType CudaT;
const int64_t normalized_axis = HandleNegativeAxis(axis, input_shape.NumDimensions());
int64_t N = input_shape.SizeToDimension(normalized_axis);
int64_t D = input_shape.SizeFromDimension(normalized_axis);
int64_t N = input_shape.SizeToDimension(axis);
int64_t D = input_shape.SizeFromDimension(axis);
auto Y_data = reinterpret_cast<CudaT*>(Y);
auto X_data = reinterpret_cast<const CudaT*>(X);
@ -112,17 +111,103 @@ template <typename T>
Status Softmax<T>::ComputeInternal(OpKernelContext* ctx) const {
const Tensor* X = ctx->Input<Tensor>(0);
const TensorShape& input_shape{X->Shape()};
const T* X_data = X->template Data<T>();
T* Y_data = ctx->Output(0, input_shape)->template MutableData<T>();
size_t rank = input_shape.NumDimensions();
Tensor* Y = ctx->Output(0, input_shape);
// special case when there is a dim value of 0 in the shape.
if (input_shape.Size() == 0)
return Status::OK();
if (log_softmax_) {
return SoftMaxComputeHelper<T, true>(X_data, input_shape, Y_data, CudnnHandle(), axis_);
} else {
return SoftMaxComputeHelper<T, false>(X_data, input_shape, Y_data, CudnnHandle(), axis_);
// handle negative and enforce axis is valid
const size_t axis = static_cast<size_t>(HandleNegativeAxis(axis_, rank));
bool is_transpose_required = false;
Tensor transposed_input;
std::vector<int64_t> transposed_input_dims;
Tensor intermediate_output; // output that the softmax implementation will write into while using transposed input
std::vector<size_t> permutation(rank);
// The "semantic" meaning of axis has changed in opset-13.
// Please compare: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softmax
// with https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-11 for detailed explanations
// To account for the opset-13 behavior, our plan will be to transpose the "axis" dim to the innermost dim
// and perform softmax and then reverse the transpose. We can skip the transposing aspect if the axis is already
// the innermost dim
if (opset_ >= 13 && axis != (rank - 1)) {
is_transpose_required = true;
}
if (is_transpose_required) {
AllocatorPtr alloc;
auto status = ctx->GetTempSpaceAllocator(&alloc);
if (!status.IsOK())
return status;
std::iota(std::begin(permutation), std::end(permutation), 0);
// swap the innermost dim with the dim corresponding to axis
permutation[axis] = rank - 1;
permutation[rank - 1] = axis;
transposed_input_dims.reserve(rank);
for (auto e : permutation) {
transposed_input_dims.push_back(input_shape[e]);
}
// Allocate a temporary tensor to hold transposed input
Tensor temp_input(X->DataType(), TensorShape(transposed_input_dims), alloc);
// Perform the transpose
ORT_RETURN_IF_ERROR(Transpose::DoTranspose(cuda_ep_->GetDeviceProp(),
CublasHandle(),
permutation, *X, temp_input));
transposed_input = std::move(temp_input);
// Allocate memory for the intermediate output
Tensor temp_output(Y->DataType(), TensorShape(transposed_input_dims), alloc);
intermediate_output = std::move(temp_output);
}
const T* X_data = nullptr;
T* Y_data = nullptr;
const TensorShape* compute_input_shape = nullptr;
if (is_transpose_required) { // use intermediate buffers to compute the softmax values
X_data = transposed_input.template Data<T>();
Y_data = intermediate_output.template MutableData<T>();
compute_input_shape = &transposed_input.Shape();
} else { // use the node input/output directly
X_data = X->template Data<T>();
Y_data = Y->template MutableData<T>();
compute_input_shape = &input_shape;
}
Status status;
if (log_softmax_) {
status = SoftMaxComputeHelper<T, true>(X_data, *compute_input_shape, Y_data, CudnnHandle(),
is_transpose_required ? static_cast<int64_t>(rank) - 1
: static_cast<int64_t>(axis));
} else {
status = SoftMaxComputeHelper<T, false>(X_data, *compute_input_shape, Y_data, CudnnHandle(),
is_transpose_required ? static_cast<int64_t>(rank) - 1
: static_cast<int64_t>(axis));
}
if (!status.IsOK())
return status;
if (is_transpose_required) {
std::vector<size_t> reverse_permutation(rank);
for (size_t i = 0, end = permutation.size(); i < end; ++i) {
reverse_permutation[permutation[i]] = i;
}
// Perform the transpose to get the axes back to the original ordering
ORT_RETURN_IF_ERROR(Transpose::DoTranspose(cuda_ep_->GetDeviceProp(),
CublasHandle(),
reverse_permutation, intermediate_output, *Y));
}
return Status::OK();
}
#define SPECIALIZED_COMPUTE(T) \

27
onnxruntime/core/providers/cuda/math/softmax.h
View file

@ -24,8 +24,28 @@ template <typename T>
class Softmax final : public CudaKernel {
public:
Softmax(const OpKernelInfo& info) : CudaKernel{info} {
info.GetAttrOrDefault("axis", &axis_, static_cast<int64_t>(1));
const auto& node = info.node();
opset_ = node.SinceVersion();
int64_t axis;
Status status = info.GetAttr<int64_t>("axis", &axis);
if (status.IsOK()) {
axis_ = gsl::narrow_cast<int>(axis);
} else {
if (opset_ < 13) {
axis_ = 1; // opset-12 and below, the default axis value is 1
} else {
axis_ = -1; // opset-13, the default axis value is -1
}
}
log_softmax_ = info.GetKernelDef().OpName() == "LogSoftmax";
// We need to cast away the const as PerThreadCublasHandle() is currently a non-const method
// TODO: Clean up the CUDAExecutionProvider interface to avoid this
cuda_ep_ = const_cast<CUDAExecutionProvider*>(
static_cast<const CUDAExecutionProvider*>(info.GetExecutionProvider()));
}
Status ComputeInternal(OpKernelContext* context) const override;
@ -33,6 +53,11 @@ class Softmax final : public CudaKernel {
private:
int64_t axis_;
bool log_softmax_;
int opset_;
// We need to access to the CUDA EP instance to get the cublas handle to use
// for transposing(if applicable)
CUDAExecutionProvider* cuda_ep_;
};
} // namespace cuda

83
onnxruntime/test/providers/cpu/math/hardmax_test.cc
View file

@ -11,13 +11,20 @@ namespace test {
static void RunTest(const std::vector<float>& x_vals,
const std::vector<float>& expected_vals,
const std::vector<int64_t>& dimensions,
int opset = 7,
int64_t axis = 1,
OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess,
const std::string& expected_err_str = "") {
OpTester test("Hardmax");
OpTester test("Hardmax", opset);
if (axis != 1) {
test.AddAttribute("axis", axis);
if (opset < 13) {
if (axis != 1) { // opset-12 and below : default axis value is 1
test.AddAttribute("axis", axis);
}
} else {
if (axis != -1) { // opset-13 : default axis value is -1
test.AddAttribute("axis", axis);
}
}
test.AddInput<float>("X", dimensions, x_vals);
@ -85,7 +92,7 @@ TEST(HardmaxOperator, ThreeDimsAxis0) {
0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0);
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 0);
}
TEST(HardmaxOperator, ThreeDimsAxis1) {
@ -108,9 +115,31 @@ TEST(HardmaxOperator, ThreeDimsAxis1) {
0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1);
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 1);
}
TEST(HardmaxOperator, ThreeDimsAxis1_opset13) {
// For the same input, opset-13's behavior is different from an earlier opset
// and we see different expected results for the same test input
std::vector<float> expected_vals = {
0.0f, 0.0f, 0.0f, 1.0f, 0.0f,
1.0f, 1.0f, 0.0f, 0.0f, 1.0f,
0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f,
1.0f, 0.0f, 0.0f, 1.0f, 1.0f,
0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 1.0f,
1.0f, 0.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 1.0f, 0.0f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 1);
}
TEST(HardmaxOperator, ThreeDimsAxis2) {
// x = <see x_vals_3dims>
// import cntk as C
@ -131,7 +160,47 @@ TEST(HardmaxOperator, ThreeDimsAxis2) {
0.0f, 0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 2);
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 2);
}
TEST(HardmaxOperator, ThreeDimsAxis2_opset13) {
std::vector<float> expected_vals = {
0.0f, 0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 2);
}
TEST(HardmaxOperator, ThreeDimsDefaultAxis_opset13) {
std::vector<float> expected_vals = {
0.0f, 0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*default axis*/ -1);
}
TEST(HardmaxOperator, ThreeDimsNegAxis2) {
@ -154,7 +223,7 @@ TEST(HardmaxOperator, ThreeDimsNegAxis2) {
0.0f, 0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f, 0.0f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ -1);
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ -1);
}
} // namespace test

100
onnxruntime/test/providers/cpu/math/logsoftmax_test.cc
View file

@ -11,15 +11,21 @@ namespace test {
static void RunTest(const std::vector<float>& x_vals,
const std::vector<float>& expected_vals,
const std::vector<int64_t>& dimensions,
int opset = 7,
int64_t axis = 1,
bool is_tensorrt_supported = true,
OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess,
const std::string& error_msg = "",
int opset = 7) {
const std::string& error_msg = "") {
OpTester tester("LogSoftmax", opset);
if (axis != 1) {
tester.AddAttribute("axis", axis);
if (opset < 13) {
if (axis != 1) { // opset-12 and below : default axis value is 1
tester.AddAttribute("axis", axis);
}
} else {
if (axis != -1) { // opset-13 : default axis value is -1
tester.AddAttribute("axis", axis);
}
}
tester.AddInput("X", dimensions, x_vals);
@ -101,7 +107,7 @@ TEST(LogSoftmaxOperator, ThreeDimsAxis0) {
-4.042971f, -4.2982683f, -3.5933442f, -4.538994f, -5.307373f,
-4.2677402f, -4.44635f, -3.5821702f, -3.8414123f, -4.267664f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0, false); // axis=0 is not supported by TensorRT
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 0, false); // axis=0 is not supported by TensorRT
}
TEST(LogSoftmaxOperator, ThreeDimsAxis1) {
@ -127,16 +133,33 @@ TEST(LogSoftmaxOperator, ThreeDimsAxis1) {
-2.9822054f, -3.2375026f, -2.5325785f, -3.4782279f, -4.246608f,
-3.2069747f, -3.3855844f, -2.5214045f, -2.7806466f, -3.206898f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1, false); // This test failed on TensorRT
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 1, false); // This test failed on TensorRT
}
TEST(LogSoftmaxOperator, ThreeDimsAxis1_opset13) {
// For the same input, opset-13's behavior is different from an earlier opset
// and we see different expected results for the same test input
std::vector<float> expected_vals = {
-1.373224f, -2.1894338f, -2.5028024f, -1.0337411f, -1.3945004f,
-0.8074181f, -0.7601002f, -2.3568683f, -1.2740996f, -1.1063602f,
-1.7799685f, -3.0920703f, -1.2943912f, -1.9011338f, -1.5291187f,
-2.0245035f, -0.9808493f, -0.5989947f, -1.5359819f, -1.5869142f,
-1.1288074f, -1.7014627f, -1.7446783f, -1.0005655f, -1.0210506f,
-1.2284245f, -2.2850897f, -1.2518314f, -2.036326f, -1.4131763f,
-1.6105566f, -0.3936059f, -0.90897894f, -1.4765173f, -1.3474687f,
-1.6925404f, -3.189349f, -1.9922893f, -1.2968583f, -1.9913039f,
-1.4780827f, -1.355593f, -2.2826173f, -1.8348985f, -2.3507955f,
-2.2716188f, -0.69089717f, -2.2606049f, -1.4299684f, -0.45124117f,
-0.9893639f, -2.0444741f, -0.92980486f, -1.6106415f, -2.6597016f,
-1.2141333f, -2.192556f, -0.9186309f, -0.9130602f, -1.6199919f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 1, false); // This test failed on TensorRT
}
TEST(LogSoftmaxOperator, ThreeDimsAxis2) {
// x = <see x_vals_3dims>
// node = onnx.helper.make_node('LogSoftmax', inputs = ['x'], outputs = ['y'], axis = 2)
// y = logsoftmax_2d(x.reshape(12, 5)).reshape(3, 4, 5)
// expect(node, inputs = [x], outputs = [y],
// name = 'test_logsoftmax_axis_2')
std::vector<float> expected_vals = {
-1.5016061f, -1.5898913f, -2.3042583f, -1.080942f, -2.0086365f,
-1.5264852f, -0.7512426f, -2.7490091f, -1.9119854f, -2.3111813f,
@ -153,9 +176,48 @@ TEST(LogSoftmaxOperator, ThreeDimsAxis2) {
-1.4430928f, -1.6983899f, -0.9934659f, -1.9391153f, -2.7074947f,
-1.8489327f, -2.027542f, -1.1633625f, -1.4226046f, -1.848856f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 2);
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 2);
}
TEST(LogSoftmaxOperator, ThreeDimsAxis2_opset13) {
std::vector<float> expected_vals = {
-1.5016061f, -1.5898913f, -2.3042583f, -1.080942f, -2.0086365f,
-1.5264852f, -0.7512426f, -2.7490091f, -1.9119854f, -2.3111813f,
-1.716058f, -2.3002353f, -0.9035546f, -1.7560422f, -1.9509623f,
-2.7323837f, -0.96080494f, -0.97994876f, -2.162681f, -2.7805486f,
-2.024213f, -1.2708496f, -1.8257477f, -1.5857526f, -1.507701f,
-1.8521607f, -1.582807f, -1.0612315f, -2.3498435f, -1.6281573f,
-3.0813656f, -0.538396f, -1.5654519f, -2.6371078f, -2.4095225f,
-1.8958019f, -2.0665917f, -1.3812149f, -1.1899012f, -1.7858102f,
-1.7220669f, -0.79976386f, -2.1365335f, -1.9536276f, -2.1888442f,
-3.2268262f, -0.84629166f, -2.8257446f, -2.259921f, -1.0005134f,
-1.4430928f, -1.6983899f, -0.9934659f, -1.9391153f, -2.7074947f,
-1.8489327f, -2.027542f, -1.1633625f, -1.4226046f, -1.848856f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 2);
}
TEST(LogSoftmaxOperator, ThreeDimsDefaultAxis_opset13) {
std::vector<float> expected_vals = {
-1.5016061f, -1.5898913f, -2.3042583f, -1.080942f, -2.0086365f,
-1.5264852f, -0.7512426f, -2.7490091f, -1.9119854f, -2.3111813f,
-1.716058f, -2.3002353f, -0.9035546f, -1.7560422f, -1.9509623f,
-2.7323837f, -0.96080494f, -0.97994876f, -2.162681f, -2.7805486f,
-2.024213f, -1.2708496f, -1.8257477f, -1.5857526f, -1.507701f,
-1.8521607f, -1.582807f, -1.0612315f, -2.3498435f, -1.6281573f,
-3.0813656f, -0.538396f, -1.5654519f, -2.6371078f, -2.4095225f,
-1.8958019f, -2.0665917f, -1.3812149f, -1.1899012f, -1.7858102f,
-1.7220669f, -0.79976386f, -2.1365335f, -1.9536276f, -2.1888442f,
-3.2268262f, -0.84629166f, -2.8257446f, -2.259921f, -1.0005134f,
-1.4430928f, -1.6983899f, -0.9934659f, -1.9391153f, -2.7074947f,
-1.8489327f, -2.027542f, -1.1633625f, -1.4226046f, -1.848856f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*default axis*/ -1);
}
TEST(LogSoftmaxOperator, ThreeDimsNegativeAxis) {
// x = <see x_vals_3dims>
// node = onnx.helper.make_node('LogSoftmax', inputs = ['x'], outputs = ['y'], axis = 2)
@ -180,7 +242,7 @@ TEST(LogSoftmaxOperator, ThreeDimsNegativeAxis) {
-1.8489327f, -2.027542f, -1.1633625f, -1.4226046f, -1.848856f};
// -1 is last axis so same as axis == 2
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ -1);
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 12, /*axis*/ -1);
}
TEST(LogSoftmaxOperator, InvalidAxis) {
@ -191,15 +253,13 @@ TEST(LogSoftmaxOperator, InvalidAxis) {
RunTest(x_vals,
expected_vals,
dimensions,
/*opset*/ 12,
/* invalid axis */ -7,
false,
false, //TensorRT parser: Assertion failed: axis >= 0 && axis < nbDims
OpTester::ExpectResult::kExpectFailure,
// ONNX has a bug in the error message generation so this is somewhat cryptic until it's fixed. Message should be:
// "[ShapeInferenceError] 'axis' must be in [-2 , 1]. Its actual value is: -7"
", 1]. Its actual value is: -7",
// latest opset so we get shape inferencing errors
// Latest valid opset for this is 12. Once opset 13 changes are implemented this can be changed back to -1
12); //TensorRT parser: Assertion failed: axis >= 0 && axis < nbDims
"[ShapeInferenceError] 'axis' must be in [-2 , 1]. Its actual value is: -7");
//", 1]. Its actual value is: -7");
}
} // namespace test

116
onnxruntime/test/providers/cpu/math/softmax_test.cc
View file

@ -11,17 +11,22 @@ namespace test {
static void RunTest(const std::vector<float>& x_vals,
const std::vector<float>& expected_vals,
const std::vector<int64_t>& dimensions,
int opset = 7,
int64_t axis = 1,
const std::unordered_set<std::string>& excluded_providers = {},
OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess,
const std::string& error_msg = "",
int opset = 7) {
const std::string& error_msg = "") {
OpTester test("Softmax", opset);
if (axis != 1) {
test.AddAttribute("axis", axis);
if (opset < 13) {
if (axis != 1) { // opset-12 and below : default axis value is 1
test.AddAttribute("axis", axis);
}
} else {
if (axis != -1) { // opset-13 : default axis value is -1
test.AddAttribute("axis", axis);
}
}
test.AddInput<float>("X", dimensions, x_vals);
test.AddOutput<float>("Y", dimensions, expected_vals);
test.Run(expect_result, error_msg, excluded_providers);
@ -93,7 +98,7 @@ TEST(SoftmaxOperator, ThreeDimsAxis0) {
0.017545262f, 0.0135920765f, 0.027506188f, 0.010684152f, 0.0049549243f,
0.01401341f, 0.011721271f, 0.027815264f, 0.021463264f, 0.014014485f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0, {kTensorrtExecutionProvider}); // Axis=0 is not supported by TensorRT
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 0, {kTensorrtExecutionProvider}); // Axis=0 is not supported by TensorRT
}
TEST(SoftmaxOperator, ThreeDimsAxis1) {
@ -119,7 +124,31 @@ TEST(SoftmaxOperator, ThreeDimsAxis1) {
0.050680935f, 0.03926183f, 0.079453886f, 0.030862054f, 0.014312706f,
0.040478885f, 0.033857856f, 0.080346674f, 0.06199841f, 0.040481992f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1, {kTensorrtExecutionProvider});
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 1, {kTensorrtExecutionProvider});
}
TEST(SoftmaxOperator, ThreeDimsAxis1_opset13) {
// For the same input, opset-13's behavior is different from an earlier opset
// and we see different expected results for the same test input
std::vector<float> expected_vals = {
0.253289f, 0.11198013f, 0.08185529f, 0.35567388f, 0.24795689f,
0.44600812f, 0.46761957f, 0.09471639f, 0.2796827f, 0.3307607f,
0.16864346f, 0.04540785f, 0.27406466f, 0.14939913f, 0.2167266f,
0.1320594f, 0.3749925f, 0.5493636f, 0.21524426f, 0.20455585f,
0.32341874f, 0.18241648f, 0.1747012f, 0.36767146f, 0.36021632f,
0.29275346f, 0.10176494f, 0.28598055f, 0.13050734f, 0.24336906f,
0.19977638f, 0.67461985f, 0.40293545f, 0.22843185f, 0.25989732f,
0.18405138f, 0.04119869f, 0.13638285f, 0.27338937f, 0.13651732f,
0.22807457f, 0.2577944f, 0.10201685f, 0.15962972f, 0.09529332f,
0.10314508f, 0.5011263f, 0.10428739f, 0.23931651f, 0.63683724f,
0.37181312f, 0.12944824f, 0.3946307f, 0.19975942f, 0.0699691f,
0.29696727f, 0.11163106f, 0.39906505f, 0.4012943f, 0.1979003f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 1,
{kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); // OpenVINO doesn't support opset-13 yet
}
TEST(SoftmaxOperator, ThreeDimsAxis2) {
@ -145,9 +174,50 @@ TEST(SoftmaxOperator, ThreeDimsAxis2) {
0.23619612f, 0.1829779f, 0.37029108f, 0.14383113f, 0.0667037f,
0.15740506f, 0.13165872f, 0.31243387f, 0.24108529f, 0.15741715f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 2);
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 2);
}
TEST(SoftmaxOperator, ThreeDimsAxis2_opset13) {
std::vector<float> expected_vals = {
0.22277209f, 0.20394778f, 0.09983283f, 0.33927578f, 0.13417149f,
0.21729809f, 0.47177994f, 0.06399124f, 0.14778666f, 0.099144064f,
0.1797734f, 0.10023525f, 0.40512702f, 0.17272712f, 0.14213723f,
0.06506401f, 0.3825848f, 0.37533033f, 0.11501635f, 0.062004484f,
0.13209775f, 0.28059313f, 0.16109712f, 0.2047936f, 0.22141843f,
0.1568978f, 0.20539774f, 0.3460294f, 0.0953841f, 0.19629094f,
0.045896534f, 0.5836837f, 0.20899355f, 0.07156797f, 0.08985819f,
0.15019783f, 0.1266166f, 0.2512731f, 0.30425128f, 0.16766116f,
0.17869644f, 0.44943509f, 0.11806339f, 0.1417589f, 0.112046175f,
0.03968324f, 0.42900288f, 0.059264507f, 0.10435873f, 0.36769062f,
0.23619612f, 0.1829779f, 0.37029108f, 0.14383113f, 0.0667037f,
0.15740506f, 0.13165872f, 0.31243387f, 0.24108529f, 0.15741715f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 2,
{kOpenVINOExecutionProvider}); // OpenVINO doesn't support opset-13 yet
}
TEST(SoftmaxOperator, ThreeDimsDefaultAxis_opset13) {
std::vector<float> expected_vals = {
0.22277209f, 0.20394778f, 0.09983283f, 0.33927578f, 0.13417149f,
0.21729809f, 0.47177994f, 0.06399124f, 0.14778666f, 0.099144064f,
0.1797734f, 0.10023525f, 0.40512702f, 0.17272712f, 0.14213723f,
0.06506401f, 0.3825848f, 0.37533033f, 0.11501635f, 0.062004484f,
0.13209775f, 0.28059313f, 0.16109712f, 0.2047936f, 0.22141843f,
0.1568978f, 0.20539774f, 0.3460294f, 0.0953841f, 0.19629094f,
0.045896534f, 0.5836837f, 0.20899355f, 0.07156797f, 0.08985819f,
0.15019783f, 0.1266166f, 0.2512731f, 0.30425128f, 0.16766116f,
0.17869644f, 0.44943509f, 0.11806339f, 0.1417589f, 0.112046175f,
0.03968324f, 0.42900288f, 0.059264507f, 0.10435873f, 0.36769062f,
0.23619612f, 0.1829779f, 0.37029108f, 0.14383113f, 0.0667037f,
0.15740506f, 0.13165872f, 0.31243387f, 0.24108529f, 0.15741715f};
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*default axis*/ -1,
{kOpenVINOExecutionProvider}); // OpenVINO doesn't support opset-13 yet
}
TEST(SoftmaxOperator, ThreeDimsNegativeAxis) {
// x = <see x_vals_3dims>
// node = onnx.helper.make_node('Softmax', inputs = ['x'], outputs = ['y'], axis = 2)
@ -172,7 +242,7 @@ TEST(SoftmaxOperator, ThreeDimsNegativeAxis) {
0.15740506f, 0.13165872f, 0.31243387f, 0.24108529f, 0.15741715f};
// -1 is last axis so same as axis == 2
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ -1);
RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 12, /*axis*/ -1);
}
TEST(SoftmaxOperator, InvalidAxis) {
@ -183,25 +253,39 @@ TEST(SoftmaxOperator, InvalidAxis) {
RunTest(x_vals,
expected_vals,
dimensions,
/*opset*/ 12,
/* invalid axis */ -10, {kTensorrtExecutionProvider},
OpTester::ExpectResult::kExpectFailure,
// bug in ONNX error message currently. Message should be
// "[ShapeInferenceError] 'axis' must be in [-2 , 1]. Its actual value is: -10"
", 1]. Its actual value is: -10",
// latest opset so we get shape inferencing errors
// Latest valid opset for this is 12. Once opset 13 changes are implemented this can be changed back to -1
12);
", 1]. Its actual value is: -10");
}
TEST(SoftmaxOperator, InvalidAxis_opset13) {
std::vector<float> x_vals = {-1.0f, 0.0f, 1.0f};
std::vector<float> expected_vals = {0.0f, 0.0f, 0.0f};
std::vector<int64_t> dimensions = {1, 3};
RunTest(x_vals,
expected_vals,
dimensions,
/*opset*/ -1, // latest opset so we get shape inferencing errors
/* invalid axis */ -10, {kTensorrtExecutionProvider, kOpenVINOExecutionProvider},
OpTester::ExpectResult::kExpectFailure,
// In opset-13, Softmax is composed as afunction of several other ops,
// and hence it breaks differently to the test above but the most important thing
// is that it breaks and this is the right behavior
"[ShapeInferenceError] axis must be in [-rank, rank-1]. input rank was 2");
}
TEST(SoftmaxOperator, DimWithZero) {
std::vector<float> x_vals = {};
std::vector<float> expected_vals = {};
std::vector<int64_t> dimensions = {1, 0}; // dim with value of 0 should be handled
RunTest(x_vals, expected_vals, dimensions, 0,
RunTest(x_vals, expected_vals, dimensions, /*opset*/ -1, /*axis*/ 0,
{kTensorrtExecutionProvider,
kNnapiExecutionProvider}, // NNAPI softmax does not support empty input
OpTester::ExpectResult::kExpectSuccess, "", 10);
kNnapiExecutionProvider} // NNAPI softmax does not support empty input
);
}
} // namespace test

41
onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc vendored
View file

@ -35,27 +35,6 @@
"^test_training_dropout.*", // NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel)
"^test_if_seq_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node If(13)
"^test_loop13_seq_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Loop(13)
"^test_hardmax_axis_0_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Hardmax(13)
"^test_hardmax_axis_1_cpu",
"^test_hardmax_axis_2_cpu",
"^test_hardmax_default_axis_cpu",
"^test_hardmax_example_cpu",
"^test_hardmax_negative_axis_cpu",
"^test_hardmax_one_hot_cpu",
"^test_logsoftmax_axis_0_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Logsoftmax(13)
"^test_logsoftmax_axis_0_expanded_cpu",
"^test_logsoftmax_axis_1_cpu",
"^test_logsoftmax_axis_1_expanded_cpu",
"^test_logsoftmax_axis_2_cpu",
"^test_logsoftmax_axis_2_expanded_cpu",
"^test_logsoftmax_default_axis_cpu",
"^test_logsoftmax_default_axis_expanded_cpu",
"^test_logsoftmax_example_1_cpu",
"^test_logsoftmax_example_1_expanded_cpu",
"^test_logsoftmax_large_number_cpu",
"^test_logsoftmax_large_number_expanded_cpu",
"^test_logsoftmax_negative_axis_cpu",
"^test_logsoftmax_negative_axis_expanded_cpu",
"^test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Resize(13)
"^test_resize_downsample_scales_cubic_cpu",
"^test_resize_downsample_scales_linear_cpu",
@ -75,21 +54,7 @@
"^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu",
"^test_resize_upsample_sizes_nearest_cpu",
"^test_resize_upsample_sizes_nearest_floor_align_corners_cpu",
"^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu",
"^test_softmax_axis_0_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Softmax(13)
"^test_softmax_axis_0_expanded_cpu",
"^test_softmax_axis_1_cpu",
"^test_softmax_axis_1_expanded_cpu",
"^test_softmax_axis_2_cpu",
"^test_softmax_axis_2_expanded_cpu",
"^test_softmax_default_axis_cpu",
"^test_softmax_default_axis_expanded_cpu",
"^test_softmax_example_cpu",
"^test_softmax_example_expanded_cpu",
"^test_softmax_large_number_cpu",
"^test_softmax_large_number_expanded_cpu",
"^test_softmax_negative_axis_cpu",
"^test_softmax_negative_axis_expanded_cpu"
"^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu"
],
"current_failing_tests_x86": [
"^test_vgg19",
@ -199,7 +164,9 @@
"^test_reduce_sum_negative_axes_keepdims*",
"^test_reduce_sum_empty_axes_input_noop*",
"^test_unsqueeze_*", // Does not support axes as input
"^test_squeeze_*" // Does not support axes as input
"^test_squeeze_*", // Does not support axes as input
"^test_logsoftmax_*", // Does not support opset-13 yet
"^test_softmax_*" // Does not support opset-13 yet
],
// ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported
"tests_with_pre_opset7_dependencies": [