From a14cd6267b186fe5fc5d6b3d3e546a604574bc1c Mon Sep 17 00:00:00 2001 From: Hariharan Seshadri Date: Wed, 11 Nov 2020 15:45:03 -0800 Subject: [PATCH] Support opset-13 specs of softmax family ops (Softmax, LogSoftmax, Hardmax) (#5707) --- .../providers/cpu/cpu_execution_provider.cc | 43 +++-- .../core/providers/cpu/math/hardmax.cc | 115 +++++++++++-- onnxruntime/core/providers/cpu/math/hardmax.h | 12 +- .../core/providers/cpu/math/softmax.cc | 153 +++++++++++++++++- onnxruntime/core/providers/cpu/math/softmax.h | 37 +++-- .../core/providers/cuda/math/softmax.cc | 105 ++++++++++-- .../core/providers/cuda/math/softmax.h | 27 +++- .../test/providers/cpu/math/hardmax_test.cc | 83 +++++++++- .../providers/cpu/math/logsoftmax_test.cc | 100 +++++++++--- .../test/providers/cpu/math/softmax_test.cc | 116 +++++++++++-- .../onnx_backend_test_series_filters.jsonc | 41 +---- 11 files changed, 687 insertions(+), 145 deletions(-) diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 73b130a1f8..3394a8f2e0 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -345,11 +345,11 @@ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOn class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float, ReduceSumSquare); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, ReduceSumSquare); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, int32_t, ReduceSumSquare); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Hardmax); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, float, LogSoftmax); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, double, LogSoftmax); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, float, Softmax); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, double, Softmax); +class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, Hardmax); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float, LogSoftmax); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, LogSoftmax); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, float, Softmax); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, double, Softmax); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Loop); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, DepthToSpace); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Scan); @@ -594,6 +594,12 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Resize); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, int32_t, Resize); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, uint8_t, Resize); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, Hardmax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, LogSoftmax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double, LogSoftmax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Softmax); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double, Softmax); + // !!PLEASE READ BELOW!! Following that, add new entries above this comment /* *** IMPORTANT! *** @@ -1109,15 +1115,15 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { float, ArgMin)>, BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, @@ -1552,6 +1558,15 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { int32_t, Resize)>, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, }; for (auto& function_table_entry : function_table) { diff --git a/onnxruntime/core/providers/cpu/math/hardmax.cc b/onnxruntime/core/providers/cpu/math/hardmax.cc index 8a3c0b0262..3c2f6461ff 100644 --- a/onnxruntime/core/providers/cpu/math/hardmax.cc +++ b/onnxruntime/core/providers/cpu/math/hardmax.cc @@ -5,48 +5,123 @@ #include "core/providers/common.h" #include "core/util/math_cpuonly.h" #include "core/util/math.h" +#include "core/providers/cpu/tensor/transpose.h" namespace onnxruntime { template <> Status Hardmax::Compute(OpKernelContext* ctx) const { const auto* X = ctx->Input(0); - const TensorShape& input_shape = X->Shape(); - const auto* Xdata = X->template Data(); + const TensorShape& X_shape = X->Shape(); + size_t rank = X_shape.NumDimensions(); + Tensor* Y = ctx->Output(0, X_shape); - auto axis = HandleNegativeAxis(axis_, input_shape.NumDimensions()); // handle negative and enforce axis is valid - size_t tmpN = input_shape.SizeToDimension(axis); - size_t tmpD = input_shape.SizeFromDimension(axis); + // special case when there is a dim value of 0 in the shape. + if (X_shape.Size() == 0) + return Status::OK(); + + // handle negative and enforce axis is valid + const size_t axis = static_cast(HandleNegativeAxis(axis_, rank)); + + bool is_transpose_required = false; + Tensor transposed_input; + std::vector transposed_input_dims; + Tensor intermediate_output; // output that the hardmax implementation will write into while using transposed input + std::vector permutation(rank); + + // The "semantic" meaning of axis has changed in opset-13. + // Please compare: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Hardmax + // with https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Hardmax-11 for detailed explanations + // To account for the opset-13 behavior, our plan will be to transpose the "axis" dim to the innermost dim + // and perform softmax and then reverse the transpose. We can skip the transposing aspect if the axis is already + // the innermost dim + if (opset_ >= 13 && axis != (rank - 1)) { + is_transpose_required = true; + } + + if (is_transpose_required) { + AllocatorPtr alloc; + auto status = ctx->GetTempSpaceAllocator(&alloc); + if (!status.IsOK()) + return status; + + std::iota(std::begin(permutation), std::end(permutation), 0); + + // swap the innermost dim with the dim corresponding to axis + permutation[axis] = rank - 1; + permutation[rank - 1] = axis; + + transposed_input_dims.reserve(rank); + for (auto e : permutation) { + transposed_input_dims.push_back(X_shape[e]); + } + + // Allocate a temporary tensor to hold transposed input + Tensor temp_input(X->DataType(), TensorShape(transposed_input_dims), alloc); + + // Perform the transpose + ORT_RETURN_IF_ERROR(TransposeBase::DoTranspose(permutation, *X, temp_input)); + transposed_input = std::move(temp_input); + + // Allocate memory for the intermediate output + Tensor temp_output(Y->DataType(), TensorShape(transposed_input_dims), alloc); + intermediate_output = std::move(temp_output); + } + + size_t tmp_N = is_transpose_required ? TensorShape(transposed_input_dims).SizeToDimension(rank - 1) : X_shape.SizeToDimension(axis); + size_t tmp_D = is_transpose_required ? TensorShape(transposed_input_dims).SizeFromDimension(rank - 1) : X_shape.SizeFromDimension(axis); // Math::RowwiseMax expects int N and D. - if (tmpN * tmpD > INT32_MAX || tmpN > INT32_MAX || tmpD > INT32_MAX) { + if (tmp_N * tmp_D > INT32_MAX || tmp_N > INT32_MAX || tmp_D > INT32_MAX) { std::ostringstream ss; - ss << "Hardmax inputs N, D and N * D must be < " << INT32_MAX << ". N=" << tmpN << ", D=" << tmpD; + ss << "Hardmax inputs N, D and N * D must be < " << INT32_MAX << ". N=" << tmp_N << ", D=" << tmp_D; std::string msg = ss.str(); return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, msg); } - const int N = gsl::narrow_cast(tmpN); - const int D = gsl::narrow_cast(tmpD); + const int N = gsl::narrow_cast(tmp_N); + const int D = gsl::narrow_cast(tmp_D); std::vector rowmax_(N); float* rowmax_data = rowmax_.data(); - math::RowwiseMax(N, D, Xdata, rowmax_data, nullptr); - Tensor* Y = ctx->Output(0, input_shape); - auto* Ydata = Y->template MutableData(); - math::Set(input_shape.Size(), 0.f, Ydata, &CPUMathUtil::Instance()); + const float* X_data = nullptr; + float* Y_data = nullptr; + + if (is_transpose_required) { // use intermediate buffers to compute the hardmax values + X_data = transposed_input.template Data(); + Y_data = intermediate_output.template MutableData(); + } else { // use the node input/output directly + X_data = X->template Data(); + Y_data = Y->template MutableData(); + } + + math::RowwiseMax(N, D, X_data, rowmax_data, nullptr); + + // Even if we had to transpose the input, it is safe to go with X_shape.Size() which computes + // the size of the buffer from the original input's shape as even if we do transpose, the size + // of the transposed buffer will be the same as the original input's buffer + math::Set(X_shape.Size(), 0.f, Y_data, &CPUMathUtil::Instance()); for (int i = 0; i < N; ++i) { for (int j = 0; j < D; ++j) { - if (Xdata[i * D + j] == rowmax_data[i]) { - Ydata[i * D + j] = 1; + if (X_data[i * D + j] == rowmax_data[i]) { + Y_data[i * D + j] = 1; break; } } } + if (is_transpose_required) { + std::vector reverse_permutation(rank); + for (size_t i = 0, end = permutation.size(); i < end; ++i) { + reverse_permutation[permutation[i]] = i; + } + // Perform the transpose to get the axes back to the original ordering + ORT_RETURN_IF_ERROR(TransposeBase::DoTranspose(reverse_permutation, intermediate_output, *Y)); + } + return Status::OK(); } @@ -58,9 +133,17 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL( Hardmax); // Opset 11 starts to support Neg Axis. -ONNX_CPU_OPERATOR_KERNEL( +ONNX_CPU_OPERATOR_VERSIONED_KERNEL( Hardmax, 11, + 12, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Hardmax); + +// Opset 13 changed the semantic meaning of the axis attribute. +ONNX_CPU_OPERATOR_KERNEL( + Hardmax, + 13, KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), Hardmax); diff --git a/onnxruntime/core/providers/cpu/math/hardmax.h b/onnxruntime/core/providers/cpu/math/hardmax.h index 9dc5dd563e..277001780b 100644 --- a/onnxruntime/core/providers/cpu/math/hardmax.h +++ b/onnxruntime/core/providers/cpu/math/hardmax.h @@ -12,12 +12,21 @@ namespace onnxruntime { template class Hardmax final : public OpKernel { public: - Hardmax(const OpKernelInfo& info) : OpKernel{info}, axis_{1} { + Hardmax(const OpKernelInfo& info) : OpKernel{info} { + const auto& node = info.node(); + opset_ = node.SinceVersion(); + int64_t axis; Status status = info.GetAttr("axis", &axis); if (status.IsOK()) { axis_ = gsl::narrow_cast(axis); + } else { + if (opset_ < 13) { + axis_ = 1; // opset-12 and below, the default axis value is 1 + } else { + axis_ = -1; // opset-13, the default axis value is -1 + } } } @@ -25,5 +34,6 @@ class Hardmax final : public OpKernel { private: int axis_; + int opset_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/softmax.cc b/onnxruntime/core/providers/cpu/math/softmax.cc index ae5490ce1c..4c8f977b2d 100644 --- a/onnxruntime/core/providers/cpu/math/softmax.cc +++ b/onnxruntime/core/providers/cpu/math/softmax.cc @@ -2,6 +2,9 @@ // Licensed under the MIT License. #include "core/providers/cpu/math/softmax.h" +#include "core/providers/cpu/tensor/transpose.h" +#include +#include namespace onnxruntime { @@ -14,9 +17,18 @@ ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( Softmax); // Opset 11 starts to support Neg Axis. -ONNX_CPU_OPERATOR_TYPED_KERNEL( +ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( Softmax, 11, + 12, + float, + KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", DataTypeImpl::GetTensorType()), + Softmax); + +// Opset 13 changed the semantic meaning of the axis attribute. +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Softmax, + 13, float, KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", DataTypeImpl::GetTensorType()), Softmax); @@ -30,9 +42,18 @@ ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( Softmax); // Opset 11 starts to support Neg Axis. -ONNX_CPU_OPERATOR_TYPED_KERNEL( +ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( Softmax, 11, + 12, + double, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Softmax); + +// Opset 13 changed the semantic meaning of the axis attribute. +ONNX_CPU_OPERATOR_TYPED_KERNEL( + Softmax, + 13, double, KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), Softmax); @@ -46,9 +67,18 @@ ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( Softmax); // Opset 11 starts to support Neg Axis. -ONNX_CPU_OPERATOR_TYPED_KERNEL( +ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( LogSoftmax, 11, + 12, + float, + KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", DataTypeImpl::GetTensorType()), + Softmax); + +// Opset 13 changed the semantic meaning of the axis attribute. +ONNX_CPU_OPERATOR_TYPED_KERNEL( + LogSoftmax, + 13, float, KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", DataTypeImpl::GetTensorType()), Softmax); @@ -62,11 +92,126 @@ ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( Softmax); // Opset 11 starts to support Neg Axis. -ONNX_CPU_OPERATOR_TYPED_KERNEL( +ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( LogSoftmax, 11, + 12, double, KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), Softmax); +// Opset 13 changed the semantic meaning of the axis attribute. +ONNX_CPU_OPERATOR_TYPED_KERNEL( + LogSoftmax, + 13, + double, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Softmax); + +// opset-12 and below +template +Status Softmax::ComputeImpl(const Tensor& input, Tensor& output, size_t axis, + concurrency::ThreadPool* thread_pool) const { + const auto& X_shape = input.Shape(); + const size_t N = X_shape.SizeToDimension(axis); + const size_t D = X_shape.SizeFromDimension(axis); + + return SoftmaxCPU(N, D, input.template Data(), output.template MutableData(), log_softmax_, thread_pool); +} + +// opset-13 and above +template +Status Softmax::ComputeImplOpset13(const Tensor& input, Tensor& output, size_t axis, + concurrency::ThreadPool* thread_pool, OpKernelContext* ctx) const { + const auto& X_shape = input.Shape(); + size_t rank = X_shape.NumDimensions(); + + bool is_transpose_required = false; + Tensor transposed_input; + std::vector transposed_input_dims; + Tensor intermediate_output; // output that the softmax implementation will write into while using transposed input + std::vector permutation(rank); + + // The "semantic" meaning of axis has changed in opset-13. + // Please compare: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softmax + // with https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-11 for detailed explanations + // To account for the opset-13 behavior, our plan will be to transpose the "axis" dim to the innermost dim + // and perform softmax and then reverse the transpose. We can skip the transposing aspect if the axis is already + // the innermost dim + if (axis != (rank - 1)) { + is_transpose_required = true; + } + + if (is_transpose_required) { + AllocatorPtr alloc; + auto status = ctx->GetTempSpaceAllocator(&alloc); + if (!status.IsOK()) + return status; + + std::iota(std::begin(permutation), std::end(permutation), 0); + + // swap the innermost dim with the dim corresponding to axis + permutation[axis] = rank - 1; + permutation[rank - 1] = axis; + + transposed_input_dims.reserve(rank); + for (auto e : permutation) { + transposed_input_dims.push_back(X_shape[e]); + } + + // Allocate a temporary tensor to hold transposed input + Tensor temp_input(input.DataType(), TensorShape(transposed_input_dims), alloc); + + // Perform the transpose + ORT_RETURN_IF_ERROR(TransposeBase::DoTranspose(permutation, input, temp_input)); + transposed_input = std::move(temp_input); + + // Allocate memory for the intermediate output + Tensor temp_output(output.DataType(), TensorShape(transposed_input_dims), alloc); + intermediate_output = std::move(temp_output); + } + + const size_t N = is_transpose_required ? TensorShape(transposed_input_dims).SizeToDimension(rank - 1) : X_shape.SizeToDimension(rank - 1); + const size_t D = is_transpose_required ? TensorShape(transposed_input_dims).SizeFromDimension(rank - 1) : X_shape.SizeFromDimension(rank - 1); + + ORT_RETURN_IF_ERROR(SoftmaxCPU(N, D, + is_transpose_required ? transposed_input.template Data() : input.template Data(), + is_transpose_required ? intermediate_output.template MutableData() : output.template MutableData(), + log_softmax_, thread_pool)); + + if (is_transpose_required) { + std::vector reverse_permutation(rank); + for (size_t i = 0, end = permutation.size(); i < end; ++i) { + reverse_permutation[permutation[i]] = i; + } + // Perform the transpose to get the axes back to the original ordering + ORT_RETURN_IF_ERROR(TransposeBase::DoTranspose(reverse_permutation, intermediate_output, output)); + } + + return Status::OK(); +} + +// compute method of Softmax +template +Status Softmax::Compute(OpKernelContext* ctx) const { + const auto* X = ctx->Input(0); + const auto& X_shape = X->Shape(); + size_t rank = X_shape.NumDimensions(); + auto* Y = ctx->Output(0, X_shape); + + // edge case. one or more dims with value of 0. nothing to do + if (X_shape.Size() == 0) { + return Status::OK(); + } + + const size_t axis = static_cast(HandleNegativeAxis(axis_, rank)); + concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool(); + + if (opset_ < 13) { + return ComputeImpl(*X, *Y, axis, thread_pool); + } else { + return ComputeImplOpset13(*X, *Y, axis, thread_pool, ctx); + } +} + } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/softmax.h b/onnxruntime/core/providers/cpu/math/softmax.h index 739614760a..dbcce64c7a 100644 --- a/onnxruntime/core/providers/cpu/math/softmax.h +++ b/onnxruntime/core/providers/cpu/math/softmax.h @@ -14,38 +14,37 @@ namespace onnxruntime { template class Softmax final : public OpKernel { public: - Softmax(const OpKernelInfo& info) : OpKernel{info}, axis_{1} { + Softmax(const OpKernelInfo& info) : OpKernel{info} { + const auto& node = info.node(); + opset_ = node.SinceVersion(); + int64_t axis; Status status = info.GetAttr("axis", &axis); if (status.IsOK()) { axis_ = gsl::narrow_cast(axis); + } else { + if (opset_ < 13) { + axis_ = 1; // opset-12 and below, the default axis value is 1 + } else { + axis_ = -1; // opset-13, the default axis value is -1 + } } log_softmax_ = info.GetKernelDef().OpName() == "LogSoftmax"; } - Status Compute(OpKernelContext* ctx) const override { - const auto* X = ctx->Input(0); - const auto& X_shape = X->Shape(); - auto* Y = ctx->Output(0, X_shape); - - // edge case. one or more dims with value of 0. nothing to do - if (X_shape.Size() == 0) { - return Status::OK(); - } - - const int64_t axis = HandleNegativeAxis(axis_, X_shape.NumDimensions()); - const size_t N = X_shape.SizeToDimension(axis); - const size_t D = X_shape.SizeFromDimension(axis); - - concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool(); - - return SoftmaxCPU(N, D, X->template Data(), Y->template MutableData(), log_softmax_, thread_pool); - } + Status Compute(OpKernelContext* ctx) const override; private: + Status ComputeImpl(const Tensor& input, Tensor& output, size_t axis, + concurrency::ThreadPool* thread_pool) const; + + Status ComputeImplOpset13(const Tensor& input, Tensor& output, size_t axis, + concurrency::ThreadPool* thread_pool, OpKernelContext* ctx) const; + int axis_; + int opset_; bool log_softmax_; }; diff --git a/onnxruntime/core/providers/cuda/math/softmax.cc b/onnxruntime/core/providers/cuda/math/softmax.cc index 15adc65514..f80eafe2fa 100644 --- a/onnxruntime/core/providers/cuda/math/softmax.cc +++ b/onnxruntime/core/providers/cuda/math/softmax.cc @@ -6,6 +6,7 @@ #include "core/providers/common.h" #include "core/providers/cuda/cudnn_common.h" #include "core/providers/cuda/shared_inc/accumulation_type.h" +#include "core/providers/cuda/tensor/transpose.h" namespace onnxruntime { namespace cuda { @@ -19,10 +20,8 @@ Status SoftMaxComputeHelper( int64_t axis) { typedef typename ToCudaType::MappedType CudaT; - const int64_t normalized_axis = HandleNegativeAxis(axis, input_shape.NumDimensions()); - - int64_t N = input_shape.SizeToDimension(normalized_axis); - int64_t D = input_shape.SizeFromDimension(normalized_axis); + int64_t N = input_shape.SizeToDimension(axis); + int64_t D = input_shape.SizeFromDimension(axis); auto Y_data = reinterpret_cast(Y); auto X_data = reinterpret_cast(X); @@ -112,17 +111,103 @@ template Status Softmax::ComputeInternal(OpKernelContext* ctx) const { const Tensor* X = ctx->Input(0); const TensorShape& input_shape{X->Shape()}; - const T* X_data = X->template Data(); - T* Y_data = ctx->Output(0, input_shape)->template MutableData(); + size_t rank = input_shape.NumDimensions(); + Tensor* Y = ctx->Output(0, input_shape); + // special case when there is a dim value of 0 in the shape. if (input_shape.Size() == 0) return Status::OK(); - if (log_softmax_) { - return SoftMaxComputeHelper(X_data, input_shape, Y_data, CudnnHandle(), axis_); - } else { - return SoftMaxComputeHelper(X_data, input_shape, Y_data, CudnnHandle(), axis_); + // handle negative and enforce axis is valid + const size_t axis = static_cast(HandleNegativeAxis(axis_, rank)); + + bool is_transpose_required = false; + Tensor transposed_input; + std::vector transposed_input_dims; + Tensor intermediate_output; // output that the softmax implementation will write into while using transposed input + std::vector permutation(rank); + + // The "semantic" meaning of axis has changed in opset-13. + // Please compare: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softmax + // with https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-11 for detailed explanations + // To account for the opset-13 behavior, our plan will be to transpose the "axis" dim to the innermost dim + // and perform softmax and then reverse the transpose. We can skip the transposing aspect if the axis is already + // the innermost dim + if (opset_ >= 13 && axis != (rank - 1)) { + is_transpose_required = true; } + + if (is_transpose_required) { + AllocatorPtr alloc; + auto status = ctx->GetTempSpaceAllocator(&alloc); + if (!status.IsOK()) + return status; + + std::iota(std::begin(permutation), std::end(permutation), 0); + + // swap the innermost dim with the dim corresponding to axis + permutation[axis] = rank - 1; + permutation[rank - 1] = axis; + + transposed_input_dims.reserve(rank); + for (auto e : permutation) { + transposed_input_dims.push_back(input_shape[e]); + } + + // Allocate a temporary tensor to hold transposed input + Tensor temp_input(X->DataType(), TensorShape(transposed_input_dims), alloc); + + // Perform the transpose + ORT_RETURN_IF_ERROR(Transpose::DoTranspose(cuda_ep_->GetDeviceProp(), + CublasHandle(), + permutation, *X, temp_input)); + transposed_input = std::move(temp_input); + + // Allocate memory for the intermediate output + Tensor temp_output(Y->DataType(), TensorShape(transposed_input_dims), alloc); + intermediate_output = std::move(temp_output); + } + + const T* X_data = nullptr; + T* Y_data = nullptr; + const TensorShape* compute_input_shape = nullptr; + + if (is_transpose_required) { // use intermediate buffers to compute the softmax values + X_data = transposed_input.template Data(); + Y_data = intermediate_output.template MutableData(); + compute_input_shape = &transposed_input.Shape(); + } else { // use the node input/output directly + X_data = X->template Data(); + Y_data = Y->template MutableData(); + compute_input_shape = &input_shape; + } + + Status status; + if (log_softmax_) { + status = SoftMaxComputeHelper(X_data, *compute_input_shape, Y_data, CudnnHandle(), + is_transpose_required ? static_cast(rank) - 1 + : static_cast(axis)); + } else { + status = SoftMaxComputeHelper(X_data, *compute_input_shape, Y_data, CudnnHandle(), + is_transpose_required ? static_cast(rank) - 1 + : static_cast(axis)); + } + + if (!status.IsOK()) + return status; + + if (is_transpose_required) { + std::vector reverse_permutation(rank); + for (size_t i = 0, end = permutation.size(); i < end; ++i) { + reverse_permutation[permutation[i]] = i; + } + // Perform the transpose to get the axes back to the original ordering + ORT_RETURN_IF_ERROR(Transpose::DoTranspose(cuda_ep_->GetDeviceProp(), + CublasHandle(), + reverse_permutation, intermediate_output, *Y)); + } + + return Status::OK(); } #define SPECIALIZED_COMPUTE(T) \ diff --git a/onnxruntime/core/providers/cuda/math/softmax.h b/onnxruntime/core/providers/cuda/math/softmax.h index 8634503fb4..497ad53b36 100644 --- a/onnxruntime/core/providers/cuda/math/softmax.h +++ b/onnxruntime/core/providers/cuda/math/softmax.h @@ -24,8 +24,28 @@ template class Softmax final : public CudaKernel { public: Softmax(const OpKernelInfo& info) : CudaKernel{info} { - info.GetAttrOrDefault("axis", &axis_, static_cast(1)); + const auto& node = info.node(); + opset_ = node.SinceVersion(); + + int64_t axis; + Status status = info.GetAttr("axis", &axis); + + if (status.IsOK()) { + axis_ = gsl::narrow_cast(axis); + } else { + if (opset_ < 13) { + axis_ = 1; // opset-12 and below, the default axis value is 1 + } else { + axis_ = -1; // opset-13, the default axis value is -1 + } + } + log_softmax_ = info.GetKernelDef().OpName() == "LogSoftmax"; + + // We need to cast away the const as PerThreadCublasHandle() is currently a non-const method + // TODO: Clean up the CUDAExecutionProvider interface to avoid this + cuda_ep_ = const_cast( + static_cast(info.GetExecutionProvider())); } Status ComputeInternal(OpKernelContext* context) const override; @@ -33,6 +53,11 @@ class Softmax final : public CudaKernel { private: int64_t axis_; bool log_softmax_; + int opset_; + + // We need to access to the CUDA EP instance to get the cublas handle to use + // for transposing(if applicable) + CUDAExecutionProvider* cuda_ep_; }; } // namespace cuda diff --git a/onnxruntime/test/providers/cpu/math/hardmax_test.cc b/onnxruntime/test/providers/cpu/math/hardmax_test.cc index b9f6bf25ac..ee5b4aaa39 100644 --- a/onnxruntime/test/providers/cpu/math/hardmax_test.cc +++ b/onnxruntime/test/providers/cpu/math/hardmax_test.cc @@ -11,13 +11,20 @@ namespace test { static void RunTest(const std::vector& x_vals, const std::vector& expected_vals, const std::vector& dimensions, + int opset = 7, int64_t axis = 1, OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, const std::string& expected_err_str = "") { - OpTester test("Hardmax"); + OpTester test("Hardmax", opset); - if (axis != 1) { - test.AddAttribute("axis", axis); + if (opset < 13) { + if (axis != 1) { // opset-12 and below : default axis value is 1 + test.AddAttribute("axis", axis); + } + } else { + if (axis != -1) { // opset-13 : default axis value is -1 + test.AddAttribute("axis", axis); + } } test.AddInput("X", dimensions, x_vals); @@ -85,7 +92,7 @@ TEST(HardmaxOperator, ThreeDimsAxis0) { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0); + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 0); } TEST(HardmaxOperator, ThreeDimsAxis1) { @@ -108,9 +115,31 @@ TEST(HardmaxOperator, ThreeDimsAxis1) { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1); + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 1); } +TEST(HardmaxOperator, ThreeDimsAxis1_opset13) { + // For the same input, opset-13's behavior is different from an earlier opset + // and we see different expected results for the same test input + + std::vector expected_vals = { + 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, + 1.0f, 1.0f, 0.0f, 0.0f, 1.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, + + 1.0f, 0.0f, 0.0f, 1.0f, 1.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 1.0f, + 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 1.0f, 0.0f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 1); +} TEST(HardmaxOperator, ThreeDimsAxis2) { // x = // import cntk as C @@ -131,7 +160,47 @@ TEST(HardmaxOperator, ThreeDimsAxis2) { 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 2); + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 2); +} + +TEST(HardmaxOperator, ThreeDimsAxis2_opset13) { + std::vector expected_vals = { + 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, + + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 2); +} + +TEST(HardmaxOperator, ThreeDimsDefaultAxis_opset13) { + std::vector expected_vals = { + 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, + + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, 0.0f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*default axis*/ -1); } TEST(HardmaxOperator, ThreeDimsNegAxis2) { @@ -154,7 +223,7 @@ TEST(HardmaxOperator, ThreeDimsNegAxis2) { 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ -1); + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ -1); } } // namespace test diff --git a/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc b/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc index ac9b53cfb3..5b65416397 100644 --- a/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc +++ b/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc @@ -11,15 +11,21 @@ namespace test { static void RunTest(const std::vector& x_vals, const std::vector& expected_vals, const std::vector& dimensions, + int opset = 7, int64_t axis = 1, bool is_tensorrt_supported = true, OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, - const std::string& error_msg = "", - int opset = 7) { + const std::string& error_msg = "") { OpTester tester("LogSoftmax", opset); - if (axis != 1) { - tester.AddAttribute("axis", axis); + if (opset < 13) { + if (axis != 1) { // opset-12 and below : default axis value is 1 + tester.AddAttribute("axis", axis); + } + } else { + if (axis != -1) { // opset-13 : default axis value is -1 + tester.AddAttribute("axis", axis); + } } tester.AddInput("X", dimensions, x_vals); @@ -101,7 +107,7 @@ TEST(LogSoftmaxOperator, ThreeDimsAxis0) { -4.042971f, -4.2982683f, -3.5933442f, -4.538994f, -5.307373f, -4.2677402f, -4.44635f, -3.5821702f, -3.8414123f, -4.267664f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0, false); // axis=0 is not supported by TensorRT + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 0, false); // axis=0 is not supported by TensorRT } TEST(LogSoftmaxOperator, ThreeDimsAxis1) { @@ -127,16 +133,33 @@ TEST(LogSoftmaxOperator, ThreeDimsAxis1) { -2.9822054f, -3.2375026f, -2.5325785f, -3.4782279f, -4.246608f, -3.2069747f, -3.3855844f, -2.5214045f, -2.7806466f, -3.206898f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1, false); // This test failed on TensorRT + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 1, false); // This test failed on TensorRT +} + +TEST(LogSoftmaxOperator, ThreeDimsAxis1_opset13) { + // For the same input, opset-13's behavior is different from an earlier opset + // and we see different expected results for the same test input + + std::vector expected_vals = { + -1.373224f, -2.1894338f, -2.5028024f, -1.0337411f, -1.3945004f, + -0.8074181f, -0.7601002f, -2.3568683f, -1.2740996f, -1.1063602f, + -1.7799685f, -3.0920703f, -1.2943912f, -1.9011338f, -1.5291187f, + -2.0245035f, -0.9808493f, -0.5989947f, -1.5359819f, -1.5869142f, + + -1.1288074f, -1.7014627f, -1.7446783f, -1.0005655f, -1.0210506f, + -1.2284245f, -2.2850897f, -1.2518314f, -2.036326f, -1.4131763f, + -1.6105566f, -0.3936059f, -0.90897894f, -1.4765173f, -1.3474687f, + -1.6925404f, -3.189349f, -1.9922893f, -1.2968583f, -1.9913039f, + + -1.4780827f, -1.355593f, -2.2826173f, -1.8348985f, -2.3507955f, + -2.2716188f, -0.69089717f, -2.2606049f, -1.4299684f, -0.45124117f, + -0.9893639f, -2.0444741f, -0.92980486f, -1.6106415f, -2.6597016f, + -1.2141333f, -2.192556f, -0.9186309f, -0.9130602f, -1.6199919f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 1, false); // This test failed on TensorRT } TEST(LogSoftmaxOperator, ThreeDimsAxis2) { - // x = - // node = onnx.helper.make_node('LogSoftmax', inputs = ['x'], outputs = ['y'], axis = 2) - // y = logsoftmax_2d(x.reshape(12, 5)).reshape(3, 4, 5) - // expect(node, inputs = [x], outputs = [y], - // name = 'test_logsoftmax_axis_2') - std::vector expected_vals = { -1.5016061f, -1.5898913f, -2.3042583f, -1.080942f, -2.0086365f, -1.5264852f, -0.7512426f, -2.7490091f, -1.9119854f, -2.3111813f, @@ -153,9 +176,48 @@ TEST(LogSoftmaxOperator, ThreeDimsAxis2) { -1.4430928f, -1.6983899f, -0.9934659f, -1.9391153f, -2.7074947f, -1.8489327f, -2.027542f, -1.1633625f, -1.4226046f, -1.848856f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 2); + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 2); } +TEST(LogSoftmaxOperator, ThreeDimsAxis2_opset13) { + std::vector expected_vals = { + -1.5016061f, -1.5898913f, -2.3042583f, -1.080942f, -2.0086365f, + -1.5264852f, -0.7512426f, -2.7490091f, -1.9119854f, -2.3111813f, + -1.716058f, -2.3002353f, -0.9035546f, -1.7560422f, -1.9509623f, + -2.7323837f, -0.96080494f, -0.97994876f, -2.162681f, -2.7805486f, + + -2.024213f, -1.2708496f, -1.8257477f, -1.5857526f, -1.507701f, + -1.8521607f, -1.582807f, -1.0612315f, -2.3498435f, -1.6281573f, + -3.0813656f, -0.538396f, -1.5654519f, -2.6371078f, -2.4095225f, + -1.8958019f, -2.0665917f, -1.3812149f, -1.1899012f, -1.7858102f, + + -1.7220669f, -0.79976386f, -2.1365335f, -1.9536276f, -2.1888442f, + -3.2268262f, -0.84629166f, -2.8257446f, -2.259921f, -1.0005134f, + -1.4430928f, -1.6983899f, -0.9934659f, -1.9391153f, -2.7074947f, + -1.8489327f, -2.027542f, -1.1633625f, -1.4226046f, -1.848856f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 2); +} + +TEST(LogSoftmaxOperator, ThreeDimsDefaultAxis_opset13) { + std::vector expected_vals = { + -1.5016061f, -1.5898913f, -2.3042583f, -1.080942f, -2.0086365f, + -1.5264852f, -0.7512426f, -2.7490091f, -1.9119854f, -2.3111813f, + -1.716058f, -2.3002353f, -0.9035546f, -1.7560422f, -1.9509623f, + -2.7323837f, -0.96080494f, -0.97994876f, -2.162681f, -2.7805486f, + + -2.024213f, -1.2708496f, -1.8257477f, -1.5857526f, -1.507701f, + -1.8521607f, -1.582807f, -1.0612315f, -2.3498435f, -1.6281573f, + -3.0813656f, -0.538396f, -1.5654519f, -2.6371078f, -2.4095225f, + -1.8958019f, -2.0665917f, -1.3812149f, -1.1899012f, -1.7858102f, + + -1.7220669f, -0.79976386f, -2.1365335f, -1.9536276f, -2.1888442f, + -3.2268262f, -0.84629166f, -2.8257446f, -2.259921f, -1.0005134f, + -1.4430928f, -1.6983899f, -0.9934659f, -1.9391153f, -2.7074947f, + -1.8489327f, -2.027542f, -1.1633625f, -1.4226046f, -1.848856f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*default axis*/ -1); +} TEST(LogSoftmaxOperator, ThreeDimsNegativeAxis) { // x = // node = onnx.helper.make_node('LogSoftmax', inputs = ['x'], outputs = ['y'], axis = 2) @@ -180,7 +242,7 @@ TEST(LogSoftmaxOperator, ThreeDimsNegativeAxis) { -1.8489327f, -2.027542f, -1.1633625f, -1.4226046f, -1.848856f}; // -1 is last axis so same as axis == 2 - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ -1); + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 12, /*axis*/ -1); } TEST(LogSoftmaxOperator, InvalidAxis) { @@ -191,15 +253,13 @@ TEST(LogSoftmaxOperator, InvalidAxis) { RunTest(x_vals, expected_vals, dimensions, + /*opset*/ 12, /* invalid axis */ -7, - false, + false, //TensorRT parser: Assertion failed: axis >= 0 && axis < nbDims OpTester::ExpectResult::kExpectFailure, // ONNX has a bug in the error message generation so this is somewhat cryptic until it's fixed. Message should be: - // "[ShapeInferenceError] 'axis' must be in [-2 , 1]. Its actual value is: -7" - ", 1]. Its actual value is: -7", - // latest opset so we get shape inferencing errors - // Latest valid opset for this is 12. Once opset 13 changes are implemented this can be changed back to -1 - 12); //TensorRT parser: Assertion failed: axis >= 0 && axis < nbDims + "[ShapeInferenceError] 'axis' must be in [-2 , 1]. Its actual value is: -7"); + //", 1]. Its actual value is: -7"); } } // namespace test diff --git a/onnxruntime/test/providers/cpu/math/softmax_test.cc b/onnxruntime/test/providers/cpu/math/softmax_test.cc index ed3d3fe9c4..6236787df8 100644 --- a/onnxruntime/test/providers/cpu/math/softmax_test.cc +++ b/onnxruntime/test/providers/cpu/math/softmax_test.cc @@ -11,17 +11,22 @@ namespace test { static void RunTest(const std::vector& x_vals, const std::vector& expected_vals, const std::vector& dimensions, + int opset = 7, int64_t axis = 1, const std::unordered_set& excluded_providers = {}, OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, - const std::string& error_msg = "", - int opset = 7) { + const std::string& error_msg = "") { OpTester test("Softmax", opset); - if (axis != 1) { - test.AddAttribute("axis", axis); + if (opset < 13) { + if (axis != 1) { // opset-12 and below : default axis value is 1 + test.AddAttribute("axis", axis); + } + } else { + if (axis != -1) { // opset-13 : default axis value is -1 + test.AddAttribute("axis", axis); + } } - test.AddInput("X", dimensions, x_vals); test.AddOutput("Y", dimensions, expected_vals); test.Run(expect_result, error_msg, excluded_providers); @@ -93,7 +98,7 @@ TEST(SoftmaxOperator, ThreeDimsAxis0) { 0.017545262f, 0.0135920765f, 0.027506188f, 0.010684152f, 0.0049549243f, 0.01401341f, 0.011721271f, 0.027815264f, 0.021463264f, 0.014014485f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0, {kTensorrtExecutionProvider}); // Axis=0 is not supported by TensorRT + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 0, {kTensorrtExecutionProvider}); // Axis=0 is not supported by TensorRT } TEST(SoftmaxOperator, ThreeDimsAxis1) { @@ -119,7 +124,31 @@ TEST(SoftmaxOperator, ThreeDimsAxis1) { 0.050680935f, 0.03926183f, 0.079453886f, 0.030862054f, 0.014312706f, 0.040478885f, 0.033857856f, 0.080346674f, 0.06199841f, 0.040481992f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1, {kTensorrtExecutionProvider}); + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 1, {kTensorrtExecutionProvider}); +} + +TEST(SoftmaxOperator, ThreeDimsAxis1_opset13) { + // For the same input, opset-13's behavior is different from an earlier opset + // and we see different expected results for the same test input + + std::vector expected_vals = { + 0.253289f, 0.11198013f, 0.08185529f, 0.35567388f, 0.24795689f, + 0.44600812f, 0.46761957f, 0.09471639f, 0.2796827f, 0.3307607f, + 0.16864346f, 0.04540785f, 0.27406466f, 0.14939913f, 0.2167266f, + 0.1320594f, 0.3749925f, 0.5493636f, 0.21524426f, 0.20455585f, + + 0.32341874f, 0.18241648f, 0.1747012f, 0.36767146f, 0.36021632f, + 0.29275346f, 0.10176494f, 0.28598055f, 0.13050734f, 0.24336906f, + 0.19977638f, 0.67461985f, 0.40293545f, 0.22843185f, 0.25989732f, + 0.18405138f, 0.04119869f, 0.13638285f, 0.27338937f, 0.13651732f, + + 0.22807457f, 0.2577944f, 0.10201685f, 0.15962972f, 0.09529332f, + 0.10314508f, 0.5011263f, 0.10428739f, 0.23931651f, 0.63683724f, + 0.37181312f, 0.12944824f, 0.3946307f, 0.19975942f, 0.0699691f, + 0.29696727f, 0.11163106f, 0.39906505f, 0.4012943f, 0.1979003f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 1, + {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); // OpenVINO doesn't support opset-13 yet } TEST(SoftmaxOperator, ThreeDimsAxis2) { @@ -145,9 +174,50 @@ TEST(SoftmaxOperator, ThreeDimsAxis2) { 0.23619612f, 0.1829779f, 0.37029108f, 0.14383113f, 0.0667037f, 0.15740506f, 0.13165872f, 0.31243387f, 0.24108529f, 0.15741715f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 2); + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 7, /*axis*/ 2); } +TEST(SoftmaxOperator, ThreeDimsAxis2_opset13) { + std::vector expected_vals = { + 0.22277209f, 0.20394778f, 0.09983283f, 0.33927578f, 0.13417149f, + 0.21729809f, 0.47177994f, 0.06399124f, 0.14778666f, 0.099144064f, + 0.1797734f, 0.10023525f, 0.40512702f, 0.17272712f, 0.14213723f, + 0.06506401f, 0.3825848f, 0.37533033f, 0.11501635f, 0.062004484f, + + 0.13209775f, 0.28059313f, 0.16109712f, 0.2047936f, 0.22141843f, + 0.1568978f, 0.20539774f, 0.3460294f, 0.0953841f, 0.19629094f, + 0.045896534f, 0.5836837f, 0.20899355f, 0.07156797f, 0.08985819f, + 0.15019783f, 0.1266166f, 0.2512731f, 0.30425128f, 0.16766116f, + + 0.17869644f, 0.44943509f, 0.11806339f, 0.1417589f, 0.112046175f, + 0.03968324f, 0.42900288f, 0.059264507f, 0.10435873f, 0.36769062f, + 0.23619612f, 0.1829779f, 0.37029108f, 0.14383113f, 0.0667037f, + 0.15740506f, 0.13165872f, 0.31243387f, 0.24108529f, 0.15741715f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*axis*/ 2, + {kOpenVINOExecutionProvider}); // OpenVINO doesn't support opset-13 yet +} + +TEST(SoftmaxOperator, ThreeDimsDefaultAxis_opset13) { + std::vector expected_vals = { + 0.22277209f, 0.20394778f, 0.09983283f, 0.33927578f, 0.13417149f, + 0.21729809f, 0.47177994f, 0.06399124f, 0.14778666f, 0.099144064f, + 0.1797734f, 0.10023525f, 0.40512702f, 0.17272712f, 0.14213723f, + 0.06506401f, 0.3825848f, 0.37533033f, 0.11501635f, 0.062004484f, + + 0.13209775f, 0.28059313f, 0.16109712f, 0.2047936f, 0.22141843f, + 0.1568978f, 0.20539774f, 0.3460294f, 0.0953841f, 0.19629094f, + 0.045896534f, 0.5836837f, 0.20899355f, 0.07156797f, 0.08985819f, + 0.15019783f, 0.1266166f, 0.2512731f, 0.30425128f, 0.16766116f, + + 0.17869644f, 0.44943509f, 0.11806339f, 0.1417589f, 0.112046175f, + 0.03968324f, 0.42900288f, 0.059264507f, 0.10435873f, 0.36769062f, + 0.23619612f, 0.1829779f, 0.37029108f, 0.14383113f, 0.0667037f, + 0.15740506f, 0.13165872f, 0.31243387f, 0.24108529f, 0.15741715f}; + + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 13, /*default axis*/ -1, + {kOpenVINOExecutionProvider}); // OpenVINO doesn't support opset-13 yet +} TEST(SoftmaxOperator, ThreeDimsNegativeAxis) { // x = // node = onnx.helper.make_node('Softmax', inputs = ['x'], outputs = ['y'], axis = 2) @@ -172,7 +242,7 @@ TEST(SoftmaxOperator, ThreeDimsNegativeAxis) { 0.15740506f, 0.13165872f, 0.31243387f, 0.24108529f, 0.15741715f}; // -1 is last axis so same as axis == 2 - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ -1); + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*opset*/ 12, /*axis*/ -1); } TEST(SoftmaxOperator, InvalidAxis) { @@ -183,25 +253,39 @@ TEST(SoftmaxOperator, InvalidAxis) { RunTest(x_vals, expected_vals, dimensions, + /*opset*/ 12, /* invalid axis */ -10, {kTensorrtExecutionProvider}, OpTester::ExpectResult::kExpectFailure, // bug in ONNX error message currently. Message should be // "[ShapeInferenceError] 'axis' must be in [-2 , 1]. Its actual value is: -10" - ", 1]. Its actual value is: -10", - // latest opset so we get shape inferencing errors - // Latest valid opset for this is 12. Once opset 13 changes are implemented this can be changed back to -1 - 12); + ", 1]. Its actual value is: -10"); } +TEST(SoftmaxOperator, InvalidAxis_opset13) { + std::vector x_vals = {-1.0f, 0.0f, 1.0f}; + std::vector expected_vals = {0.0f, 0.0f, 0.0f}; + std::vector dimensions = {1, 3}; + + RunTest(x_vals, + expected_vals, + dimensions, + /*opset*/ -1, // latest opset so we get shape inferencing errors + /* invalid axis */ -10, {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}, + OpTester::ExpectResult::kExpectFailure, + // In opset-13, Softmax is composed as afunction of several other ops, + // and hence it breaks differently to the test above but the most important thing + // is that it breaks and this is the right behavior + "[ShapeInferenceError] axis must be in [-rank, rank-1]. input rank was 2"); +} TEST(SoftmaxOperator, DimWithZero) { std::vector x_vals = {}; std::vector expected_vals = {}; std::vector dimensions = {1, 0}; // dim with value of 0 should be handled - RunTest(x_vals, expected_vals, dimensions, 0, + RunTest(x_vals, expected_vals, dimensions, /*opset*/ -1, /*axis*/ 0, {kTensorrtExecutionProvider, - kNnapiExecutionProvider}, // NNAPI softmax does not support empty input - OpTester::ExpectResult::kExpectSuccess, "", 10); + kNnapiExecutionProvider} // NNAPI softmax does not support empty input + ); } } // namespace test diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc index 5d61909136..5891a37ec0 100644 --- a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc +++ b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc @@ -35,27 +35,6 @@ "^test_training_dropout.*", // NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel) "^test_if_seq_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node If(13) "^test_loop13_seq_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Loop(13) - "^test_hardmax_axis_0_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Hardmax(13) - "^test_hardmax_axis_1_cpu", - "^test_hardmax_axis_2_cpu", - "^test_hardmax_default_axis_cpu", - "^test_hardmax_example_cpu", - "^test_hardmax_negative_axis_cpu", - "^test_hardmax_one_hot_cpu", - "^test_logsoftmax_axis_0_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Logsoftmax(13) - "^test_logsoftmax_axis_0_expanded_cpu", - "^test_logsoftmax_axis_1_cpu", - "^test_logsoftmax_axis_1_expanded_cpu", - "^test_logsoftmax_axis_2_cpu", - "^test_logsoftmax_axis_2_expanded_cpu", - "^test_logsoftmax_default_axis_cpu", - "^test_logsoftmax_default_axis_expanded_cpu", - "^test_logsoftmax_example_1_cpu", - "^test_logsoftmax_example_1_expanded_cpu", - "^test_logsoftmax_large_number_cpu", - "^test_logsoftmax_large_number_expanded_cpu", - "^test_logsoftmax_negative_axis_cpu", - "^test_logsoftmax_negative_axis_expanded_cpu", "^test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Resize(13) "^test_resize_downsample_scales_cubic_cpu", "^test_resize_downsample_scales_linear_cpu", @@ -75,21 +54,7 @@ "^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu", "^test_resize_upsample_sizes_nearest_cpu", "^test_resize_upsample_sizes_nearest_floor_align_corners_cpu", - "^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu", - "^test_softmax_axis_0_cpu", // NOT_IMPLEMENTED : Could not find an implementation for the node Softmax(13) - "^test_softmax_axis_0_expanded_cpu", - "^test_softmax_axis_1_cpu", - "^test_softmax_axis_1_expanded_cpu", - "^test_softmax_axis_2_cpu", - "^test_softmax_axis_2_expanded_cpu", - "^test_softmax_default_axis_cpu", - "^test_softmax_default_axis_expanded_cpu", - "^test_softmax_example_cpu", - "^test_softmax_example_expanded_cpu", - "^test_softmax_large_number_cpu", - "^test_softmax_large_number_expanded_cpu", - "^test_softmax_negative_axis_cpu", - "^test_softmax_negative_axis_expanded_cpu" + "^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu" ], "current_failing_tests_x86": [ "^test_vgg19", @@ -199,7 +164,9 @@ "^test_reduce_sum_negative_axes_keepdims*", "^test_reduce_sum_empty_axes_input_noop*", "^test_unsqueeze_*", // Does not support axes as input - "^test_squeeze_*" // Does not support axes as input + "^test_squeeze_*", // Does not support axes as input + "^test_logsoftmax_*", // Does not support opset-13 yet + "^test_softmax_*" // Does not support opset-13 yet ], // ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported "tests_with_pre_opset7_dependencies": [