saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add SpaceToDepth and DepthToSpace CUDA NHWC Ops (#19646)

Browse source 
### Description
- Adding CUDA NHWC support for SpaceToDepth and DepthToSpace
- Add a new test which verifies that swizzling SpaceToDepth swizzling
for the H axis is correct.
- If CUDA NHWC is enabled, run all tests on the CUDA EP with NHWC as
well.

### Motivation and Context
Adding more NHWC operations to avoid layout transformations when using
the CUDA EP for more efficiency.
This commit is contained in:
Markus Tavenrath 2024-03-06 21:35:55 +01:00 committed by GitHub
parent 8bd1335d00
commit f2dc725b33
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 348 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
include/onnxruntime/core/graph/constants.h
View file

@ -31,6 +31,7 @@ constexpr size_t kMaxExecutionProviderNameLen = 30;
constexpr const char* kCpuExecutionProvider = "CPUExecutionProvider";
constexpr const char* kCudaExecutionProvider = "CUDAExecutionProvider";
constexpr const char* kCudaNHWCExecutionProvider = "CUDANHWCExecutionProvider";
constexpr const char* kDnnlExecutionProvider = "DnnlExecutionProvider";
constexpr const char* kOpenVINOExecutionProvider = "OpenVINOExecutionProvider";
constexpr const char* kVitisAIExecutionProvider = "VitisAIExecutionProvider";

1
onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_schemas.cc
View file

@ -106,6 +106,7 @@ void OpSet_Internal_NHWC_ONNX::ForEachSchema(const std::function<void(ONNX_NAMES
REGISTER_NHWC_SCHEMA_WITH_ACTIVATION(fn, BatchNormalization, 14);
REGISTER_NHWC_SCHEMA_WITH_ACTIVATION(fn, BatchNormalization, 15);
REGISTER_NHWC_SCHEMA(fn, DepthToSpace, 1);
REGISTER_NHWC_SCHEMA(fn, DepthToSpace, 11);
REGISTER_NHWC_SCHEMA(fn, DepthToSpace, 13);

3
onnxruntime/core/optimizer/layout_transformation/layout_transformation.cc
View file

@ -43,7 +43,8 @@ const std::unordered_set<std::string_view>& GetCUDALayoutSensitiveOps() {
"GlobalAveragePool",
"AveragePool",
"GridSample",
};
"DepthToSpace",
"SpaceToDepth"};
}();
return cuda_nhwc_ops;
}

16
onnxruntime/core/providers/cpu/tensor/space_depth_ops.h
View file

@ -14,6 +14,7 @@ class SpaceDepthBase {
"Attribute blocksize is not set.");
}
template <bool IsNHWC = false>
Status InputValidationsAndOutputDimsCalc(const Tensor& input,
int64_t& batch,
int64_t& input_depth, int64_t& input_height, int64_t& input_width,
@ -27,9 +28,15 @@ class SpaceDepthBase {
}
batch = input_shape[0];
input_depth = input_shape[1];
input_height = input_shape[2];
input_width = input_shape[3];
if constexpr (IsNHWC) {
input_depth = input_shape[3];
input_height = input_shape[1];
input_width = input_shape[2];
} else {
input_depth = input_shape[1];
input_height = input_shape[2];
input_width = input_shape[3];
}
if (is_space_to_depth) { // SpaceToDepth op
if ((input_height % this->blocksize_) != 0) {
@ -46,7 +53,8 @@ class SpaceDepthBase {
} else { // DepthToSpace op
if ((input_depth % (blocksize_ * blocksize_) != 0)) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "DepthToSpace requires input depth to be a multiple of (block_size * blok_size)");
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"DepthToSpace requires input depth to be a multiple of (block_size * block_size)");
}
output_depth = input_depth / blocksize_ / blocksize_;

16
onnxruntime/core/providers/cuda/cuda_nhwc_kernels.cc
View file

@ -86,6 +86,11 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalN
BatchNormalization);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain, 15, MLFloat16,
BatchNormalization);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain, 1, 10, DepthToSpace);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain, 11, 12, DepthToSpace);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain, 13, DepthToSpace);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain, 1, 12, SpaceToDepth);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain, 13, SpaceToDepth);
Status RegisterCudaNhwcKernels(KernelRegistry& kernel_registry) {
static const BuildKernelCreateInfoFn nhwc_function_table[] = {
@ -171,6 +176,17 @@ Status RegisterCudaNhwcKernels(KernelRegistry& kernel_registry) {
kCudaExecutionProvider, kMSInternalNHWCDomain, 1, 10, float, ConvTranspose)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(
kCudaExecutionProvider, kMSInternalNHWCDomain, 1, 10, MLFloat16, ConvTranspose)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain,
1, 10, DepthToSpace)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain,
11, 12, DepthToSpace)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain,
13, DepthToSpace)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain,
1, 12, SpaceToDepth)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSInternalNHWCDomain,
13, SpaceToDepth)>,
};
for (auto& function_table_entry : nhwc_function_table) {

202
onnxruntime/core/providers/cuda/tensor/space_depth_ops.cc
View file

@ -20,7 +20,22 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
SpaceToDepth);
SpaceToDepth<LAYOUT_NCHW>);
#ifdef ENABLE_CUDA_NHWC_OPS
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
SpaceToDepth,
kMSInternalNHWCDomain,
1,
12,
kCudaExecutionProvider,
(*KernelDefBuilder::Create())
.TypeConstraint("T",
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
SpaceToDepth<LAYOUT_NHWC>);
#endif
ONNX_OPERATOR_KERNEL_EX(
SpaceToDepth,
@ -32,7 +47,21 @@ ONNX_OPERATOR_KERNEL_EX(
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
SpaceToDepth);
SpaceToDepth<LAYOUT_NCHW>);
#ifdef ENABLE_CUDA_NHWC_OPS
ONNX_OPERATOR_KERNEL_EX(
SpaceToDepth,
kMSInternalNHWCDomain,
13,
kCudaExecutionProvider,
(*KernelDefBuilder::Create())
.TypeConstraint("T",
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
SpaceToDepth<LAYOUT_NHWC>);
#endif
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
DepthToSpace,
@ -45,7 +74,22 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
DepthToSpace);
DepthToSpace<LAYOUT_NCHW>);
#ifdef ENABLE_CUDA_NHWC_OPS
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
DepthToSpace,
kMSInternalNHWCDomain,
1,
10,
kCudaExecutionProvider,
(*KernelDefBuilder::Create())
.TypeConstraint("T",
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
DepthToSpace<LAYOUT_NHWC>);
#endif
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
DepthToSpace,
@ -58,7 +102,22 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
DepthToSpace);
DepthToSpace<LAYOUT_NCHW>);
#ifdef ENABLE_CUDA_NHWC_OPS
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
DepthToSpace,
kMSInternalNHWCDomain,
11,
12,
kCudaExecutionProvider,
(*KernelDefBuilder::Create())
.TypeConstraint("T",
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
DepthToSpace<LAYOUT_NHWC>);
#endif
ONNX_OPERATOR_KERNEL_EX(
DepthToSpace,
@ -70,23 +129,35 @@ ONNX_OPERATOR_KERNEL_EX(
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
DepthToSpace);
DepthToSpace<LAYOUT_NCHW>);
#ifdef ENABLE_CUDA_NHWC_OPS
ONNX_OPERATOR_KERNEL_EX(
DepthToSpace,
kMSInternalNHWCDomain,
13,
kCudaExecutionProvider,
(*KernelDefBuilder::Create())
.TypeConstraint("T",
{DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
DepthToSpace<LAYOUT_NHWC>);
#endif
static Status SpaceDepthOpCudaImpl(const cudaDeviceProp& prop,
cudaStream_t stream,
const cublasHandle_t cublas_handle,
const Tensor& input, Tensor& output,
const std::vector<size_t>& permutation,
const int64_t batch_size,
const int64_t in_dim1, const int64_t in_dim2, const int64_t in_dim3,
const int64_t in_dim4, const int64_t in_dim5,
const TensorShape& virtual_input_shape,
const TensorShape& virtual_output_shape) {
TensorShape virtual_input_shape{batch_size, in_dim1, in_dim2, in_dim3, in_dim4, in_dim5};
return Transpose::DoTranspose(prop, stream, cublas_handle, permutation, input, output,
&virtual_input_shape, &virtual_output_shape);
}
Status SpaceToDepth::ComputeInternal(OpKernelContext* context) const {
template <bool Layout>
Status SpaceToDepth<Layout>::ComputeInternal(OpKernelContext* context) const {
const auto* tensor_pointer = context->Input<Tensor>(0);
if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch");
const Tensor& input = *tensor_pointer;
@ -101,29 +172,44 @@ Status SpaceToDepth::ComputeInternal(OpKernelContext* context) const {
int64_t output_height = -1;
int64_t output_width = -1;
ORT_RETURN_IF_ERROR(InputValidationsAndOutputDimsCalc(input,
batch,
input_depth, input_height, input_width,
output_depth, output_height, output_width,
true));
ORT_RETURN_IF_ERROR(
InputValidationsAndOutputDimsCalc<Layout == LAYOUT_NHWC>(input,
batch,
input_depth, input_height, input_width,
output_depth, output_height, output_width,
true));
// We use the "actual" output shape to construct the output tensor
Tensor& output = *context->Output(0, {batch, output_depth, output_height, output_width});
Tensor& output = (Layout == LAYOUT_NCHW)
? *context->Output(0, {batch, output_depth, output_height, output_width})
: *context->Output(0, {batch, output_height, output_width, output_depth});
TensorShape virtual_input_shape = (Layout == LAYOUT_NCHW)
? TensorShape{batch, input_depth, input_height / blocksize_,
blocksize_, input_width / blocksize_, blocksize_}
: TensorShape{batch, input_height / blocksize_, blocksize_,
input_width / blocksize_, blocksize_, input_depth};
// We will pass in the "virtual" output shape to be used by DoTranspose() in SpaceDepthOpCudaImpl(...)
TensorShape virtual_output_shape{batch, blocksize_, blocksize_, input_depth,
input_height / blocksize_, input_width / blocksize_};
TensorShape virtual_output_shape = (Layout == LAYOUT_NCHW)
? TensorShape{batch, blocksize_, blocksize_, input_depth,
input_height / blocksize_, input_width / blocksize_}
: TensorShape{batch, input_height / blocksize_, input_width / blocksize_,
blocksize_, blocksize_, input_depth};
std::vector<size_t> permutation = {0, 3, 5, 1, 2, 4};
std::vector<size_t> permutation = (Layout == LAYOUT_NCHW)
? std::vector<size_t>{0, 3, 5, 1, 2, 4}
: std::vector<size_t>{0, 1, 3, 2, 4, 5};
ORT_RETURN_IF_ERROR(SpaceDepthOpCudaImpl(GetDeviceProp(), Stream(context), GetCublasHandle(context), input, output, permutation, batch,
input_depth, input_height / blocksize_, blocksize_, input_width / blocksize_, blocksize_,
virtual_output_shape));
ORT_RETURN_IF_ERROR(
SpaceDepthOpCudaImpl(GetDeviceProp(), Stream(context), GetCublasHandle(context), input, output, permutation,
virtual_input_shape, virtual_output_shape));
return Status::OK();
}
Status DepthToSpace::ComputeInternal(OpKernelContext* context) const {
template <bool Layout>
Status DepthToSpace<Layout>::ComputeInternal(OpKernelContext* context) const {
const auto* tensor_pointer = context->Input<Tensor>(0);
if (tensor_pointer == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch");
const Tensor& input = *tensor_pointer;
@ -138,46 +224,56 @@ Status DepthToSpace::ComputeInternal(OpKernelContext* context) const {
int64_t output_height = -1;
int64_t output_width = -1;
ORT_RETURN_IF_ERROR(InputValidationsAndOutputDimsCalc(input,
batch,
input_depth, input_height, input_width,
output_depth, output_height, output_width,
false));
ORT_RETURN_IF_ERROR(
InputValidationsAndOutputDimsCalc<Layout == LAYOUT_NHWC>(input,
batch,
input_depth, input_height, input_width,
output_depth, output_height, output_width,
false));
// We use the "actual" output shape to construct the output tensor
Tensor& output = *context->Output(0, {batch, output_depth, output_height, output_width});
Tensor& output = (Layout == LAYOUT_NCHW)
? *context->Output(0, {batch, output_depth, output_height, output_width})
: *context->Output(0, {batch, output_height, output_width, output_depth});
// We will pass in the "virtual" output shape to be used by DoTranspose() in SpaceDepthOpCudaImpl(...)
TensorShape virtual_output_shape{batch, input_depth / blocksize_ / blocksize_,
input_height, blocksize_, input_width, blocksize_};
std::vector<size_t> permutation;
permutation.reserve(6);
permutation.push_back(0);
int64_t virtual_input_depth = input_depth / blocksize_ / blocksize_;
TensorShape virtual_input_shape;
// cdr only here!
if (is_dcr_) {
permutation.push_back(3);
permutation.push_back(4);
permutation.push_back(1);
permutation.push_back(5);
permutation.push_back(2);
virtual_input_shape = (Layout == LAYOUT_NCHW)
? TensorShape{batch, blocksize_, blocksize_,
virtual_input_depth, input_height, input_width}
: TensorShape{batch, input_height, input_width,
blocksize_, blocksize_, virtual_input_depth};
} else {
permutation.push_back(1);
permutation.push_back(4);
permutation.push_back(2);
permutation.push_back(5);
permutation.push_back(3);
virtual_input_shape = (Layout == LAYOUT_NCHW)
? TensorShape{batch, virtual_input_depth, blocksize_,
blocksize_, input_height, input_width}
: TensorShape{batch, input_height, input_width,
virtual_input_depth, blocksize_, blocksize_};
}
int64_t dim1 = is_dcr_ ? blocksize_ : input_depth / blocksize_ / blocksize_;
int64_t dim3 = is_dcr_ ? input_depth / blocksize_ / blocksize_ : blocksize_;
// We will pass in the "virtual" output shape to be used by DoTranspose() in SpaceDepthOpCudaImpl(...)
TensorShape virtual_output_shape = (Layout == LAYOUT_NCHW)
? TensorShape{batch, virtual_input_depth, input_height,
blocksize_, input_width, blocksize_}
: TensorShape{batch, input_height, blocksize_,
input_width, blocksize_, virtual_input_depth};
std::vector<size_t> permutation;
if (is_dcr_) {
permutation = (Layout == LAYOUT_NCHW)
? std::vector<size_t>({0, 3, 4, 1, 5, 2})
: std::vector<size_t>({0, 1, 3, 2, 4, 5});
} else {
permutation = std::vector<size_t>({0, 1, 4, 2, 5, 3});
}
ORT_RETURN_IF_ERROR(SpaceDepthOpCudaImpl(GetDeviceProp(), Stream(context), GetCublasHandle(context), input, output,
permutation,
batch,
dim1, blocksize_, dim3, input_height, input_width,
virtual_output_shape));
permutation, virtual_input_shape, virtual_output_shape));
return Status::OK();
}

2
onnxruntime/core/providers/cuda/tensor/space_depth_ops.h
View file

@ -9,6 +9,7 @@
namespace onnxruntime {
namespace cuda {
template <bool Layout>
class SpaceToDepth final : public CudaKernel, SpaceDepthBase {
public:
explicit SpaceToDepth(const OpKernelInfo& info) : CudaKernel(info), SpaceDepthBase(info) {
@ -17,6 +18,7 @@ class SpaceToDepth final : public CudaKernel, SpaceDepthBase {
Status ComputeInternal(OpKernelContext* context) const override;
};
template <bool Layout>
class DepthToSpace final : public CudaKernel, SpaceDepthBase {
public:
explicit DepthToSpace(const OpKernelInfo& info) : CudaKernel(info), SpaceDepthBase(info) {

17
onnxruntime/test/contrib_ops/gridsample_test.cc
View file

@ -32,7 +32,7 @@ TEST(GridsampleContribOpTest, gridsample_default) {
3.8000f, 7.9000f, 8.7000f, 9.5000f, 10.3000f, 5.3000f,
5.4000f, 11.1000f, 11.9000f, 12.7000f, 13.5000f, 6.9000f,
3.0000f, 6.1500f, 6.5500f, 6.9500f, 7.3500f, 3.7500f});
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(GridsampleContribOpTest, gridsample_paddingmode_zeros) {
@ -45,7 +45,7 @@ TEST(GridsampleContribOpTest, gridsample_paddingmode_zeros) {
5.0000f, 5.0000f, 10.0000f, 10.0000f});
test.AddAttribute("padding_mode", "zeros");
test.AddOutput<float>("Y", {1, 1, 2, 4}, {0.0000f, 0.0000f, 1.7000f, 0.0000f, 0.0000f, 1.7000f, 0.0000f, 0.0000f});
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(GridsampleContribOpTest, gridsample_paddingmode_border) {
@ -58,7 +58,7 @@ TEST(GridsampleContribOpTest, gridsample_paddingmode_border) {
5.0000f, 5.0000f, 10.0000f, 10.0000f});
test.AddAttribute("padding_mode", "border");
test.AddOutput<float>("Y", {1, 1, 2, 4}, {0.0000f, 0.0000f, 1.7000f, 5.0000f, 5.0000f, 1.7000f, 5.0000f, 5.0000f});
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(GridsampleContribOpTest, gridsample_paddingmode_reflection) {
@ -71,7 +71,8 @@ TEST(GridsampleContribOpTest, gridsample_paddingmode_reflection) {
5.0000f, 5.0000f, 10.0000f, 10.0000f});
test.AddAttribute("padding_mode", "reflection");
test.AddOutput<float>("Y", {1, 1, 2, 4}, {2.5000f, 0.0000f, 1.7000f, 2.5000f, 2.5000f, 1.7000f, 5.0000f, 2.5000f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kQnnExecutionProvider}); // Accuracy issue for QNN
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaNHWCExecutionProvider, kQnnExecutionProvider}); // Accuracy issue for QNN
}
TEST(GridsampleContribOpTest, gridsample_aligncorners_true) {
@ -86,7 +87,7 @@ TEST(GridsampleContribOpTest, gridsample_aligncorners_true) {
test.AddAttribute("mode", "bilinear");
test.AddAttribute("align_corners", align_corners);
test.AddOutput<float>("Y", {1, 1, 2, 4}, {0.0000f, 1.2500f, 2.0000f, 2.5000f, 2.5000f, 2.0000f, 3.7500f, 5.0000f});
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(GridsampleContribOpTest, gridsample_mode_bilinear) {
@ -99,7 +100,7 @@ TEST(GridsampleContribOpTest, gridsample_mode_bilinear) {
0.5000f, 0.5000f, 1.0000f, 1.0000f});
test.AddAttribute("mode", "bilinear");
test.AddOutput<float>("Y", {1, 1, 2, 4}, {0.0000f, 0.5000f, 1.7000f, 2.5000f, 2.5000f, 1.7000f, 4.5000f, 1.2500f});
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(GridsampleContribOpTest, gridsample_mode_nearest) {
@ -112,7 +113,7 @@ TEST(GridsampleContribOpTest, gridsample_mode_nearest) {
0.5000f, 0.5000f, 1.0000f, 1.0000f});
test.AddAttribute("mode", "nearest");
test.AddOutput<float>("Y", {1, 1, 2, 4}, {0.f, 0.f, 2.f, 2.f, 2.f, 2.f, 5.f, 0.f});
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(GridsampleContribOpTest, gridsample_mode_bicubic) {
@ -125,7 +126,7 @@ TEST(GridsampleContribOpTest, gridsample_mode_bicubic) {
0.5000f, 0.5000f, 1.0000f, 1.0000f});
test.AddAttribute("mode", "bicubic");
test.AddOutput<float>("Y", {1, 1, 2, 4}, {-0.1406f, 0.3828f, 1.7556f, 2.9688f, 2.9688f, 1.7556f, 5.1445f, 1.3906f});
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
} // namespace test

7
onnxruntime/test/providers/base_tester.cc
View file

@ -622,6 +622,9 @@ void BaseTester::RunWithConfig(size_t* number_of_pre_packed_weights_counter,
static const std::string all_provider_types[] = {
kCpuExecutionProvider,
kCudaExecutionProvider,
#ifdef ENABLE_CUDA_NHWC_OPS
kCudaNHWCExecutionProvider,
#endif
kDnnlExecutionProvider,
kTensorrtExecutionProvider,
kOpenVINOExecutionProvider,
@ -650,6 +653,10 @@ void BaseTester::RunWithConfig(size_t* number_of_pre_packed_weights_counter,
execution_provider = DefaultCpuExecutionProvider();
else if (provider_type == onnxruntime::kCudaExecutionProvider)
execution_provider = DefaultCudaExecutionProvider();
#ifdef ENABLE_CUDA_NHWC_OPS
else if (provider_type == onnxruntime::kCudaNHWCExecutionProvider)
execution_provider = DefaultCudaNHWCExecutionProvider();
#endif
else if (provider_type == onnxruntime::kDnnlExecutionProvider)
execution_provider = DefaultDnnlExecutionProvider();
else if (provider_type == onnxruntime::kOpenVINOExecutionProvider)

12
onnxruntime/test/providers/cpu/generator/random_test.cc
View file

@ -36,7 +36,8 @@ TEST(Random, RandomNormal2DDouble) {
// The expected_output is generated using std lib, which is used by CPU kernel only.
// So we need to exclude other EPs here. Ditto for other places.
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kRocmExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider});
}
void RunRandomNormalLike3DFloat(bool infer_dtype = false) {
@ -72,7 +73,8 @@ void RunRandomNormalLike3DFloat(bool infer_dtype = false) {
test.AddOutput<float>("Y", dims, expected_output);
// TensorRT does not support manual seed overrides and there will be result mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kRocmExecutionProvider, kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kTensorrtExecutionProvider});
}
TEST(Random, RandomNormalLike3DDouble) {
@ -109,7 +111,8 @@ TEST(Random, RandomUniform1DFloat) {
test.AddOutput<float>("Y", dims, expected_output);
// TensorRT does not support manual seed overrides and there will be result mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kRocmExecutionProvider, kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kTensorrtExecutionProvider});
}
void RunRandomUniformLikeTest(bool infer_dtype = false) {
@ -142,7 +145,8 @@ void RunRandomUniformLikeTest(bool infer_dtype = false) {
test.AddOutput<double>("Y", dims, expected_output);
// TensorRT does not support seed parameter and there will be result mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kRocmExecutionProvider, kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kTensorrtExecutionProvider});
}
TEST(Random, RandomUniformLike2DDouble) {

6
onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc
View file

@ -917,7 +917,7 @@ TEST(BatchNormTest, ForwardTrainingTestWithSavedOutputsOpset9) {
// exclude TRT and OpenVINO for same reasons as seen in TestBatchNorm()
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
// TODO(mtavenrath) flakiness of running_mean for CUDA has been fixed, the delta of running_var is still ~0.1
{kCudaExecutionProvider, kRocmExecutionProvider,
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider,
kTensorrtExecutionProvider, kOpenVINOExecutionProvider, kDnnlExecutionProvider});
}
@ -945,7 +945,7 @@ TEST(BatchNormTest, ForwardTrainingTestOpset14) {
// exclude CUDA Execution Provider due to flakiness
// exclude TRT and OpenVINO for same reasons as seen in TestBatchNorm()
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kRocmExecutionProvider,
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider,
kTensorrtExecutionProvider, kOpenVINOExecutionProvider, kDnnlExecutionProvider});
}
@ -972,7 +972,7 @@ TEST(BatchNormTest, ForwardTrainingTestOpset15) {
// Same exclusions as the opset 14 test
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kRocmExecutionProvider,
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider,
kTensorrtExecutionProvider, kOpenVINOExecutionProvider, kDnnlExecutionProvider});
}
#endif // BATCHNORM_INCLUDE_TRAINING_SUPPORT

2
onnxruntime/test/providers/cpu/nn/conv_op_test.cc
View file

@ -59,6 +59,8 @@ void TestConvOp(const ConvOpAndTestAttributes& attributes,
std::unordered_set<std::string> excluded_providers(attributes.excluded_providers);
// Disable TensorRT because weight as input is not supported
excluded_providers.insert(kTensorrtExecutionProvider);
// Disable CUDA NHWC execution provider as it is currently flaky
excluded_providers.insert(kCudaNHWCExecutionProvider);
// QNN SDK 2.10.0 has a bug that breaks support for dynamic bias inputs.
excluded_providers.insert(kQnnExecutionProvider);

15
onnxruntime/test/providers/cpu/nn/conv_transpose_op_test.cc
View file

@ -75,7 +75,8 @@ void TestConvTransposeOp(const ConvTransposeOpAttributes& attributes,
const vector<int64_t>& expected_output_shape,
OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess,
const std::string& err_str = "",
const std::unordered_set<std::string>& excluded_provider_types = {kTensorrtExecutionProvider, kQnnExecutionProvider}) {
const std::unordered_set<std::string>& excluded_provider_types =
{kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kQnnExecutionProvider}) {
std::unordered_set<std::string> extra_exclude_openvino_for_initializer_filter = excluded_provider_types;
extra_exclude_openvino_for_initializer_filter.insert(kOpenVINOExecutionProvider);
TestConvTransposeOpInitializer(attributes, inputs, input_shapes, expected_output, expected_output_shape,
@ -409,7 +410,8 @@ TEST(ConvTransposeTest, ConvTranspose_2D_OutputShape_2) {
vector<int64_t> Y_shape = {1, 1, 1, 14};
auto expected_vals = {1.0f, 2.0f, 5.0f, 11.0f, 19.0f, 28.0f, 37.0f, 46.0f, 55.0f, 64.0f, 63.0f, 51.0f, 27.0f, 10.0f};
TestConvTransposeOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape,
OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider, kQnnExecutionProvider});
OpTester::ExpectResult::kExpectSuccess, "",
{kOpenVINOExecutionProvider, kCudaNHWCExecutionProvider, kQnnExecutionProvider});
}
TEST(ConvTransposeTest, ConvTranspose_2D_OutputShapeWithBatchSize) {
@ -434,7 +436,8 @@ TEST(ConvTransposeTest, ConvTranspose_2D_OutputShapeWithBatchSize) {
auto expected_vals = {1.0f, 2.0f, 5.0f, 11.0f, 19.0f, 28.0f, 37.0f, 46.0f, 55.0f, 64.0f, 63.0f, 51.0f, 27.0f, 10.0f,
11.0f, 32.0f, 65.0f, 91.0f, 109.0f, 118.0f, 127.0f, 136.0f, 145.0f, 154.0f, 143.0f, 111.0f, 57.0f, 20.0f};
TestConvTransposeOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape,
OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider, kQnnExecutionProvider});
OpTester::ExpectResult::kExpectSuccess, "",
{kOpenVINOExecutionProvider, kCudaNHWCExecutionProvider, kQnnExecutionProvider});
}
TEST(ConvTransposeTest, ConvTranspose_InvalidKernelShape) {
@ -871,7 +874,8 @@ TEST(ConvTransposeTest, DimWithZero) {
TestConvTransposeOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape,
OpTester::ExpectResult::kExpectSuccess, "",
{kTensorrtExecutionProvider, kAclExecutionProvider, kQnnExecutionProvider});
{kCudaNHWCExecutionProvider, kTensorrtExecutionProvider,
kAclExecutionProvider, kQnnExecutionProvider});
}
TEST(ConvTransposeTest, ConvTranspose_3D) {
@ -1005,7 +1009,8 @@ TEST(ConvTransposeTest, ConvTranspose_3D) {
TestConvTransposeOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape,
OpTester::ExpectResult::kExpectSuccess, "",
{kTensorrtExecutionProvider, kCudaExecutionProvider, kQnnExecutionProvider});
{kTensorrtExecutionProvider, kCudaExecutionProvider,
kCudaNHWCExecutionProvider, kQnnExecutionProvider});
}
TEST(ConvTransposeTest, ConvTranspose_1D_AsymmetricPads) {

86
onnxruntime/test/providers/cpu/nn/pool_op_test.cc
View file

@ -57,7 +57,8 @@ TEST(PoolTest, MaxPool) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // TensorRT: result differs
// TensorRT: result differs
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
// Only CUDA kernel has float 16 support
@ -115,7 +116,8 @@ TEST(PoolTest, MaxPool_F16) {
test.AddInput<MLFloat16>("X", x_dims, f_X);
test.AddOutput<MLFloat16>("Y", expected_dims, f_Y);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // TensorRT: Assertion `!attrs.count("pads")' failed
// TensorRT: Assertion `!attrs.count("pads")' failed
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
#endif
@ -167,7 +169,9 @@ static void MaxPool_8_WithIndexTest(bool has_index, int64_t storage_order = 0) {
storage_order == 0 ? test.AddOutput<int64_t>("Indices", expected_dims, expected_indices_row)
: test.AddOutput<int64_t>("Indices", expected_dims, expected_indices_col);
}
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kDnnlExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider, kArmNNExecutionProvider, kOpenVINOExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kDnnlExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider,
kAclExecutionProvider, kArmNNExecutionProvider, kOpenVINOExecutionProvider});
}
TEST(PoolTest, MaxPool_8_With_Index) {
@ -196,7 +200,7 @@ TEST(PoolTest, MaxPool1D) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
static void MaxPool1D_8_WithIndexTest(int64_t storage_order) {
@ -217,7 +221,8 @@ static void MaxPool1D_8_WithIndexTest(int64_t storage_order) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.AddOutput<int64_t>("Indices", expected_dims, expected_indices);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kAclExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider});
}
TEST(PoolTest, MaxPool1D_8_With_Index) {
@ -243,7 +248,8 @@ static void MaxPool1D_12_WithIndexTest_int8(int64_t storage_order) {
test.AddInput<int8_t>("X", x_dims, x_vals);
test.AddOutput<int8_t>("Y", expected_dims, expected_vals);
test.AddOutput<int64_t>("Indices", expected_dims, expected_indices);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kAclExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider});
}
static void MaxPool1D_12_WithIndexTest_uint8(int64_t storage_order) {
@ -264,7 +270,8 @@ static void MaxPool1D_12_WithIndexTest_uint8(int64_t storage_order) {
test.AddInput<uint8_t>("X", x_dims, x_vals);
test.AddOutput<uint8_t>("Y", expected_dims, expected_vals);
test.AddOutput<int64_t>("Indices", expected_dims, expected_indices);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kAclExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider});
}
TEST(PoolTest, MaxPool1D_12_With_Index_8bits) {
@ -302,9 +309,9 @@ TEST(PoolTest, MaxPool2D_uint8) {
test.AddOutput<uint8_t>("Output", output_shape, output);
#if defined(OPENVINO_CONFIG_GPU_FP32) || defined(OPENVINO_CONFIG_GPU_FP16)
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kOpenVINOExecutionProvider});
#else
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
#endif
}
@ -330,7 +337,7 @@ TEST(PoolTest, MaxPool_10_Dilation_1d) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(PoolTest, MaxPool_DefaultDilations) {
@ -350,7 +357,7 @@ TEST(PoolTest, MaxPool_DefaultDilations) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(PoolTest, MaxPool_DefaultDilations_int8) {
@ -370,7 +377,7 @@ TEST(PoolTest, MaxPool_DefaultDilations_int8) {
test.AddInput<int8_t>("X", x_dims, x_vals);
test.AddOutput<int8_t>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(PoolTest, MaxPool_DefaultDilations_uint8) {
@ -390,7 +397,7 @@ TEST(PoolTest, MaxPool_DefaultDilations_uint8) {
test.AddInput<uint8_t>("X", x_dims, x_vals);
test.AddOutput<uint8_t>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(PoolTest, MaxPool_10_DilationPadding_1d) {
@ -416,7 +423,7 @@ TEST(PoolTest, MaxPool_10_DilationPadding_1d) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
}
TEST(PoolTest, MaxPool_10_Dilation_2d) {
@ -444,7 +451,7 @@ TEST(PoolTest, MaxPool_10_Dilation_2d) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(PoolTest, MaxPool_10_Dilation_2d_int8) {
@ -472,7 +479,7 @@ TEST(PoolTest, MaxPool_10_Dilation_2d_int8) {
test.AddInput<int8_t>("X", x_dims, x_vals);
test.AddOutput<int8_t>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(PoolTest, MaxPool_10_DilationPadding_2d) {
@ -500,7 +507,7 @@ TEST(PoolTest, MaxPool_10_DilationPadding_2d) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
}
TEST(PoolTest, MaxPool_10_Dilation_Ceil0_2d) {
@ -528,7 +535,8 @@ TEST(PoolTest, MaxPool_10_Dilation_Ceil0_2d) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kAclExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider});
}
TEST(PoolTest, MaxPool_12_Dilation_Ceil0_2d_int8) {
@ -556,7 +564,8 @@ TEST(PoolTest, MaxPool_12_Dilation_Ceil0_2d_int8) {
test.AddInput<int8_t>("X", x_dims, x_vals);
test.AddOutput<int8_t>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kAclExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider});
}
TEST(PoolTest, MaxPool_10_Dilation_Ceil1_2d) {
@ -585,7 +594,8 @@ TEST(PoolTest, MaxPool_10_Dilation_Ceil1_2d) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kAclExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider});
}
TEST(PoolTest, MaxPool_10_DilationPadding_3d) {
@ -621,7 +631,7 @@ TEST(PoolTest, MaxPool_10_DilationPadding_3d) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
}
TEST(PoolTest, GlobalMaxPool) {
@ -697,7 +707,7 @@ TEST(PoolTest, GlobalMaxPool) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(PoolTest, GlobalMaxPool3D) {
@ -773,7 +783,7 @@ TEST(PoolTest, GlobalMaxPool3D) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(PoolTest, AveragePool) {
@ -854,7 +864,7 @@ TEST(PoolTest, AveragePool) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(PoolTest, AveragePool_IncludePadPixel) {
@ -878,7 +888,7 @@ TEST(PoolTest, AveragePool_IncludePadPixel) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
// test 'strides' attribute not specified
@ -897,7 +907,7 @@ TEST(PoolTest, AveragePool_DefaultStrides) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(PoolTest, AveragePool_10_ceil1_2d) {
@ -920,7 +930,8 @@ TEST(PoolTest, AveragePool_10_ceil1_2d) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kAclExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider});
}
TEST(PoolTest, AveragePool_19_dilation_2d) {
@ -944,7 +955,7 @@ TEST(PoolTest, AveragePool_19_dilation_2d) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kAclExecutionProvider, kOpenVINOExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kAclExecutionProvider, kOpenVINOExecutionProvider});
}
TEST(PoolTest, GlobalAveragePool) {
@ -1020,7 +1031,7 @@ TEST(PoolTest, GlobalAveragePool) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(PoolTest, GlobalAveragePool_Large_128) {
@ -1033,7 +1044,7 @@ TEST(PoolTest, GlobalAveragePool_Large_128) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals,
/*sort_output=*/false, /*rel_error=*/1e-3f, /*abs_error=*/1e-2f);
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(PoolTest, GlobalAveragePool_Large_256) {
@ -1046,7 +1057,7 @@ TEST(PoolTest, GlobalAveragePool_Large_256) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals,
/*sort_output=*/false, /*rel_error=*/1e-3f, /*abs_error=*/1e-2f);
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(PoolTest, LpPool) {
@ -1353,7 +1364,7 @@ TEST(PoolTest, LpPool) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
// test data generated with lp_pool_test_generator.py
@ -1385,7 +1396,7 @@ TEST(PoolTest, LpPool1d) {
// https://docs.nvidia.com/deeplearning/tensorrt/api/c_api/classnvinfer1_1_1_i_network_definition.html#a94f434942252e6d98ac17705c06ce060
// TensorRT does not support 1d pooling
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
y_count++;
}
}
@ -1417,7 +1428,7 @@ TEST(PoolTest, LpPool2d) {
test.AddAttribute("kernel_shape", kernel_sizes[kernel_size_count]);
test.AddOutput<float>("Y", y_sizes[y_count], ys[y_count]);
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
y_count++;
}
}
@ -1435,7 +1446,7 @@ TEST(PoolTest, LpPoolCeilMode) {
// https://docs.nvidia.com/deeplearning/tensorrt/api/c_api/classnvinfer1_1_1_i_network_definition.html#a94f434942252e6d98ac17705c06ce060
// TensorRT does not support 1d pooling
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(PoolTest, GlobalLpPool) {
@ -1690,7 +1701,7 @@ TEST(PoolTest, GlobalLpPool) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider});
}
TEST(PoolTest, MaxPoolDimWithZeroForN) {
@ -1707,7 +1718,8 @@ TEST(PoolTest, MaxPoolDimWithZeroForN) {
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kQnnExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kQnnExecutionProvider});
}
} // namespace test

3
onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc
View file

@ -3541,6 +3541,7 @@ TEST(ReductionOpTest, ReduceDimWithZero1) {
{
kCoreMLExecutionProvider,
kCudaExecutionProvider,
kCudaNHWCExecutionProvider,
kDnnlExecutionProvider,
kMIGraphXExecutionProvider,
kOpenVINOExecutionProvider,
@ -3591,6 +3592,7 @@ TEST(ReductionOpTest, ReduceDimWithZero2) {
{
kCoreMLExecutionProvider,
kCudaExecutionProvider,
kCudaNHWCExecutionProvider,
kDnnlExecutionProvider,
kMIGraphXExecutionProvider,
kOpenVINOExecutionProvider,
@ -5779,6 +5781,7 @@ void test_empty_set(const std::string& op, int opset, bool axes_as_input, float
{
kCoreMLExecutionProvider,
kCudaExecutionProvider,
kCudaNHWCExecutionProvider,
kDmlExecutionProvider,
kDnnlExecutionProvider,
kMIGraphXExecutionProvider,

7
onnxruntime/test/providers/cpu/rnn/rnn_op_test.cc
View file

@ -744,7 +744,9 @@ TEST(RNNTest, RNN_invalid_sequence_lens) {
test.AddOutput<float>("Y_h", Y_h_dims, Y_h_data);
// the CUDA RNN version allows the invalid sequence lengths, so disable testing on CUDA and TensorRT
test.Run(OpTester::ExpectResult::kExpectFailure, error_msg, {kCudaExecutionProvider, kTensorrtExecutionProvider, kOpenVINOExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectFailure, error_msg,
{kCudaExecutionProvider, kCudaNHWCExecutionProvider,
kTensorrtExecutionProvider, kOpenVINOExecutionProvider});
};
// should batch batch_size to be valid
@ -842,7 +844,8 @@ TEST(RNNTest, RNN_bidirectional_with_sequence_lens) {
test.AddOutput<float>("Y_h", Y_h_dims, Y_h_data);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kTensorrtExecutionProvider, kOpenVINOExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kTensorrtExecutionProvider, kOpenVINOExecutionProvider});
}
TEST(RNNTest, RNN_with_invalid_activation_load_failure) {

2
onnxruntime/test/providers/cpu/tensor/gather_elements_op_test.cc
View file

@ -383,7 +383,7 @@ TEST(GatherElementsOpTest, IndicesOutOfBounds) {
// skip openvino which will not throw error message but will ensure no out-of-bound access
// skip TensorRT because it doesn't support out of bounds indices
test.Run(OpTester::ExpectResult::kExpectFailure, "",
{kCudaExecutionProvider, kRocmExecutionProvider, kOpenVINOExecutionProvider,
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kOpenVINOExecutionProvider,
kTensorrtExecutionProvider, kDmlExecutionProvider});
}

22
onnxruntime/test/providers/cpu/tensor/resize_op_test.cc
View file

@ -102,7 +102,7 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_with_extr
// TensorRT: results mismatch
// ROCm: results mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
}
TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_with_extrapolation_uint8) {
@ -132,7 +132,8 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_with_extr
test.AddOutput<uint8_t>("Y", {N, static_cast<int64_t>(H * scales[1]), static_cast<int64_t>(W * scales[2]), C}, Y);
// CUDA: result mismatch due to not implementing NHWC support
// ROCm: results mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kRocmExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider});
}
TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_with_extrapolation_int8) {
@ -192,7 +193,7 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_without_e
// DML: results mismatch
test.Run(
OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kRocmExecutionProvider, kDmlExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kDmlExecutionProvider});
}
TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_without_extrapolation_int8) {
@ -267,7 +268,7 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear) {
// CUDA: result mismatch due to not implementing NHWC support
// ROCm: results mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kRocmExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider});
}
TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_uint8) {
@ -291,7 +292,8 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_uint8) {
test.AddOutput<uint8_t>("Y", {N, static_cast<int64_t>(H * scales[1]), static_cast<int64_t>(W * scales[2]), C}, Y);
// CUDA: result mismatch due to not implementing NHWC support
// ROCm: results mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kRocmExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider});
}
TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_int8) {
@ -439,7 +441,8 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_align_corners_uin
test.AddOutput<uint8_t>("Y", {N, static_cast<int64_t>(H * scales[1]), static_cast<int64_t>(W * scales[2]), C}, Y);
// CUDA: result mismatch due to not implementing NHWC support
// ROCm: results mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kRocmExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider});
};
run_test(false);
@ -539,7 +542,7 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_pytorch_half_pixe
// ROCm: results mismatch
// DML: results mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kRocmExecutionProvider, kDmlExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kDmlExecutionProvider});
}
TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_pytorch_half_pixel_int8) {
@ -650,7 +653,8 @@ TEST(ResizeOpTest, NhwcResizeOpLinearUpSampleTest_4DBilinear_asymmetric_uint8) {
Y, false, .0f, 1.0f);
// CUDA: result mismatch due to not implementing NHWC support
// ROCm: results mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kRocmExecutionProvider});
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider});
};
run_test(false);
@ -1913,6 +1917,8 @@ void TestAntialiasing(std::map<std::string, std::string> attributes,
});
// TensorRT 8.5 supports operators up to Opset 17. Temporarily exclude TensorRT EP due to accuracy issue.
excluded_eps.insert(kTensorrtExecutionProvider);
// Test is flaky on kCudaNHWCExecutionProvider
excluded_eps.insert(kCudaNHWCExecutionProvider);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", excluded_eps);
}

7
onnxruntime/test/providers/cpu/tensor/scatter_op_test.cc
View file

@ -268,7 +268,7 @@ static void scatter_invalid_index(const char* op_name, int op_version) {
test.AddOutput<float>("y", {4, 2, 1}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 5.0f, 0.0f});
test.Run(OpTester::ExpectResult::kExpectFailure,
"indices element out of data bounds, idx=4 must be within the inclusive range [-4,3]",
{kCudaExecutionProvider, kTensorrtExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider});
}
TEST(Scatter, InvalidIndex) {
@ -291,9 +291,10 @@ static void scatter_bool_with_axis_tests(const char* op_name, int op_version) {
test.AddOutput<bool>("y", {1, 5}, {false, true, false, false, false});
#if defined(OPENVINO_CONFIG_GPU_FP32) || defined(OPENVINO_CONFIG_GPU_FP16)
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kOpenVINOExecutionProvider}); // OpenVINO: Disabled due to failure for GPU
{kCudaNHWCExecutionProvider, kOpenVINOExecutionProvider}); // OpenVINO: Disabled due to failure for GPU
#else
test.Run();
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaNHWCExecutionProvider}); // OpenVINO: Disabled due to failure for GPU
#endif
}

47
onnxruntime/test/providers/cpu/tensor/space_depth_ops_test.cc
View file

@ -108,6 +108,53 @@ TEST(TensorOpTest, SpaceToDepthTest_2) {
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kQnnExecutionProvider});
}
TEST(TensorOpTest, SpaceToDepthTest_3) {
// Test swizzling with H_output > 1
OpTester test("SpaceToDepth");
constexpr int64_t blocksize = 2;
test.AddAttribute("blocksize", blocksize);
constexpr int64_t N = 1, C = 2, H = 4, W = 8;
const std::vector<float> X = {
0.0f, 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f,
1.0f, 1.1f, 1.2f, 1.3f, 1.4f, 1.5f, 1.6f, 1.7f,
2.0f, 2.1f, 2.2f, 2.3f, 2.4f, 2.5f, 2.6f, 2.7f,
3.0f, 3.1f, 3.2f, 3.3f, 3.4f, 3.5f, 3.6f, 3.7f,
4.0f, 4.1f, 4.2f, 4.3f, 4.4f, 4.5f, 4.6f, 4.7f,
5.0f, 5.1f, 5.2f, 5.3f, 5.4f, 5.5f, 5.6f, 5.7f,
6.0f, 6.1f, 6.2f, 6.3f, 6.4f, 6.5f, 6.6f, 6.7f,
7.0f, 7.1f, 7.2f, 7.3f, 7.4f, 7.5f, 7.6f, 7.7f};
test.AddInput<float>("input", {N, C, H, W}, X);
const std::vector<float> result = {
0.0f, 0.2f, 0.4f, 0.6f,
2.0f, 2.2f, 2.4f, 2.6f,
4.0f, 4.2f, 4.4f, 4.6f,
6.0f, 6.2f, 6.4f, 6.6f,
0.1f, 0.3f, 0.5f, 0.7f,
2.1f, 2.3f, 2.5f, 2.7f,
4.1f, 4.3f, 4.5f, 4.7f,
6.1f, 6.3f, 6.5f, 6.7f,
1.0f, 1.2f, 1.4f, 1.6f,
3.0f, 3.2f, 3.4f, 3.6f,
5.0f, 5.2f, 5.4f, 5.6f,
7.0f, 7.2f, 7.4f, 7.6f,
1.1f, 1.3f, 1.5f, 1.7f,
3.1f, 3.3f, 3.5f, 3.7f,
5.1f, 5.3f, 5.5f, 5.7f,
7.1f, 7.3f, 7.5f, 7.7f};
test.AddOutput<float>("output", {N, C * blocksize * blocksize, H / blocksize, W / blocksize}, result);
test.Run();
}
TEST(TensorOpTest, DepthToSpaceTest_1) {
OpTester test("DepthToSpace", 7); // create an opset 7 model
constexpr int64_t blocksize = 2;

6
onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc
View file

@ -692,7 +692,7 @@ TEST(UpsampleOpTest, NhwcUpsampleOp4D1CBilinearTest) {
// TensorRT: results mismatch
// ROCm: results mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
}
TEST(UpsampleOpTest, NhwcUpsampleOp4DBilinearTest) {
@ -766,7 +766,7 @@ TEST(UpsampleOpTest, NhwcUpsampleOp4DBilinearTest) {
// TensorRT: results mismatch
// ROCm: results mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
}
TEST(UpsampleOpTest, UpsampleOp2DBilinearTest) {
@ -886,7 +886,7 @@ TEST(UpsampleOpTest, NhwcUpsampleOp4DBilinearTest_int32) {
// TensorRT: results mismatch
// ROCm: results mismatch
test.Run(OpTester::ExpectResult::kExpectSuccess, "",
{kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
{kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider});
}
TEST(UpsampleOpTest, UpsampleOpNearestTest_1D) {