diff --git a/onnxruntime/core/framework/kernel_def_builder.cc b/onnxruntime/core/framework/kernel_def_builder.cc index 9aa8ecf787..6496b94ac5 100644 --- a/onnxruntime/core/framework/kernel_def_builder.cc +++ b/onnxruntime/core/framework/kernel_def_builder.cc @@ -24,7 +24,10 @@ inline bool AreVectorsOverlap(const std::vector& v1, const std::vector& v2 } } // namespace -//TODO: Tell user why it has conflicts +// TODO: Tell user why it has conflicts +// TODO: Investigate why IsConflict() was not triggered when there were duplicate Tile CUDA +// kernels registered. Removing `InputMemoryType(1)` in the kernel definition +// triggered the conflict. bool KernelDef::IsConflict(const KernelDef& other) const { if (op_name_ != other.OpName() || provider_type_ != other.Provider()) return false; diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc index b157ecea21..20a29fd0bb 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc @@ -67,7 +67,7 @@ CUDAExecutionProvider::PerThreadContext::PerThreadContext(OrtDevice::DeviceId de [](OrtDevice::DeviceId id) { return onnxruntime::make_unique(id, CUDA); }, - cuda_mem_limit, + cuda_mem_limit, arena_extend_strategy}); // CUDA malloc/free is expensive so always use an arena @@ -105,19 +105,17 @@ CUDAExecutionProvider::PerThreadContext::~PerThreadContext() { void CUDAExecutionProvider::UpdateProviderOptionsInfo() { UnorderedMapStringToString options; - options["device_id"] = std::to_string(device_id_); - options["cuda_mem_limit"] = std::to_string(cuda_mem_limit_); + options["device_id"] = std::to_string(device_id_); + options["cuda_mem_limit"] = std::to_string(cuda_mem_limit_); std::string strategy; if (arena_extend_strategy_ == ArenaExtendStrategy::kNextPowerOfTwo) { strategy = "kNextPowerOfTwo"; - } - else if (arena_extend_strategy_ == ArenaExtendStrategy::kSameAsRequested) { + } else if (arena_extend_strategy_ == ArenaExtendStrategy::kSameAsRequested) { strategy = "kSameAsRequested"; + } else { + strategy = "unknown"; } - else { - strategy = "unknown"; - } - options["arena_extend_strategy"] = strategy; + options["arena_extend_strategy"] = strategy; IExecutionProvider::SetProviderOptions(options); } @@ -347,9 +345,6 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 9, double, MatMul); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 9, MLFloat16, MatMul); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 10, int8_t, MatMulInteger); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Tile); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Tile); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Tile); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Elu); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Elu); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Elu); @@ -581,9 +576,9 @@ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kO class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 5, Reshape); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, 4, Reshape_1); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, Shape); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, Tile); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, Tile); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, MLFloat16, Tile); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, Tile); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, Tile); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, Tile); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, Transpose); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, float, InstanceNormalization); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 6, double, InstanceNormalization); @@ -834,9 +829,6 @@ static Status RegisterCudaKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, @@ -1067,9 +1059,9 @@ static Status RegisterCudaKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/onnxruntime/core/providers/cuda/tensor/tile.cc b/onnxruntime/core/providers/cuda/tensor/tile.cc index 697c8293ea..746a6d80cb 100644 --- a/onnxruntime/core/providers/cuda/tensor/tile.cc +++ b/onnxruntime/core/providers/cuda/tensor/tile.cc @@ -8,21 +8,22 @@ using namespace onnxruntime::common; namespace onnxruntime { namespace cuda { -#define REGISTER_KERNEL_TYPED(T) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - Tile, \ - kOnnxDomain, \ - 6, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder() \ - .InputMemoryType(1) \ - .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ - .TypeConstraint("T1", DataTypeImpl::GetTensorType()), \ - Tile); +ONNX_OPERATOR_KERNEL_EX( + Tile, + kOnnxDomain, + 6, + kCudaExecutionProvider, + KernelDefBuilder() + .InputMemoryType(1) + .TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}) + .TypeConstraint("T1", DataTypeImpl::GetTensorType()), + Tile); -template -Status Tile::ComputeInternal(OpKernelContext* ctx) const { +Status Tile::ComputeInternal(OpKernelContext* ctx) const { auto& input_tensor = *ctx->Input(0); auto& repeats_tensor = *ctx->Input(1); int32_t rank = static_cast(input_tensor.Shape().NumDimensions()); @@ -41,8 +42,8 @@ Status Tile::ComputeInternal(OpKernelContext* ctx) const { TensorShape outputShape(output_dims); auto& output_tensor = *ctx->Output(0, outputShape); - T* output_data = output_tensor.template MutableData(); - const T* input_data = input_tensor.template Data(); + void* output_data = output_tensor.MutableDataRaw(); + const void* input_data = input_tensor.DataRaw(); TensorPitches input_pitches(input_shape); TArray input_strides(input_pitches); @@ -58,27 +59,47 @@ Status Tile::ComputeInternal(OpKernelContext* ctx) const { fdm_output_strides[i] = fast_divmod(static_cast(output_pitches[i])); } + static_assert(sizeof(float) == sizeof(int32_t), "Float and Int32 are of different sizes"); + static_assert(sizeof(double) == sizeof(int64_t), "Double and Int64 are of different sizes"); + if (output_tensor.Shape().Size() > 0) { - TileImpl( - rank, - fdm_input_shape, - input_strides, - reinterpret_cast::MappedType*>(input_data), - fdm_output_strides, - reinterpret_cast::MappedType*>(output_data), - output_tensor.Shape().Size()); + if (input_tensor.IsDataType() || + input_tensor.IsDataType()) { + TileImpl( + rank, + fdm_input_shape, + input_strides, + reinterpret_cast::MappedType*>(input_data), + fdm_output_strides, + reinterpret_cast::MappedType*>(output_data), + output_tensor.Shape().Size()); + } else if (input_tensor.IsDataType() || + input_tensor.IsDataType()) { + TileImpl( + rank, + fdm_input_shape, + input_strides, + reinterpret_cast::MappedType*>(input_data), + fdm_output_strides, + reinterpret_cast::MappedType*>(output_data), + output_tensor.Shape().Size()); + } else if (input_tensor.IsDataType()) { + TileImpl( + rank, + fdm_input_shape, + input_strides, + reinterpret_cast::MappedType*>(input_data), + fdm_output_strides, + reinterpret_cast::MappedType*>(output_data), + output_tensor.Shape().Size()); + } else { + // Won't hit this as the kernel doesn't claim support for any type that will trigger this + ORT_THROW("Tile doesn't have an implementation yet for the type: ", input_tensor.DataType()); + } } return Status::OK(); } -#define SPECIALIZED_COMPUTE(T) \ - REGISTER_KERNEL_TYPED(T) \ - template Status Tile::ComputeInternal(OpKernelContext* ctx) const; - -SPECIALIZED_COMPUTE(float) -SPECIALIZED_COMPUTE(double) -SPECIALIZED_COMPUTE(MLFloat16) - } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/tile.h b/onnxruntime/core/providers/cuda/tensor/tile.h index e4c215ddb2..97709b429e 100644 --- a/onnxruntime/core/providers/cuda/tensor/tile.h +++ b/onnxruntime/core/providers/cuda/tensor/tile.h @@ -7,7 +7,7 @@ namespace onnxruntime { namespace cuda { -template + struct Tile final : CudaKernel { Tile(const OpKernelInfo& info) : CudaKernel(info) { }