From a841d17d067bc09b8fc87076a4be3322a32c55b0 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Mon, 12 Apr 2021 14:02:08 -0700 Subject: [PATCH] More build options tested, converted the training ops over. --- cmake/onnxruntime_framework.cmake | 4 + cmake/onnxruntime_providers.cmake | 21 +- cmake/onnxruntime_python.cmake | 10 + cmake/onnxruntime_unittests.cmake | 5 +- .../core/framework/kernel_def_builder.h | 7 + include/onnxruntime/core/framework/ml_value.h | 2 + .../providers/cuda/cuda_provider_factory.h | 23 + onnxruntime/contrib_ops/cpu/bert/attention.cc | 8 - .../contrib_ops/cpu/bert/attention_base.h | 25 +- .../cpu/bert/longformer_attention_base.cc | 10 - .../cpu/bert/longformer_attention_base.h | 15 +- .../contrib_ops/cuda/bert/attention.cc | 1 - onnxruntime/contrib_ops/cuda/bert/attention.h | 1 - .../contrib_ops/cuda/bert/embed_layer_norm.cc | 3 - .../contrib_ops/cuda/bert/embed_layer_norm.h | 1 - .../contrib_ops/cuda/bert/fast_gelu.cc | 1 - .../cuda/bert/longformer_attention.cc | 1 - .../contrib_ops/cuda/bert/skip_layer_norm.cc | 3 - .../contrib_ops/cuda/cuda_contrib_kernels.cc | 161 +- .../contrib_ops/cuda/cuda_contrib_kernels.h | 1 - onnxruntime/contrib_ops/cuda/fused_conv.cc | 4 +- onnxruntime/contrib_ops/cuda/inverse.cc | 2 +- onnxruntime/contrib_ops/cuda/layer_norm.cc | 8 +- onnxruntime/contrib_ops/cuda/layer_norm.h | 1 - .../contrib_ops/cuda/math/bias_softmax.cc | 1 - .../cuda/math/bias_softmax_impl.cu | 1 - .../quantization/attention_quantization.cc | 2 - .../core/framework/feeds_fetches_manager.h | 2 + .../core/framework/ort_value_tensor_slicer.cc | 4 + .../core/framework/ort_value_tensor_slicer.h | 2 + .../core/framework/provider_bridge_ort.cc | 255 +++- onnxruntime/core/framework/tensorprotoutils.h | 21 +- onnxruntime/core/platform/env_var_utils.h | 4 + onnxruntime/core/providers/common.h | 2 + .../core/providers/cpu/controlflow/if.h | 3 - .../core/providers/cpu/controlflow/loop.cc | 7 + .../core/providers/cpu/controlflow/loop.h | 2 + .../core/providers/cpu/controlflow/scan.h | 4 +- .../core/providers/cpu/controlflow/utils.h | 1 - .../cpu/generator/constant_of_shape_base.h | 16 +- onnxruntime/core/providers/cpu/math/einsum.h | 4 +- .../math/einsum_utils/einsum_auxiliary_ops.cc | 19 +- .../math/einsum_utils/einsum_auxiliary_ops.h | 16 +- .../einsum_compute_preprocessor.h | 3 +- .../core/providers/cpu/nn/batch_norm_helper.h | 2 + .../core/providers/cpu/nn/conv_attributes.h | 2 +- .../cpu/nn/conv_transpose_attributes.h | 2 +- .../providers/cpu/nn/instance_norm_helper.h | 2 + .../core/providers/cpu/nn/pool_attributes.h | 4 +- onnxruntime/core/providers/cpu/nn/pool_base.h | 7 +- .../object_detection/non_max_suppression.h | 9 +- .../providers/cpu/reduction/reduction_ops.cc | 18 +- .../providers/cpu/reduction/reduction_ops.h | 6 +- .../providers/cpu/tensor/reshape_helper.h | 4 - onnxruntime/core/providers/cpu/tensor/tile.h | 2 + .../core/providers/cuda/controlflow/if.cc | 20 +- .../core/providers/cuda/controlflow/if.h | 6 +- .../core/providers/cuda/controlflow/loop.cc | 31 +- .../core/providers/cuda/controlflow/loop.h | 5 +- .../core/providers/cuda/controlflow/scan.cc | 25 +- .../core/providers/cuda/controlflow/scan.h | 6 +- .../core/providers/cuda/cuda_allocator.cc | 2 - onnxruntime/core/providers/cuda/cuda_call.cc | 3 - onnxruntime/core/providers/cuda/cuda_common.h | 1 - .../providers/cuda/cuda_execution_provider.cc | 6 - .../providers/cuda/cuda_execution_provider.h | 1 - onnxruntime/core/providers/cuda/cuda_fence.cc | 1 - onnxruntime/core/providers/cuda/cuda_fence.h | 3 - onnxruntime/core/providers/cuda/cuda_fwd.h | 2 - onnxruntime/core/providers/cuda/cuda_kernel.h | 3 - .../providers/cuda/cuda_provider_factory.cc | 144 +- .../core/providers/cuda/cudnn_common.h | 4 +- .../cuda/generator/constant_of_shape.cc | 19 +- .../cuda/generator/constant_of_shape.h | 2 - .../core/providers/cuda/generator/range.cc | 14 +- onnxruntime/core/providers/cuda/math/clip.cc | 24 +- .../core/providers/cuda/math/cumsum.cc | 12 +- .../core/providers/cuda/math/einsum.cc | 73 +- onnxruntime/core/providers/cuda/math/einsum.h | 5 +- .../math/einsum_utils/einsum_auxiliary_ops.cc | 4 +- .../math/einsum_utils/einsum_auxiliary_ops.h | 8 +- onnxruntime/core/providers/cuda/math/gemm.cc | 10 +- .../core/providers/cuda/math/matmul.cc | 8 +- .../providers/cuda/math/matmul_integer.cc | 8 +- .../core/providers/cuda/math/softmax.cc | 159 +- onnxruntime/core/providers/cuda/math/topk.cc | 8 +- .../core/providers/cuda/math/topk_impl.cu | 1 - .../cuda/math/variadic_elementwise_ops.cc | 18 +- .../core/providers/cuda/nn/batch_norm.cc | 6 +- .../core/providers/cuda/nn/batch_norm.h | 1 - onnxruntime/core/providers/cuda/nn/conv.cc | 38 +- onnxruntime/core/providers/cuda/nn/conv.h | 1 - .../core/providers/cuda/nn/conv_transpose.cc | 34 +- .../core/providers/cuda/nn/conv_transpose.h | 2 +- onnxruntime/core/providers/cuda/nn/dropout.cc | 23 +- .../core/providers/cuda/nn/instance_norm.cc | 4 +- onnxruntime/core/providers/cuda/nn/lrn.cc | 36 +- onnxruntime/core/providers/cuda/nn/pool.cc | 44 +- onnxruntime/core/providers/cuda/nn/pool.h | 5 +- onnxruntime/core/providers/cuda/nn/shrink.cc | 4 +- .../object_detection/non_max_suppression.cc | 18 +- .../non_max_suppression_impl.cu | 2 - .../non_max_suppression_impl.h | 3 - .../cuda/object_detection/roialign.cc | 9 +- .../cuda/object_detection/roialign_impl.h | 2 - .../cuda/reduction/reduction_functions.cc | 1 - .../providers/cuda/reduction/reduction_ops.cc | 184 ++- .../providers/cuda/reduction/reduction_ops.h | 1 - .../core/providers/cuda/rnn/cudnn_rnn_base.cc | 2 - onnxruntime/core/providers/cuda/rnn/gru.cc | 2 - onnxruntime/core/providers/cuda/rnn/lstm.cc | 2 - onnxruntime/core/providers/cuda/rnn/rnn.cc | 2 - .../providers/cuda/shared_inc/cuda_utils.h | 1 - .../core/providers/cuda/tensor/compress.cc | 2 - .../core/providers/cuda/tensor/expand.h | 1 - .../core/providers/cuda/tensor/expand_impl.h | 2 - .../core/providers/cuda/tensor/eye_like.cc | 1 - .../core/providers/cuda/tensor/eye_like.h | 1 - .../core/providers/cuda/tensor/flatten.cc | 1 - .../core/providers/cuda/tensor/flatten.h | 1 - .../core/providers/cuda/tensor/gather.cc | 1 - .../core/providers/cuda/tensor/gather.h | 1 - .../providers/cuda/tensor/gather_elements.cc | 1 - .../providers/cuda/tensor/gather_elements.h | 2 - .../core/providers/cuda/tensor/gather_nd.h | 1 - .../core/providers/cuda/tensor/identity_op.h | 1 - .../core/providers/cuda/tensor/nonzero_op.cc | 1 - .../core/providers/cuda/tensor/nonzero_op.h | 1 - .../core/providers/cuda/tensor/onehot.cc | 1 - .../core/providers/cuda/tensor/onehot.h | 1 - onnxruntime/core/providers/cuda/tensor/pad.h | 1 - .../providers/cuda/tensor/quantize_linear.cc | 2 - .../providers/cuda/tensor/quantize_linear.cuh | 1 - .../providers/cuda/tensor/quantize_linear.h | 1 - .../core/providers/cuda/tensor/reshape.h | 1 - .../providers/cuda/tensor/reverse_sequence.cc | 2 - .../providers/cuda/tensor/reverse_sequence.h | 1 - .../providers/cuda/tensor/scatter_elements.cc | 1 - .../providers/cuda/tensor/scatter_elements.h | 1 - .../core/providers/cuda/tensor/scatter_nd.h | 1 - .../core/providers/cuda/tensor/slice.h | 1 - .../core/providers/cuda/tensor/split.cc | 1 - .../core/providers/cuda/tensor/split.h | 1 - .../core/providers/cuda/tensor/squeeze.h | 1 - onnxruntime/core/providers/cuda/tensor/tile.h | 1 - .../core/providers/cuda/tensor/transpose.h | 1 - .../core/providers/cuda/tensor/unsqueeze.h | 1 - .../core/providers/cuda/tensor/upsample.h | 1 - .../core/providers/cuda/tensor/where.h | 1 - .../providers/op_kernel_type_control_utils.h | 2 + .../openvino/openvino_provider_factory.cc | 2 +- .../openvino/ov_versions/data_ops.cc | 1352 ++++++++--------- .../providers/openvino/ov_versions/data_ops.h | 51 +- .../providers/openvino/ov_versions/utils.cc | 4 +- .../providers/openvino/ov_versions/utils.h | 2 +- .../providers/shared_library/provider_api.h | 85 +- .../provider_bridge_provider.cc | 119 +- .../shared_library/provider_interfaces.h | 336 +++- .../tensorrt/tensorrt_execution_provider.cc | 9 +- .../framework/cuda/allocator_cuda_test.cc | 2 +- .../test/framework/cuda/fence_cuda_test.cc | 2 +- onnxruntime/test/framework/test_utils.cc | 30 - onnxruntime/test/framework/test_utils.h | 16 +- .../cuda/reduction_functions_test.cc | 2 +- .../test/providers/provider_test_utils.h | 1 + .../providers/tensorrt/tensorrt_basic_test.cc | 12 +- onnxruntime/test/util/default_providers.cc | 6 +- .../orttraining/core/session/tensor_helper.cc | 47 +- .../core/session/training_session.cc | 9 +- orttraining/orttraining/models/bert/main.cc | 26 +- orttraining/orttraining/models/gpt2/main.cc | 25 +- orttraining/orttraining/models/mnist/main.cc | 21 +- .../test/framework/slice_concatenate_test.cc | 14 +- .../test/graph/gradient_graph_builder_test.cc | 171 +-- .../test/model/training_runner_test.cc | 6 +- .../session/training_session_test_utils.cc | 4 +- .../cpu/reduction/reduction_ops_test.cc | 8 +- .../test/training_ops/cuda/cuda_utils_test.cc | 3 +- .../training_ops/cpu/loss/reduction_type.h | 2 + .../cuda/activation/activations_grad.cc | 2 +- .../cuda/activation/bias_gelu_grad.cc | 4 +- .../training_ops/cuda/controlflow/group.cc | 7 +- .../training_ops/cuda/controlflow/record.cc | 6 +- .../training_ops/cuda/controlflow/wait.cc | 6 +- .../cuda/cuda_training_kernels.cc | 2 +- .../training_ops/cuda/cuda_training_kernels.h | 5 +- .../loss/softmax_cross_entropy_loss_impl.cc | 19 +- .../loss/softmax_cross_entropy_loss_impl.cu | 3 +- .../cuda/loss/softmaxcrossentropy_impl.cc | 18 +- .../cuda/loss/softmaxcrossentropy_impl.cu | 3 +- .../cuda/loss/softmaxcrossentropy_impl.h | 1 - .../training_ops/cuda/math/div_grad.cc | 16 +- .../training_ops/cuda/math/isfinite.cc | 4 +- .../cuda/math/mixed_precision_scale.cc | 31 +- .../training_ops/cuda/math/scale.cc | 30 +- .../training_ops/cuda/math/softmax_grad.cc | 78 +- .../training_ops/cuda/nn/batch_norm_grad.cc | 16 +- .../training_ops/cuda/nn/dropout.cc | 37 +- .../training_ops/cuda/nn/layer_norm.cc | 22 +- .../training_ops/cuda/optimizer/adam.cc | 8 +- .../cuda/optimizer/gradient_control.cc | 6 +- .../training_ops/cuda/optimizer/lamb.cc | 8 +- .../training_ops/cuda/optimizer/sg.cc | 2 +- .../training_ops/cuda/reduction/all.cc | 16 +- .../cuda/reduction/reduction_all.cc | 21 +- .../cuda/reduction/reduction_ops.cc | 5 +- .../training_ops/cuda/tensor/concat.cc | 3 +- .../training_ops/cuda/tensor/concat.h | 3 +- .../cuda/tensor/gather_elements_grad.cc | 4 +- .../training_ops/cuda/tensor/gather_grad.cc | 13 +- .../cuda/tensor/gather_nd_grad.cc | 28 +- .../training_ops/cuda/tensor/slice_grad.cc | 12 +- .../training_ops/cuda/tensor/split.cc | 4 +- .../training_ops/cuda/tensor/view.cc | 4 +- 214 files changed, 2532 insertions(+), 2133 deletions(-) diff --git a/cmake/onnxruntime_framework.cmake b/cmake/onnxruntime_framework.cmake index f47b678720..b7193b894a 100644 --- a/cmake/onnxruntime_framework.cmake +++ b/cmake/onnxruntime_framework.cmake @@ -37,6 +37,10 @@ target_include_directories(onnxruntime_framework PRIVATE ${ONNXRUNTIME_ROOT} ${e else() target_include_directories(onnxruntime_framework PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) endif() +# Needed for the provider interface, as it includes training headers when training is enabled +if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS) + target_include_directories(onnxruntime_framework PRIVATE ${ORTTRAINING_ROOT}) +endif() onnxruntime_add_include_to_target(onnxruntime_framework onnxruntime_common onnx onnx_proto protobuf::libprotobuf flatbuffers) set_target_properties(onnxruntime_framework PROPERTIES FOLDER "ONNXRuntime") # need onnx to build to create headers that this project includes diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 5b39ab3956..67d84ecf77 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -218,6 +218,8 @@ if (onnxruntime_USE_CUDA) file(GLOB_RECURSE onnxruntime_providers_cuda_cc_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/core/providers/cuda/*.h" "${ONNXRUNTIME_ROOT}/core/providers/cuda/*.cc" + ) + file(GLOB_RECURSE onnxruntime_providers_cuda_shared_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.h" "${ONNXRUNTIME_ROOT}/core/providers/shared_library/*.cc" ) @@ -226,8 +228,8 @@ if (onnxruntime_USE_CUDA) "${ONNXRUNTIME_ROOT}/core/providers/cuda/*.cuh" ) - source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_cu_srcs}) - set(onnxruntime_providers_cuda_src ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_cu_srcs}) + source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_shared_srcs} ${onnxruntime_providers_cuda_cu_srcs}) + set(onnxruntime_providers_cuda_src ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_shared_srcs} ${onnxruntime_providers_cuda_cu_srcs}) # disable contrib ops conditionally if(NOT onnxruntime_DISABLE_CONTRIB_OPS) @@ -420,6 +422,11 @@ if (onnxruntime_USE_DNNL) target_compile_definitions(onnxruntime_providers_dnnl PRIVATE USE_DNNL_GPU_OCL=1) endif() + # Needed for the provider interface, as it includes training headers when training is enabled + if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS) + target_include_directories(onnxruntime_providers_dnnl PRIVATE ${ORTTRAINING_ROOT}) + endif() + if(APPLE) set_property(TARGET onnxruntime_providers_dnnl APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/core/providers/dnnl/exported_symbols.lst") target_link_libraries(onnxruntime_providers_dnnl PRIVATE nsync_cpp) @@ -507,6 +514,11 @@ if (onnxruntime_USE_TENSORRT) target_compile_options(onnxruntime_providers_tensorrt INTERFACE /wd4267 /wd4244 /wd4996 /wd4456) endif() + # Needed for the provider interface, as it includes training headers when training is enabled + if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS) + target_include_directories(onnxruntime_providers_tensorrt PRIVATE ${ORTTRAINING_ROOT}) + endif() + if(APPLE) set_property(TARGET onnxruntime_providers_tensorrt APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/core/providers/tensorrt/exported_symbols.lst") target_link_libraries(onnxruntime_providers_tensorrt PRIVATE nsync_cpp) @@ -615,6 +627,11 @@ if (onnxruntime_USE_OPENVINO) target_compile_options(onnxruntime_providers_openvino PUBLIC /wd4275 /wd4100 /wd4005 /wd4244 /wd4267) endif() + # Needed for the provider interface, as it includes training headers when training is enabled + if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS) + target_include_directories(onnxruntime_providers_openvino PRIVATE ${ORTTRAINING_ROOT}) + endif() + if(APPLE) set_property(TARGET onnxruntime_providers_openvino APPEND_STRING PROPERTY LINK_FLAGS "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/core/providers/openvino/exported_symbols.lst") elseif(UNIX) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index ce333c3cab..ca1ee65171 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -404,6 +404,16 @@ if (onnxruntime_USE_OPENVINO) ) endif() +if (onnxruntime_USE_CUDA) + add_custom_command( + TARGET onnxruntime_pybind11_state POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + $ + $ + $/onnxruntime/capi/ + ) +endif() + if (onnxruntime_USE_TVM) add_custom_command( TARGET onnxruntime_pybind11_state POST_BUILD diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index c6d2a100cc..6f9da04c19 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -382,7 +382,7 @@ endif() set (onnxruntime_test_providers_dependencies ${onnxruntime_EXTERNAL_DEPENDENCIES}) if(onnxruntime_USE_CUDA) - list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_cuda) + list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_cuda onnxruntime_providers_shared) endif() if(onnxruntime_USE_DNNL) @@ -451,8 +451,7 @@ set(ONNXRUNTIME_TEST_LIBS onnxruntime_session ${ONNXRUNTIME_INTEROP_TEST_LIBS} ${onnxruntime_libs} - ${PROVIDERS_CUDA} - # TENSORRT, DNNL, and OpenVINO are explicitly linked at runtime + # CUDA, TENSORRT, DNNL, and OpenVINO are explicitly linked at runtime ${PROVIDERS_MIGRAPHX} ${PROVIDERS_NUPHAR} ${PROVIDERS_NNAPI} diff --git a/include/onnxruntime/core/framework/kernel_def_builder.h b/include/onnxruntime/core/framework/kernel_def_builder.h index 965d553eb9..873be7f445 100644 --- a/include/onnxruntime/core/framework/kernel_def_builder.h +++ b/include/onnxruntime/core/framework/kernel_def_builder.h @@ -285,6 +285,13 @@ class KernelDefBuilder { return *this; } + KernelDefBuilder& InputMemoryType(OrtMemType type, const std::vector& input_indexes) { + for (auto input_index : input_indexes) { + kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, type)); + } + return *this; + } + /** Specify that this kernel provides an output arg in certain memory type (instead of the default, device memory). diff --git a/include/onnxruntime/core/framework/ml_value.h b/include/onnxruntime/core/framework/ml_value.h index 27ffa58875..f7e1c924db 100644 --- a/include/onnxruntime/core/framework/ml_value.h +++ b/include/onnxruntime/core/framework/ml_value.h @@ -4,11 +4,13 @@ #pragma once #include +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/common/exceptions.h" #include "core/framework/allocator.h" #include "core/framework/data_types.h" #include "core/framework/tensor.h" +#endif namespace onnxruntime { class SparseTensor; diff --git a/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h b/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h index 81b5477b3c..35a179b0f1 100644 --- a/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h +++ b/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h @@ -4,6 +4,29 @@ #include "onnxruntime_c_api.h" #ifdef __cplusplus +namespace onnxruntime { +class IAllocator; +class IDataTransfer; +} // namespace onnxruntime + +struct ProviderInfo_CUDA { + virtual OrtStatus* SetCurrentGpuDeviceId(_In_ int device_id) = 0; + virtual OrtStatus* GetCurrentGpuDeviceId(_In_ int* device_id) = 0; + + virtual std::unique_ptr CreateCUDAAllocator(int16_t device_id, const char* name) = 0; + virtual std::unique_ptr CreateCUDAPinnedAllocator(int16_t device_id, const char* name) = 0; + virtual std::unique_ptr CreateGPUDataTransfer(void* stream) = 0; + + virtual void cuda__Impl_Cast(void* stream, const int64_t* input_data, int32_t* output_data, size_t count) = 0; + virtual void cuda__Impl_Cast(void* stream, const int32_t* input_data, int64_t* output_data, size_t count) = 0; + + virtual bool CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; + virtual bool CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; + + virtual void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const OrtMemoryInfo& dst_location, const OrtMemoryInfo& src_location) = 0; + virtual void cudaMemcpy_HostToDevice(void* dst, const void* src, size_t count) = 0; +}; + extern "C" { #endif diff --git a/onnxruntime/contrib_ops/cpu/bert/attention.cc b/onnxruntime/contrib_ops/cpu/bert/attention.cc index 1d3a2c2bc6..898eaf2b41 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention.cc +++ b/onnxruntime/contrib_ops/cpu/bert/attention.cc @@ -40,14 +40,6 @@ ONNX_OPERATOR_TYPED_KERNEL_EX( .TypeConstraint("T", DataTypeImpl::GetTensorType()), Attention); -AttentionBase::AttentionBase(const OpKernelInfo& info) { - int64_t num_heads = 0; - ORT_ENFORCE(info.GetAttr("num_heads", &num_heads).IsOK() && num_heads > 0); - num_heads_ = static_cast(num_heads); - - is_unidirectional_ = info.GetAttrOrDefault("unidirectional", 0) == 1; -} - Status AttentionBase::CheckInputs(const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_base.h b/onnxruntime/contrib_ops/cpu/bert/attention_base.h index 3421f14999..f14268af3e 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention_base.h +++ b/onnxruntime/contrib_ops/cpu/bert/attention_base.h @@ -10,15 +10,7 @@ namespace onnxruntime { namespace contrib { class AttentionBase { - protected: - AttentionBase(const OpKernelInfo& info); - - Status CheckInputs(const TensorShape& input_shape, - const TensorShape& weights_shape, - const TensorShape& bias_shape, - const Tensor*& mask_index, // For dummy mask with shape (1, 1) or (batch_size, 1), it will be updated to nullptr. - const Tensor* past) const; - + public: // This check function is specifically used in cuda Status CheckInputs(const TensorShape& input_shape, const TensorShape& weights_shape, @@ -34,6 +26,21 @@ class AttentionBase { int sequence_length, int& past_sequence_length) const; + protected: + AttentionBase(const OpKernelInfo& info) { + int64_t num_heads = 0; + ORT_ENFORCE(info.GetAttr("num_heads", &num_heads).IsOK() && num_heads > 0); + num_heads_ = static_cast(num_heads); + + is_unidirectional_ = info.GetAttrOrDefault("unidirectional", 0) == 1; + } + + Status CheckInputs(const TensorShape& input_shape, + const TensorShape& weights_shape, + const TensorShape& bias_shape, + const Tensor*& mask_index, // For dummy mask with shape (1, 1) or (batch_size, 1), it will be updated to nullptr. + const Tensor* past) const; + int num_heads_; // number of attention heads bool is_unidirectional_; // whether every token can only attend to previous tokens. }; diff --git a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc index 385653991f..6a6685ac3f 100644 --- a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc +++ b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc @@ -6,16 +6,6 @@ namespace onnxruntime { namespace contrib { -LongformerAttentionBase::LongformerAttentionBase(const OpKernelInfo& info) { - int64_t num_heads = 0; - ORT_ENFORCE(info.GetAttr("num_heads", &num_heads).IsOK() && num_heads > 0); - num_heads_ = static_cast(num_heads); - - int64_t window = 0; - ORT_ENFORCE(info.GetAttr("window", &window).IsOK() && window > 0); - window_ = static_cast(window); -} - Status LongformerAttentionBase::CheckInputs(const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, diff --git a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h index 00cc451073..1f974db5cd 100644 --- a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h +++ b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h @@ -10,9 +10,7 @@ namespace onnxruntime { namespace contrib { class LongformerAttentionBase { - protected: - LongformerAttentionBase(const OpKernelInfo& info); - + public: Status CheckInputs(const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, @@ -21,6 +19,17 @@ class LongformerAttentionBase { const TensorShape& global_bias_shape, const TensorShape& global_shape) const; + protected: + LongformerAttentionBase(const OpKernelInfo& info) { + int64_t num_heads = 0; + ORT_ENFORCE(info.GetAttr("num_heads", &num_heads).IsOK() && num_heads > 0); + num_heads_ = static_cast(num_heads); + + int64_t window = 0; + ORT_ENFORCE(info.GetAttr("window", &window).IsOK() && window > 0); + window_ = static_cast(window); + } + int num_heads_; // Number of attention heads int window_; // Attention windows length (W). It is half (one-sided) of total window size. }; diff --git a/onnxruntime/contrib_ops/cuda/bert/attention.cc b/onnxruntime/contrib_ops/cuda/bert/attention.cc index 01d97fc4a5..8cc92094c4 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention.cc +++ b/onnxruntime/contrib_ops/cuda/bert/attention.cc @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -//#include "core/framework/tensorprotoutils.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/shared_inc/fpgeneric.h" #include "attention.h" diff --git a/onnxruntime/contrib_ops/cuda/bert/attention.h b/onnxruntime/contrib_ops/cuda/bert/attention.h index e9dd6f116b..2aec22cc87 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention.h +++ b/onnxruntime/contrib_ops/cuda/bert/attention.h @@ -3,7 +3,6 @@ #pragma once -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "contrib_ops/cpu/bert/attention_base.h" diff --git a/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.cc b/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.cc index 21cea0421b..0fd880b87b 100644 --- a/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.cc +++ b/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.cc @@ -1,9 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -//#include "core/providers/common.h" -//#include "core/framework/tensorprotoutils.h" -//#include "onnx/defs/tensor_proto_util.h" #include "core/providers/cuda/cuda_common.h" #include "contrib_ops/cpu/bert/embed_layer_norm_helper.h" #include "embed_layer_norm.h" diff --git a/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.h b/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.h index 633d651504..8749dc925e 100644 --- a/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.h +++ b/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #pragma once -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc index eaeae10ec0..a45b5509b5 100644 --- a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc +++ b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc @@ -3,7 +3,6 @@ #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/cudnn_common.h" -//#include "core/framework/tensorprotoutils.h" #include "fast_gelu.h" #include "fast_gelu_impl.h" #include "contrib_ops/cpu/bert/bias_gelu_helper.h" diff --git a/onnxruntime/contrib_ops/cuda/bert/longformer_attention.cc b/onnxruntime/contrib_ops/cuda/bert/longformer_attention.cc index 9187c14c8f..c21adcfc72 100644 --- a/onnxruntime/contrib_ops/cuda/bert/longformer_attention.cc +++ b/onnxruntime/contrib_ops/cuda/bert/longformer_attention.cc @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -//#include "core/framework/tensorprotoutils.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/shared_inc/fpgeneric.h" #include "core/platform/env_var_utils.h" diff --git a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc index dbe95e6c89..dd975ca90f 100644 --- a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc +++ b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm.cc @@ -1,10 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -//#include "core/providers/common.h" #include "core/providers/cuda/cuda_common.h" -//#include "core/framework/tensorprotoutils.h" -//#include "onnx/defs/tensor_proto_util.h" #include "skip_layer_norm.h" #include "skip_layer_norm_impl.h" diff --git a/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc b/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc index cf7fdf5b14..e6fe461533 100644 --- a/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc @@ -3,7 +3,6 @@ #include "core/providers/shared_library/provider_api.h" #include "core/providers/cuda/cuda_common.h" -//#include "core/framework/kernel_registry.h" using namespace onnxruntime::common; @@ -18,9 +17,9 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1 class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, BiasGelu); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, BiasGelu); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, BiasGelu); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, TransposeMatMul); // backward compatibility -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, TransposeMatMul); // backward compatibility -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, TransposeMatMul); // backward compatibility +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, TransposeMatMul); // backward compatibility +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, TransposeMatMul); // backward compatibility +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, TransposeMatMul); // backward compatibility class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, FusedMatMul); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, FusedMatMul); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, FusedMatMul); @@ -84,7 +83,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1 #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, FastGelu); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, TransposeMatMul); // backward compatibility +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, TransposeMatMul); // backward compatibility class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, FusedMatMul); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, BFloat16_float, LayerNormalization); #endif @@ -97,87 +96,87 @@ KernelCreateInfo BuildKernelCreateInfo() { Status RegisterCudaContribKernels(KernelRegistry& kernel_registry) { static const BuildKernelCreateInfoFn function_table[] = { - BuildKernelCreateInfo, //default entry to avoid the list become empty after ops-reducing - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, // backward compatibility - BuildKernelCreateInfo, // backward compatibility - BuildKernelCreateInfo, // backward compatibility - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, //default entry to avoid the list become empty after ops-reducing + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, // backward compatibility + BuildKernelCreateInfo, // backward compatibility + BuildKernelCreateInfo, // backward compatibility + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, - // These ops were experimental ops in onnx domain which have been removed now. We add them here as - // contrib ops to maintain backward compatibility - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, + // These ops were experimental ops in onnx domain which have been removed now. We add them here as + // contrib ops to maintain backward compatibility + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 - BuildKernelCreateInfo, - // TransposedMatMul is still here for backward compatibility - BuildKernelCreateInfo, // backward compatibility - BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, + // TransposedMatMul is still here for backward compatibility + BuildKernelCreateInfo, // backward compatibility + BuildKernelCreateInfo, + BuildKernelCreateInfo, #endif - BuildKernelCreateInfo, + BuildKernelCreateInfo, }; for (auto& function_table_entry : function_table) { diff --git a/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.h b/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.h index dad8aa9b7a..d047e8fc78 100644 --- a/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.h +++ b/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #pragma once -//#include "core/framework/kernel_registry.h" namespace onnxruntime { namespace contrib { diff --git a/onnxruntime/contrib_ops/cuda/fused_conv.cc b/onnxruntime/contrib_ops/cuda/fused_conv.cc index a45e784d8d..02fd74061a 100644 --- a/onnxruntime/contrib_ops/cuda/fused_conv.cc +++ b/onnxruntime/contrib_ops/cuda/fused_conv.cc @@ -119,9 +119,9 @@ ONNX_OPERATOR_TYPED_KERNEL_EX( 1, float, kCudaExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), FusedConv); } // namespace cuda } // namespace contrib -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/inverse.cc b/onnxruntime/contrib_ops/cuda/inverse.cc index f75ae8a1e6..4007fb8faf 100644 --- a/onnxruntime/contrib_ops/cuda/inverse.cc +++ b/onnxruntime/contrib_ops/cuda/inverse.cc @@ -28,7 +28,7 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("T", BuildKernelDefConstraints()), Inverse); diff --git a/onnxruntime/contrib_ops/cuda/layer_norm.cc b/onnxruntime/contrib_ops/cuda/layer_norm.cc index bbd292ffa9..c9c4e8ff2c 100644 --- a/onnxruntime/contrib_ops/cuda/layer_norm.cc +++ b/onnxruntime/contrib_ops/cuda/layer_norm.cc @@ -3,8 +3,6 @@ #include "layer_norm.h" #include "layer_norm_impl.h" - -#include "core/providers/common.h" #include "core/providers/cuda/cuda_common.h" namespace onnxruntime { @@ -18,7 +16,7 @@ namespace cuda { 1, \ T##_##U, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("U", DataTypeImpl::GetTensorType()), \ LayerNorm); \ @@ -28,7 +26,7 @@ namespace cuda { 1, \ T##_##U, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("U", DataTypeImpl::GetTensorType()), \ LayerNorm); @@ -59,7 +57,7 @@ Status LayerNorm::ComputeInternal(OpKernelContext* ctx) const auto X_data = reinterpret_cast(X->template Data()); auto scale_data = reinterpret_cast(scale->template Data()); - auto bias_data = (simplified || (nullptr == bias)) ? nullptr: reinterpret_cast(bias->template Data()); + auto bias_data = (simplified || (nullptr == bias)) ? nullptr : reinterpret_cast(bias->template Data()); const TensorShape& x_shape = X->Shape(); const int64_t axis = HandleNegativeAxis(axis_, x_shape.NumDimensions()); diff --git a/onnxruntime/contrib_ops/cuda/layer_norm.h b/onnxruntime/contrib_ops/cuda/layer_norm.h index 4f4d95fc99..5d6dd6ff1f 100644 --- a/onnxruntime/contrib_ops/cuda/layer_norm.h +++ b/onnxruntime/contrib_ops/cuda/layer_norm.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #pragma once -#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc b/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc index cbf364a12f..05bbd57d9f 100644 --- a/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc +++ b/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc @@ -3,7 +3,6 @@ #include "core/providers/cuda/cuda_common.h" #include "contrib_ops/cuda/math/bias_softmax.h" -//#include "core/providers/common.h" using namespace onnxruntime; using namespace onnxruntime::cuda; diff --git a/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu b/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu index 104934af18..a3e0189a2b 100644 --- a/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu +++ b/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu @@ -6,7 +6,6 @@ #include #include -//#include "core/providers/common.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/cudnn_common.h" #include "core/providers/cuda/cu_inc/binary_elementwise_impl.cuh" diff --git a/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc b/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc index 89f131f076..383f69ebd1 100644 --- a/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc +++ b/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc @@ -4,8 +4,6 @@ #include "attention_quantization.h" #include "attention_quantization_impl.cuh" #include "contrib_ops/cuda/bert/attention_impl.h" -//#include "core/framework/tensorprotoutils.h" -//#include "core/providers/common.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/shared_inc/fpgeneric.h" #include "core/providers/cuda/shared_inc/integer_gemm.h" diff --git a/onnxruntime/core/framework/feeds_fetches_manager.h b/onnxruntime/core/framework/feeds_fetches_manager.h index 2110b8b5a5..889e3594c5 100644 --- a/onnxruntime/core/framework/feeds_fetches_manager.h +++ b/onnxruntime/core/framework/feeds_fetches_manager.h @@ -6,7 +6,9 @@ #include #include +#ifndef SHARED_PROVIDER #include "core/framework/ml_value.h" +#endif namespace onnxruntime { class ExecutionProviders; diff --git a/onnxruntime/core/framework/ort_value_tensor_slicer.cc b/onnxruntime/core/framework/ort_value_tensor_slicer.cc index 661b3d1109..21d7d93f02 100644 --- a/onnxruntime/core/framework/ort_value_tensor_slicer.cc +++ b/onnxruntime/core/framework/ort_value_tensor_slicer.cc @@ -80,8 +80,12 @@ void OrtValueTensorSlicer::Iterator::MaterializeMLValue() const { // // TODO: Ideally we could avoid the overhead of creating a new Tensor (mainly cost of copying type and shape info) // and would simply update Tensor::p_data_ given all other info remains constant for each slice. +#ifndef SHARED_PROVIDER auto sub_tensor = onnxruntime::make_unique(tensor_data_type_, per_iteration_shape_, const_cast(tensor_slice_data_raw), *tensor_location_); +#else + auto sub_tensor = Tensor::Create(tensor_data_type_, per_iteration_shape_, const_cast(tensor_slice_data_raw), *tensor_location_); +#endif auto ml_tensor = DataTypeImpl::GetType(); current_ = OrtValue{sub_tensor.release(), ml_tensor, ml_tensor->GetDeleteFunc()}; } diff --git a/onnxruntime/core/framework/ort_value_tensor_slicer.h b/onnxruntime/core/framework/ort_value_tensor_slicer.h index a7a9c90462..3069e79e9a 100644 --- a/onnxruntime/core/framework/ort_value_tensor_slicer.h +++ b/onnxruntime/core/framework/ort_value_tensor_slicer.h @@ -7,9 +7,11 @@ #include #include +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/framework/ml_value.h" #include "core/framework/tensor.h" +#endif namespace onnxruntime { diff --git a/onnxruntime/core/framework/provider_bridge_ort.cc b/onnxruntime/core/framework/provider_bridge_ort.cc index 9ac4589881..be9f3fbd60 100644 --- a/onnxruntime/core/framework/provider_bridge_ort.cc +++ b/onnxruntime/core/framework/provider_bridge_ort.cc @@ -19,12 +19,14 @@ void Foo() {} #include "core/session/abi_session_options_impl.h" #include "core/session/ort_apis.h" #include "core/util/math.h" +#include "core/framework/tensorprotoutils.h" +#include "core/framework/fallback_cpu_capability.h" +#include "core/framework/random_generator.h" #include "core/providers/cpu/tensor/gatherbase.h" #include "core/providers/cpu/tensor/gather_elements.h" #include "core/providers/cpu/tensor/unsqueeze.h" #include "core/providers/cpu/tensor/slice.h" -#include "core/framework/fallback_cpu_capability.h" #include "core/providers/cpu/tensor/split.h" #include "core/providers/cpu/tensor/size.h" #include "core/providers/cpu/tensor/scatter_nd.h" @@ -32,12 +34,27 @@ void Foo() {} #include "core/providers/cpu/tensor/concatbase.h" #include "core/providers/cpu/tensor/gatherbase.h" #include "core/providers/cpu/tensor/onehot.h" +#include "core/providers/cpu/tensor/tile.h" +#include "core/providers/cpu/controlflow/if.h" +#include "core/providers/cpu/controlflow/loop.h" +#include "core/providers/cpu/controlflow/scan.h" +#include "core/providers/cpu/math/einsum.h" +#include "core/providers/cpu/math/cumsum.h" +#include "core/providers/cpu/object_detection/non_max_suppression.h" -#ifdef USE_TENSORRT -#include "core/providers/cuda/cuda_allocator.h" -#include "core/providers/cuda/gpu_data_transfer.h" -#include "core/providers/cuda/math/unary_elementwise_ops_impl.h" -#include "core/providers/cuda/cuda_common.h" +#ifndef DISABLE_CONTRIB_OPS +#include "contrib_ops/cpu/bert/bias_gelu_helper.h" +#include "contrib_ops/cpu/bert/embed_layer_norm_helper.h" +#include "contrib_ops/cpu/bert/longformer_attention_base.h" +#include "contrib_ops/cpu/bert/attention_base.h" +#endif + +#ifdef ENABLE_TRAINING +#include "orttraining/training_ops/cpu/controlflow/group.h" +#include "orttraining/training_ops/cpu/controlflow/record.h" +#include "orttraining/training_ops/cpu/controlflow/wait.h" +#include "orttraining/training_ops/cpu/loss/softmax_cross_entropy_loss.h" +#include "orttraining/training_ops/cpu/tensor/split.h" #endif namespace ONNX_NAMESPACE { @@ -82,6 +99,8 @@ namespace onnxruntime { ProviderHost* g_host{}; +ProviderInfo_CUDA* GetProviderInfo_CUDA(); + struct TensorShapeProto_Dimension_Iterator_Impl : TensorShapeProto_Dimension_Iterator { TensorShapeProto_Dimension_Iterator_Impl(google::protobuf::internal::RepeatedPtrIterator&& v) : v_{std::move(v)} {} @@ -100,8 +119,9 @@ struct NodeAttributes_Iterator_Impl : NodeAttributes_Iterator { void operator++() override { v_.operator++(); } const std::pair& operator*() const override { return v_.operator*(); } - // const std::string& first() const override { return v_->first; } - // const Provider_AttributeProto& second() const override { return v_->second; } + + const std::string& first() const override { return v_->first; } + const ONNX_NAMESPACE::AttributeProto& second() const override { return v_->second; } NodeAttributes::const_iterator v_; }; @@ -145,6 +165,7 @@ struct ProviderHostImpl : ProviderHost { DataTypeImpl__GetTensorType_uint64 = &DataTypeImpl::GetTensorType; DataTypeImpl__GetTensorType_float = &DataTypeImpl::GetTensorType; DataTypeImpl__GetTensorType_double = &DataTypeImpl::GetTensorType; + DataTypeImpl__GetTensorType_BFloat16 = &DataTypeImpl::GetTensorType; DataTypeImpl__GetTensorType_MLFloat16 = &DataTypeImpl::GetTensorType; } @@ -156,29 +177,19 @@ struct ProviderHostImpl : ProviderHost { return onnxruntime::make_unique(memory_info); }; -#ifdef USE_TENSORRT - std::unique_ptr CreateCUDAAllocator(int16_t device_id, const char* name) override { - return onnxruntime::make_unique(device_id, name); - } + void* CPUAllocator__Alloc(CPUAllocator* p, size_t size) override { return p->Alloc(size); } + void CPUAllocator__Free(CPUAllocator* p, void* allocation) override { return p->Free(allocation); } - std::unique_ptr CreateCUDAPinnedAllocator(int16_t device_id, const char* name) override { - return onnxruntime::make_unique(device_id, name); - } +#ifdef USE_CUDA + std::unique_ptr CreateCUDAAllocator(int16_t device_id, const char* name) override { return GetProviderInfo_CUDA()->CreateCUDAAllocator(device_id, name); } + std::unique_ptr CreateCUDAPinnedAllocator(int16_t device_id, const char* name) override { return GetProviderInfo_CUDA()->CreateCUDAPinnedAllocator(device_id, name); } + std::unique_ptr CreateGPUDataTransfer(void* stream) override { return GetProviderInfo_CUDA()->CreateGPUDataTransfer(stream); } - std::unique_ptr CreateGPUDataTransfer(void* stream) override { - return onnxruntime::make_unique(static_cast(stream)); - } + void cuda__Impl_Cast(void* stream, const int64_t* input_data, int32_t* output_data, size_t count) override { return GetProviderInfo_CUDA()->cuda__Impl_Cast(stream, input_data, output_data, count); } + void cuda__Impl_Cast(void* stream, const int32_t* input_data, int64_t* output_data, size_t count) override { return GetProviderInfo_CUDA()->cuda__Impl_Cast(stream, input_data, output_data, count); } - void cuda__Impl_Cast(void* stream, const int64_t* input_data, int32_t* output_data, size_t count) override { - return cuda::Impl_Cast(static_cast(stream), input_data, output_data, count); - } - - void cuda__Impl_Cast(void* stream, const int32_t* input_data, int64_t* output_data, size_t count) override { - return cuda::Impl_Cast(static_cast(stream), input_data, output_data, count); - } - - bool CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg); } - bool CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg); } + bool CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return GetProviderInfo_CUDA()->CudaCall_false(retCode, exprString, libName, successCode, msg); } + bool CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return GetProviderInfo_CUDA()->CudaCall_true(retCode, exprString, libName, successCode, msg); } #endif std::string GetEnvironmentVar(const std::string& var_name) override { @@ -189,6 +200,10 @@ struct ProviderHostImpl : ProviderHost { return const_cast(&logging::LoggingManager::DefaultLogger()); } + OrtStatus* CreateStatus(OrtErrorCode code, _In_ const char* msg) noexcept override { + return OrtApis::CreateStatus(code, msg); + } + std::unordered_set GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph, const std::string& provider_type, const std::vector& kernel_registries, @@ -199,21 +214,35 @@ struct ProviderHostImpl : ProviderHost { int32_t PrimitiveDataTypeBase__GetDataType(const PrimitiveDataTypeBase* p) override { return p->GetDataType(); } const char* DataTypeImpl__ToString(MLDataType type) override { return DataTypeImpl::ToString(type); } + bool DataTypeImpl__IsTensorType(const DataTypeImpl* p) override { return p->IsTensorType(); } + bool DataTypeImpl__IsTensorSequenceType(const DataTypeImpl* p) override { return p->IsTensorSequenceType(); } + bool DataTypeImpl__IsSparseTensorType(const DataTypeImpl* p) override { return p->IsSparseTensorType(); } + DeleteFunc DataTypeImpl__GetDeleteFunc(const DataTypeImpl* p) override { return p->GetDeleteFunc(); } const std::vector& DataTypeImpl__AllFixedSizeTensorTypes() override { return DataTypeImpl::AllFixedSizeTensorTypes(); } const std::vector& DataTypeImpl__AllTensorTypes() override { return DataTypeImpl::AllTensorTypes(); } const std::vector& DataTypeImpl__AllIEEEFloatTensorTypes() override { return DataTypeImpl::AllIEEEFloatTensorTypes(); } size_t DataTypeImpl__Size(const DataTypeImpl* p) override { return p->Size(); } const PrimitiveDataTypeBase* DataTypeImpl__AsPrimitiveDataType(const DataTypeImpl* p) override { return p->AsPrimitiveDataType(); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ bool* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ float* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ double* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ MLFloat16* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int8_t* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint8_t* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int16_t* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint16_t* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int32_t* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint32_t* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int64_t* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint64_t* p_data, size_t expected_size) override { return utils::UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + void* HeapAllocate(size_t size) override { return new uint8_t[size]; } void HeapFree(void* p) override { delete[] reinterpret_cast(p); } std::vector GetStackTrace() override { return onnxruntime::GetStackTrace(); } uint16_t math__floatToHalf(float f) override { return math::floatToHalf(f); } - - AutoPadType StringToAutoPadType(const std::string& str) override { return onnxruntime::StringToAutoPadType(str); } - - int64_t HandleNegativeAxis(int64_t axis, int64_t tensor_rank) override { return onnxruntime::HandleNegativeAxis(axis, tensor_rank); } + float math__halfToFloat(uint16_t h) override { return math::halfToFloat(h); } void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file, const char* function, uint32_t line) override { return ::onnxruntime::LogRuntimeError(session_id, status, file, function, line); @@ -284,6 +313,7 @@ struct ProviderHostImpl : ProviderHost { const int64_t& int64s__Get(const ONNX_NAMESPACE::int64s* p, int index) override { return p->Get(index); } // TypeProto_Tensor + bool TypeProto_Tensor__has_shape(const ONNX_NAMESPACE::TypeProto_Tensor* p) override { return p->has_shape(); } const ONNX_NAMESPACE::TensorShapeProto& TypeProto_Tensor__shape(const ONNX_NAMESPACE::TypeProto_Tensor* p) override { return p->shape(); } ONNX_NAMESPACE::TensorShapeProto* TypeProto_Tensor__mutable_shape(ONNX_NAMESPACE::TypeProto_Tensor* p) override { return p->mutable_shape(); } int32_t TypeProto_Tensor__elem_type(const ONNX_NAMESPACE::TypeProto_Tensor* p) override { return p->elem_type(); } @@ -291,6 +321,7 @@ struct ProviderHostImpl : ProviderHost { // TypeProto const ONNX_NAMESPACE::TypeProto_Tensor& TypeProto__tensor_type(const ONNX_NAMESPACE::TypeProto* p) override { return p->tensor_type(); } ONNX_NAMESPACE::TypeProto_Tensor* TypeProto__mutable_tensor_type(ONNX_NAMESPACE::TypeProto* p) override { return p->mutable_tensor_type(); } + int TypeProto__value_case(const ONNX_NAMESPACE::TypeProto* p) override { return p->value_case(); } // AttributeProto std::unique_ptr AttributeProto__construct() override { return onnxruntime::make_unique(); } @@ -346,10 +377,19 @@ struct ProviderHostImpl : ProviderHost { void ModelProto__set_ir_version(ONNX_NAMESPACE::ModelProto* p, int64_t value) override { p->set_ir_version(value); } // TensorProto + std::unique_ptr TensorProto__construct() override { return onnxruntime::make_unique(); } void TensorProto__operator_delete(ONNX_NAMESPACE::TensorProto* p) override { delete p; } void TensorProto__operator_assign(ONNX_NAMESPACE::TensorProto* p, const ONNX_NAMESPACE::TensorProto& v) override { *p = v; } + bool TensorProto__has_name(const ONNX_NAMESPACE::TensorProto* p) override { return p->has_name(); } + int TensorProto__dims_size(const ONNX_NAMESPACE::TensorProto* p) override { return p->dims_size(); } + const ONNX_NAMESPACE::int64s& TensorProto__dims(const ONNX_NAMESPACE::TensorProto* p) override { return p->dims(); } bool TensorProto__has_data_location(const ONNX_NAMESPACE::TensorProto* p) override { return p->has_data_location(); } int TensorProto__data_location(const ONNX_NAMESPACE::TensorProto* p) override { return p->data_location(); } + bool TensorProto__has_raw_data(const ONNX_NAMESPACE::TensorProto* p) override { return p->has_raw_data(); } + const std::string& TensorProto__raw_data(const ONNX_NAMESPACE::TensorProto* p) override { return p->raw_data(); } + int32_t TensorProto__data_type(const ONNX_NAMESPACE::TensorProto* p) override { return p->data_type(); } + + bool TensorProto_DataType_IsValid(int value) override { return ONNX_NAMESPACE::TensorProto::DataType_IsValid(value); } // TensorProtos ONNX_NAMESPACE::TensorProto* TensorProtos__Add(ONNX_NAMESPACE::TensorProtos* p) override { return p->Add(); } @@ -444,10 +484,13 @@ struct ProviderHostImpl : ProviderHost { void KernelDefBuilder__TypeConstraint(KernelDefBuilder* p, const char* arg_name, MLDataType supported_type) override { p->TypeConstraint(arg_name, supported_type); } void KernelDefBuilder__TypeConstraint(KernelDefBuilder* p, const char* arg_name, const std::vector& supported_types) override { p->TypeConstraint(arg_name, supported_types); } void KernelDefBuilder__InputMemoryType(KernelDefBuilder* p, OrtMemType type, int input_index) override { p->InputMemoryType(type, input_index); } + void KernelDefBuilder__InputMemoryType(KernelDefBuilder* p, OrtMemType type, const std::vector& input_indexes) override { p->InputMemoryType(type, input_indexes); } void KernelDefBuilder__OutputMemoryType(KernelDefBuilder* p, OrtMemType type, int input_index) override { p->OutputMemoryType(type, input_index); } void KernelDefBuilder__ExecQueueId(KernelDefBuilder* p, int queue_id) override { p->ExecQueueId(queue_id); } void KernelDefBuilder__MayInplace(KernelDefBuilder* p, int input_index, int output_index) override { p->MayInplace(input_index, output_index); } void KernelDefBuilder__Alias(KernelDefBuilder* p, int input_index, int output_index) override { p->Alias(input_index, output_index); } + void KernelDefBuilder__Alias(KernelDefBuilder* p, const std::vector>& aliases) override { p->Alias(aliases); } + void KernelDefBuilder__VariadicAlias(KernelDefBuilder* p, int input_offset, int output_offset) override { p->VariadicAlias(input_offset, output_offset); } std::unique_ptr KernelDefBuilder__Build(KernelDefBuilder* p) override { return p->Build(); } @@ -571,7 +614,7 @@ struct ProviderHostImpl : ProviderHost { const std::vector& GraphViewer__GetOutputs(const GraphViewer* p) noexcept override { return p->GetOutputs(); } const std::vector& GraphViewer__GetValueInfo(const GraphViewer* p) noexcept override { return p->GetValueInfo(); } - const Provider_InitializedTensorSet& GraphViewer__GetAllInitializedTensors(const GraphViewer* p) override { return p->GetAllInitializedTensors(); } + const InitializedTensorSet& GraphViewer__GetAllInitializedTensors(const GraphViewer* p) override { return p->GetAllInitializedTensors(); } bool GraphViewer__GetInitializedTensor(const GraphViewer* p, const std::string& tensor_name, const ONNX_NAMESPACE::TensorProto*& value) override { return p->GetInitializedTensor(tensor_name, value); } const std::unordered_map& GraphViewer__DomainToVersionMap(const GraphViewer* p) override { return p->DomainToVersionMap(); } @@ -587,9 +630,13 @@ struct ProviderHostImpl : ProviderHost { // OpKernelContext const Tensor* OpKernelContext__Input_Tensor(const OpKernelContext* p, int index) override { return p->Input(index); } + const Tensor& OpKernelContext__RequiredInput_Tensor(const OpKernelContext* p, int index) override { return p->RequiredInput(index); } Tensor* OpKernelContext__Output(OpKernelContext* p, int index, const TensorShape& shape) override { return p->Output(index, shape); } + Tensor& OpKernelContext__RequiredOutput(OpKernelContext* p, int index, const TensorShape& shape) override { return p->RequiredOutput(index, shape); } int OpKernelContext__InputCount(const OpKernelContext* p) override { return p->InputCount(); } int OpKernelContext__OutputCount(const OpKernelContext* p) override { return p->OutputCount(); } + Status OpKernelContext__GetTempSpaceAllocator(const OpKernelContext* p, AllocatorPtr* output) override { return p->GetTempSpaceAllocator(output); } + bool OpKernelContext__GetUseDeterministicCompute(const OpKernelContext* p) override { return p->GetUseDeterministicCompute(); } // OpKernelInfo std::unique_ptr CopyOpKernelInfo(const OpKernelInfo& info) override { return onnxruntime::CopyOpKernelInfo(info); } @@ -598,6 +645,7 @@ struct ProviderHostImpl : ProviderHost { Status OpKernelInfo__GetAttr_int64(const OpKernelInfo* p, const std::string& name, int64_t* value) override { return p->GetAttr(name, value); } Status OpKernelInfo__GetAttr_float(const OpKernelInfo* p, const std::string& name, float* value) override { return p->GetAttr(name, value); } Status OpKernelInfo__GetAttr_string(const OpKernelInfo* p, const std::string& name, std::string* value) override { return p->GetAttr(name, value); } + Status OpKernelInfo__GetAttr_TensorProto(const OpKernelInfo* p, const std::string& name, ONNX_NAMESPACE::TensorProto* value) override { return p->GetAttr(name, value); } Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) override { return p->GetAttrs(name, values); } Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) override { return p->GetAttrs(name, values); } Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) override { return p->GetAttrs(name, values); } @@ -614,6 +662,7 @@ struct ProviderHostImpl : ProviderHost { // Tensor std::unique_ptr Tensor__construct(MLDataType p_type, const TensorShape& shape, std::shared_ptr allocator) override { return onnxruntime::make_unique(p_type, shape, allocator); } + std::unique_ptr Tensor__construct(MLDataType p_type, const TensorShape& shape, void* p_data, const OrtMemoryInfo& alloc, ptrdiff_t offset) override { return onnxruntime::make_unique(p_type, shape, p_data, alloc, offset); } void Tensor__operator_delete(Tensor* p) override { delete p; } bool* Tensor__MutableData_bool(Tensor* p) override { return p->MutableData(); } @@ -627,6 +676,7 @@ struct ProviderHostImpl : ProviderHost { uint64_t* Tensor__MutableData_uint64(Tensor* p) override { return p->MutableData(); } float* Tensor__MutableData_float(Tensor* p) override { return p->MutableData(); } double* Tensor__MutableData_double(Tensor* p) override { return p->MutableData(); } + BFloat16* Tensor__MutableData_BFloat16(Tensor* p) override { return p->MutableData(); } MLFloat16* Tensor__MutableData_MLFloat16(Tensor* p) override { return p->MutableData(); } const bool* Tensor__Data_bool(const Tensor* p) override { return p->Data(); } @@ -640,21 +690,16 @@ struct ProviderHostImpl : ProviderHost { const uint64_t* Tensor__Data_uint64(const Tensor* p) override { return p->Data(); } const float* Tensor__Data_float(const Tensor* p) override { return p->Data(); } const double* Tensor__Data_double(const Tensor* p) override { return p->Data(); } + const BFloat16* Tensor__Data_BFloat16(const Tensor* p) override { return p->Data(); } const MLFloat16* Tensor__Data_MLFloat16(const Tensor* p) override { return p->Data(); } + gsl::span Tensor__DataAsSpan_int64(const Tensor* p) override { return p->DataAsSpan(); } + void* Tensor__MutableDataRaw(Tensor* p, MLDataType type) override { return p->MutableDataRaw(type); } const void* Tensor__DataRaw(const Tensor* p, MLDataType type) override { return p->DataRaw(type); } - void* Tensor__MutableDataRaw(Tensor* p) noexcept override { return p->MutableDataRaw(); } const void* Tensor__DataRaw(const Tensor* p) noexcept override { return p->DataRaw(); } - const TensorShape& Tensor__Shape(const Tensor* p) override { return p->Shape(); } - size_t Tensor__SizeInBytes(const Tensor* p) override { return p->SizeInBytes(); } - const OrtMemoryInfo& Tensor__Location(const Tensor* p) override { return p->Location(); } - - int32_t Tensor__GetElementType(const Tensor* p) override { return p->GetElementType(); } - MLDataType Tensor__DataType(const Tensor* p) override { return p->DataType(); } - bool Tensor__IsDataType_bool(const Tensor* p) noexcept override { return p->IsDataType(); } bool Tensor__IsDataType_int8(const Tensor* p) noexcept override { return p->IsDataType(); } bool Tensor__IsDataType_uint8(const Tensor* p) noexcept override { return p->IsDataType(); } @@ -669,16 +714,34 @@ struct ProviderHostImpl : ProviderHost { bool Tensor__IsDataType_MLFloat16(const Tensor* p) noexcept override { return p->IsDataType(); } bool Tensor__IsDataTypeString(const Tensor* p) noexcept override { return p->IsDataTypeString(); } + const TensorShape& Tensor__Shape(const Tensor* p) override { return p->Shape(); } + void Tensor__Reshape(Tensor* p, const TensorShape& new_shape) override { return p->Reshape(new_shape); } + void Tensor__SetByteOffset(Tensor* p, ptrdiff_t byte_offset) override { p->SetByteOffset(byte_offset); } + ptrdiff_t Tensor__ByteOffset(const Tensor* p) override { return p->ByteOffset(); } + size_t Tensor__SizeInBytes(const Tensor* p) override { return p->SizeInBytes(); } + const OrtMemoryInfo& Tensor__Location(const Tensor* p) override { return p->Location(); } + int32_t Tensor__GetElementType(const Tensor* p) override { return p->GetElementType(); } + MLDataType Tensor__DataType(const Tensor* p) override { return p->DataType(); } + // GatherElements Status GatherElements__ValidateInputShapes(const TensorShape& input_data_shape, const TensorShape& indices_shape, int64_t axis) override { return GatherElements::ValidateInputShapes(input_data_shape, indices_shape, axis); } + // cumsum.cc + Status cumsum_op__GetAxis(const Tensor* axis_tensor, int64_t input_rank, int64_t& axis_out) override { return cumsum_op::GetAxis(axis_tensor, input_rank, axis_out); } + + // TileOp + bool TileOp__IsTileMemcpy(const TensorShape& input_shape, const int64_t* repeats, size_t rank, bool& is_batched_memcpy, size_t& num_of_elements_per_batch, size_t& num_of_copies_per_batch, size_t& num_of_batch_copies) override { return TileOp::IsTileMemcpy(input_shape, repeats, rank, is_batched_memcpy, num_of_elements_per_batch, num_of_copies_per_batch, num_of_batch_copies); } + + // ROI + Status CheckROIAlignValidInput(const Tensor* X_ptr, const Tensor* rois_ptr, const Tensor* batch_indices_ptr) override { return CheckROIAlignValidInput(X_ptr, rois_ptr, batch_indices_ptr); } + + Status NonMaxSuppressionBase__PrepareCompute(OpKernelContext* ctx, PrepareContext& pc) override { return NonMaxSuppressionBase::PrepareCompute(ctx, pc); } + Status NonMaxSuppressionBase__GetThresholdsFromInputs(const PrepareContext& pc, int64_t& max_output_boxes_per_class, float& iou_threshold, float& score_threshold) override { return NonMaxSuppressionBase::GetThresholdsFromInputs(pc, max_output_boxes_per_class, iou_threshold, score_threshold); } + // From onehot.h Status ValidateInputs(const Tensor* depth, const Tensor* values) override { return onnxruntime::ValidateInputs(depth, values); } Status PrepareOutputShape(const Tensor* indices, const int64_t depth_val, const int64_t axis, int64_t& prefix_dim_size, int64_t& suffix_dim_size, std::vector& output_shape) override { return onnxruntime::PrepareOutputShape(indices, depth_val, axis, prefix_dim_size, suffix_dim_size, output_shape); } - // From Providers/Common.h - bool IsScalarOr1ElementVector(const Tensor* input) override { return onnxruntime::IsScalarOr1ElementVector(input); } - // From cpu/tensor/unsqueeze.h Status UnsqueezeBase__PrepareCompute(const UnsqueezeBase* p, OpKernelContext* ctx, UnsqueezeBase__Prepare& prepare) override { return p->PrepareCompute(ctx, reinterpret_cast(prepare)); } // From cpu/tensor/slice.h @@ -718,9 +781,63 @@ struct ProviderHostImpl : ProviderHost { // From cpu/tensor/gatherbase.h Status GatherBase__PrepareForCompute(const GatherBase* p, OpKernelContext* context, GatherBase__Prepare& prepare) override { return p->PrepareForCompute(context, reinterpret_cast(prepare)); } + PhiloxGenerator& PhiloxGenerator__Default() override { return PhiloxGenerator::Default(); } + + Status Einsum__Compute(const Einsum* p, OpKernelContext* context) override { return p->Compute(context); } + + // EinsumComputePreprocessor + void EinsumComputePreprocessor__operator_delete(EinsumComputePreprocessor* p) override { delete p; } + virtual std::unique_ptr EinsumComputePreprocessor__Create(EinsumEquationPreprocessor& equation_preprocessor, + const std::vector& inputs, + AllocatorPtr allocator, + void* einsum_cuda_assets) override { return onnxruntime::make_unique(equation_preprocessor, inputs, allocator, einsum_cuda_assets); } + + virtual Status EinsumComputePreprocessor__Run(EinsumComputePreprocessor* p) override { return p->Run(); } + virtual void EinsumComputePreprocessor__SetDeviceHelpers(EinsumComputePreprocessor* p, const EinsumOp::DeviceHelpers::Diagonal& diagonal_func, const EinsumOp::DeviceHelpers::Transpose& transpose_func) override { return p->SetDeviceHelpers(diagonal_func, transpose_func); } + + // EinsumTypedComputeProcessor + void EinsumTypedComputeProcessor__operator_delete(EinsumTypedComputeProcessor* p) override { delete p; } + void EinsumTypedComputeProcessor__operator_delete(EinsumTypedComputeProcessor* p) override { delete p; } + void EinsumTypedComputeProcessor__operator_delete(EinsumTypedComputeProcessor* p) override { delete p; } + std::unique_ptr> EinsumTypedComputeProcessor_float__Create(OpKernelContext* context, AllocatorPtr allocator, concurrency::ThreadPool* tp, EinsumComputePreprocessor& einsum_compute_preprocessor, void* einsum_cuda_assets) override { return onnxruntime::make_unique>(context, allocator, tp, einsum_compute_preprocessor, einsum_cuda_assets); } + std::unique_ptr> EinsumTypedComputeProcessor_double__Create(OpKernelContext* context, AllocatorPtr allocator, concurrency::ThreadPool* tp, EinsumComputePreprocessor& einsum_compute_preprocessor, void* einsum_cuda_assets) override { return onnxruntime::make_unique>(context, allocator, tp, einsum_compute_preprocessor, einsum_cuda_assets); } + std::unique_ptr> EinsumTypedComputeProcessor_MLFloat16__Create(OpKernelContext* context, AllocatorPtr allocator, concurrency::ThreadPool* tp, EinsumComputePreprocessor& einsum_compute_preprocessor, void* einsum_cuda_assets) override { return onnxruntime::make_unique>(context, allocator, tp, einsum_compute_preprocessor, einsum_cuda_assets); } + void EinsumTypedComputeProcessor__SetDeviceHelpers(EinsumTypedComputeProcessor* p, const EinsumOp::DeviceHelpers::Transpose& device_transpose_func, const EinsumOp::DeviceHelpers::MatMul& device_matmul_func, const EinsumOp::DeviceHelpers::ReduceSum& device_reduce_sum_func, const EinsumOp::DeviceHelpers::DataCopy& device_data_copy_func) override { return p->SetDeviceHelpers(device_transpose_func, device_matmul_func, device_reduce_sum_func, device_data_copy_func); } + void EinsumTypedComputeProcessor__SetDeviceHelpers(EinsumTypedComputeProcessor* p, const EinsumOp::DeviceHelpers::Transpose& device_transpose_func, const EinsumOp::DeviceHelpers::MatMul& device_matmul_func, const EinsumOp::DeviceHelpers::ReduceSum& device_reduce_sum_func, const EinsumOp::DeviceHelpers::DataCopy& device_data_copy_func) override { return p->SetDeviceHelpers(device_transpose_func, device_matmul_func, device_reduce_sum_func, device_data_copy_func); } + void EinsumTypedComputeProcessor__SetDeviceHelpers(EinsumTypedComputeProcessor* p, const EinsumOp::DeviceHelpers::Transpose& device_transpose_func, const EinsumOp::DeviceHelpers::MatMul& device_matmul_func, const EinsumOp::DeviceHelpers::ReduceSum& device_reduce_sum_func, const EinsumOp::DeviceHelpers::DataCopy& device_data_copy_func) override { return p->SetDeviceHelpers(device_transpose_func, device_matmul_func, device_reduce_sum_func, device_data_copy_func); } + Status EinsumTypedComputeProcessor__Run(EinsumTypedComputeProcessor* p) override { return p->Run(); } + Status EinsumTypedComputeProcessor__Run(EinsumTypedComputeProcessor* p) override { return p->Run(); } + Status EinsumTypedComputeProcessor__Run(EinsumTypedComputeProcessor* p) override { return p->Run(); } + // AllocatorManager void AllocatorManager__InsertAllocator(AllocatorManager* p, AllocatorPtr allocator) override { p->InsertAllocator(allocator); } - AllocatorPtr AllocatorManager__GetAllocator(AllocatorManager* p, int id, OrtMemType mem_type) override { return p->GetAllocator(id, mem_type); }; + AllocatorPtr AllocatorManager__GetAllocator(const AllocatorManager* p, int id, OrtMemType mem_type) override { return p->GetAllocator(id, mem_type); }; + + std::unique_ptr CreateOpKernel_CPU_If(const OpKernelInfo& info) override { return onnxruntime::make_unique(info); } + std::unique_ptr CreateOpKernel_CPU_Loop(const OpKernelInfo& info, const void* concat_output_func, void* stream) override { return Loop::Create(info, *reinterpret_cast(concat_output_func), stream); } + std::unique_ptr CreateOpKernel_CPU_Scan_8(const OpKernelInfo& info) override { return onnxruntime::make_unique>(info); } + std::unique_ptr CreateOpKernel_CPU_Scan_9(const OpKernelInfo& info) override { return onnxruntime::make_unique>(info); } + + // ContribOps +#ifndef DISABLE_CONTRIB_OPS + Status embed_layer_norm__CheckInputs(const OpKernelContext* context) override { return contrib::embed_layer_norm::CheckInputs(context); } + Status bias_gelu_helper__CheckInputs(const OpKernelContext* context) override { return contrib::bias_gelu_helper::CheckInputs(context); } + Status LongformerAttentionBase__CheckInputs(const contrib::LongformerAttentionBase* p, const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, const TensorShape& mask_shape, const TensorShape& global_weights_shape, const TensorShape& global_bias_shape, const TensorShape& global_shape) override { return p->CheckInputs(input_shape, weights_shape, bias_shape, mask_shape, global_weights_shape, global_bias_shape, global_shape); } + + Status AttentionBase__CheckInputs(const contrib::AttentionBase* p, const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, const Tensor*& mask_index, const Tensor* past, const int max_threads_per_block) override { return p->CheckInputs(input_shape, weights_shape, bias_shape, mask_index, past, max_threads_per_block); } + Tensor* AttentionBase__GetPresent(const contrib::AttentionBase* p, OpKernelContext* context, const Tensor* past, int batch_size, int head_size, int sequence_length, int& past_sequence_length) override { return p->GetPresent(context, past, batch_size, head_size, sequence_length, past_sequence_length); } +#endif + +#ifdef ENABLE_TRAINING + void contrib__record_event_in_tensor(const Tensor& event_id_tensor) override { return contrib::record_event_in_tensor(event_id_tensor); } + void contrib__wait_event_in_tensor(const Tensor& event_id_tensor) override { return contrib::wait_event_in_tensor(event_id_tensor); } + Status contrib__Group__Compute(const contrib::Group* p, OpKernelContext* context) override { return p->Compute(context); } + Status contrib__PassThrough__Compute(const contrib::PassThrough* p, OpKernelContext* context) override { return p->Compute(context); } + void contrib__VerifyLogitWeightAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, const TensorShape* weight_shape) override { contrib::VerifyLogitWeightAndLabelShape(logit_shape, label_shape, weight_shape); } + void contrib__GetNDCFromLogitAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, int64_t& N_D, int64_t& C) override { contrib::GetNDCFromLogitAndLabelShape(logit_shape, label_shape, N_D, C); } + void contrib__GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, std::vector& new_shape, std::vector& permutations) override { contrib::GetPermutationAndShape(ncd_to_ndc, tensor_shape, new_shape, permutations); } + Status contrib__PrepareForTrainingCompute(const TensorShape& input_shape, int num_outputs, int64_t& axis, int& before_dims, int& after_dims_including_split_axis, int& after_dims_excluding_split, std::vector& split_sizes) override { return contrib::PrepareForTrainingCompute(input_shape, num_outputs, axis, before_dims, after_dims_including_split_axis, after_dims_excluding_split, split_sizes); } +#endif } provider_host_; @@ -820,6 +937,14 @@ void UnloadSharedProviders() { s_library_shared.Unload(); } +// Used by test code +std::unique_ptr CreateCUDAPinnedAllocator(int16_t device_id, const char* name) { + if (auto* info = onnxruntime::GetProviderInfo_CUDA()) + return info->CreateCUDAPinnedAllocator(device_id, name); + + return nullptr; +} + std::shared_ptr CreateExecutionProviderFactory_Cuda(const OrtCUDAProviderOptions* provider_options) { if (auto provider = s_library_cuda.Get()) return provider->CreateExecutionProviderFactory(provider_options); @@ -855,12 +980,35 @@ std::shared_ptr CreateExecutionProviderFactory_OpenVI return nullptr; } -const ProviderInfo_OpenVINO* GetProviderInfo_OpenVINO() { +ProviderInfo_OpenVINO* GetProviderInfo_OpenVINO() { if (auto provider = s_library_openvino.Get()) - return reinterpret_cast(provider->GetInfo()); + return reinterpret_cast(provider->GetInfo()); return nullptr; } +ProviderInfo_CUDA* GetProviderInfo_CUDA() { + if (auto provider = s_library_cuda.Get()) + return reinterpret_cast(provider->GetInfo()); + return nullptr; +} + +void CopyGpuToCpu( + void* dst_ptr, + const void* src_ptr, + const size_t size, + const OrtMemoryInfo& dst_location, + const OrtMemoryInfo& src_location) { + if (auto* info = onnxruntime::GetProviderInfo_CUDA()) + return info->CopyGpuToCpu(dst_ptr, src_ptr, size, dst_location, src_location); + ORT_THROW("GPU-to-CPU copy is not implemented."); +} + +void cudaMemcpy_HostToDevice(void* dst, const void* src, size_t count) { + if (auto* info = onnxruntime::GetProviderInfo_CUDA()) + return info->cudaMemcpy_HostToDevice(dst, src, count); + ORT_THROW("cudaMemcpy_HostToDevice is not implemented."); +} + } // namespace onnxruntime ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessionOptions* options, int use_arena) { @@ -916,6 +1064,18 @@ ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessi return OrtApis::SessionOptionsAppendExecutionProvider_CUDA(options, &provider_options); } +ORT_API_STATUS_IMPL(OrtApis::SetCurrentGpuDeviceId, _In_ int device_id) { + if (auto* info = onnxruntime::GetProviderInfo_CUDA()) + return info->SetCurrentGpuDeviceId(device_id); + return nullptr; +} + +ORT_API_STATUS_IMPL(OrtApis::GetCurrentGpuDeviceId, _In_ int* device_id) { + if (auto* info = onnxruntime::GetProviderInfo_CUDA()) + return info->GetCurrentGpuDeviceId(device_id); + return nullptr; +} + ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options) { auto factory = onnxruntime::CreateExecutionProviderFactory_Cuda(cuda_options); if (!factory) { @@ -925,4 +1085,3 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA, _In_ Or options->provider_factories.push_back(factory); return nullptr; } - diff --git a/onnxruntime/core/framework/tensorprotoutils.h b/onnxruntime/core/framework/tensorprotoutils.h index f429349c5f..9f79968fdd 100644 --- a/onnxruntime/core/framework/tensorprotoutils.h +++ b/onnxruntime/core/framework/tensorprotoutils.h @@ -6,6 +6,7 @@ #include #include +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/common/path.h" #include "core/common/status.h" @@ -26,15 +27,17 @@ bool operator==(const TensorShapeProto_Dimension& l, const TensorShapeProto_Dime bool operator!=(const TensorShapeProto_Dimension& l, const TensorShapeProto_Dimension& r); } // namespace ONNX_NAMESPACE +#endif namespace onnxruntime { -class Tensor; namespace utils { +#ifndef SHARED_PROVIDER + TensorShape GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::TensorShapeProto& tensor_shape_proto); std::vector GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto); - /** +/** * deserialize a TensorProto into a preallocated memory buffer. * \param tensor_proto_path A local file path of where the 'input' was loaded from. Can be NULL if the tensor proto doesn't * have any external data or it was loaded from current working dir. This path could be either a @@ -51,8 +54,8 @@ common::Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_prot * @return */ common::Status TensorProtoToTensor(const Env& env, const ORTCHAR_T* model_path, - const ONNX_NAMESPACE::TensorProto& tensor_proto, - Tensor& tensor); + const ONNX_NAMESPACE::TensorProto& tensor_proto, + Tensor& tensor); /** Creates a TensorProto from a Tensor. @param[in] tensor the Tensor whose data and shape will be used to create the TensorProto. @@ -99,6 +102,7 @@ common::Status DenseTensorToSparseTensorProto(const ONNX_NAMESPACE::TensorProto& const Path& model_path, ONNX_NAMESPACE::SparseTensorProto& sparse); #endif // !ORT_MINIMAL_BUILD +#endif inline bool HasDimValue(const ONNX_NAMESPACE::TensorShapeProto_Dimension& dim) { return dim.value_case() == ONNX_NAMESPACE::TensorShapeProto_Dimension::kDimValue; @@ -122,10 +126,12 @@ inline bool HasShape(const ONNX_NAMESPACE::TypeProto_Tensor& ten_proto) { return ten_proto.has_shape(); } +#ifndef SHARED_PROVIDER inline bool HasShape(const ONNX_NAMESPACE::TypeProto_SparseTensor& ten_proto) { // XXX: Figure out how do in proto3 return ten_proto.has_shape(); } +#endif inline bool HasRawData(const ONNX_NAMESPACE::TensorProto& ten_proto) { // Can not be UNDEFINED and can not be STRING but test for STRING is usually performed separately @@ -145,6 +151,7 @@ inline bool HasDataType(const ONNX_NAMESPACE::TensorProto& ten_proto) { return ten_proto.data_type() != ONNX_NAMESPACE::TensorProto::UNDEFINED; } +#ifndef SHARED_PROVIDER inline bool HasName(const ONNX_NAMESPACE::TensorProto& ten_proto) { return ten_proto.has_name(); // XXX } @@ -168,11 +175,13 @@ inline bool HasKeyType(const ONNX_NAMESPACE::TypeProto_Map& map_proto) { inline bool HasValueType(const ONNX_NAMESPACE::TypeProto_Map& map_proto) { return map_proto.value_type().value_case() != ONNX_NAMESPACE::TypeProto::VALUE_NOT_SET; } +#endif inline bool HasType(const ONNX_NAMESPACE::ValueInfoProto& vi_proto) { return vi_proto.type().value_case() != ONNX_NAMESPACE::TypeProto::VALUE_NOT_SET; } +#ifndef SHARED_PROVIDER inline bool HasName(const ONNX_NAMESPACE::ValueInfoProto& vi_proto) { return vi_proto.has_name(); // XXX: Figure out proto3 way } @@ -184,6 +193,7 @@ inline bool HasDomain(const ONNX_NAMESPACE::TypeProto_Opaque& op_proto) { inline bool HasName(const ONNX_NAMESPACE::TypeProto_Opaque& op_proto) { return !op_proto.name().empty(); } +#endif inline bool HasType(const ONNX_NAMESPACE::AttributeProto& at_proto) { return at_proto.type() != ONNX_NAMESPACE::AttributeProto::AttributeType::AttributeProto_AttributeType_UNDEFINED; @@ -229,6 +239,7 @@ inline bool HasGraphs(const ONNX_NAMESPACE::AttributeProto& at_proto) { return at_proto.type() == ONNX_NAMESPACE::AttributeProto::AttributeType::AttributeProto_AttributeType_GRAPHS; } +#ifndef SHARED_PROVIDER inline bool HasName(const ONNX_NAMESPACE::AttributeProto& at_proto) { return at_proto.has_name(); // XXX: Fugure out proto3 } @@ -249,6 +260,7 @@ inline bool HasName(const ONNX_NAMESPACE::NodeProto& node_proto) { //XXX: Figure out proto3 style return node_proto.has_name(); } +#endif // UnpackTensor from raw data or the type specific data field. Does not handle external data. // If the tensor does not contain raw data then raw_data should be nullptr and raw_data_len should be 0. @@ -276,6 +288,5 @@ common::Status UnpackInitializerData(const ONNX_NAMESPACE::TensorProto& initiali const Path& model_path, std::unique_ptr& unpacked_tensor, size_t& tensor_byte_size) ORT_MUST_USE_RESULT; - } // namespace utils } // namespace onnxruntime diff --git a/onnxruntime/core/platform/env_var_utils.h b/onnxruntime/core/platform/env_var_utils.h index 3c8fba76bf..61d086a28a 100644 --- a/onnxruntime/core/platform/env_var_utils.h +++ b/onnxruntime/core/platform/env_var_utils.h @@ -14,7 +14,11 @@ namespace onnxruntime { */ template optional ParseEnvironmentVariable(const std::string& name) { +#ifndef SHARED_PROVIDER const std::string value_str = Env::Default().GetEnvironmentVar(name); +#else + const std::string value_str = GetEnvironmentVar(name); +#endif if (value_str.empty()) { return {}; } diff --git a/onnxruntime/core/providers/common.h b/onnxruntime/core/providers/common.h index 3688133c6d..fca366b295 100644 --- a/onnxruntime/core/providers/common.h +++ b/onnxruntime/core/providers/common.h @@ -6,8 +6,10 @@ #include #include +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/framework/tensor.h" +#endif namespace onnxruntime { diff --git a/onnxruntime/core/providers/cpu/controlflow/if.h b/onnxruntime/core/providers/cpu/controlflow/if.h index dfadc5ea55..adb5a78a24 100644 --- a/onnxruntime/core/providers/cpu/controlflow/if.h +++ b/onnxruntime/core/providers/cpu/controlflow/if.h @@ -5,9 +5,6 @@ #include #include "gsl/gsl" -//#include "core/common/common.h" -//#include "core/framework/feeds_fetches_manager.h" -//#include "core/framework/op_kernel.h" #include "core/providers/cpu/controlflow/utils.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cpu/controlflow/loop.cc b/onnxruntime/core/providers/cpu/controlflow/loop.cc index a2b7c4b961..9dcde1a337 100644 --- a/onnxruntime/core/providers/cpu/controlflow/loop.cc +++ b/onnxruntime/core/providers/cpu/controlflow/loop.cc @@ -259,6 +259,13 @@ Loop::Loop(const OpKernelInfo& info) : IControlFlowKernel(info) { stream_ = nullptr; } +std::unique_ptr Loop::Create(const OpKernelInfo& info, const ConcatOutput& concat_output_func, void* stream) { + auto result = make_unique(info); + result->SetConcatOutputFunc(concat_output_func); + result->SetComputeStream(stream); + return result; +} + // we need this to be in the .cc so 'unique_ptr info_' can be handled Loop::~Loop() = default; diff --git a/onnxruntime/core/providers/cpu/controlflow/loop.h b/onnxruntime/core/providers/cpu/controlflow/loop.h index 73f863b22d..0094e90b1d 100644 --- a/onnxruntime/core/providers/cpu/controlflow/loop.h +++ b/onnxruntime/core/providers/cpu/controlflow/loop.h @@ -32,6 +32,8 @@ class Loop : public controlflow::IControlFlowKernel { using ConcatOutput = std::function& per_iteration_output, void* output, size_t output_size_in_bytes)>; + static std::unique_ptr Create(const OpKernelInfo& info, const ConcatOutput& concat_output_func, void* stream); + protected: // derived class can provide implementation for handling concatenation of Loop output on a different device void SetConcatOutputFunc(const ConcatOutput& concat_output_func) { concat_output_func_ = concat_output_func; } diff --git a/onnxruntime/core/providers/cpu/controlflow/scan.h b/onnxruntime/core/providers/cpu/controlflow/scan.h index 01bcfcd564..f58161a450 100644 --- a/onnxruntime/core/providers/cpu/controlflow/scan.h +++ b/onnxruntime/core/providers/cpu/controlflow/scan.h @@ -5,8 +5,11 @@ #include #include "gsl/gsl" +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/framework/op_kernel.h" +#endif + #include "core/framework/feeds_fetches_manager.h" #include "core/providers/cpu/controlflow/utils.h" #include "core/framework/ort_value_tensor_slicer.h" @@ -56,7 +59,6 @@ class Scan : public controlflow::IControlFlowKernel { struct Info; ~Scan(); - protected: void SetDeviceHelpers(const scan::detail::DeviceHelpers& device_helpers) { device_helpers_ = device_helpers; // copy } diff --git a/onnxruntime/core/providers/cpu/controlflow/utils.h b/onnxruntime/core/providers/cpu/controlflow/utils.h index 1c54e1716b..0f4d00152a 100644 --- a/onnxruntime/core/providers/cpu/controlflow/utils.h +++ b/onnxruntime/core/providers/cpu/controlflow/utils.h @@ -7,7 +7,6 @@ #include #include -//#include "core/common/common.h" #include "core/framework/feeds_fetches_manager.h" #ifndef SHARED_PROVIDER #include "core/framework/op_kernel.h" diff --git a/onnxruntime/core/providers/cpu/generator/constant_of_shape_base.h b/onnxruntime/core/providers/cpu/generator/constant_of_shape_base.h index 2e72f97e69..f105e2762f 100644 --- a/onnxruntime/core/providers/cpu/generator/constant_of_shape_base.h +++ b/onnxruntime/core/providers/cpu/generator/constant_of_shape_base.h @@ -3,6 +3,7 @@ #pragma once +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/common/type_list.h" #include "core/framework/data_types.h" @@ -10,6 +11,7 @@ #include "core/framework/op_kernel.h" #include "core/framework/tensorprotoutils.h" #include "core/providers/op_kernel_type_control_utils.h" +#endif namespace onnxruntime { @@ -25,11 +27,17 @@ template class ConstantOfShapeBase { protected: ConstantOfShapeBase(const OpKernelInfo& info) { +#ifndef SHARED_PROVIDER ONNX_NAMESPACE::TensorProto t_proto; - if (info.GetAttr("value", &t_proto).IsOK()) { - ORT_ENFORCE(t_proto.dims_size() == 1, "Must have a single dimension"); - ORT_ENFORCE(t_proto.dims()[0] == 1, "Must have a single dimension of 1"); - SetValueFromTensorProto(t_proto); + auto* t_proto_p = &t_proto; +#else + auto t_proto = ONNX_NAMESPACE::TensorProto::Create(); + auto* t_proto_p = t_proto.get(); +#endif + if (info.GetAttr("value", t_proto_p).IsOK()) { + ORT_ENFORCE(t_proto_p->dims_size() == 1, "Must have a single dimension"); + ORT_ENFORCE(t_proto_p->dims()[0] == 1, "Must have a single dimension of 1"); + SetValueFromTensorProto(*t_proto_p); } else { float f_value = 0.f; SetValue(sizeof(float), reinterpret_cast(&f_value)); diff --git a/onnxruntime/core/providers/cpu/math/einsum.h b/onnxruntime/core/providers/cpu/math/einsum.h index 139eed91cc..8497c9c020 100644 --- a/onnxruntime/core/providers/cpu/math/einsum.h +++ b/onnxruntime/core/providers/cpu/math/einsum.h @@ -3,10 +3,12 @@ #pragma once +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/framework/op_kernel.h" -#include "einsum_utils/einsum_compute_preprocessor.h" #include "einsum_utils/einsum_typed_compute_processor.h" +#endif +#include "einsum_utils/einsum_compute_preprocessor.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.cc b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.cc index ab9f9e1c35..5c4d46f13f 100644 --- a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.cc +++ b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.cc @@ -49,10 +49,10 @@ Status MatMul(const T* input_1_data, const T* input_2_data, T* output_data, // CPU specific ReduceSum helper template -Tensor ReduceSum(const Tensor& input, const std::vector& reduce_axes, - bool keep_dims, AllocatorPtr allocator, - const TensorShape* input_shape_override, - concurrency::ThreadPool* tp, void* /*einsum_cuda_assets*/) { +std::unique_ptr ReduceSum(const Tensor& input, const std::vector& reduce_axes, + bool keep_dims, AllocatorPtr allocator, + const TensorShape* input_shape_override, + concurrency::ThreadPool* tp, void* /*einsum_cuda_assets*/) { return onnxruntime::ReduceSum::Impl(input, reduce_axes, allocator, tp, keep_dims, input_shape_override); @@ -352,8 +352,7 @@ std::unique_ptr ReduceSum(const Tensor& input, const std::vector& device_reduce_sum_func) { TensorShape overriden_shape(input_shape_override); - auto output = device_reduce_sum_func(input, reduce_axes, true, allocator, &overriden_shape, tp, einsum_cuda_assets); - return onnxruntime::make_unique(std::move(output)); + return device_reduce_sum_func(input, reduce_axes, true, allocator, &overriden_shape, tp, einsum_cuda_assets); } // Explicit template instantiations of functions @@ -371,7 +370,7 @@ template std::unique_ptr MatMul( AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets, const DeviceHelpers::MatMul& device_matmul_func); -template Tensor DeviceHelpers::CpuDeviceHelpers::ReduceSum( +template std::unique_ptr DeviceHelpers::CpuDeviceHelpers::ReduceSum( const Tensor& input, const std::vector& reduce_axes, bool keep_dims, AllocatorPtr allocator, const TensorShape* input_shape_override, @@ -395,7 +394,7 @@ template std::unique_ptr MatMul( AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets, const DeviceHelpers::MatMul& device_matmul_func); -template Tensor DeviceHelpers::CpuDeviceHelpers::ReduceSum( +template std::unique_ptr DeviceHelpers::CpuDeviceHelpers::ReduceSum( const Tensor& input, const std::vector& reduce_axes, bool keep_dims, AllocatorPtr allocator, const TensorShape* input_shape_override, @@ -420,7 +419,7 @@ template std::unique_ptr MatMul( AllocatorPtr allocator, concurrency::ThreadPool* tp, void* einsum_cuda_assets, const DeviceHelpers::MatMul& device_matmul_func); -template Tensor DeviceHelpers::CpuDeviceHelpers::ReduceSum( +template std::unique_ptr DeviceHelpers::CpuDeviceHelpers::ReduceSum( const Tensor& input, const std::vector& reduce_axes, bool keep_dims, AllocatorPtr allocator, const TensorShape* input_shape_override, @@ -439,7 +438,7 @@ template Status DeviceHelpers::CpuDeviceHelpers::MatMul( size_t num_batches, size_t M, size_t K, size_t N, concurrency::ThreadPool* tp, void* einsum_cuda_assets); -template Tensor DeviceHelpers::CpuDeviceHelpers::ReduceSum( +template std::unique_ptr DeviceHelpers::CpuDeviceHelpers::ReduceSum( const Tensor& input, const std::vector& reduce_axes, bool keep_dims, AllocatorPtr allocator, const TensorShape* input_shape_override, diff --git a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.h b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.h index ef6c04af07..dd21654c4a 100644 --- a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.h +++ b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_auxiliary_ops.h @@ -41,10 +41,10 @@ using MatMul = std::function -using ReduceSum = std::function& reduce_axes, - bool keep_dims, AllocatorPtr allocator, - const TensorShape* input_shape_override, - concurrency::ThreadPool* tp, void* einsum_cuda_assets)>; +using ReduceSum = std::function(const Tensor& input, const std::vector& reduce_axes, + bool keep_dims, AllocatorPtr allocator, + const TensorShape* input_shape_override, + concurrency::ThreadPool* tp, void* einsum_cuda_assets)>; // Diagonal op // Diagonal - A specialized implementation somewhat similar to Torch's Diagonal op @@ -73,10 +73,10 @@ Status MatMul(const T* input_1_data, const T* input_2_data, T* output_data, void* einsum_cuda_assets); template -Tensor ReduceSum(const Tensor& input, const std::vector& reduce_axes, - bool keep_dims, AllocatorPtr allocator, - const TensorShape* input_shape_override, - concurrency::ThreadPool* tp, void* einsum_cuda_assets); +std::unique_ptr ReduceSum(const Tensor& input, const std::vector& reduce_axes, + bool keep_dims, AllocatorPtr allocator, + const TensorShape* input_shape_override, + concurrency::ThreadPool* tp, void* einsum_cuda_assets); std::unique_ptr Diagonal(const Tensor& input, int64_t dim_1, int64_t dim_2, AllocatorPtr allocator, void* einsum_cuda_assets); diff --git a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.h b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.h index 7c7b25f2a7..955ff4a920 100644 --- a/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.h +++ b/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.h @@ -104,6 +104,7 @@ struct EinsumEquationPreprocessor { // Subscript labels (letter) and subcript indices (a unique id to the letter) are interchangeable // This is a pre-processor class that maps subscript labels to a dimension value, etc. +#ifndef SHARED_PROVIDER class EinsumComputePreprocessor final { public: explicit EinsumComputePreprocessor(EinsumEquationPreprocessor& equation_preprocessor, @@ -224,5 +225,5 @@ class EinsumComputePreprocessor final { // Holds EP-specific assets required for (auxiliary) ops that need to be executed on non-CPU EPs void* einsum_ep_assets_; }; - +#endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/batch_norm_helper.h b/onnxruntime/core/providers/cpu/nn/batch_norm_helper.h index 9add18710c..92efe04472 100644 --- a/onnxruntime/core/providers/cpu/nn/batch_norm_helper.h +++ b/onnxruntime/core/providers/cpu/nn/batch_norm_helper.h @@ -3,8 +3,10 @@ #pragma once +#ifndef SHARED_PROVIDER #include "core/common/status.h" #include "core/framework/tensor.h" +#endif #include namespace onnxruntime { diff --git a/onnxruntime/core/providers/cpu/nn/conv_attributes.h b/onnxruntime/core/providers/cpu/nn/conv_attributes.h index 2383bcbe0b..d7041d4145 100644 --- a/onnxruntime/core/providers/cpu/nn/conv_attributes.h +++ b/onnxruntime/core/providers/cpu/nn/conv_attributes.h @@ -14,7 +14,7 @@ namespace onnxruntime { // A helper struct holding attributes for Conv-family ops struct ConvAttributes { - explicit ConvAttributes(const OpNodeProtoHelper& info) { + explicit ConvAttributes(const OpKernelInfo& info) { std::string auto_pad_str; auto status = info.GetAttr("auto_pad", &auto_pad_str); if (status.IsOK()) { diff --git a/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h b/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h index 07a36ccccc..59ef6ee538 100644 --- a/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h +++ b/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h @@ -22,7 +22,7 @@ namespace onnxruntime { struct ConvTransposeAttributes : public ConvAttributes { - explicit ConvTransposeAttributes(const OpNodeProtoHelper& info) + explicit ConvTransposeAttributes(const OpKernelInfo& info) : ConvAttributes(info), output_padding(info.GetAttrsOrDefault("output_padding")), output_shape(info.GetAttrsOrDefault("output_shape")) { diff --git a/onnxruntime/core/providers/cpu/nn/instance_norm_helper.h b/onnxruntime/core/providers/cpu/nn/instance_norm_helper.h index bdf6f90b38..48e54ac7ee 100644 --- a/onnxruntime/core/providers/cpu/nn/instance_norm_helper.h +++ b/onnxruntime/core/providers/cpu/nn/instance_norm_helper.h @@ -3,8 +3,10 @@ #pragma once +#ifndef SHARED_PROVIDER #include "core/common/status.h" #include "core/framework/tensor.h" +#endif #include namespace onnxruntime { diff --git a/onnxruntime/core/providers/cpu/nn/pool_attributes.h b/onnxruntime/core/providers/cpu/nn/pool_attributes.h index 8f14f16d88..1aa0de8353 100644 --- a/onnxruntime/core/providers/cpu/nn/pool_attributes.h +++ b/onnxruntime/core/providers/cpu/nn/pool_attributes.h @@ -4,10 +4,12 @@ #pragma once #include +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/framework/op_node_proto_helper.h" #include "core/framework/tensor_shape.h" #include "core/providers/common.h" +#endif namespace onnxruntime { @@ -17,7 +19,7 @@ struct PoolAttributes { return op_name == "GlobalAveragePool" || op_name == "GlobalMaxPool" || op_name == "GlobalLpPool"; } - PoolAttributes(const OpNodeProtoHelper& info, + PoolAttributes(const OpKernelInfo& info, const std::string& op_name, int start_version) : global_pooling(IsGlobalPooling(op_name)) { if (global_pooling) { diff --git a/onnxruntime/core/providers/cpu/nn/pool_base.h b/onnxruntime/core/providers/cpu/nn/pool_base.h index 4e2c9910b9..d03754eec9 100644 --- a/onnxruntime/core/providers/cpu/nn/pool_base.h +++ b/onnxruntime/core/providers/cpu/nn/pool_base.h @@ -4,10 +4,12 @@ #pragma once #include +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/framework/op_kernel.h" -#include "core/providers/cpu/nn/pool_attributes.h" #include "core/util/math.h" +#endif +#include "core/providers/cpu/nn/pool_attributes.h" #include "core/mlas/inc/mlas.h" namespace onnxruntime { @@ -106,8 +108,7 @@ class PoolBase { protected: PoolBase(const OpKernelInfo& info) - : op_name_(info.GetKernelDef().OpName().rfind("QLinear", 0) != 0 ? - info.GetKernelDef().OpName() : info.GetKernelDef().OpName().substr(7)), + : op_name_(info.GetKernelDef().OpName().rfind("QLinear", 0) != 0 ? info.GetKernelDef().OpName() : info.GetKernelDef().OpName().substr(7)), pool_attrs_(info, op_name_, GetStartVersion(info)) { } diff --git a/onnxruntime/core/providers/cpu/object_detection/non_max_suppression.h b/onnxruntime/core/providers/cpu/object_detection/non_max_suppression.h index f6465ea3ee..e349a53cbd 100644 --- a/onnxruntime/core/providers/cpu/object_detection/non_max_suppression.h +++ b/onnxruntime/core/providers/cpu/object_detection/non_max_suppression.h @@ -17,16 +17,17 @@ class NonMaxSuppressionBase { ORT_ENFORCE(0 == center_point_box_ || 1 == center_point_box_, "center_point_box only support 0 or 1"); } + int64_t GetCenterPointBox() const { + return center_point_box_; + } + + public: static Status PrepareCompute(OpKernelContext* ctx, PrepareContext& pc); static Status GetThresholdsFromInputs(const PrepareContext& pc, int64_t& max_output_boxes_per_class, float& iou_threshold, float& score_threshold); - int64_t GetCenterPointBox() const { - return center_point_box_; - } - private: int64_t center_point_box_; }; diff --git a/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc b/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc index ca89296dc2..63b32438ad 100644 --- a/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc +++ b/onnxruntime/core/providers/cpu/reduction/reduction_ops.cc @@ -586,9 +586,9 @@ Status ReduceSum::Compute(OpKernelContext* ctx) const { } template -Tensor ReduceSum::Impl(const Tensor& input, const std::vector& reduce_axes, - AllocatorPtr allocator, concurrency::ThreadPool* tp, bool keep_dims, - const TensorShape* input_shape_override) { +std::unique_ptr ReduceSum::Impl(const Tensor& input, const std::vector& reduce_axes, + AllocatorPtr allocator, concurrency::ThreadPool* tp, bool keep_dims, + const TensorShape* input_shape_override) { std::vector axes; auto reduced_dims = input.Shape().GetDims(); std::vector output_shape; @@ -597,10 +597,10 @@ Tensor ReduceSum::Impl(const Tensor& input, const std::vector& reduc SetupForReduce(&input, reduce_axes, axes, new_input_shape, output_shape, empty_reduce, input_shape_override); if (empty_reduce) { - Tensor output(input.DataType(), keep_dims ? output_shape : std::vector(), allocator); + auto output = make_unique(input.DataType(), keep_dims ? output_shape : std::vector(), allocator); if (new_input_shape.Size() == 1) { const T* from_data = input.template Data(); - T* to_data = output.template MutableData(); + T* to_data = output->template MutableData(); *to_data = *from_data; } else { ORT_ENFORCE(keep_dims, @@ -613,15 +613,15 @@ Tensor ReduceSum::Impl(const Tensor& input, const std::vector& reduc if (keep_dims) { ResultsNoTransposePrepareForReduce last_results; - Tensor output(input.DataType(), output_shape, allocator); - NoTransposeReduce>(&output, new_input_shape, input, axes, tp, last_results); + auto output = make_unique(input.DataType(), output_shape, allocator); + NoTransposeReduce>(output.get(), new_input_shape, input, axes, tp, last_results); return output; } else { ResultsNoTransposePrepareForReduce last_results; std::vector dropped_axes; DropDimensions(output_shape, axes, dropped_axes); - Tensor output(input.DataType(), dropped_axes, allocator); - NoTransposeReduce>(&output, new_input_shape, input, axes, tp, last_results); + auto output = make_unique(input.DataType(), dropped_axes, allocator); + NoTransposeReduce>(output.get(), new_input_shape, input, axes, tp, last_results); return output; } } diff --git a/onnxruntime/core/providers/cpu/reduction/reduction_ops.h b/onnxruntime/core/providers/cpu/reduction/reduction_ops.h index 145f524350..498ba6587a 100644 --- a/onnxruntime/core/providers/cpu/reduction/reduction_ops.h +++ b/onnxruntime/core/providers/cpu/reduction/reduction_ops.h @@ -472,9 +472,9 @@ class ReduceSum final : public ReduceKernel { // For external calls requiring ReduceSum implementation - will return the reduced output. //`input_shape_override` overrides the shape of `input` for compute purposes. - static Tensor Impl(const Tensor& input, const std::vector& reduce_axes, - AllocatorPtr allocator, concurrency::ThreadPool* tp, bool keep_dims, - const TensorShape* input_shape_override = nullptr); + static std::unique_ptr Impl(const Tensor& input, const std::vector& reduce_axes, + AllocatorPtr allocator, concurrency::ThreadPool* tp, bool keep_dims, + const TensorShape* input_shape_override = nullptr); }; template diff --git a/onnxruntime/core/providers/cpu/tensor/reshape_helper.h b/onnxruntime/core/providers/cpu/tensor/reshape_helper.h index be9812ea88..995a99f9bf 100644 --- a/onnxruntime/core/providers/cpu/tensor/reshape_helper.h +++ b/onnxruntime/core/providers/cpu/tensor/reshape_helper.h @@ -3,10 +3,6 @@ #pragma once -//#include "core/common/common.h" -//#include "core/framework/tensor.h" -//#include "gsl/gsl" - namespace onnxruntime { // Verify and convert unknown dim during reshape diff --git a/onnxruntime/core/providers/cpu/tensor/tile.h b/onnxruntime/core/providers/cpu/tensor/tile.h index 093ee74969..c9479c01ad 100644 --- a/onnxruntime/core/providers/cpu/tensor/tile.h +++ b/onnxruntime/core/providers/cpu/tensor/tile.h @@ -1,9 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/framework/op_kernel.h" #include "core/util/math_cpuonly.h" +#endif namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/controlflow/if.cc b/onnxruntime/core/providers/cuda/controlflow/if.cc index eaf63b6050..f34ace718c 100644 --- a/onnxruntime/core/providers/cuda/controlflow/if.cc +++ b/onnxruntime/core/providers/cuda/controlflow/if.cc @@ -1,7 +1,7 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/providers/shared_library/provider_api.h" #include "core/providers/cuda/controlflow/if.h" #include "core/providers/cuda/cuda_fwd.h" @@ -16,8 +16,8 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(If, kOnnxDomain, 1, 10, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) // 'cond' needs to be on CPU + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) // 'cond' needs to be on CPU .TypeConstraint("B", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), If); @@ -28,8 +28,8 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(If, kOnnxDomain, 11, 12, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) // 'cond' needs to be on CPU + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) // 'cond' needs to be on CPU .TypeConstraint("B", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), If); @@ -41,22 +41,24 @@ ONNX_OPERATOR_KERNEL_EX(If, kOnnxDomain, 13, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) // 'cond' needs to be on CPU + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) // 'cond' needs to be on CPU .TypeConstraint("B", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), If); +If::If(const OpKernelInfo& info) : OpKernel(info), if_cpu_{CreateOpKernel_CPU_If(info)} { +} + Status If::Compute(OpKernelContext* ctx) const { // call the base CPU version. // we have this CUDA implementation so the inputs/outputs stay on GPU where possible. // the logic to run the subgraph must be on CPU either way. // technically we don't need this override of Compute, but it will be optimized out and it's easier to debug // that this implementation is being called with it. - auto status = onnxruntime::If::Compute(ctx); + auto status = if_cpu_->Compute(ctx); return status; } } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/controlflow/if.h b/onnxruntime/core/providers/cuda/controlflow/if.h index f182bbeba2..081109f182 100644 --- a/onnxruntime/core/providers/cuda/controlflow/if.h +++ b/onnxruntime/core/providers/cuda/controlflow/if.h @@ -14,11 +14,13 @@ class SessionState; namespace cuda { // Use the CPU implementation for the logic -class If final : public onnxruntime::If { +class If final : public OpKernel { public: - If(const OpKernelInfo& info) : onnxruntime::If(info) {} + If(const OpKernelInfo& info); Status Compute(OpKernelContext* ctx) const override; + + std::unique_ptr if_cpu_; }; } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/controlflow/loop.cc b/onnxruntime/core/providers/cuda/controlflow/loop.cc index bf4d325639..05e0a81cf2 100644 --- a/onnxruntime/core/providers/cuda/controlflow/loop.cc +++ b/onnxruntime/core/providers/cuda/controlflow/loop.cc @@ -1,10 +1,11 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/providers/shared_library/provider_api.h" #include "core/providers/cuda/controlflow/loop.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/cuda_fwd.h" +#include "core/framework/ml_value.h" using namespace ONNX_NAMESPACE; using namespace onnxruntime::common; @@ -16,9 +17,9 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(Loop, kOnnxDomain, 1, 10, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) // 'M' needs to be on CPU - .InputMemoryType(1) // 'cond' needs to be on CPU + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) // 'M' needs to be on CPU + .InputMemoryType(OrtMemTypeCPUInput, 1) // 'cond' needs to be on CPU .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("B", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), @@ -29,9 +30,9 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(Loop, kOnnxDomain, 11, 12, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) // 'M' needs to be on CPU - .InputMemoryType(1) // 'cond' needs to be on CPU + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) // 'M' needs to be on CPU + .InputMemoryType(OrtMemTypeCPUInput, 1) // 'cond' needs to be on CPU .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("B", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), @@ -44,9 +45,9 @@ ONNX_OPERATOR_KERNEL_EX(Loop, kOnnxDomain, 13, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) // 'M' needs to be on CPU - .InputMemoryType(1) // 'cond' needs to be on CPU + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) // 'M' needs to be on CPU + .InputMemoryType(OrtMemTypeCPUInput, 1) // 'cond' needs to be on CPU .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("B", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), @@ -70,7 +71,7 @@ static Status ConcatenateGpuOutput(void* stream, std::vector& per_iter } CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(cur_output, iteration_data.DataRaw(), bytes_per_iteration, - cudaMemcpyDeviceToDevice, static_cast(stream))); + cudaMemcpyDeviceToDevice, static_cast(stream))); cur_output = static_cast((static_cast(cur_output) + bytes_per_iteration)); } @@ -81,9 +82,8 @@ static Status ConcatenateGpuOutput(void* stream, std::vector& per_iter return Status::OK(); } -Loop::Loop(const OpKernelInfo& info) : onnxruntime::Loop(info) { - SetConcatOutputFunc(ConcatenateGpuOutput); - SetComputeStream(static_cast(info.GetExecutionProvider()->GetComputeStream())); +Loop::Loop(const OpKernelInfo& info) : OpKernel(info) { + cpu_loop_ = onnxruntime::Loop::Create(info, ConcatenateGpuOutput, static_cast(info.GetExecutionProvider()->GetComputeStream())); } Status Loop::Compute(OpKernelContext* ctx) const { @@ -92,10 +92,9 @@ Status Loop::Compute(OpKernelContext* ctx) const { // the logic to run the subgraph must be on CPU either way. // technically we don't need this override of Compute, but it will be optimized out and it's easier to debug // that this implementation is being called with it. - auto status = onnxruntime::Loop::Compute(ctx); + auto status = cpu_loop_->Compute(ctx); return status; } } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/controlflow/loop.h b/onnxruntime/core/providers/cuda/controlflow/loop.h index 4f3ccb7cd6..d71e52eecd 100644 --- a/onnxruntime/core/providers/cuda/controlflow/loop.h +++ b/onnxruntime/core/providers/cuda/controlflow/loop.h @@ -10,11 +10,14 @@ namespace onnxruntime { namespace cuda { // Use the CPU implementation for the logic -class Loop final : public onnxruntime::Loop { +class Loop final : public OpKernel { public: Loop(const OpKernelInfo& info); Status Compute(OpKernelContext* ctx) const override; + + private: + std::unique_ptr cpu_loop_; }; } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/controlflow/scan.cc b/onnxruntime/core/providers/cuda/controlflow/scan.cc index 848911e806..b29dc6d254 100644 --- a/onnxruntime/core/providers/cuda/controlflow/scan.cc +++ b/onnxruntime/core/providers/cuda/controlflow/scan.cc @@ -1,11 +1,13 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/providers/shared_library/provider_api.h" #include "core/providers/cuda/controlflow/scan.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/tensor/transpose.h" +#include "core/framework/ml_value.h" +#include "core/framework/ort_value_tensor_slicer.cc" using namespace ONNX_NAMESPACE; using namespace onnxruntime::common; @@ -14,7 +16,7 @@ namespace onnxruntime { namespace cuda { template <> -Scan<8>::Scan(const OpKernelInfo& info) : onnxruntime::Scan<8>(info) { +Scan<8>::Scan(const OpKernelInfo& info) : OpKernel(info), scan_cpu_{static_cast*>(CreateOpKernel_CPU_Scan_8(info).release())} { scan::detail::DeviceHelpers helpers; helpers.set_data_to_zero_func = [](void* data, size_t size_in_bytes) -> Status { @@ -23,11 +25,11 @@ Scan<8>::Scan(const OpKernelInfo& info) : onnxruntime::Scan<8>(info) { }; // copy into base class - SetDeviceHelpers(helpers); + scan_cpu_->SetDeviceHelpers(helpers); } template <> -Scan<9>::Scan(const OpKernelInfo& info) : onnxruntime::Scan<9>(info) { +Scan<9>::Scan(const OpKernelInfo& info) : OpKernel(info), scan_cpu_{static_cast*>(CreateOpKernel_CPU_Scan_9(info).release())} { scan::detail::DeviceHelpers helpers; helpers.transpose_func = [this](const std::vector& permutations, const Tensor& input, Tensor& output) { @@ -38,7 +40,7 @@ Scan<9>::Scan(const OpKernelInfo& info) : onnxruntime::Scan<9>(info) { }; // copy into base class - SetDeviceHelpers(helpers); + scan_cpu_->SetDeviceHelpers(helpers); } template <> @@ -48,7 +50,7 @@ Status Scan<8>::Compute(OpKernelContext* ctx) const { // the logic to run the subgraph must be on CPU either way. // technically we don't need this override of Compute, but it will be optimized out and it's easier to debug // that this implementation is being called with it. - auto status = onnxruntime::Scan<8>::Compute(ctx); + auto status = scan_cpu_->Compute(ctx); return status; } @@ -59,7 +61,7 @@ Status Scan<9>::Compute(OpKernelContext* ctx) const { // the logic to run the subgraph must be on CPU either way. // technically we don't need this override of Compute, but it will be optimized out and it's easier to debug // that this implementation is being called with it. - auto status = onnxruntime::Scan<9>::Compute(ctx); + auto status = scan_cpu_->Compute(ctx); return status; } @@ -67,8 +69,8 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(Scan, kOnnxDomain, 8, 8, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) // 'sequence_lens' needs to be on CPU + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) // 'sequence_lens' needs to be on CPU .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllTensorTypes()), Scan<8>); @@ -77,7 +79,7 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(Scan, kOnnxDomain, 9, 10, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), Scan<9>); @@ -87,11 +89,10 @@ ONNX_OPERATOR_KERNEL_EX(Scan, kOnnxDomain, 11, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("V", DataTypeImpl::AllFixedSizeTensorTypes()), Scan<9>); } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/controlflow/scan.h b/onnxruntime/core/providers/cuda/controlflow/scan.h index 577f3d8fc2..eadce19a98 100644 --- a/onnxruntime/core/providers/cuda/controlflow/scan.h +++ b/onnxruntime/core/providers/cuda/controlflow/scan.h @@ -5,7 +5,6 @@ #include #include "gsl/gsl" -#include "core/common/common.h" #include "core/providers/cpu/controlflow/scan.h" namespace onnxruntime { @@ -15,11 +14,14 @@ namespace cuda { // Use the CPU implementation for the logic template -class Scan final : public onnxruntime::Scan { +class Scan final : public OpKernel { public: Scan(const OpKernelInfo& info); Status Compute(OpKernelContext* ctx) const override; + + private: + std::unique_ptr> scan_cpu_; }; } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_allocator.cc b/onnxruntime/core/providers/cuda/cuda_allocator.cc index 50704a37bb..9dad792efc 100644 --- a/onnxruntime/core/providers/cuda/cuda_allocator.cc +++ b/onnxruntime/core/providers/cuda/cuda_allocator.cc @@ -4,7 +4,6 @@ #include "cuda_allocator.h" #include "cuda_common.h" #include "core/framework/allocatormgr.h" -//#include "core/framework/session_state.h" #include "cuda_fence.h" #include "gpu_data_transfer.h" @@ -71,7 +70,6 @@ void* CUDAExternalAllocator::Alloc(size_t size) { // review(codemzs): ORT_ENFORCE does not seem appropiate. ORT_ENFORCE(p != nullptr); - } return p; diff --git a/onnxruntime/core/providers/cuda/cuda_call.cc b/onnxruntime/core/providers/cuda/cuda_call.cc index 4e66d1edf1..794e0cdd3a 100644 --- a/onnxruntime/core/providers/cuda/cuda_call.cc +++ b/onnxruntime/core/providers/cuda/cuda_call.cc @@ -3,9 +3,6 @@ #include "core/providers/shared_library/provider_api.h" #include "shared_inc/cuda_call.h" -//#include "core/common/common.h" -//#include "core/common/status.h" -//#include "core/common/logging/logging.h" #ifdef _WIN32 #else // POSIX diff --git a/onnxruntime/core/providers/cuda/cuda_common.h b/onnxruntime/core/providers/cuda/cuda_common.h index d1bb2a109e..44028659d0 100644 --- a/onnxruntime/core/providers/cuda/cuda_common.h +++ b/onnxruntime/core/providers/cuda/cuda_common.h @@ -10,7 +10,6 @@ #include "core/providers/cuda/shared_inc/cuda_call.h" #include "core/providers/cuda/shared_inc/fast_divmod.h" #include "gsl/gsl" -//#include "core/util/math.h" // Can't include "core/util/math.h" in a provider, so this is the part we need for cuda: namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc index c04bbaab89..47645b47a5 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc @@ -3,12 +3,6 @@ #include "core/providers/shared_library/provider_api.h" #include "core/providers/cuda/cuda_execution_provider.h" -//#include "core/framework/compute_capability.h" -//#include "core/framework/fallback_cpu_capability.h" -//#include "core/framework/kernel_registry.h" -//#include "core/framework/memcpy.h" -//#include "core/framework/provider_options_utils.h" -//#include "core/graph/graph_utils.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/cuda_allocator.h" #include "core/providers/cuda/cuda_fence.h" diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.h b/onnxruntime/core/providers/cuda/cuda_execution_provider.h index f8436be9a2..ac4a1633e6 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.h +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.h @@ -6,7 +6,6 @@ #include #include -//#include "core/graph/constants.h" #include "core/framework/allocatormgr.h" #include "core/framework/arena_extend_strategy.h" #include "core/framework/execution_provider.h" diff --git a/onnxruntime/core/providers/cuda/cuda_fence.cc b/onnxruntime/core/providers/cuda/cuda_fence.cc index 95404319f9..74462650a5 100644 --- a/onnxruntime/core/providers/cuda/cuda_fence.cc +++ b/onnxruntime/core/providers/cuda/cuda_fence.cc @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -//#include "core/graph/constants.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/gpu_data_transfer.h" #include "core/providers/cuda/cuda_fence.h" diff --git a/onnxruntime/core/providers/cuda/cuda_fence.h b/onnxruntime/core/providers/cuda/cuda_fence.h index cd20c0298c..1d0c5b3512 100644 --- a/onnxruntime/core/providers/cuda/cuda_fence.h +++ b/onnxruntime/core/providers/cuda/cuda_fence.h @@ -3,9 +3,6 @@ #pragma once -//#include "core/framework/tensor.h" -//#include "core/graph/basic_types.h" -//#include "core/providers/cuda/cuda_common.h" #include "core/framework/fence.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/cuda_fwd.h b/onnxruntime/core/providers/cuda/cuda_fwd.h index 082672327f..4f2cc556c0 100644 --- a/onnxruntime/core/providers/cuda/cuda_fwd.h +++ b/onnxruntime/core/providers/cuda/cuda_fwd.h @@ -3,8 +3,6 @@ #pragma once -//#include "core/framework/op_kernel.h" - namespace onnxruntime { namespace cuda { template diff --git a/onnxruntime/core/providers/cuda/cuda_kernel.h b/onnxruntime/core/providers/cuda/cuda_kernel.h index 060fceacba..f6fe4807b7 100644 --- a/onnxruntime/core/providers/cuda/cuda_kernel.h +++ b/onnxruntime/core/providers/cuda/cuda_kernel.h @@ -3,9 +3,6 @@ #pragma once -//#include "core/common/status.h" -//#include "core/framework/data_transfer_manager.h" -//#include "core/framework/op_kernel.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/cuda_execution_provider.h" #include "core/providers/cuda/cuda_fwd.h" diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc index 945b6a3206..0e7219813d 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc @@ -11,9 +11,9 @@ #include "core/common/make_unique.h" #include "core/providers/cuda/cuda_execution_provider.h" -//#include "core/session/abi_session_options_impl.h" -//#include "core/session/onnxruntime_c_api.h" -//#include "core/session/ort_apis.h" +#include "core/providers/cuda/cuda_allocator.h" +#include "core/providers/cuda/gpu_data_transfer.h" +#include "core/providers/cuda/math/unary_elementwise_ops_impl.h" using namespace onnxruntime; @@ -40,72 +40,86 @@ std::shared_ptr CreateExecutionProviderFactory_CUDA(c } // namespace onnxruntime -#if 0 -ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id) { - CUDAExecutionProviderInfo info{}; - info.device_id = gsl::narrow(device_id); - - options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CUDA(info)); - - return nullptr; -} - -ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA, - _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options) { - CUDAExecutionProviderInfo info{}; - info.device_id = gsl::narrow(cuda_options->device_id); - info.cuda_mem_limit = cuda_options->cuda_mem_limit; - info.arena_extend_strategy = static_cast(cuda_options->arena_extend_strategy); - info.cudnn_conv_algo_search = cuda_options->cudnn_conv_algo_search; - info.do_copy_in_default_stream = cuda_options->do_copy_in_default_stream; - info.has_user_compute_stream = cuda_options->has_user_compute_stream; - info.user_compute_stream = cuda_options->user_compute_stream; - options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CUDA(info)); - - return nullptr; -} - -ORT_API_STATUS_IMPL(OrtApis::SetCurrentGpuDeviceId, _In_ int device_id) { - int num_devices; - auto cuda_err = cudaGetDeviceCount(&num_devices); - if (cuda_err != cudaSuccess) { - return CreateStatus(ORT_FAIL, "Failed to set device id since cudaGetDeviceCount failed."); - } - - if (device_id >= num_devices) { - std::ostringstream ostr; - ostr << "Invalid device id. Device id should be less than total number of devices (" << num_devices << ")"; - return CreateStatus(ORT_INVALID_ARGUMENT, ostr.str().c_str()); - } - - cuda_err = cudaSetDevice(device_id); - if (cuda_err != cudaSuccess) { - return CreateStatus(ORT_FAIL, "Failed to set device id."); - } - return nullptr; -} - -ORT_API_STATUS_IMPL(OrtApis::GetCurrentGpuDeviceId, _In_ int* device_id) { - auto cuda_err = cudaGetDevice(device_id); - if (cuda_err != cudaSuccess) { - return CreateStatus(ORT_FAIL, "Failed to get device id."); - } - return nullptr; -} -#endif - namespace onnxruntime { -#if 0 struct ProviderInfo_CUDA_Impl : ProviderInfo_CUDA { - std::vector GetAvailableDevices() const override { - InferenceEngine::Core ie_core; - return ie_core.GetAvailableDevices(); + OrtStatus* SetCurrentGpuDeviceId(_In_ int device_id) override { + int num_devices; + auto cuda_err = cudaGetDeviceCount(&num_devices); + if (cuda_err != cudaSuccess) { + return CreateStatus(ORT_FAIL, "Failed to set device id since cudaGetDeviceCount failed."); + } + + if (device_id >= num_devices) { + std::ostringstream ostr; + ostr << "Invalid device id. Device id should be less than total number of devices (" << num_devices << ")"; + return CreateStatus(ORT_INVALID_ARGUMENT, ostr.str().c_str()); + } + + cuda_err = cudaSetDevice(device_id); + if (cuda_err != cudaSuccess) { + return CreateStatus(ORT_FAIL, "Failed to set device id."); + } + return nullptr; } + + OrtStatus* GetCurrentGpuDeviceId(_In_ int* device_id) override { + auto cuda_err = cudaGetDevice(device_id); + if (cuda_err != cudaSuccess) { + return CreateStatus(ORT_FAIL, "Failed to get device id."); + } + return nullptr; + } + + std::unique_ptr CreateCUDAAllocator(int16_t device_id, const char* name) override { + return onnxruntime::make_unique(device_id, name); + } + + std::unique_ptr CreateCUDAPinnedAllocator(int16_t device_id, const char* name) override { + return onnxruntime::make_unique(device_id, name); + } + + std::unique_ptr CreateGPUDataTransfer(void* stream) override { + return onnxruntime::make_unique(static_cast(stream)); + } + + void cuda__Impl_Cast(void* stream, const int64_t* input_data, int32_t* output_data, size_t count) override { + return cuda::Impl_Cast(static_cast(stream), input_data, output_data, count); + } + + void cuda__Impl_Cast(void* stream, const int32_t* input_data, int64_t* output_data, size_t count) override { + return cuda::Impl_Cast(static_cast(stream), input_data, output_data, count); + } + + bool CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg); } + bool CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg); } + + void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const OrtMemoryInfo& dst_location, const OrtMemoryInfo& src_location) override { + ORT_ENFORCE(dst_location.device.Type() == OrtDevice::CPU); + + // Current CUDA device. + int device; + CUDA_CALL(cudaGetDevice(&device)); + + if (device != src_location.id) { + // Need to switch to the allocating device. + CUDA_CALL(cudaSetDevice(src_location.id)); + // Copy from GPU to CPU. + CUDA_CALL(cudaMemcpy(dst_ptr, src_ptr, size, cudaMemcpyDeviceToHost)); + // Switch back to current device. + CUDA_CALL(cudaSetDevice(device)); + } else { + // Copy from GPU to CPU. + CUDA_CALL(cudaMemcpy(dst_ptr, src_ptr, size, cudaMemcpyDeviceToHost)); + } + } + + // Used only by slice_concatenate_test.cc + void cudaMemcpy_HostToDevice(void* dst, const void* src, size_t count) override { cudaMemcpy(dst, src, count, cudaMemcpyHostToDevice); } + } g_info; -#endif struct CUDA_Provider : Provider { - // const void* GetInfo() override { return &g_info; } + void* GetInfo() override { return &g_info; } std::shared_ptr CreateExecutionProviderFactory(const void* void_params) override { auto params = reinterpret_cast(void_params); @@ -116,12 +130,13 @@ struct CUDA_Provider : Provider { info.arena_extend_strategy = static_cast(params->arena_extend_strategy); info.cudnn_conv_algo_search = params->cudnn_conv_algo_search; info.do_copy_in_default_stream = params->do_copy_in_default_stream; + info.has_user_compute_stream = params->has_user_compute_stream; + info.user_compute_stream = params->user_compute_stream; return std::make_shared(info); } void Shutdown() override { - // openvino_ep::BackendManager::ReleaseGlobalContext(); } } g_provider; @@ -134,4 +149,3 @@ ORT_API(onnxruntime::Provider*, GetProvider) { return &onnxruntime::g_provider; } } - diff --git a/onnxruntime/core/providers/cuda/cudnn_common.h b/onnxruntime/core/providers/cuda/cudnn_common.h index 96b2eeda7b..b9337d9127 100644 --- a/onnxruntime/core/providers/cuda/cudnn_common.h +++ b/onnxruntime/core/providers/cuda/cudnn_common.h @@ -5,8 +5,6 @@ #include -//#include "core/common/logging/logging.h" -//#include "core/framework/tensor.h" #include "core/providers/cuda/cuda_common.h" namespace onnxruntime { @@ -127,7 +125,7 @@ struct Consts { }; #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 -template<> +template <> struct Consts { static const float Zero; static const float One; diff --git a/onnxruntime/core/providers/cuda/generator/constant_of_shape.cc b/onnxruntime/core/providers/cuda/generator/constant_of_shape.cc index 156972fb6a..bdcd3ff48e 100644 --- a/onnxruntime/core/providers/cuda/generator/constant_of_shape.cc +++ b/onnxruntime/core/providers/cuda/generator/constant_of_shape.cc @@ -1,10 +1,6 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - #include "constant_of_shape.h" -#include "core/providers/common.h" -#include "gsl/gsl" using namespace ::onnxruntime::common; using namespace ONNX_NAMESPACE; @@ -16,8 +12,8 @@ ONNX_OPERATOR_KERNEL_EX( kOnnxDomain, 9, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) .TypeConstraint("T1", DataTypeImpl::GetTensorType()) .TypeConstraint("T2", DataTypeImpl::AllFixedSizeTensorTypes()), ConstantOfShape); @@ -30,11 +26,11 @@ Status ConstantOfShape::ComputeInternal(OpKernelContext* ctx) const { const void* value_ptr = GetValuePtr(); const auto element_size = output_tensor->DataType()->Size(); -#define CASE(TYPE) \ - case sizeof(TYPE): \ - if (size > 0) { \ - cuda::Fill(Stream(), reinterpret_cast(output_data), *(reinterpret_cast(value_ptr)), size); \ - } \ +#define CASE(TYPE) \ + case sizeof(TYPE): \ + if (size > 0) { \ + cuda::Fill(Stream(), reinterpret_cast(output_data), *(reinterpret_cast(value_ptr)), size); \ + } \ break; switch (element_size) { @@ -52,4 +48,3 @@ Status ConstantOfShape::ComputeInternal(OpKernelContext* ctx) const { } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/generator/constant_of_shape.h b/onnxruntime/core/providers/cuda/generator/constant_of_shape.h index a0870d915a..99c5da0615 100644 --- a/onnxruntime/core/providers/cuda/generator/constant_of_shape.h +++ b/onnxruntime/core/providers/cuda/generator/constant_of_shape.h @@ -3,8 +3,6 @@ #pragma once -#include "core/common/common.h" -#include "core/framework/data_types.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/generator/constant_of_shape_base.h" #include "core/providers/cuda/shared_inc/cuda_utils.h" diff --git a/onnxruntime/core/providers/cuda/generator/range.cc b/onnxruntime/core/providers/cuda/generator/range.cc index d4539bde2b..5b2afa01fc 100644 --- a/onnxruntime/core/providers/cuda/generator/range.cc +++ b/onnxruntime/core/providers/cuda/generator/range.cc @@ -1,8 +1,7 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/framework/tensorprotoutils.h" +#include "core/providers/shared_library/provider_api.h" #include "core/providers/cuda/cuda_common.h" #include "range.h" #include "range_impl.h" @@ -19,10 +18,10 @@ ONNX_OPERATOR_KERNEL_EX( kOnnxDomain, 11, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) // start - .InputMemoryType(1) // limit - .InputMemoryType(2) // delta + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) // start + .InputMemoryType(OrtMemTypeCPUInput, 1) // limit + .InputMemoryType(OrtMemTypeCPUInput, 2) // delta .TypeConstraint("T", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), @@ -52,7 +51,7 @@ static Status ComputeRange(cudaStream_t stream, OpKernelContext* ctx) { delta_tensor_ptr->Shape()); } - // Start, Limit and Delta are stored in CPU. + // Start, Limit and Delta are stored in CPU. T start = *(start_tensor.template Data()); T limit = *(limit_tensor.template Data()); @@ -105,4 +104,3 @@ Status Range::ComputeInternal(OpKernelContext* ctx) const { } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/math/clip.cc b/onnxruntime/core/providers/cuda/math/clip.cc index 4d5fc01df1..a2d6015ef7 100644 --- a/onnxruntime/core/providers/cuda/math/clip.cc +++ b/onnxruntime/core/providers/cuda/math/clip.cc @@ -1,8 +1,6 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/providers/common.h" #include "core/providers/cuda/math/clip.h" #include "core/providers/cuda/math/clip_impl.h" @@ -16,7 +14,7 @@ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( 10, float, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("T", DataTypeImpl::GetTensorType()), Clip_6); @@ -25,9 +23,9 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX( kOnnxDomain, 11, 11, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(1) - .InputMemoryType(2) + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 1) + .InputMemoryType(OrtMemTypeCPUInput, 2) .TypeConstraint("T", DataTypeImpl::GetTensorType()), Clip); @@ -36,9 +34,9 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX( kOnnxDomain, 12, 12, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(1) - .InputMemoryType(2) + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 1) + .InputMemoryType(OrtMemTypeCPUInput, 2) .TypeConstraint("T", BuildKernelDefConstraints()), Clip); @@ -47,13 +45,12 @@ ONNX_OPERATOR_KERNEL_EX( kOnnxDomain, 13, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(1) - .InputMemoryType(2) + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 1) + .InputMemoryType(OrtMemTypeCPUInput, 2) .TypeConstraint("T", BuildKernelDefConstraints()), Clip); - template Status Clip_6::ComputeInternal(OpKernelContext* ctx) const { const Tensor& X = *ctx->Input(0); @@ -132,4 +129,3 @@ Status Clip::ComputeInternal(OpKernelContext* ctx) const { } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/math/cumsum.cc b/onnxruntime/core/providers/cuda/math/cumsum.cc index 9d6e06ecf2..7e496ef928 100644 --- a/onnxruntime/core/providers/cuda/math/cumsum.cc +++ b/onnxruntime/core/providers/cuda/math/cumsum.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -15,8 +14,8 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX( kOnnxDomain, 11, 13, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(1) // 'axis' needs to be on CPU + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 1) // 'axis' needs to be on CPU .TypeConstraint("T", std::vector{ DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), @@ -32,8 +31,8 @@ ONNX_OPERATOR_KERNEL_EX( kOnnxDomain, 14, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(1) // 'axis' needs to be on CPU + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 1) // 'axis' needs to be on CPU .TypeConstraint("T", std::vector{ DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), @@ -41,7 +40,7 @@ ONNX_OPERATOR_KERNEL_EX( DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType()}) // MLFloat16 is added in opset 14 + DataTypeImpl::GetTensorType()}) // MLFloat16 is added in opset 14 .TypeConstraint("T2", std::vector{DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), CumSum); @@ -143,4 +142,3 @@ Status CumSum::ComputeInternal(OpKernelContext* ctx) const { } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/math/einsum.cc b/onnxruntime/core/providers/cuda/math/einsum.cc index dcdced3224..0cee71f62a 100644 --- a/onnxruntime/core/providers/cuda/math/einsum.cc +++ b/onnxruntime/core/providers/cuda/math/einsum.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -6,6 +5,13 @@ namespace onnxruntime { +// This function must exist due to the C++ base class constructor needing this to be defined for the vtable, but it is never called. +Status Einsum::DeviceCompute(OpKernelContext* context, const std::vector& inputs, + AllocatorPtr allocator, concurrency::ThreadPool* tp) const { + assert(false); + return Status::OK(); +} + namespace cuda { ONNX_OPERATOR_KERNEL_EX( @@ -13,11 +19,7 @@ ONNX_OPERATOR_KERNEL_EX( kOnnxDomain, 12, kCudaExecutionProvider, - KernelDefBuilder().TypeConstraint("T", - std::vector{ - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType(), - DataTypeImpl::GetTensorType()}), + (*KernelDefBuilder::Create()).TypeConstraint("T", std::vector{DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), Einsum); Status Einsum::Compute(OpKernelContext* context) const { @@ -31,46 +33,46 @@ Status Einsum::DeviceCompute(OpKernelContext* context, const std::vectorSetDeviceHelpers(EinsumOp::DeviceHelpers::CudaDeviceHelpers::Diagonal, + EinsumOp::DeviceHelpers::CudaDeviceHelpers::Transpose); // Compute all required metadata to be used at Einsum compute time and return error status code if one was generated - ORT_RETURN_IF_ERROR(einsum_compute_preprocessor.Run()); + ORT_RETURN_IF_ERROR(einsum_compute_preprocessor->Run()); // EinsumComputeProcessor section - if (inputs[0]->IsDataType()) { - auto einsum_compute_processor = EinsumTypedComputeProcessor(context, allocator, tp, - einsum_compute_preprocessor, - &einsum_cuda_assets); + auto einsum_compute_processor = EinsumTypedComputeProcessor::Create(context, allocator, tp, + *einsum_compute_preprocessor, + &einsum_cuda_assets); - einsum_compute_processor.SetDeviceHelpers(EinsumOp::DeviceHelpers::CudaDeviceHelpers::Transpose, - EinsumOp::DeviceHelpers::CudaDeviceHelpers::MatMul, - EinsumOp::DeviceHelpers::CudaDeviceHelpers::ReduceSum, - EinsumOp::DeviceHelpers::CudaDeviceHelpers::DataCopy); - return einsum_compute_processor.Run(); + einsum_compute_processor->SetDeviceHelpers(EinsumOp::DeviceHelpers::CudaDeviceHelpers::Transpose, + EinsumOp::DeviceHelpers::CudaDeviceHelpers::MatMul, + EinsumOp::DeviceHelpers::CudaDeviceHelpers::ReduceSum, + EinsumOp::DeviceHelpers::CudaDeviceHelpers::DataCopy); + return einsum_compute_processor->Run(); } else if (inputs[0]->IsDataType()) { - auto einsum_compute_processor = EinsumTypedComputeProcessor(context, allocator, tp, - einsum_compute_preprocessor, - &einsum_cuda_assets); + auto einsum_compute_processor = EinsumTypedComputeProcessor::Create(context, allocator, tp, + *einsum_compute_preprocessor, + &einsum_cuda_assets); // Set device specific methods (CPU methods) to be used during processing - einsum_compute_processor.SetDeviceHelpers(EinsumOp::DeviceHelpers::CudaDeviceHelpers::Transpose, - EinsumOp::DeviceHelpers::CudaDeviceHelpers::MatMul, - EinsumOp::DeviceHelpers::CudaDeviceHelpers::ReduceSum, - EinsumOp::DeviceHelpers::CudaDeviceHelpers::DataCopy); - return einsum_compute_processor.Run(); + einsum_compute_processor->SetDeviceHelpers(EinsumOp::DeviceHelpers::CudaDeviceHelpers::Transpose, + EinsumOp::DeviceHelpers::CudaDeviceHelpers::MatMul, + EinsumOp::DeviceHelpers::CudaDeviceHelpers::ReduceSum, + EinsumOp::DeviceHelpers::CudaDeviceHelpers::DataCopy); + return einsum_compute_processor->Run(); } else if (inputs[0]->IsDataType()) { - auto einsum_compute_processor = EinsumTypedComputeProcessor(context, allocator, tp, - einsum_compute_preprocessor, - &einsum_cuda_assets); + auto einsum_compute_processor = EinsumTypedComputeProcessor::Create(context, allocator, tp, + *einsum_compute_preprocessor, + &einsum_cuda_assets); - einsum_compute_processor.SetDeviceHelpers(EinsumOp::DeviceHelpers::CudaDeviceHelpers::Transpose, - EinsumOp::DeviceHelpers::CudaDeviceHelpers::MatMul, - EinsumOp::DeviceHelpers::CudaDeviceHelpers::ReduceSum, - EinsumOp::DeviceHelpers::CudaDeviceHelpers::DataCopy); - return einsum_compute_processor.Run(); + einsum_compute_processor->SetDeviceHelpers(EinsumOp::DeviceHelpers::CudaDeviceHelpers::Transpose, + EinsumOp::DeviceHelpers::CudaDeviceHelpers::MatMul, + EinsumOp::DeviceHelpers::CudaDeviceHelpers::ReduceSum, + EinsumOp::DeviceHelpers::CudaDeviceHelpers::DataCopy); + return einsum_compute_processor->Run(); } return ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, @@ -81,4 +83,3 @@ Status Einsum::DeviceCompute(OpKernelContext* context, const std::vector( size_t num_batches, size_t M, size_t K, size_t N, concurrency::ThreadPool* tp, void* einsum_cuda_assets); -template Tensor DeviceHelpers::CudaDeviceHelpers::ReduceSum( +template std::unique_ptr DeviceHelpers::CudaDeviceHelpers::ReduceSum( const Tensor& input, const std::vector& reduce_axes, bool keep_dims, AllocatorPtr allocator, const TensorShape* input_shape_override, @@ -190,4 +189,3 @@ template Tensor DeviceHelpers::CudaDeviceHelpers::ReduceSum( } // namespace EinsumOp } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.h b/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.h index 797f8f301e..6ba139f2da 100644 --- a/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.h +++ b/onnxruntime/core/providers/cuda/math/einsum_utils/einsum_auxiliary_ops.h @@ -47,10 +47,10 @@ Status MatMul(const T* input_1_data, const T* input_2_data, T* output_data, void* einsum_cuda_assets); template -Tensor ReduceSum(const Tensor& input, const std::vector& reduce_axes, - bool keep_dims, AllocatorPtr allocator, - const TensorShape* input_shape_override, - concurrency::ThreadPool* /*tp*/, void* einsum_cuda_assets); +std::unique_ptr ReduceSum(const Tensor& input, const std::vector& reduce_axes, + bool keep_dims, AllocatorPtr allocator, + const TensorShape* input_shape_override, + concurrency::ThreadPool* /*tp*/, void* einsum_cuda_assets); std::unique_ptr Diagonal(const Tensor& input, int64_t dim_1, int64_t dim_2, AllocatorPtr allocator, void* einsum_cuda_assets); diff --git a/onnxruntime/core/providers/cuda/math/gemm.cc b/onnxruntime/core/providers/cuda/math/gemm.cc index 1bdec3d558..def63e4db2 100644 --- a/onnxruntime/core/providers/cuda/math/gemm.cc +++ b/onnxruntime/core/providers/cuda/math/gemm.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -18,7 +17,7 @@ namespace cuda { 8, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ Gemm); \ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ @@ -28,7 +27,7 @@ namespace cuda { 10, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ Gemm); \ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ @@ -38,7 +37,7 @@ namespace cuda { 12, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ Gemm); \ ONNX_OPERATOR_TYPED_KERNEL_EX( \ @@ -47,7 +46,7 @@ namespace cuda { 13, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ Gemm); @@ -147,4 +146,3 @@ Status Gemm::ComputeInternal(OpKernelContext* ctx) const { } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/math/matmul.cc b/onnxruntime/core/providers/cuda/math/matmul.cc index 6319634a4f..1a233aec96 100644 --- a/onnxruntime/core/providers/cuda/math/matmul.cc +++ b/onnxruntime/core/providers/cuda/math/matmul.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -17,7 +16,7 @@ namespace cuda { 1, 8, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ MatMul); \ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ @@ -26,7 +25,7 @@ namespace cuda { 9, 12, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ MatMul); \ ONNX_OPERATOR_TYPED_KERNEL_EX( \ @@ -35,7 +34,7 @@ namespace cuda { 13, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ MatMul); @@ -197,4 +196,3 @@ Status MatMul::ComputeInternal(OpKernelContext* ctx) const { } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/math/matmul_integer.cc b/onnxruntime/core/providers/cuda/math/matmul_integer.cc index d52435e402..57e835da20 100644 --- a/onnxruntime/core/providers/cuda/math/matmul_integer.cc +++ b/onnxruntime/core/providers/cuda/math/matmul_integer.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -19,9 +18,9 @@ ONNX_OPERATOR_TYPED_KERNEL_EX( 10, int8_t, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(2) - .InputMemoryType(3) + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 2) + .InputMemoryType(OrtMemTypeCPUInput, 3) .TypeConstraint("T1", DataTypeImpl::GetTensorType()) .TypeConstraint("T2", DataTypeImpl::GetTensorType()) .TypeConstraint("T3", DataTypeImpl::GetTensorType()), @@ -113,4 +112,3 @@ Status MatMulInteger::ComputeInternal(OpKernelContext* ctx) cons } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/math/softmax.cc b/onnxruntime/core/providers/cuda/math/softmax.cc index 017070bd76..a9f2f61f63 100644 --- a/onnxruntime/core/providers/cuda/math/softmax.cc +++ b/onnxruntime/core/providers/cuda/math/softmax.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -51,7 +50,7 @@ Status SoftMaxComputeHelper( return Status::OK(); } -#define SPECIALIZED_SOFTMAX_HELPER_IMPL(T) \ +#define SPECIALIZED_SOFTMAX_HELPER_IMPL(T) \ template Status SoftMaxComputeHelper(cudaStream_t stream, const T* input, const TensorShape& shape, T* Y, cudnnHandle_t handle, int64_t axis); \ template Status SoftMaxComputeHelper(cudaStream_t stream, const T* input, const TensorShape& shape, T* Y, cudnnHandle_t handle, int64_t axis); @@ -61,81 +60,81 @@ SPECIALIZED_SOFTMAX_HELPER_IMPL(MLFloat16) #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 // cudnnSoftmaxForward/Backward doesn't support BFloat16. -#define SPECIALIZED_SOFTMAX_HELPER_IMPL_BFloat16(is_log_softmax) \ - template <> \ - Status SoftMaxComputeHelper( \ - cudaStream_t stream, \ - const BFloat16* X, \ - const TensorShape& input_shape, \ - BFloat16* Y, \ - cudnnHandle_t, \ - int64_t axis) { \ - typedef typename ToCudaType::MappedType CudaT; \ - int64_t N = input_shape.SizeToDimension(axis); \ - int64_t D = input_shape.SizeFromDimension(axis); \ - auto Y_data = reinterpret_cast(Y); \ - auto X_data = reinterpret_cast(X); \ - dispatch_softmax_forward, is_log_softmax>( \ - stream, Y_data, X_data, gsl::narrow_cast(D), gsl::narrow_cast(D), gsl::narrow_cast(N)); \ - return Status::OK(); \ +#define SPECIALIZED_SOFTMAX_HELPER_IMPL_BFloat16(is_log_softmax) \ + template <> \ + Status SoftMaxComputeHelper( \ + cudaStream_t stream, \ + const BFloat16* X, \ + const TensorShape& input_shape, \ + BFloat16* Y, \ + cudnnHandle_t, \ + int64_t axis) { \ + typedef typename ToCudaType::MappedType CudaT; \ + int64_t N = input_shape.SizeToDimension(axis); \ + int64_t D = input_shape.SizeFromDimension(axis); \ + auto Y_data = reinterpret_cast(Y); \ + auto X_data = reinterpret_cast(X); \ + dispatch_softmax_forward, is_log_softmax>( \ + stream, Y_data, X_data, gsl::narrow_cast(D), gsl::narrow_cast(D), gsl::narrow_cast(N)); \ + return Status::OK(); \ } SPECIALIZED_SOFTMAX_HELPER_IMPL_BFloat16(true) -SPECIALIZED_SOFTMAX_HELPER_IMPL_BFloat16(false) + SPECIALIZED_SOFTMAX_HELPER_IMPL_BFloat16(false) #endif -#define REGISTER_KERNEL_TYPED(T) \ - ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ - Softmax, \ - kOnnxDomain, \ - 1, 10, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - Softmax); \ - ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ - Softmax, \ - kOnnxDomain, \ - 11, 12, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - Softmax); \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - Softmax, \ - kOnnxDomain, \ - 13, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - Softmax); \ - ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ - LogSoftmax, \ - kOnnxDomain, \ - 1, 10, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - Softmax); \ - ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ - LogSoftmax, \ - kOnnxDomain, \ - 11, 12, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - Softmax); \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - LogSoftmax, \ - kOnnxDomain, \ - 13, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + Softmax, \ + kOnnxDomain, \ + 1, 10, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Softmax); \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + Softmax, \ + kOnnxDomain, \ + 11, 12, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Softmax); \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Softmax, \ + kOnnxDomain, \ + 13, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Softmax); \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + LogSoftmax, \ + kOnnxDomain, \ + 1, 10, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Softmax); \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + LogSoftmax, \ + kOnnxDomain, \ + 11, 12, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Softmax); \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + LogSoftmax, \ + kOnnxDomain, \ + 13, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ Softmax); -template -Status Softmax::ComputeInternal(OpKernelContext* ctx) const { + template + Status Softmax::ComputeInternal(OpKernelContext* ctx) const { const Tensor* X = ctx->Input(0); const TensorShape& input_shape{X->Shape()}; size_t rank = input_shape.NumDimensions(); @@ -149,9 +148,9 @@ Status Softmax::ComputeInternal(OpKernelContext* ctx) const { const size_t axis = static_cast(HandleNegativeAxis(axis_, rank)); bool is_transpose_required = false; - Tensor transposed_input; + std::unique_ptr transposed_input; std::vector transposed_input_dims; - Tensor intermediate_output; // output that the softmax implementation will write into while using transposed input + std::unique_ptr intermediate_output; // output that the softmax implementation will write into while using transposed input std::vector permutation(rank); // The "semantic" meaning of axis has changed in opset-13. @@ -182,18 +181,17 @@ Status Softmax::ComputeInternal(OpKernelContext* ctx) const { } // Allocate a temporary tensor to hold transposed input - Tensor temp_input(X->DataType(), TensorShape(transposed_input_dims), alloc); + auto temp_input = Tensor::Create(X->DataType(), TensorShape(transposed_input_dims), alloc); // Perform the transpose ORT_RETURN_IF_ERROR(Transpose::DoTranspose(cuda_ep_->GetDeviceProp(), Stream(), CublasHandle(), - permutation, *X, temp_input)); + permutation, *X, *temp_input)); transposed_input = std::move(temp_input); // Allocate memory for the intermediate output - Tensor temp_output(Y->DataType(), TensorShape(transposed_input_dims), alloc); - intermediate_output = std::move(temp_output); + intermediate_output = Tensor::Create(Y->DataType(), TensorShape(transposed_input_dims), alloc); } const T* X_data = nullptr; @@ -201,9 +199,9 @@ Status Softmax::ComputeInternal(OpKernelContext* ctx) const { const TensorShape* compute_input_shape = nullptr; if (is_transpose_required) { // use intermediate buffers to compute the softmax values - X_data = transposed_input.template Data(); - Y_data = intermediate_output.template MutableData(); - compute_input_shape = &transposed_input.Shape(); + X_data = transposed_input->template Data(); + Y_data = intermediate_output->template MutableData(); + compute_input_shape = &transposed_input->Shape(); } else { // use the node input/output directly X_data = X->template Data(); Y_data = Y->template MutableData(); @@ -233,7 +231,7 @@ Status Softmax::ComputeInternal(OpKernelContext* ctx) const { ORT_RETURN_IF_ERROR(Transpose::DoTranspose(cuda_ep_->GetDeviceProp(), Stream(), CublasHandle(), - reverse_permutation, intermediate_output, *Y)); + reverse_permutation, *intermediate_output, *Y)); } return Status::OK(); @@ -252,4 +250,3 @@ SPECIALIZED_COMPUTE(BFloat16) } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/math/topk.cc b/onnxruntime/core/providers/cuda/math/topk.cc index 9d5fd6f67b..6baaaf2a6d 100644 --- a/onnxruntime/core/providers/cuda/math/topk.cc +++ b/onnxruntime/core/providers/cuda/math/topk.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -13,7 +12,7 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX( kOnnxDomain, 1, 9, kCudaExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), TopK); ONNX_OPERATOR_VERSIONED_KERNEL_EX( @@ -21,7 +20,7 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX( kOnnxDomain, 10, 10, kCudaExecutionProvider, - KernelDefBuilder().InputMemoryType(1).TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()).TypeConstraint("I", DataTypeImpl::GetTensorType()), + (*KernelDefBuilder::Create()).InputMemoryType(OrtMemTypeCPUInput, 1).TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()).TypeConstraint("I", DataTypeImpl::GetTensorType()), TopK); ONNX_OPERATOR_KERNEL_EX( @@ -29,7 +28,7 @@ ONNX_OPERATOR_KERNEL_EX( kOnnxDomain, 11, kCudaExecutionProvider, - KernelDefBuilder().InputMemoryType(1).TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()).TypeConstraint("I", DataTypeImpl::GetTensorType()), + (*KernelDefBuilder::Create()).InputMemoryType(OrtMemTypeCPUInput, 1).TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()).TypeConstraint("I", DataTypeImpl::GetTensorType()), TopK); template @@ -104,4 +103,3 @@ Status TopK::ComputeInternal(OpKernelContext* ctx) const { } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/math/topk_impl.cu b/onnxruntime/core/providers/cuda/math/topk_impl.cu index 20c88d8fc5..9f81f428dd 100644 --- a/onnxruntime/core/providers/cuda/math/topk_impl.cu +++ b/onnxruntime/core/providers/cuda/math/topk_impl.cu @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "topk_impl.h" -//#include "core/framework/data_types.h" #include "core/providers/cuda/cu_inc/common.cuh" #include "device_atomic_functions.h" #include "cub/cub.cuh" diff --git a/onnxruntime/core/providers/cuda/math/variadic_elementwise_ops.cc b/onnxruntime/core/providers/cuda/math/variadic_elementwise_ops.cc index d41c9e43b8..da1683f448 100644 --- a/onnxruntime/core/providers/cuda/math/variadic_elementwise_ops.cc +++ b/onnxruntime/core/providers/cuda/math/variadic_elementwise_ops.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -214,13 +213,13 @@ const auto k_hfd_datatypes = // kernel registration -#define REGISTER_KERNEL(name, impl_class, version, datatypes) \ - ONNX_OPERATOR_KERNEL_EX( \ - name, \ - kOnnxDomain, \ - version, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", datatypes), \ +#define REGISTER_KERNEL(name, impl_class, version, datatypes) \ + ONNX_OPERATOR_KERNEL_EX( \ + name, \ + kOnnxDomain, \ + version, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", datatypes), \ impl_class) #define REGISTER_VERSIONED_KERNEL(name, impl_class, start_version, end_version, datatypes) \ @@ -229,7 +228,7 @@ const auto k_hfd_datatypes = kOnnxDomain, \ start_version, end_version, \ kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", datatypes), \ + (*KernelDefBuilder::Create()).TypeConstraint("T", datatypes), \ impl_class) REGISTER_KERNEL(Sum, SumOp, 13, k_hfd_datatypes) @@ -249,4 +248,3 @@ REGISTER_VERSIONED_KERNEL(Max, MaxOp, 6, 11, k_hfd_datatypes) } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/nn/batch_norm.cc b/onnxruntime/core/providers/cuda/nn/batch_norm.cc index c9e062be4a..56d297429b 100644 --- a/onnxruntime/core/providers/cuda/nn/batch_norm.cc +++ b/onnxruntime/core/providers/cuda/nn/batch_norm.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -19,7 +18,7 @@ namespace cuda { 7, 8, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ BatchNorm); \ ONNX_OPERATOR_TYPED_KERNEL_EX( \ @@ -28,7 +27,7 @@ namespace cuda { 9, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ BatchNorm); @@ -165,4 +164,3 @@ SPECIALIZED_COMPUTE(MLFloat16) } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/nn/batch_norm.h b/onnxruntime/core/providers/cuda/nn/batch_norm.h index 57baae17a6..62241ef52d 100644 --- a/onnxruntime/core/providers/cuda/nn/batch_norm.h +++ b/onnxruntime/core/providers/cuda/nn/batch_norm.h @@ -3,7 +3,6 @@ #pragma once -#include "gsl/gsl" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cuda/cudnn_common.h" diff --git a/onnxruntime/core/providers/cuda/nn/conv.cc b/onnxruntime/core/providers/cuda/nn/conv.cc index a830e6e641..ac105c350c 100644 --- a/onnxruntime/core/providers/cuda/nn/conv.cc +++ b/onnxruntime/core/providers/cuda/nn/conv.cc @@ -1,8 +1,6 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/providers/common.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/nn/conv.h" #include "core/providers/cuda/shared_inc/fpgeneric.h" @@ -13,22 +11,22 @@ namespace cuda { // Op Set 11 for Conv only update document to clearify default dilations and strides value. // which are already convered by op set 11 cpu versoin, so simply add declaration. -#define REGISTER_KERNEL_TYPED(T) \ - ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ - Conv, \ - kOnnxDomain, \ - 1, 10, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - Conv); \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - Conv, \ - kOnnxDomain, \ - 11, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + Conv, \ + kOnnxDomain, \ + 1, 10, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + Conv); \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Conv, \ + kOnnxDomain, \ + 11, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ Conv); REGISTER_KERNEL_TYPED(float) @@ -36,8 +34,7 @@ REGISTER_KERNEL_TYPED(double) REGISTER_KERNEL_TYPED(MLFloat16) Status SliceOutUnwantedOutputSection(cudaStream_t stream, - const void* input_data, - const std::vector& input_dims, + const void* input_data, const std::vector& input_dims, void* output_data, const std::vector& output_dims, std::vector starts, @@ -359,4 +356,3 @@ Status CudnnConvolutionDescriptor::Set( } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/nn/conv.h b/onnxruntime/core/providers/cuda/nn/conv.h index 03e60f934c..ef0a1f92dc 100644 --- a/onnxruntime/core/providers/cuda/nn/conv.h +++ b/onnxruntime/core/providers/cuda/nn/conv.h @@ -3,7 +3,6 @@ #pragma once -//#include "core/common/common.h" #include "core/platform/ort_mutex.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cuda/cudnn_common.h" diff --git a/onnxruntime/core/providers/cuda/nn/conv_transpose.cc b/onnxruntime/core/providers/cuda/nn/conv_transpose.cc index 8bfb87bc8c..428179ed8f 100644 --- a/onnxruntime/core/providers/cuda/nn/conv_transpose.cc +++ b/onnxruntime/core/providers/cuda/nn/conv_transpose.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -9,22 +8,22 @@ namespace cuda { // Op Set 11 for ConvTranspose only update document to clearify default dilations and strides value. // which are already covered by op set 11 cpu version, so simply add declaration. -#define REGISTER_KERNEL_TYPED(T) \ - ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ - ConvTranspose, \ - kOnnxDomain, \ - 1, 10, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - ConvTranspose); \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - ConvTranspose, \ - kOnnxDomain, \ - 11, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + ConvTranspose, \ + kOnnxDomain, \ + 1, 10, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + ConvTranspose); \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + ConvTranspose, \ + kOnnxDomain, \ + 11, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ ConvTranspose); REGISTER_KERNEL_TYPED(float) @@ -208,4 +207,3 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/nn/conv_transpose.h b/onnxruntime/core/providers/cuda/nn/conv_transpose.h index 6e70827710..165d548d27 100644 --- a/onnxruntime/core/providers/cuda/nn/conv_transpose.h +++ b/onnxruntime/core/providers/cuda/nn/conv_transpose.h @@ -7,7 +7,7 @@ #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cuda/cudnn_common.h" #include "core/providers/cuda/nn/conv.h" -//#include "core/providers/cpu/nn/conv_transpose_attributes.h" +#include "core/providers/cpu/nn/conv_transpose_attributes.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/nn/dropout.cc b/onnxruntime/core/providers/cuda/nn/dropout.cc index f489f37088..0d682885f9 100644 --- a/onnxruntime/core/providers/cuda/nn/dropout.cc +++ b/onnxruntime/core/providers/cuda/nn/dropout.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -8,10 +7,11 @@ namespace onnxruntime { namespace cuda { #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 -#define ALL_IEEE_FLOAT_TENSOR_TYPES {DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType()} +#define ALL_IEEE_FLOAT_TENSOR_TYPES \ + { DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType() } #else #define ALL_IEEE_FLOAT_TENSOR_TYPES DataTypeImpl::AllIEEEFloatTensorTypes() #endif @@ -21,12 +21,12 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX( kOnnxDomain, 12, 12, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("T", DataTypeImpl::AllIEEEFloatTensorTypes()) .TypeConstraint("T1", DataTypeImpl::AllIEEEFloatTensorTypes()) .TypeConstraint("T2", DataTypeImpl::GetTensorType()) - .InputMemoryType(1) - .InputMemoryType(2), + .InputMemoryType(OrtMemTypeCPUInput, 1) + .InputMemoryType(OrtMemTypeCPUInput, 2), Dropout); ONNX_OPERATOR_KERNEL_EX( @@ -34,14 +34,13 @@ ONNX_OPERATOR_KERNEL_EX( kOnnxDomain, 13, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("T", ALL_IEEE_FLOAT_TENSOR_TYPES) .TypeConstraint("T1", ALL_IEEE_FLOAT_TENSOR_TYPES) .TypeConstraint("T2", DataTypeImpl::GetTensorType()) - .InputMemoryType(1) - .InputMemoryType(2), + .InputMemoryType(OrtMemTypeCPUInput, 1) + .InputMemoryType(OrtMemTypeCPUInput, 2), Dropout); } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/nn/instance_norm.cc b/onnxruntime/core/providers/cuda/nn/instance_norm.cc index b9dec7e730..3360db8f25 100644 --- a/onnxruntime/core/providers/cuda/nn/instance_norm.cc +++ b/onnxruntime/core/providers/cuda/nn/instance_norm.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -17,7 +16,7 @@ namespace cuda { 6, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ InstanceNorm); @@ -154,4 +153,3 @@ Status InstanceNorm::ComputeInternal(OpKernelContext* p_op_kernel_context) co } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/nn/lrn.cc b/onnxruntime/core/providers/cuda/nn/lrn.cc index 16cbf361a0..33e1e15e6a 100644 --- a/onnxruntime/core/providers/cuda/nn/lrn.cc +++ b/onnxruntime/core/providers/cuda/nn/lrn.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -7,25 +6,25 @@ namespace onnxruntime { namespace cuda { -#define REGISTER_KERNEL_VERSIONED_TYPED(START_VER, END_VER, T) \ - ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ - LRN, \ - kOnnxDomain, \ - START_VER, \ - END_VER, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define REGISTER_KERNEL_VERSIONED_TYPED(START_VER, END_VER, T) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + LRN, \ + kOnnxDomain, \ + START_VER, \ + END_VER, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ LRN); -#define REGISTER_KERNEL_TYPED(VER, T) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - LRN, \ - kOnnxDomain, \ - VER, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define REGISTER_KERNEL_TYPED(VER, T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + LRN, \ + kOnnxDomain, \ + VER, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ LRN); REGISTER_KERNEL_VERSIONED_TYPED(1, 12, float) @@ -111,4 +110,3 @@ Status CudnnLRNDescriptor::Set(uint32_t N, double alpha, double beta, double K) } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/nn/pool.cc b/onnxruntime/core/providers/cuda/nn/pool.cc index 46a54143ee..b9ee3975e1 100644 --- a/onnxruntime/core/providers/cuda/nn/pool.cc +++ b/onnxruntime/core/providers/cuda/nn/pool.cc @@ -1,9 +1,8 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/providers/shared_library/provider_api.h" #include "core/providers/cuda/nn/pool.h" -#include "core/providers/common.h" #include "core/providers/cuda/cudnn_common.h" #include "core/providers/cuda/nn/max_pool_with_index.h" #include "core/providers/cuda/math/unary_elementwise_ops_impl.h" @@ -12,28 +11,27 @@ using namespace onnxruntime::common; namespace onnxruntime { namespace cuda { -#define POOLING_KERNEL(op_name, data_type, pool_type, since_version) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - op_name, \ - kOnnxDomain, \ - since_version, \ - data_type, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define POOLING_KERNEL(op_name, data_type, pool_type, since_version) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + op_name, \ + kOnnxDomain, \ + since_version, \ + data_type, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ Pool); -#define POOLING_KERNEL_VERSIONED(op_name, data_type, pool_type, since_version, end_version) \ - ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ - op_name, \ - kOnnxDomain, \ - since_version, \ - end_version, \ - data_type, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()).TypeConstraint("I", DataTypeImpl::GetTensorType()), \ +#define POOLING_KERNEL_VERSIONED(op_name, data_type, pool_type, since_version, end_version) \ + ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX( \ + op_name, \ + kOnnxDomain, \ + since_version, \ + end_version, \ + data_type, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()).TypeConstraint("I", DataTypeImpl::GetTensorType()), \ Pool); - POOLING_KERNEL_VERSIONED(AveragePool, float, AveragePool, 7, 9) POOLING_KERNEL_VERSIONED(AveragePool, double, AveragePool, 7, 9) POOLING_KERNEL_VERSIONED(AveragePool, MLFloat16, AveragePool, 7, 9) @@ -65,7 +63,6 @@ POOLING_KERNEL(MaxPool, MLFloat16, MaxPool<8>, 12) POOLING_KERNEL(MaxPool, int8_t, MaxPool<8>, 12) POOLING_KERNEL(MaxPool, uint8_t, MaxPool<8>, 12) - POOLING_KERNEL(GlobalMaxPool, float, MaxPool<1>, 1) POOLING_KERNEL(GlobalMaxPool, double, MaxPool<1>, 1) POOLING_KERNEL(GlobalMaxPool, MLFloat16, MaxPool<1>, 1) @@ -168,8 +165,8 @@ Status Pool::ComputeInternal(OpKernelContext* context) const { cudnnPoolingMode_t mode = CUDNN_POOLING_MAX; if (PoolType::type == onnxruntime::PoolType::kAveragePool) { - mode = pool_attrs_.count_include_pad ? CUDNN_POOLING_AVERAGE_COUNT_INCLUDE_PADDING - : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING; + mode = pool_attrs_.count_include_pad ? CUDNN_POOLING_AVERAGE_COUNT_INCLUDE_PADDING + : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING; } CudnnPoolingDescriptor pooling_desc; ORT_RETURN_IF_ERROR(pooling_desc.Set(mode, kernel_shape, pads, strides)); @@ -259,4 +256,3 @@ Status Pool>::ComputeInternal(OpKernelContext* context) const { } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/nn/pool.h b/onnxruntime/core/providers/cuda/nn/pool.h index 32969e8520..fb223c18d2 100644 --- a/onnxruntime/core/providers/cuda/nn/pool.h +++ b/onnxruntime/core/providers/cuda/nn/pool.h @@ -3,7 +3,6 @@ #pragma once -#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cuda/cudnn_common.h" #include "core/providers/cpu/nn/pool_base.h" @@ -14,7 +13,7 @@ namespace cuda { template class Pool : public CudaKernel, public PoolBase { public: - Pool(OpKernelInfo info) : CudaKernel(info), PoolBase(info) {} + Pool(const OpKernelInfo& info) : CudaKernel(info), PoolBase(info) {} Status ComputeInternal(OpKernelContext* context) const override; }; @@ -22,7 +21,7 @@ class Pool : public CudaKernel, public PoolBase { template class Pool> final : public Pool> { public: - Pool(OpKernelInfo info) : Pool>(info) {} + Pool(const OpKernelInfo& info) : Pool>(info) {} Status ComputeInternal(OpKernelContext* context) const override; }; diff --git a/onnxruntime/core/providers/cuda/nn/shrink.cc b/onnxruntime/core/providers/cuda/nn/shrink.cc index cf96dd3cb3..e526f58a46 100644 --- a/onnxruntime/core/providers/cuda/nn/shrink.cc +++ b/onnxruntime/core/providers/cuda/nn/shrink.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -17,7 +16,7 @@ namespace cuda { 9, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .MayInplace(0, 0) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ Shrink); @@ -53,4 +52,3 @@ SHRINK_REGISTER_KERNEL(int64_t) } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/object_detection/non_max_suppression.cc b/onnxruntime/core/providers/cuda/object_detection/non_max_suppression.cc index dd6e4db95b..9bf91e2e2b 100644 --- a/onnxruntime/core/providers/cuda/object_detection/non_max_suppression.cc +++ b/onnxruntime/core/providers/cuda/object_detection/non_max_suppression.cc @@ -1,4 +1,3 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. @@ -15,10 +14,10 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX( kOnnxDomain, 10, 10, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(2) - .InputMemoryType(3) - .InputMemoryType(4), + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 2) + .InputMemoryType(OrtMemTypeCPUInput, 3) + .InputMemoryType(OrtMemTypeCPUInput, 4), NonMaxSuppression); ONNX_OPERATOR_KERNEL_EX( @@ -26,10 +25,10 @@ ONNX_OPERATOR_KERNEL_EX( kOnnxDomain, 11, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(2) - .InputMemoryType(3) - .InputMemoryType(4), + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 2) + .InputMemoryType(OrtMemTypeCPUInput, 3) + .InputMemoryType(OrtMemTypeCPUInput, 4), NonMaxSuppression); Status NonMaxSuppression::ComputeInternal(OpKernelContext* ctx) const { @@ -137,4 +136,3 @@ Status NonMaxSuppression::ComputeInternal(OpKernelContext* ctx) const { } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu b/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu index b90d248eac..6e79409ebb 100644 --- a/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu +++ b/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu @@ -20,8 +20,6 @@ limitations under the License. #include "core/providers/cuda/cu_inc/common.cuh" #include "core/providers/cuda/cuda_common.h" -//#include "core/framework/tensor.h" - #include #include diff --git a/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.h b/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.h index 803589e065..aa1b78e53d 100644 --- a/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.h +++ b/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.h @@ -4,9 +4,6 @@ #pragma once #include #include "core/providers/cuda/shared_inc/cuda_utils.h" -//#include "core/common/common.h" -//#include "core/framework/allocator.h" -//#include "core/framework/data_types.h" #include #include "core/providers/cpu/object_detection/non_max_suppression_helper.h" diff --git a/onnxruntime/core/providers/cuda/object_detection/roialign.cc b/onnxruntime/core/providers/cuda/object_detection/roialign.cc index 8e4c69f328..7bf97dc7b5 100644 --- a/onnxruntime/core/providers/cuda/object_detection/roialign.cc +++ b/onnxruntime/core/providers/cuda/object_detection/roialign.cc @@ -1,5 +1,4 @@ -#if 0 -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include "roialign.h" @@ -15,7 +14,7 @@ namespace cuda { 10, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("T2", DataTypeImpl::GetTensorType()), \ RoiAlign); @@ -60,8 +59,7 @@ Status RoiAlign::ComputeInternal(OpKernelContext* context) const { num_roi_cols, reinterpret_cast::MappedType*>(Y.template MutableData()), this->mode_ == RoiAlignMode::avg, - batch_indices_ptr->template Data() - ); + batch_indices_ptr->template Data()); } return Status::OK(); @@ -77,4 +75,3 @@ SPECIALIZED_COMPUTE(double) } // namespace cuda }; // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/object_detection/roialign_impl.h b/onnxruntime/core/providers/cuda/object_detection/roialign_impl.h index 291bc700cf..4ff9d4cb1d 100644 --- a/onnxruntime/core/providers/cuda/object_detection/roialign_impl.h +++ b/onnxruntime/core/providers/cuda/object_detection/roialign_impl.h @@ -4,8 +4,6 @@ #pragma once #include #include "core/providers/cuda/shared_inc/cuda_utils.h" -//#include "core/framework/data_types.h" -//#include "core/common/common.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/reduction/reduction_functions.cc b/onnxruntime/core/providers/cuda/reduction/reduction_functions.cc index b5297ece9e..955df6d9a1 100644 --- a/onnxruntime/core/providers/cuda/reduction/reduction_functions.cc +++ b/onnxruntime/core/providers/cuda/reduction/reduction_functions.cc @@ -11,7 +11,6 @@ #include "core/common/optional.h" #include "core/framework/tensor_shape.h" -//#include "core/providers/common.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc b/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc index 94f8176f2a..e3b9066bcd 100644 --- a/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc +++ b/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc @@ -1,14 +1,9 @@ -#if 0 // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -#include "core/providers/cuda/reduction/reduction_ops.h" - -//#include "core/framework/data_types_internal.h" -//#include "core/framework/op_kernel_context_internal.h" -//#include "core/providers/common.h" #include "core/providers/cpu/tensor/utils.h" +#include "core/providers/cuda/reduction/reduction_ops.h" #include "core/providers/cuda/cudnn_common.h" #include "core/providers/cuda/math/binary_elementwise_ops_impl.h" #include "core/providers/cuda/math/binary_elementwise_ops.h" @@ -688,88 +683,88 @@ Status ReduceKernel::ComputeImpl(OpKernelContext* ctx, cudnnRe calculate_log_, calculate_sqt_, log_sum_exp_, fast_reduction); } -#define SPECIALIZED_REDUCEKERNEL_COMPUTEIMPL(T) \ - template <> \ - template <> \ - Status ReduceKernel::ComputeImpl( \ - OpKernelContext * ctx, cudnnReduceTensorOp_t cudnn_reduce_op) const { \ - typedef typename ToCudaType::MappedType CudaT; \ - const Tensor* X = ctx->Input(0); \ - std::vector axes; \ - size_t num_inputs = ctx->InputCount(); \ - if (num_inputs == 2) { \ - const Tensor* axes_tensor = ctx->Input(1); \ - ORT_ENFORCE(axes_tensor != nullptr, "Axes input is null"); \ - ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 1, "An axes tensor must be a vector tensor."); \ - auto nDims = static_cast(axes_tensor->Shape()[0]); \ - const auto* data = axes_tensor->template Data(); \ - axes.assign(data, data + nDims); \ - } else { \ - axes.assign(axes_.begin(), axes_.end()); \ - } \ - \ - if (axes.empty() && noop_with_empty_axes_) { \ - auto* Y = ctx->Output(0, X->Shape()); \ - CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(Y->template MutableData(), X->template Data(), X->SizeInBytes(), \ - cudaMemcpyDeviceToDevice, Stream())); \ - return Status::OK(); \ - } \ - \ - PrepareReduceMetadata prepare_reduce_metadata; \ - ORT_RETURN_IF_ERROR(PrepareForReduce(X, keepdims_, axes, prepare_reduce_metadata)); \ - \ - Tensor* Y = ctx->Output(0, prepare_reduce_metadata.squeezed_output_dims); \ - \ - int64_t input_count = prepare_reduce_metadata.input_count; \ - int64_t output_count = prepare_reduce_metadata.output_count; \ - std::vector& input_dims_cudnn = prepare_reduce_metadata.input_dims_cudnn; \ - std::vector& output_dims_cudnn = prepare_reduce_metadata.output_dims_cudnn; \ - \ - if (input_count == 0) { \ - assert(Y->Shape().Size() == 0); \ - return Status::OK(); \ - } \ - \ - if (input_count == output_count) { \ - if (Y->template MutableData() != X->template Data()) { \ - CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(Y->template MutableData(), X->template Data(), \ - input_count * sizeof(T), cudaMemcpyDeviceToDevice, Stream())); \ - } \ - return Status::OK(); \ - } \ - \ - CUDA_RETURN_IF_ERROR(cudaMemsetAsync(Y->MutableDataRaw(), 0, Y->SizeInBytes(), Stream())); \ - \ - size_t indices_bytes = 0; \ - size_t workspace_bytes = 0; \ - CudnnTensor input_tensor; \ - CudnnTensor output_tensor; \ - CudnnReduceDescriptor reduce_desc; \ - \ - cudnnDataType_t cudnn_type_X = CUDNN_DATA_FLOAT; \ - IAllocatorUniquePtr temp_X = GetScratchBuffer(input_count); \ +#define SPECIALIZED_REDUCEKERNEL_COMPUTEIMPL(T) \ + template <> \ + template <> \ + Status ReduceKernel::ComputeImpl( \ + OpKernelContext * ctx, cudnnReduceTensorOp_t cudnn_reduce_op) const { \ + typedef typename ToCudaType::MappedType CudaT; \ + const Tensor* X = ctx->Input(0); \ + std::vector axes; \ + size_t num_inputs = ctx->InputCount(); \ + if (num_inputs == 2) { \ + const Tensor* axes_tensor = ctx->Input(1); \ + ORT_ENFORCE(axes_tensor != nullptr, "Axes input is null"); \ + ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 1, "An axes tensor must be a vector tensor."); \ + auto nDims = static_cast(axes_tensor->Shape()[0]); \ + const auto* data = axes_tensor->template Data(); \ + axes.assign(data, data + nDims); \ + } else { \ + axes.assign(axes_.begin(), axes_.end()); \ + } \ + \ + if (axes.empty() && noop_with_empty_axes_) { \ + auto* Y = ctx->Output(0, X->Shape()); \ + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(Y->template MutableData(), X->template Data(), X->SizeInBytes(), \ + cudaMemcpyDeviceToDevice, Stream())); \ + return Status::OK(); \ + } \ + \ + PrepareReduceMetadata prepare_reduce_metadata; \ + ORT_RETURN_IF_ERROR(PrepareForReduce(X, keepdims_, axes, prepare_reduce_metadata)); \ + \ + Tensor* Y = ctx->Output(0, prepare_reduce_metadata.squeezed_output_dims); \ + \ + int64_t input_count = prepare_reduce_metadata.input_count; \ + int64_t output_count = prepare_reduce_metadata.output_count; \ + std::vector& input_dims_cudnn = prepare_reduce_metadata.input_dims_cudnn; \ + std::vector& output_dims_cudnn = prepare_reduce_metadata.output_dims_cudnn; \ + \ + if (input_count == 0) { \ + assert(Y->Shape().Size() == 0); \ + return Status::OK(); \ + } \ + \ + if (input_count == output_count) { \ + if (Y->template MutableData() != X->template Data()) { \ + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(Y->template MutableData(), X->template Data(), \ + input_count * sizeof(T), cudaMemcpyDeviceToDevice, Stream())); \ + } \ + return Status::OK(); \ + } \ + \ + CUDA_RETURN_IF_ERROR(cudaMemsetAsync(Y->MutableDataRaw(), 0, Y->SizeInBytes(), Stream())); \ + \ + size_t indices_bytes = 0; \ + size_t workspace_bytes = 0; \ + CudnnTensor input_tensor; \ + CudnnTensor output_tensor; \ + CudnnReduceDescriptor reduce_desc; \ + \ + cudnnDataType_t cudnn_type_X = CUDNN_DATA_FLOAT; \ + IAllocatorUniquePtr temp_X = GetScratchBuffer(input_count); \ Impl_Cast(Stream(), reinterpret_cast(X->template Data()), temp_X.get(), X->Shape().Size()); \ - \ - ORT_RETURN_IF_ERROR(reduce_desc.Set(cudnn_reduce_op, cudnn_type_X, CUDNN_REDUCE_TENSOR_FLATTENED_INDICES)); \ - ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_cudnn, cudnn_type_X)); \ - ORT_RETURN_IF_ERROR(output_tensor.Set(output_dims_cudnn, cudnn_type_X)); \ - CUDNN_RETURN_IF_ERROR( \ - cudnnGetReductionIndicesSize(CudnnHandle(), reduce_desc, input_tensor, output_tensor, &indices_bytes)); \ - CUDNN_RETURN_IF_ERROR( \ - cudnnGetReductionWorkspaceSize(CudnnHandle(), reduce_desc, input_tensor, output_tensor, &workspace_bytes)); \ - IAllocatorUniquePtr indices_cuda = GetScratchBuffer(indices_bytes); \ - IAllocatorUniquePtr workspace_cuda = GetScratchBuffer(workspace_bytes); \ - \ - const auto one = Consts::One; \ - const auto zero = Consts::Zero; \ - auto temp_Y = GetScratchBuffer(output_count); \ - CUDNN_RETURN_IF_ERROR(cudnnReduceTensor(CudnnHandle(), reduce_desc, indices_cuda.get(), indices_bytes, \ - workspace_cuda.get(), workspace_bytes, &one, input_tensor, temp_X.get(), \ - &zero, output_tensor, temp_Y.get())); \ - \ + \ + ORT_RETURN_IF_ERROR(reduce_desc.Set(cudnn_reduce_op, cudnn_type_X, CUDNN_REDUCE_TENSOR_FLATTENED_INDICES)); \ + ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_cudnn, cudnn_type_X)); \ + ORT_RETURN_IF_ERROR(output_tensor.Set(output_dims_cudnn, cudnn_type_X)); \ + CUDNN_RETURN_IF_ERROR( \ + cudnnGetReductionIndicesSize(CudnnHandle(), reduce_desc, input_tensor, output_tensor, &indices_bytes)); \ + CUDNN_RETURN_IF_ERROR( \ + cudnnGetReductionWorkspaceSize(CudnnHandle(), reduce_desc, input_tensor, output_tensor, &workspace_bytes)); \ + IAllocatorUniquePtr indices_cuda = GetScratchBuffer(indices_bytes); \ + IAllocatorUniquePtr workspace_cuda = GetScratchBuffer(workspace_bytes); \ + \ + const auto one = Consts::One; \ + const auto zero = Consts::Zero; \ + auto temp_Y = GetScratchBuffer(output_count); \ + CUDNN_RETURN_IF_ERROR(cudnnReduceTensor(CudnnHandle(), reduce_desc, indices_cuda.get(), indices_bytes, \ + workspace_cuda.get(), workspace_bytes, &one, input_tensor, temp_X.get(), \ + &zero, output_tensor, temp_Y.get())); \ + \ Impl_Cast(Stream(), temp_Y.get(), reinterpret_cast(Y->template MutableData()), output_count); \ - \ - return Status::OK(); \ + \ + return Status::OK(); \ } SPECIALIZED_REDUCEKERNEL_COMPUTEIMPL(int32_t) @@ -887,10 +882,10 @@ Status ReduceKernel::ComputeImpl namespace ReductionOps { template -Tensor ReduceCompute(CUDAExecutionProvider& cuda_ep, cudnnReduceTensorOp_t cudnn_reduce_op, AllocatorPtr allocator, - const Tensor& input, const std::vector& axes, - bool keep_dims, bool calculate_log, bool calculate_sqt, bool log_sum_exp, - bool fast_reduction, const TensorShape* input_shape_override) { +std::unique_ptr ReduceCompute(CUDAExecutionProvider& cuda_ep, cudnnReduceTensorOp_t cudnn_reduce_op, AllocatorPtr allocator, + const Tensor& input, const std::vector& axes, + bool keep_dims, bool calculate_log, bool calculate_sqt, bool log_sum_exp, + bool fast_reduction, const TensorShape* input_shape_override) { PrepareReduceMetadata prepare_reduce_metadata; auto status = PrepareForReduce(&input, keep_dims, @@ -904,7 +899,7 @@ Tensor ReduceCompute(CUDAExecutionProvider& cuda_ep, cudnnReduceTensorOp_t cudnn auto output = Tensor::Create(input.DataType(), prepare_reduce_metadata.squeezed_output_dims, allocator); - status = ReduceComputeCore(cuda_ep, input, prepare_reduce_metadata, output, cudnn_reduce_op, axes, + status = ReduceComputeCore(cuda_ep, input, prepare_reduce_metadata, *output, cudnn_reduce_op, axes, calculate_log, calculate_sqt, log_sum_exp, fast_reduction, input_shape_override); if (!status.IsOK()) { @@ -916,21 +911,21 @@ Tensor ReduceCompute(CUDAExecutionProvider& cuda_ep, cudnnReduceTensorOp_t cudnn // Explicit template instantiation (needed to be used in einsum_auxiliary_ops.cc) -template Tensor ReduceCompute( +template std::unique_ptr ReduceCompute( CUDAExecutionProvider& cuda_ep, cudnnReduceTensorOp_t cudnn_reduce_op, AllocatorPtr allocator, const Tensor& input, const std::vector& axes, bool keep_dims, bool calculate_log, bool calculate_sqt, bool log_sum_exp, bool fast_reduction, const TensorShape* input_shape_override); -template Tensor ReduceCompute( +template std::unique_ptr ReduceCompute( CUDAExecutionProvider& cuda_ep, cudnnReduceTensorOp_t cudnn_reduce_op, AllocatorPtr allocator, const Tensor& input, const std::vector& axes, bool keep_dims, bool calculate_log, bool calculate_sqt, bool log_sum_exp, bool fast_reduction, const TensorShape* input_shape_override); -template Tensor ReduceCompute( +template std::unique_ptr ReduceCompute( CUDAExecutionProvider& cuda_ep, cudnnReduceTensorOp_t cudnn_reduce_op, AllocatorPtr allocator, const Tensor& input, const std::vector& axes, @@ -1004,4 +999,3 @@ REGISTER_KERNEL_INT32(ReduceProd) } // namespace cuda } // namespace onnxruntime -#endif diff --git a/onnxruntime/core/providers/cuda/reduction/reduction_ops.h b/onnxruntime/core/providers/cuda/reduction/reduction_ops.h index 5feb65b1be..61a71f5e07 100644 --- a/onnxruntime/core/providers/cuda/reduction/reduction_ops.h +++ b/onnxruntime/core/providers/cuda/reduction/reduction_ops.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #pragma once -//#include "core/common/optional.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/reduction/reduction_ops.h" #include "core/providers/cuda/reduction/reduction_functions.h" diff --git a/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc index 0dd7bc5b54..1dcbebf695 100644 --- a/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc +++ b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc @@ -4,7 +4,6 @@ #include "core/providers/shared_library/provider_api.h" #include "cudnn_rnn_base.h" #include "rnn_impl.h" -//#include "core/providers/cpu/rnn/rnn_helpers.h" namespace onnxruntime { namespace cuda { @@ -103,7 +102,6 @@ Status CudnnRnnBase::ReorganizeWeights(const Tensor* W, const Tensor* R, cons // TODO! refine allocation size for each case. cudaMemset(reorganized_w_data.get(), 0, w_size * sizeof(T)); - const T* W_data = W->template Data(); const T* R_data = R->template Data(); const T* B_data = B == nullptr ? nullptr : B->template Data(); diff --git a/onnxruntime/core/providers/cuda/rnn/gru.cc b/onnxruntime/core/providers/cuda/rnn/gru.cc index 8f2ff4e4b9..dc6aaf4b55 100644 --- a/onnxruntime/core/providers/cuda/rnn/gru.cc +++ b/onnxruntime/core/providers/cuda/rnn/gru.cc @@ -4,9 +4,7 @@ #include "core/providers/shared_library/provider_api.h" #include "gru.h" #include "rnn_impl.h" -//#include "core/providers/common.h" #include "core/providers/cuda/cudnn_common.h" -//#include "core/providers/cpu/math/gemm_helper.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/rnn/lstm.cc b/onnxruntime/core/providers/cuda/rnn/lstm.cc index 9fa32a4bcc..f1e1943a85 100644 --- a/onnxruntime/core/providers/cuda/rnn/lstm.cc +++ b/onnxruntime/core/providers/cuda/rnn/lstm.cc @@ -3,8 +3,6 @@ #include "core/providers/shared_library/provider_api.h" #include "lstm.h" -//#include "core/providers/cpu/rnn/rnn_helpers.h" -//#include "core/providers/common.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/rnn/rnn.cc b/onnxruntime/core/providers/cuda/rnn/rnn.cc index a311ac1d26..00e231e1ad 100644 --- a/onnxruntime/core/providers/cuda/rnn/rnn.cc +++ b/onnxruntime/core/providers/cuda/rnn/rnn.cc @@ -4,9 +4,7 @@ #include "core/providers/shared_library/provider_api.h" #include "rnn.h" #include "rnn_impl.h" -//#include "core/providers/common.h" #include "core/providers/cuda/cudnn_common.h" -//#include "core/providers/cpu/math/gemm_helper.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h b/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h index ad4fe05503..31cb05564a 100644 --- a/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h +++ b/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h @@ -10,7 +10,6 @@ #include #include -//#include "core/common/common.h" #include "core/providers/cuda/shared_inc/fast_divmod.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/compress.cc b/onnxruntime/core/providers/cuda/tensor/compress.cc index a587991559..5c9537fdce 100644 --- a/onnxruntime/core/providers/cuda/tensor/compress.cc +++ b/onnxruntime/core/providers/cuda/tensor/compress.cc @@ -2,8 +2,6 @@ // Licensed under the MIT License. #include "compress.h" -//#include "core/providers/cpu/tensor/utils.h" -//#include "core/providers/common.h" #include "compress_impl.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/expand.h b/onnxruntime/core/providers/cuda/tensor/expand.h index 1ace05d962..4cf4c14e61 100644 --- a/onnxruntime/core/providers/cuda/tensor/expand.h +++ b/onnxruntime/core/providers/cuda/tensor/expand.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/expand_impl.h b/onnxruntime/core/providers/cuda/tensor/expand_impl.h index a02091aefe..f5bae955a9 100644 --- a/onnxruntime/core/providers/cuda/tensor/expand_impl.h +++ b/onnxruntime/core/providers/cuda/tensor/expand_impl.h @@ -4,8 +4,6 @@ #pragma once #include #include "core/providers/cuda/shared_inc/cuda_utils.h" -//#include "core/framework/data_types.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_common.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/eye_like.cc b/onnxruntime/core/providers/cuda/tensor/eye_like.cc index dac3325bab..0fa051fa58 100644 --- a/onnxruntime/core/providers/cuda/tensor/eye_like.cc +++ b/onnxruntime/core/providers/cuda/tensor/eye_like.cc @@ -4,7 +4,6 @@ #include "eye_like.h" #include "eye_like_impl.h" #include "core/providers/cpu/tensor/utils.h" -//#include "core/framework/tensorprotoutils.h" #include "core/providers/cuda/shared_inc/fast_divmod.h" using namespace onnxruntime::common; diff --git a/onnxruntime/core/providers/cuda/tensor/eye_like.h b/onnxruntime/core/providers/cuda/tensor/eye_like.h index 1883daf655..7abb1cb889 100644 --- a/onnxruntime/core/providers/cuda/tensor/eye_like.h +++ b/onnxruntime/core/providers/cuda/tensor/eye_like.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/flatten.cc b/onnxruntime/core/providers/cuda/tensor/flatten.cc index 61e38a9f0c..1d19a7fdf9 100644 --- a/onnxruntime/core/providers/cuda/tensor/flatten.cc +++ b/onnxruntime/core/providers/cuda/tensor/flatten.cc @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "flatten.h" -//#include "core/providers/common.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/tensor/flatten.h b/onnxruntime/core/providers/cuda/tensor/flatten.h index 0cd351103f..d83929ed7c 100644 --- a/onnxruntime/core/providers/cuda/tensor/flatten.h +++ b/onnxruntime/core/providers/cuda/tensor/flatten.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/gather.cc b/onnxruntime/core/providers/cuda/tensor/gather.cc index 5ceb451f33..189055cab7 100644 --- a/onnxruntime/core/providers/cuda/tensor/gather.cc +++ b/onnxruntime/core/providers/cuda/tensor/gather.cc @@ -4,7 +4,6 @@ #include "core/providers/cuda/tensor/gather_impl.h" #include "core/providers/cuda/tensor/gather.h" #include "core/providers/cpu/tensor/utils.h" -//#include "core/providers/common.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/tensor/gather.h b/onnxruntime/core/providers/cuda/tensor/gather.h index cebe3396cf..c4a2a71f5a 100644 --- a/onnxruntime/core/providers/cuda/tensor/gather.h +++ b/onnxruntime/core/providers/cuda/tensor/gather.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/tensor/gatherbase.h" diff --git a/onnxruntime/core/providers/cuda/tensor/gather_elements.cc b/onnxruntime/core/providers/cuda/tensor/gather_elements.cc index 952856abd9..08312899de 100644 --- a/onnxruntime/core/providers/cuda/tensor/gather_elements.cc +++ b/onnxruntime/core/providers/cuda/tensor/gather_elements.cc @@ -4,7 +4,6 @@ #include "gather_elements.h" #include "gather_elements_impl.h" #include "core/providers/cpu/tensor/utils.h" -//#include "core/providers/common.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/tensor/gather_elements.h b/onnxruntime/core/providers/cuda/tensor/gather_elements.h index f861aae6d1..9038c35d8d 100644 --- a/onnxruntime/core/providers/cuda/tensor/gather_elements.h +++ b/onnxruntime/core/providers/cuda/tensor/gather_elements.h @@ -3,8 +3,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" -//#include "core/providers/cpu/tensor/gather_elements.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/gather_nd.h b/onnxruntime/core/providers/cuda/tensor/gather_nd.h index 0421dde986..fb619cd1fd 100644 --- a/onnxruntime/core/providers/cuda/tensor/gather_nd.h +++ b/onnxruntime/core/providers/cuda/tensor/gather_nd.h @@ -4,7 +4,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/identity_op.h b/onnxruntime/core/providers/cuda/tensor/identity_op.h index bcc25c2142..4bac78d042 100644 --- a/onnxruntime/core/providers/cuda/tensor/identity_op.h +++ b/onnxruntime/core/providers/cuda/tensor/identity_op.h @@ -3,7 +3,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/nonzero_op.cc b/onnxruntime/core/providers/cuda/tensor/nonzero_op.cc index 37c107eeae..f4f4004986 100644 --- a/onnxruntime/core/providers/cuda/tensor/nonzero_op.cc +++ b/onnxruntime/core/providers/cuda/tensor/nonzero_op.cc @@ -4,7 +4,6 @@ #include "nonzero_op.h" #include "nonzero_impl.h" #include "core/providers/cpu/tensor/utils.h" -//#include "core/providers/common.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/tensor/nonzero_op.h b/onnxruntime/core/providers/cuda/tensor/nonzero_op.h index 8bc71e12fd..cac48e0ee8 100644 --- a/onnxruntime/core/providers/cuda/tensor/nonzero_op.h +++ b/onnxruntime/core/providers/cuda/tensor/nonzero_op.h @@ -3,7 +3,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/onehot.cc b/onnxruntime/core/providers/cuda/tensor/onehot.cc index 604ca981d2..1ca9c3216f 100644 --- a/onnxruntime/core/providers/cuda/tensor/onehot.cc +++ b/onnxruntime/core/providers/cuda/tensor/onehot.cc @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/cuda/tensor/onehot.h" -//#include "core/providers/cpu/tensor/onehot.h" using namespace onnxruntime::common; diff --git a/onnxruntime/core/providers/cuda/tensor/onehot.h b/onnxruntime/core/providers/cuda/tensor/onehot.h index 035cc1dfda..6d13524055 100644 --- a/onnxruntime/core/providers/cuda/tensor/onehot.h +++ b/onnxruntime/core/providers/cuda/tensor/onehot.h @@ -3,7 +3,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/pad.h b/onnxruntime/core/providers/cuda/tensor/pad.h index a11f1d8135..b206a35995 100644 --- a/onnxruntime/core/providers/cuda/tensor/pad.h +++ b/onnxruntime/core/providers/cuda/tensor/pad.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/tensor/padbase.h" diff --git a/onnxruntime/core/providers/cuda/tensor/quantize_linear.cc b/onnxruntime/core/providers/cuda/tensor/quantize_linear.cc index 3d1c1ad91f..92c1c7d98c 100644 --- a/onnxruntime/core/providers/cuda/tensor/quantize_linear.cc +++ b/onnxruntime/core/providers/cuda/tensor/quantize_linear.cc @@ -4,8 +4,6 @@ #include "quantize_linear.h" #include "quantize_linear.cuh" -//#include "core/providers/common.h" - namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/tensor/quantize_linear.cuh b/onnxruntime/core/providers/cuda/tensor/quantize_linear.cuh index fa573c8ba5..3c11e3b9a5 100644 --- a/onnxruntime/core/providers/cuda/tensor/quantize_linear.cuh +++ b/onnxruntime/core/providers/cuda/tensor/quantize_linear.cuh @@ -4,7 +4,6 @@ #pragma once #include "quantize_linear.h" -//#include "core/providers/cpu/math/matmul_helper.h" #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/shared_inc/cuda_utils.h" diff --git a/onnxruntime/core/providers/cuda/tensor/quantize_linear.h b/onnxruntime/core/providers/cuda/tensor/quantize_linear.h index 67c02ae233..fe50e9f273 100644 --- a/onnxruntime/core/providers/cuda/tensor/quantize_linear.h +++ b/onnxruntime/core/providers/cuda/tensor/quantize_linear.h @@ -4,7 +4,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "gsl/gsl" diff --git a/onnxruntime/core/providers/cuda/tensor/reshape.h b/onnxruntime/core/providers/cuda/tensor/reshape.h index d5185aa7d3..76161041b3 100644 --- a/onnxruntime/core/providers/cuda/tensor/reshape.h +++ b/onnxruntime/core/providers/cuda/tensor/reshape.h @@ -4,7 +4,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "gsl/gsl" #include "core/providers/cpu/tensor/reshape_helper.h" diff --git a/onnxruntime/core/providers/cuda/tensor/reverse_sequence.cc b/onnxruntime/core/providers/cuda/tensor/reverse_sequence.cc index 4721247d4d..7739ce18f9 100644 --- a/onnxruntime/core/providers/cuda/tensor/reverse_sequence.cc +++ b/onnxruntime/core/providers/cuda/tensor/reverse_sequence.cc @@ -5,8 +5,6 @@ #include "reverse_sequence_impl.h" #include "core/providers/cpu/tensor/utils.h" -//#include "core/providers/common.h" -//#include "core/framework/utils.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/tensor/reverse_sequence.h b/onnxruntime/core/providers/cuda/tensor/reverse_sequence.h index 1364bec4f4..84730c3bd8 100644 --- a/onnxruntime/core/providers/cuda/tensor/reverse_sequence.h +++ b/onnxruntime/core/providers/cuda/tensor/reverse_sequence.h @@ -4,7 +4,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/scatter_elements.cc b/onnxruntime/core/providers/cuda/tensor/scatter_elements.cc index 9e57b0d3f8..0df77f6734 100755 --- a/onnxruntime/core/providers/cuda/tensor/scatter_elements.cc +++ b/onnxruntime/core/providers/cuda/tensor/scatter_elements.cc @@ -4,7 +4,6 @@ #include "scatter_elements.h" #include "scatter_elements_impl.h" #include "core/providers/cpu/tensor/utils.h" -//#include "core/providers/common.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/tensor/scatter_elements.h b/onnxruntime/core/providers/cuda/tensor/scatter_elements.h index 5aefcfa4e4..bbf33b84a1 100755 --- a/onnxruntime/core/providers/cuda/tensor/scatter_elements.h +++ b/onnxruntime/core/providers/cuda/tensor/scatter_elements.h @@ -3,7 +3,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/cuda/tensor/scatter_nd.h b/onnxruntime/core/providers/cuda/tensor/scatter_nd.h index 7ef4adae0f..bececcf29b 100644 --- a/onnxruntime/core/providers/cuda/tensor/scatter_nd.h +++ b/onnxruntime/core/providers/cuda/tensor/scatter_nd.h @@ -4,7 +4,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/tensor/scatter_nd.h" diff --git a/onnxruntime/core/providers/cuda/tensor/slice.h b/onnxruntime/core/providers/cuda/tensor/slice.h index 49f5b55beb..87607bd4cc 100644 --- a/onnxruntime/core/providers/cuda/tensor/slice.h +++ b/onnxruntime/core/providers/cuda/tensor/slice.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/tensor/slice.h" #include "core/providers/cpu/tensor/utils.h" diff --git a/onnxruntime/core/providers/cuda/tensor/split.cc b/onnxruntime/core/providers/cuda/tensor/split.cc index 9ff2b6855e..c617d1a2f0 100644 --- a/onnxruntime/core/providers/cuda/tensor/split.cc +++ b/onnxruntime/core/providers/cuda/tensor/split.cc @@ -4,7 +4,6 @@ #include "core/providers/cuda/tensor/split.h" #include "core/providers/cuda/tensor/split_impl.h" #include "core/providers/cpu/tensor/utils.h" -//#include "core/providers/common.h" namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/providers/cuda/tensor/split.h b/onnxruntime/core/providers/cuda/tensor/split.h index 8e4c49ad27..bc01d11c72 100644 --- a/onnxruntime/core/providers/cuda/tensor/split.h +++ b/onnxruntime/core/providers/cuda/tensor/split.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/tensor/split.h" diff --git a/onnxruntime/core/providers/cuda/tensor/squeeze.h b/onnxruntime/core/providers/cuda/tensor/squeeze.h index 7d8b3f9288..da4cdc5d3e 100644 --- a/onnxruntime/core/providers/cuda/tensor/squeeze.h +++ b/onnxruntime/core/providers/cuda/tensor/squeeze.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/tensor/squeeze.h" diff --git a/onnxruntime/core/providers/cuda/tensor/tile.h b/onnxruntime/core/providers/cuda/tensor/tile.h index a7be3b3808..a1deb53a00 100644 --- a/onnxruntime/core/providers/cuda/tensor/tile.h +++ b/onnxruntime/core/providers/cuda/tensor/tile.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/tensor/tile.h" diff --git a/onnxruntime/core/providers/cuda/tensor/transpose.h b/onnxruntime/core/providers/cuda/tensor/transpose.h index b0e40d5f6f..ab996ba151 100644 --- a/onnxruntime/core/providers/cuda/tensor/transpose.h +++ b/onnxruntime/core/providers/cuda/tensor/transpose.h @@ -5,7 +5,6 @@ #include "core/providers/shared_library/provider_api.h" #include "gsl/gsl" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/tensor/transpose.h" diff --git a/onnxruntime/core/providers/cuda/tensor/unsqueeze.h b/onnxruntime/core/providers/cuda/tensor/unsqueeze.h index c83a862fdf..a6bc6e9508 100644 --- a/onnxruntime/core/providers/cuda/tensor/unsqueeze.h +++ b/onnxruntime/core/providers/cuda/tensor/unsqueeze.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/tensor/unsqueeze.h" diff --git a/onnxruntime/core/providers/cuda/tensor/upsample.h b/onnxruntime/core/providers/cuda/tensor/upsample.h index 0512bf3b3a..06f98e821d 100644 --- a/onnxruntime/core/providers/cuda/tensor/upsample.h +++ b/onnxruntime/core/providers/cuda/tensor/upsample.h @@ -4,7 +4,6 @@ #pragma once #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cpu/tensor/upsample.h" diff --git a/onnxruntime/core/providers/cuda/tensor/where.h b/onnxruntime/core/providers/cuda/tensor/where.h index 3659f1447c..0ca0681852 100644 --- a/onnxruntime/core/providers/cuda/tensor/where.h +++ b/onnxruntime/core/providers/cuda/tensor/where.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/providers/shared_library/provider_api.h" -//#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/op_kernel_type_control_utils.h b/onnxruntime/core/providers/op_kernel_type_control_utils.h index 760a21158e..09f3ad6be4 100644 --- a/onnxruntime/core/providers/op_kernel_type_control_utils.h +++ b/onnxruntime/core/providers/op_kernel_type_control_utils.h @@ -5,7 +5,9 @@ #include "boost/mp11.hpp" +#ifndef SHARED_PROVIDER #include "core/framework/data_types.h" +#endif namespace onnxruntime { namespace utils { diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 9ef16dcbbd..3c5b5dc159 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -46,7 +46,7 @@ struct ProviderInfo_OpenVINO_Impl : ProviderInfo_OpenVINO { } g_info; struct OpenVINO_Provider : Provider { - const void* GetInfo() override { return &g_info; } + void* GetInfo() override { return &g_info; } std::shared_ptr CreateExecutionProviderFactory(const void* void_params) override { auto& params = *reinterpret_cast(void_params); diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc index 3e0780a24b..10f72d42aa 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc @@ -5,8 +5,8 @@ #include "core/providers/shared_library/provider_api.h" #include "../backend_utils.h" #include "../backend_manager.h" -#include -#include +#include +#include #include "data_ops.h" #include "capabilities.h" #include "utils.h" @@ -29,749 +29,714 @@ namespace onnxruntime { namespace openvino_ep { std::set ops_supported_only_in_model = { - "Cast", - "Concat", - "ConstantOfShape", - "Dropout", - "Expand", - "EyeLike", - "Exp", - "GatherND", - "Identity", - "NonMaxSuppression", - "NonZero", - "Not", - "OneHot", - "Pad", - "Range", - "ReduceMin", - "Resize", - "Round", - "Shape", - "Split", - "TopK" -}; + "Cast", + "Concat", + "ConstantOfShape", + "Dropout", + "Expand", + "EyeLike", + "Exp", + "GatherND", + "Identity", + "NonMaxSuppression", + "NonZero", + "Not", + "OneHot", + "Pad", + "Range", + "ReduceMin", + "Resize", + "Round", + "Shape", + "Split", + "TopK"}; std::vector supported_op_mode = { - {"Abs", V_2020_4,{"CPU", "GPU"}}, - {"Acos", V_2020_4,{"CPU"}}, - {"Acosh", V_2020_4,{"CPU"}}, - {"Add", V_2020_4,{"All"}}, - {"And", V_2020_4,{"All"}}, - {"ArgMax", V_2020_4,{"CPU"}}, - {"ArgMax", V_2021_1,{"All"}}, - {"ArgMin", V_2020_4,{"CPU"}}, - {"ArgMin", V_2021_2,{"CPU","MYRIAD"}}, - {"Asin", V_2020_4,{"CPU", "GPU"}}, - {"Asinh", V_2020_4,{"CPU", "GPU"}}, - {"Atan", V_2020_4,{"CPU", "GPU"}}, - {"Atanh", V_2020_4,{"CPU"}}, - {"AveragePool", V_2020_4,{"All"}}, - {"BatchNormalization", V_2020_4,{"All"}}, - {"Cast", V_2020_4,{"All"}}, - {"Ceil", V_2020_4,{"GPU"}}, - {"Ceil", V_2021_2,{"GPU","MYRIAD"}}, - {"Clip", V_2020_4,{"All"}}, - {"Concat", V_2020_4,{"All"}}, - {"Constant", V_2020_4,{"All"}}, - {"ConstantOfShape", V_2020_4,{"All"}}, - {"Conv", V_2020_4,{"All"}}, - {"ConvTranspose", V_2020_4,{"All"}}, - {"Cos", V_2020_4,{"CPU"}}, - {"Cosh", V_2020_4,{"CPU"}}, - {"DepthToSpace", V_2020_4,{"All"}}, - {"Div", V_2020_4,{"All"}}, - {"Dropout", V_2020_4,{"All"}}, - {"Elu", V_2020_4,{"All"}}, - {"Equal", V_2020_4,{"All"}}, - {"Erf", V_2020_4,{"All"}}, - {"Exp", V_2020_4,{"All"}}, - {"Expand", V_2021_1,{"MYRIAD"}}, - {"Flatten", V_2020_4,{"All"}}, - {"Floor", V_2020_4,{"All"}}, - {"Gather", V_2020_4,{"All"}}, - {"GatherND", V_2021_2,{"MYRIAD"}}, - {"Gemm", V_2020_4,{"All"}}, - {"GlobalAveragePool", V_2020_4,{"All"}}, - {"GlobalLpPool", V_2020_4,{"CPU", "GPU"}}, - {"Greater", V_2020_4,{"All"}}, - {"Identity", V_2020_4,{"All"}}, - {"InstanceNormalization", V_2020_4,{"All"}}, - {"HardSigmoid", V_2020_4,{"CPU", "GPU"}}, - {"LeakyRelu", V_2020_4,{"All"}}, - {"Less", V_2020_4,{"All"}}, - {"Log", V_2020_4,{"All"}}, - {"LRN", V_2020_4,{"All"}}, - {"LSTM", V_2020_4,{"All"}}, - {"MatMul", V_2020_4,{"All"}}, - {"Max", V_2020_4,{"All"}}, - {"MaxPool", V_2020_4,{"All"}}, - {"Mean", V_2020_4,{"All"}}, - {"Min", V_2020_4,{"All"}}, - {"Mul", V_2020_4,{"All"}}, - {"Neg", V_2020_4,{"All"}}, - {"NonMaxSuppression", V_2021_1,{"All"}}, - {"NonZero", V_2021_1,{"CPU", "MYRIAD"}}, - {"Not", V_2021_1,{"All"}}, - {"Not", V_2020_4,{"CPU", "GPU"}}, - {"OneHot", V_2020_4,{"All"}}, - {"Pad", V_2020_4,{"All"}}, - {"Pow", V_2020_4,{"All"}}, - {"PRelu", V_2020_4,{"All"}}, - {"Range", V_2021_2,{"MYRIAD"}}, - {"Reciprocal", V_2020_4,{"All"}}, - {"ReduceLogSum", V_2020_4,{"CPU", "MYRIAD"}}, - {"ReduceMax", V_2020_4 ,{"All"}}, - {"ReduceMean", V_2020_4,{"All"}}, - {"ReduceMin", V_2020_4,{"All"}}, - {"ReduceProd", V_2020_4,{"CPU"}}, - {"ReduceSum", V_2020_4,{"All"}}, - {"ReduceSumSquare", V_2020_4,{"CPU", "MYRIAD"}}, - {"Relu", V_2020_4,{"All"}}, - {"Resize", V_2020_4,{"CPU"}}, - {"Reshape", V_2020_4,{"All"}}, - {"RoiAlign", V_2021_1,{"All"}}, - {"Round", V_2021_2,{"MYRIAD"}}, - {"Scatter", V_2021_1,{"MYRIAD"}}, - {"ScatterElements", V_2021_2,{"MYRIAD"}}, - {"Selu", V_2020_4,{"CPU", "GPU"}}, - {"Shape", V_2020_4,{"All"}}, - {"Sigmoid", V_2020_4,{"All"}}, - {"Sign", V_2020_4,{"CPU"}}, - {"Sign", V_2020_4,{"CPU"}}, - {"Sinh", V_2020_4,{"CPU"}}, - {"SinFloat", V_2020_4,{"MYRIAD"}}, - {"Slice", V_2020_4,{"All"}}, - {"Softmax", V_2020_4,{"All"}}, - {"SpaceToDepth", V_2020_4,{"All"}}, - {"Split", V_2020_4,{"All"}}, - {"Sqrt", V_2020_4,{"All"}}, - {"Squeeze", V_2020_4,{"All"}}, - {"Softsign", V_2020_4,{"CPU"}}, - {"Sub", V_2020_4,{"All"}}, - {"Sum", V_2020_4,{"All"}}, - {"Tan", V_2020_4,{"CPU", "GPU"}}, - {"Tanh", V_2020_4,{"All"}}, - {"Transpose", V_2020_4,{"All"}}, - {"TopK", V_2020_4,{"All"}}, - {"Unsqueeze", V_2020_4,{"All"}}, - {"Upsample", V_2021_1,{"CPU"}}, - {"Where", V_2021_2,{"MYRIAD"}}, + {"Abs", V_2020_4, {"CPU", "GPU"}}, + {"Acos", V_2020_4, {"CPU"}}, + {"Acosh", V_2020_4, {"CPU"}}, + {"Add", V_2020_4, {"All"}}, + {"And", V_2020_4, {"All"}}, + {"ArgMax", V_2020_4, {"CPU"}}, + {"ArgMax", V_2021_1, {"All"}}, + {"ArgMin", V_2020_4, {"CPU"}}, + {"ArgMin", V_2021_2, {"CPU", "MYRIAD"}}, + {"Asin", V_2020_4, {"CPU", "GPU"}}, + {"Asinh", V_2020_4, {"CPU", "GPU"}}, + {"Atan", V_2020_4, {"CPU", "GPU"}}, + {"Atanh", V_2020_4, {"CPU"}}, + {"AveragePool", V_2020_4, {"All"}}, + {"BatchNormalization", V_2020_4, {"All"}}, + {"Cast", V_2020_4, {"All"}}, + {"Ceil", V_2020_4, {"GPU"}}, + {"Ceil", V_2021_2, {"GPU", "MYRIAD"}}, + {"Clip", V_2020_4, {"All"}}, + {"Concat", V_2020_4, {"All"}}, + {"Constant", V_2020_4, {"All"}}, + {"ConstantOfShape", V_2020_4, {"All"}}, + {"Conv", V_2020_4, {"All"}}, + {"ConvTranspose", V_2020_4, {"All"}}, + {"Cos", V_2020_4, {"CPU"}}, + {"Cosh", V_2020_4, {"CPU"}}, + {"DepthToSpace", V_2020_4, {"All"}}, + {"Div", V_2020_4, {"All"}}, + {"Dropout", V_2020_4, {"All"}}, + {"Elu", V_2020_4, {"All"}}, + {"Equal", V_2020_4, {"All"}}, + {"Erf", V_2020_4, {"All"}}, + {"Exp", V_2020_4, {"All"}}, + {"Expand", V_2021_1, {"MYRIAD"}}, + {"Flatten", V_2020_4, {"All"}}, + {"Floor", V_2020_4, {"All"}}, + {"Gather", V_2020_4, {"All"}}, + {"GatherND", V_2021_2, {"MYRIAD"}}, + {"Gemm", V_2020_4, {"All"}}, + {"GlobalAveragePool", V_2020_4, {"All"}}, + {"GlobalLpPool", V_2020_4, {"CPU", "GPU"}}, + {"Greater", V_2020_4, {"All"}}, + {"Identity", V_2020_4, {"All"}}, + {"InstanceNormalization", V_2020_4, {"All"}}, + {"HardSigmoid", V_2020_4, {"CPU", "GPU"}}, + {"LeakyRelu", V_2020_4, {"All"}}, + {"Less", V_2020_4, {"All"}}, + {"Log", V_2020_4, {"All"}}, + {"LRN", V_2020_4, {"All"}}, + {"LSTM", V_2020_4, {"All"}}, + {"MatMul", V_2020_4, {"All"}}, + {"Max", V_2020_4, {"All"}}, + {"MaxPool", V_2020_4, {"All"}}, + {"Mean", V_2020_4, {"All"}}, + {"Min", V_2020_4, {"All"}}, + {"Mul", V_2020_4, {"All"}}, + {"Neg", V_2020_4, {"All"}}, + {"NonMaxSuppression", V_2021_1, {"All"}}, + {"NonZero", V_2021_1, {"CPU", "MYRIAD"}}, + {"Not", V_2021_1, {"All"}}, + {"Not", V_2020_4, {"CPU", "GPU"}}, + {"OneHot", V_2020_4, {"All"}}, + {"Pad", V_2020_4, {"All"}}, + {"Pow", V_2020_4, {"All"}}, + {"PRelu", V_2020_4, {"All"}}, + {"Range", V_2021_2, {"MYRIAD"}}, + {"Reciprocal", V_2020_4, {"All"}}, + {"ReduceLogSum", V_2020_4, {"CPU", "MYRIAD"}}, + {"ReduceMax", V_2020_4, {"All"}}, + {"ReduceMean", V_2020_4, {"All"}}, + {"ReduceMin", V_2020_4, {"All"}}, + {"ReduceProd", V_2020_4, {"CPU"}}, + {"ReduceSum", V_2020_4, {"All"}}, + {"ReduceSumSquare", V_2020_4, {"CPU", "MYRIAD"}}, + {"Relu", V_2020_4, {"All"}}, + {"Resize", V_2020_4, {"CPU"}}, + {"Reshape", V_2020_4, {"All"}}, + {"RoiAlign", V_2021_1, {"All"}}, + {"Round", V_2021_2, {"MYRIAD"}}, + {"Scatter", V_2021_1, {"MYRIAD"}}, + {"ScatterElements", V_2021_2, {"MYRIAD"}}, + {"Selu", V_2020_4, {"CPU", "GPU"}}, + {"Shape", V_2020_4, {"All"}}, + {"Sigmoid", V_2020_4, {"All"}}, + {"Sign", V_2020_4, {"CPU"}}, + {"Sign", V_2020_4, {"CPU"}}, + {"Sinh", V_2020_4, {"CPU"}}, + {"SinFloat", V_2020_4, {"MYRIAD"}}, + {"Slice", V_2020_4, {"All"}}, + {"Softmax", V_2020_4, {"All"}}, + {"SpaceToDepth", V_2020_4, {"All"}}, + {"Split", V_2020_4, {"All"}}, + {"Sqrt", V_2020_4, {"All"}}, + {"Squeeze", V_2020_4, {"All"}}, + {"Softsign", V_2020_4, {"CPU"}}, + {"Sub", V_2020_4, {"All"}}, + {"Sum", V_2020_4, {"All"}}, + {"Tan", V_2020_4, {"CPU", "GPU"}}, + {"Tanh", V_2020_4, {"All"}}, + {"Transpose", V_2020_4, {"All"}}, + {"TopK", V_2020_4, {"All"}}, + {"Unsqueeze", V_2020_4, {"All"}}, + {"Upsample", V_2021_1, {"CPU"}}, + {"Where", V_2021_2, {"MYRIAD"}}, }; void DataOps::populate_types_supported() { + supported_types_initializer_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BOOL)); + supported_types_initializer_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)); + supported_types_initializer_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); + supported_types_initializer_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); + supported_types_initializer_.insert(std::make_pair(V_2021_1, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)); - supported_types_initializer_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BOOL)); - supported_types_initializer_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)); - supported_types_initializer_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); - supported_types_initializer_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); - supported_types_initializer_.insert(std::make_pair(V_2021_1, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)); + supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BOOL)); + supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)); + supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8)); + supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8)); + supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16)); + supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); + supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); + supported_types_vpu_.insert(std::make_pair(V_2021_1, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)); - supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BOOL)); - supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)); - supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8)); - supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8)); - supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16)); - supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); - supported_types_vpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); - supported_types_vpu_.insert(std::make_pair(V_2021_1, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)); + supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BOOL)); + supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)); + supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); + supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16)); + supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8)); + supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8)); + supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); + supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8)); + supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); - supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BOOL)); - supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)); - supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); - supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16)); - supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8)); - supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8)); - supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); - supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8)); - supported_types_cpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); - - supported_types_gpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)); - supported_types_gpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); - supported_types_gpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); - supported_types_gpu_.insert(std::make_pair(V_2021_1, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)); + supported_types_gpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)); + supported_types_gpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); + supported_types_gpu_.insert(std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); + supported_types_gpu_.insert(std::make_pair(V_2021_1, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)); } void DataOps::populate_op_mode_supported() { + no_dimension_supported_.push_back({"Unsqueeze", V_2020_4, {"All"}}); + no_dimension_supported_.push_back({"Squeeze", V_2020_4, {"All"}}); + no_dimension_supported_.push_back({"Cast", V_2020_4, {"All"}}); + no_dimension_supported_.push_back({"Gather", V_2020_4, {"All"}}); + no_dimension_supported_.push_back({"Mul", V_2020_4, {"All"}}); + no_dimension_supported_.push_back({"Sub", V_2020_4, {"All"}}); + no_dimension_supported_.push_back({"Min", V_2020_4, {"All"}}); + no_dimension_supported_.push_back({"Div", V_2020_4, {"All"}}); + no_dimension_supported_.push_back({"Floor", V_2020_4, {"All"}}); + no_dimension_supported_.push_back({"Where", V_2021_2, {"All"}}); + no_dimension_supported_.push_back({"Range", V_2021_2, {"All"}}); + no_dimension_supported_.push_back({"ArgMin", V_2021_2, {"Myriad"}}); + no_dimension_supported_.push_back({"Max", V_2021_2, {"Myriad"}}); + no_dimension_supported_.push_back({"Add", V_2021_2, {"Myriad"}}); + no_dimension_supported_.push_back({"Less", V_2021_2, {"Myriad"}}); + no_dimension_supported_.push_back({"Greater", V_2021_2, {"Myriad"}}); + no_dimension_supported_.push_back({"Clip", V_2021_2, {"Myriad"}}); + no_dimension_supported_.push_back({"Resize", V_2021_2, {"Myriad"}}); + no_dimension_supported_.push_back({"Equal", V_2021_2, {"Myriad"}}); - no_dimension_supported_.push_back({"Unsqueeze", V_2020_4,{"All"}}); - no_dimension_supported_.push_back({"Squeeze", V_2020_4,{"All"}}); - no_dimension_supported_.push_back({"Cast", V_2020_4,{"All"}}); - no_dimension_supported_.push_back({"Gather", V_2020_4,{"All"}}); - no_dimension_supported_.push_back({"Mul", V_2020_4,{"All"}}); - no_dimension_supported_.push_back({"Sub", V_2020_4,{"All"}}); - no_dimension_supported_.push_back({"Min", V_2020_4,{"All"}}); - no_dimension_supported_.push_back({"Div", V_2020_4,{"All"}}); - no_dimension_supported_.push_back({"Floor", V_2020_4,{"All"}}); - no_dimension_supported_.push_back({"Where", V_2021_2,{"All"}}); - no_dimension_supported_.push_back({"Range", V_2021_2,{"All"}}); - no_dimension_supported_.push_back({"ArgMin", V_2021_2,{"Myriad"}}); - no_dimension_supported_.push_back({"Max", V_2021_2,{"Myriad"}}); - no_dimension_supported_.push_back({"Add", V_2021_2,{"Myriad"}}); - no_dimension_supported_.push_back({"Less", V_2021_2,{"Myriad"}}); - no_dimension_supported_.push_back({"Greater", V_2021_2,{"Myriad"}}); - no_dimension_supported_.push_back({"Clip", V_2021_2,{"Myriad"}}); - no_dimension_supported_.push_back({"Resize", V_2021_2,{"Myriad"}}); - no_dimension_supported_.push_back({"Equal", V_2021_2,{"Myriad"}}); + subgraph_supported_.push_back({"Mul", V_2020_4, {"All"}}); + subgraph_supported_.push_back({"Transpose", V_2020_4, {"All"}}); + subgraph_supported_.push_back({"Unsqueeze", V_2020_4, {"All"}}); + subgraph_supported_.push_back({"Cast", V_2020_4, {"All"}}); + subgraph_supported_.push_back({"Concat", V_2020_4, {"All"}}); + subgraph_supported_.push_back({"Gather", V_2020_4, {"All"}}); + subgraph_supported_.push_back({"Div", V_2020_4, {"Myriad"}}); + subgraph_supported_.push_back({"Sub", V_2020_4, {"Myriad"}}); + subgraph_supported_.push_back({"Identity", V_2021_1, {"CPU"}}); + subgraph_supported_.push_back({"Div", V_2021_1, {"CPU"}}); + subgraph_supported_.push_back({"Sub", V_2021_1, {"CPU"}}); - - subgraph_supported_.push_back({"Mul", V_2020_4,{"All"}}); - subgraph_supported_.push_back({"Transpose", V_2020_4,{"All"}}); - subgraph_supported_.push_back({"Unsqueeze", V_2020_4,{"All"}}); - subgraph_supported_.push_back({"Cast", V_2020_4,{"All"}}); - subgraph_supported_.push_back({"Concat", V_2020_4,{"All"}}); - subgraph_supported_.push_back({"Gather", V_2020_4,{"All"}}); - subgraph_supported_.push_back({"Div", V_2020_4,{"Myriad"}}); - subgraph_supported_.push_back({"Sub", V_2020_4,{"Myriad"}}); - subgraph_supported_.push_back({"Identity", V_2021_1,{"CPU"}}); - subgraph_supported_.push_back({"Div", V_2021_1,{"CPU"}}); - subgraph_supported_.push_back({"Sub", V_2021_1,{"CPU"}}); - - //populate unsupportedmode_t { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - for (size_t i = 0; i < node->InputDefs().size(); i++) { - if (node->InputDefs()[i]->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) - return true; - } - return false; - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + for (size_t i = 0; i < node->InputDefs().size(); i++) { + if (node->InputDefs()[i]->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) + return true; + } + return false; + }}; op_list_.insert({"Abs", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - //tensor type does not support select last index - auto& attributes = node->GetAttributes(); - auto last_index_arg = attributes.count("select_last_index") > 0 ? attributes.at("select_last_index").i() : 0; - if (last_index_arg != 0) - return true; - // tensor type supports float as input for argmax and argmin - if (node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) - return true; - return false; - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + //tensor type does not support select last index + auto& attributes = node->GetAttributes(); + auto last_index_arg = attributes.count("select_last_index") > 0 ? attributes.at("select_last_index").i() : 0; + if (last_index_arg != 0) + return true; + // tensor type supports float as input for argmax and argmin + if (node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) + return true; + return false; + }}; op_list_.insert({"ArgMax", obj}); op_list_.insert({"ArgMin", obj}); } { UnsupportedOpMode obj = {{V_2020_4}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - // ceil_mode attribute is not supported in nGraph - const auto& attributes = node->GetAttributes(); - auto ceil_attr = attributes.find("ceil_mode"); - // default value of ceil_mode (0) is supported. - if (ceil_attr != attributes.end() && ceil_attr->second().i() != 0) return true; - return (!dimension_unsupported(node)); - } - }; + [this](const Node* node, const InitializedTensorSet&) { + // ceil_mode attribute is not supported in nGraph + const auto& attributes = node->GetAttributes(); + auto ceil_attr = attributes.find("ceil_mode"); + // default value of ceil_mode (0) is supported. + if (ceil_attr != attributes.end() && ceil_attr->second().i() != 0) return true; + return (!dimension_unsupported(node)); + }}; op_list_.insert({"AveragePool", obj}); } { - UnsupportedOpMode obj = {{V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - //auto pad null value is not supported - const auto& attributes = node->GetAttributes(); - auto auto_attr = attributes.find("auto_pad"); - if (auto_attr->second().s() == "") { - return true; - } - // default value of ceil_mode (0) is supported. - auto ceil_attr = attributes.find("ceil_mode"); - if (ceil_attr != attributes.end() && ceil_attr->second().i() != 0) return true; - return (!dimension_unsupported(node)); - } - }; + UnsupportedOpMode obj = {{V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + //auto pad null value is not supported + const auto& attributes = node->GetAttributes(); + auto auto_attr = attributes.find("auto_pad"); + if (auto_attr->second().s() == "") { + return true; + } + // default value of ceil_mode (0) is supported. + auto ceil_attr = attributes.find("ceil_mode"); + if (ceil_attr != attributes.end() && ceil_attr->second().i() != 0) return true; + return (!dimension_unsupported(node)); + }}; op_list_.insert({"AveragePool", obj}); - } + } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - //Only float 16, float and double data types are supported - const bool data_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; - const bool data_is_float16 = node->InputDefs()[0]->Type()->find("float16") != std::string::npos; - const bool data_is_double = node->InputDefs()[0]->Type()->find("double") != std::string::npos; - return !(data_is_float || data_is_float16 || data_is_double); - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + //Only float 16, float and double data types are supported + const bool data_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; + const bool data_is_float16 = node->InputDefs()[0]->Type()->find("float16") != std::string::npos; + const bool data_is_double = node->InputDefs()[0]->Type()->find("double") != std::string::npos; + return !(data_is_float || data_is_float16 || data_is_double); + }}; op_list_.insert({"Clip", obj}); } - { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& initializers) { - if (GetInputCount(node, initializers) > 1) - return true; - return false; - } }; + { + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet& initializers) { + if (GetInputCount(node, initializers) > 1) + return true; + return false; + }}; op_list_.insert({"Conv", obj}); op_list_.insert({"ConvTranspose", obj}); } - { - UnsupportedOpMode obj = {{V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& ) { - auto& attributes = node->GetAttributes(); - if (attributes.count("auto_pad") == 0 || attributes.at("auto_pad").s() == "") { - return true; - } - return false; - } }; + { + UnsupportedOpMode obj = {{V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + auto& attributes = node->GetAttributes(); + if (attributes.count("auto_pad") == 0 || attributes.at("auto_pad").s() == "") { + return true; + } + return false; + }}; op_list_.insert({"Conv", obj}); op_list_.insert({"ConvTranspose", obj}); } - { - UnsupportedOpMode obj = {{V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& ) { - if (device_id_.find("MYRIAD") != std::string::npos) { - const auto& input_arg = node->InputDefs()[0]; - auto shape = input_arg->Shape(); - if ((shape != nullptr) && (shape->dim(0).value_case() != shape->dim(0).kDimValue)) { - return true; - } - } - return false; - } }; + { + UnsupportedOpMode obj = {{V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + if (device_id_.find("MYRIAD") != std::string::npos) { + const auto& input_arg = node->InputDefs()[0]; + auto shape = input_arg->Shape(); + if ((shape != nullptr) && (shape->dim(0).value_case() != shape->dim(0).kDimValue)) { + return true; + } + } + return false; + }}; op_list_.insert({"ConvTranspose", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& initializers) { - // all ConvInteger zero points need to be constants - if (node->InputDefs().size() == 3) { - return (initializers.find(node->InputDefs()[2]->Name()) == initializers.end()); - } else if (node->InputDefs().size() == 4) { - return initializers.find(node->InputDefs()[2]->Name()) == initializers.end() || - initializers.find(node->InputDefs()[3]->Name()) == initializers.end(); - } - return false; - } - }; - op_list_.insert({"ConvInteger", obj}); + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet& initializers) { + // all ConvInteger zero points need to be constants + if (node->InputDefs().size() == 3) { + return (initializers.find(node->InputDefs()[2]->Name()) == initializers.end()); + } else if (node->InputDefs().size() == 4) { + return initializers.find(node->InputDefs()[2]->Name()) == initializers.end() || + initializers.find(node->InputDefs()[3]->Name()) == initializers.end(); + } + return false; + }}; + op_list_.insert({"ConvInteger", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - using onnx_dtype = ONNX_NAMESPACE::TensorProto_DataType; - auto supportedOps = std::set>{ - {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT}, - {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_FLOAT}, - {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_INT8}, - {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_FLOAT}, - {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8}, - {onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_INT8}, - {onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_UINT8}, - {onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_INT8}, - {onnx_dtype::TensorProto_DataType_INT32, onnx_dtype::TensorProto_DataType_INT32, onnx_dtype::TensorProto_DataType_INT32}, - {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_FLOAT}, - {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8}}; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1}, + [this](const Node* node, const InitializedTensorSet&) { + using onnx_dtype = ONNX_NAMESPACE::TensorProto_DataType; + auto supportedOps = std::set>{ + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT}, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_FLOAT}, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_INT8}, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_FLOAT}, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8}, + {onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_INT8}, + {onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_UINT8}, + {onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_INT8}, + {onnx_dtype::TensorProto_DataType_INT32, onnx_dtype::TensorProto_DataType_INT32, onnx_dtype::TensorProto_DataType_INT32}, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_FLOAT}, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8}}; - if (node->OpType() == "Equal") { - supportedOps.insert(std::vector{onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_INT32, onnx_dtype::TensorProto_DataType_INT32}), - supportedOps.insert(std::vector{onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT}); - } + if (node->OpType() == "Equal") { + supportedOps.insert(std::vector{onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_INT32, onnx_dtype::TensorProto_DataType_INT32}), + supportedOps.insert(std::vector{onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT}); + } - onnx_dtype input_0_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - onnx_dtype input_1_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); - onnx_dtype output_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - const std::vector typePair{output_data_type, input_0_data_type, input_1_data_type}; - const auto match = supportedOps.find(typePair); - if (match == supportedOps.end()) - return true; - else - return false; - } - }; - op_list_.insert({"Equal", obj}); - op_list_.insert({"And", obj}); + onnx_dtype input_0_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + onnx_dtype input_1_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); + onnx_dtype output_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + const std::vector typePair{output_data_type, input_0_data_type, input_1_data_type}; + const auto match = supportedOps.find(typePair); + if (match == supportedOps.end()) + return true; + else + return false; + }}; + op_list_.insert({"Equal", obj}); + op_list_.insert({"And", obj}); } { - UnsupportedOpMode obj = {{V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& ) { - if (device_id_.find("GPU") != std::string::npos) { - const auto& input = node->InputDefs()[0]; - auto graph_inputs = graph_viewer_.GetInputs(); - auto it = find(graph_inputs.begin(), graph_inputs.end(), input); - if (it != graph_inputs.end()) { - const auto& indices_arg = node->InputDefs()[1]; - if (indices_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64) - return true; - } - } - return false; - } - }; + UnsupportedOpMode obj = {{V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + if (device_id_.find("GPU") != std::string::npos) { + const auto& input = node->InputDefs()[0]; + auto graph_inputs = graph_viewer_.GetInputs(); + auto it = find(graph_inputs.begin(), graph_inputs.end(), input); + if (it != graph_inputs.end()) { + const auto& indices_arg = node->InputDefs()[1]; + if (indices_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64) + return true; + } + } + return false; + }}; op_list_.insert({"Gather", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - const auto& input = node->InputDefs()[0]; - const auto& output = node->OutputDefs()[0]; - auto graph_inputs = this->graph_viewer_.GetInputs(); - auto graph_outputs = this->graph_viewer_.GetOutputs(); - auto input_it = find(graph_inputs.begin(), graph_inputs.end(), input); - auto output_it = find(graph_outputs.begin(), graph_outputs.end(), output); - if (input_it != graph_inputs.end() && output_it != graph_outputs.end()) - return true; - return false; - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + const auto& input = node->InputDefs()[0]; + const auto& output = node->OutputDefs()[0]; + auto graph_inputs = this->graph_viewer_.GetInputs(); + auto graph_outputs = this->graph_viewer_.GetOutputs(); + auto input_it = find(graph_inputs.begin(), graph_inputs.end(), input); + auto output_it = find(graph_outputs.begin(), graph_outputs.end(), output); + if (input_it != graph_inputs.end() && output_it != graph_outputs.end()) + return true; + return false; + }}; op_list_.insert({"Identity", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - //MaxPool "indices" output is not currently supported. - if (node->OutputDefs().size() > 1) - return true; - const auto& attributes = node->GetAttributes(); - // default value of ceil_mode (0) is supported. - auto ceil_attr = attributes.find("ceil_mode"); - if (ceil_attr != attributes.end() && ceil_attr->second().i() != 0) - return true; - auto auto_attr = attributes.find("auto_pad"); - //auto pad null value is not supported - if (auto_attr->second().s() == "") - return true; - // dilations attrs are not supported in nGraph - if (attributes.find("dilations") != attributes.end()) - return true; - return(!this->dimension_unsupported(node)); - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + //MaxPool "indices" output is not currently supported. + if (node->OutputDefs().size() > 1) + return true; + const auto& attributes = node->GetAttributes(); + // default value of ceil_mode (0) is supported. + auto ceil_attr = attributes.find("ceil_mode"); + if (ceil_attr != attributes.end() && ceil_attr->second().i() != 0) + return true; + auto auto_attr = attributes.find("auto_pad"); + //auto pad null value is not supported + if (auto_attr->second().s() == "") + return true; + // dilations attrs are not supported in nGraph + if (attributes.find("dilations") != attributes.end()) + return true; + return (!this->dimension_unsupported(node)); + }}; op_list_.insert({"MaxPool", obj}); } { - UnsupportedOpMode obj = {{V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& initializers) { - if (device_id_.find("MYRIAD") == std::string::npos) { - if (GetInputCount(node, initializers) == 1) - return true; - } - return false; - } - }; + UnsupportedOpMode obj = {{V_2021_2}, + [this](const Node* node, const InitializedTensorSet& initializers) { + if (device_id_.find("MYRIAD") == std::string::npos) { + if (GetInputCount(node, initializers) == 1) + return true; + } + return false; + }}; op_list_.insert({"Max", obj}); op_list_.insert({"Min", obj}); op_list_.insert({"Mean", obj}); op_list_.insert({"Sum", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1}, - [this](const Node* node, const Provider_InitializedTensorSet& initializers) { - if (GetInputCount(node, initializers) == 1) - return true; - return false; - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1}, + [this](const Node* node, const InitializedTensorSet& initializers) { + if (GetInputCount(node, initializers) == 1) + return true; + return false; + }}; op_list_.insert({"Mean", obj}); op_list_.insert({"Sum", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1}, - [this](const Node* node, const Provider_InitializedTensorSet& initializers) { - if (GetInputCount(node, initializers) == 1) - return true; - for (size_t i = 0; i < node->InputDefs().size(); i++) { - auto dtype = node->InputDefs()[i]->TypeAsProto()->tensor_type().elem_type(); - if (dtype == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8 || - dtype == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16) - return true; - } - return false; - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1}, + [this](const Node* node, const InitializedTensorSet& initializers) { + if (GetInputCount(node, initializers) == 1) + return true; + for (size_t i = 0; i < node->InputDefs().size(); i++) { + auto dtype = node->InputDefs()[i]->TypeAsProto()->tensor_type().elem_type(); + if (dtype == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8 || + dtype == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16) + return true; + } + return false; + }}; op_list_.insert({"Max", obj}); op_list_.insert({"Min", obj}); - } - { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - //All matmuls except float have computation missmatch - const bool A_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; - const bool B_is_float = node->InputDefs()[1]->Type()->find("float") != std::string::npos; - return (A_is_float && B_is_float) ? false : true; - } - }; - op_list_.insert({"MatMul", obj}); } - { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& initializers) { - // all MatMulInteger zero points need to be constants - if (node->InputDefs().size() == 3) { - // not found in initializers -> not const - return initializers.find(node->InputDefs()[2]->Name()) == initializers.end(); - } else if (node->InputDefs().size() == 4) { - // not found in initializers -> not const - return ((initializers.find(node->InputDefs()[2]->Name()) == initializers.end()) || - (initializers.find(node->InputDefs()[2]->Name()) == initializers.end())) ; - } - return false; - } - }; + { + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + //All matmuls except float have computation missmatch + const bool A_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; + const bool B_is_float = node->InputDefs()[1]->Type()->find("float") != std::string::npos; + return (A_is_float && B_is_float) ? false : true; + }}; + op_list_.insert({"MatMul", obj}); + } + { + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet& initializers) { + // all MatMulInteger zero points need to be constants + if (node->InputDefs().size() == 3) { + // not found in initializers -> not const + return initializers.find(node->InputDefs()[2]->Name()) == initializers.end(); + } else if (node->InputDefs().size() == 4) { + // not found in initializers -> not const + return ((initializers.find(node->InputDefs()[2]->Name()) == initializers.end()) || + (initializers.find(node->InputDefs()[2]->Name()) == initializers.end())); + } + return false; + }}; op_list_.insert({"MatMulInteger", obj}); } - { - UnsupportedOpMode obj = {{V_2020_4,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - //Only FP32 data type is allowed - auto& attributes = node->GetAttributes(); - auto fmod = attributes.count("fmod") > 0 ? attributes.at("fmod").i() : 0; - if (fmod != 1) return true; - //Only FP32 data type is allowed - for (const auto& input : node->InputDefs()) { - if (input->Type()->find("float") == std::string::npos) - return true; - } - return false; - } - }; + { + UnsupportedOpMode obj = {{V_2020_4, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + //Only FP32 data type is allowed + auto& attributes = node->GetAttributes(); + auto fmod = attributes.count("fmod") > 0 ? attributes.at("fmod").i() : 0; + if (fmod != 1) return true; + //Only FP32 data type is allowed + for (const auto& input : node->InputDefs()) { + if (input->Type()->find("float") == std::string::npos) + return true; + } + return false; + }}; op_list_.insert({"Mod", obj}); } { UnsupportedOpMode obj = {{V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - auto graph_outputs = graph_viewer_.GetOutputs(); - const auto& output = node->OutputDefs()[0]; - auto output_it = find(graph_outputs.begin(), graph_outputs.end(), output); - if (output_it != graph_outputs.end()) - return true; - return false; - } - }; + [this](const Node* node, const InitializedTensorSet&) { + auto graph_outputs = graph_viewer_.GetOutputs(); + const auto& output = node->OutputDefs()[0]; + auto output_it = find(graph_outputs.begin(), graph_outputs.end(), output); + if (output_it != graph_outputs.end()) + return true; + return false; + }}; op_list_.insert({"NonMaxSuppression", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - //Only supported if the data type of both inputs is same - auto x_data_type = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - auto y_data_type = node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); - return x_data_type != y_data_type; - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1}, + [this](const Node* node, const InitializedTensorSet&) { + //Only supported if the data type of both inputs is same + auto x_data_type = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + auto y_data_type = node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); + return x_data_type != y_data_type; + }}; op_list_.insert({"Pow", obj}); } { UnsupportedOpMode obj = {{V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - if (device_id_.find("GPU") != std::string::npos) { - auto x_data_type = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - auto y_data_type = node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); - return x_data_type != y_data_type; - } - //currently both inputs with int32 or int64 datatype are not supported - const bool A_is_int32 = node->InputDefs()[0]->Type()->find("int32") != std::string::npos; - const bool B_is_int32 = node->InputDefs()[1]->Type()->find("int32") != std::string::npos; - const bool A_is_int64 = node->InputDefs()[0]->Type()->find("int64") != std::string::npos; - const bool B_is_int64 = node->InputDefs()[1]->Type()->find("int64") != std::string::npos; - if((A_is_int32 && B_is_int32) || (A_is_int64 && B_is_int64)) - return true; - return false; - } - }; + [this](const Node* node, const InitializedTensorSet&) { + if (device_id_.find("GPU") != std::string::npos) { + auto x_data_type = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + auto y_data_type = node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); + return x_data_type != y_data_type; + } + //currently both inputs with int32 or int64 datatype are not supported + const bool A_is_int32 = node->InputDefs()[0]->Type()->find("int32") != std::string::npos; + const bool B_is_int32 = node->InputDefs()[1]->Type()->find("int32") != std::string::npos; + const bool A_is_int64 = node->InputDefs()[0]->Type()->find("int64") != std::string::npos; + const bool B_is_int64 = node->InputDefs()[1]->Type()->find("int64") != std::string::npos; + if ((A_is_int32 && B_is_int32) || (A_is_int64 && B_is_int64)) + return true; + return false; + }}; op_list_.insert({"Pow", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& initializers) { - auto slope = node->InputDefs()[1]; - //PRelu slope has to be an initializer or needs to come from a constant node - if (initializers.count(slope->Name())) - return false; - else { - for (auto input_node = node->InputNodesBegin(); input_node != node->InputNodesEnd(); ++input_node) { - if (GetInputCount(this->graph_viewer_.GetNode((*input_node).Index()), initializers) == 0) - return false; - } } - return true; - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet& initializers) { + auto slope = node->InputDefs()[1]; + //PRelu slope has to be an initializer or needs to come from a constant node + if (initializers.count(slope->Name())) + return false; + else { + for (auto input_node = node->InputNodesBegin(); input_node != node->InputNodesEnd(); ++input_node) { + if (GetInputCount(this->graph_viewer_.GetNode((*input_node).Index()), initializers) == 0) + return false; + } + } + return true; + }}; op_list_.insert({"PRelu", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& initializers) { - bool non_const_zero_point = false; - // check if any of the zero points is NOT in the initializers list - non_const_zero_point |= initializers.find(node->InputDefs()[2]->Name()) == initializers.end(); - non_const_zero_point |= initializers.find(node->InputDefs()[5]->Name()) == initializers.end(); - non_const_zero_point |= initializers.find(node->InputDefs()[7]->Name()) == initializers.end(); - // QLinearMatMul is not supported if any of the zero points is a dynamic input - return non_const_zero_point; - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet& initializers) { + bool non_const_zero_point = false; + // check if any of the zero points is NOT in the initializers list + non_const_zero_point |= initializers.find(node->InputDefs()[2]->Name()) == initializers.end(); + non_const_zero_point |= initializers.find(node->InputDefs()[5]->Name()) == initializers.end(); + non_const_zero_point |= initializers.find(node->InputDefs()[7]->Name()) == initializers.end(); + // QLinearMatMul is not supported if any of the zero points is a dynamic input + return non_const_zero_point; + }}; op_list_.insert({"QLinearMatMul", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - //Only FP32, INT32 and U8 data types are supported - const bool data_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; - const bool data_is_int32 = node->InputDefs()[0]->Type()->find("int32") != std::string::npos; - const bool data_is_u8 = node->InputDefs()[0]->Type()->find("uint8") != std::string::npos; - return !(data_is_float || data_is_int32 || data_is_u8); - } - }; - op_list_.insert({"ReduceMin", obj}); + UnsupportedOpMode obj = {{V_2020_4, V_2021_1}, + [this](const Node* node, const InitializedTensorSet&) { + //Only FP32, INT32 and U8 data types are supported + const bool data_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; + const bool data_is_int32 = node->InputDefs()[0]->Type()->find("int32") != std::string::npos; + const bool data_is_u8 = node->InputDefs()[0]->Type()->find("uint8") != std::string::npos; + return !(data_is_float || data_is_int32 || data_is_u8); + }}; + op_list_.insert({"ReduceMin", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - //Resize opset 11 is not supported - if (node->InputDefs().size() > 2) - return true; - return false; - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1}, + [this](const Node* node, const InitializedTensorSet&) { + //Resize opset 11 is not supported + if (node->InputDefs().size() > 2) + return true; + return false; + }}; op_list_.insert({"Resize", obj}); } { - UnsupportedOpMode obj = {{V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - const auto& attributes = node->GetAttributes(); - auto axis_attr = attributes.find("axis"); - //Negative axis is not supported - if (axis_attr->second().i() < 0) - return true; - return false; - } - }; + UnsupportedOpMode obj = {{V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + const auto& attributes = node->GetAttributes(); + auto axis_attr = attributes.find("axis"); + //Negative axis is not supported + if (axis_attr->second().i() < 0) + return true; + return false; + }}; op_list_.insert({"Scatter", obj}); - op_list_.insert({"ScatterElements", obj}); + op_list_.insert({"ScatterElements", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1, V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& initializers) { - //start, end, axes need to be a initializer - bool cond_for_slice = false; - const auto& data_arg = node->InputDefs()[0]; - auto graph_inputs = graph_viewer_.GetInputs(); + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet& initializers) { + //start, end, axes need to be a initializer + bool cond_for_slice = false; + const auto& data_arg = node->InputDefs()[0]; + auto graph_inputs = graph_viewer_.GetInputs(); - auto it = find(graph_inputs.begin(), graph_inputs.end(), data_arg); - if (it != graph_inputs.end()) { - if (node->InputDefs().size() > 1) { - const auto& start_arg = node->InputDefs()[1]; - const auto& end_arg = node->InputDefs()[2]; - cond_for_slice |= initializers.find(start_arg->Name()) == initializers.end(); - cond_for_slice |= initializers.find(end_arg->Name()) == initializers.end(); - } - if (node->InputDefs().size() > 3) { - const auto& axes_arg = node->InputDefs()[3]; - cond_for_slice |= initializers.find(axes_arg->Name()) == initializers.end(); - } - } + auto it = find(graph_inputs.begin(), graph_inputs.end(), data_arg); + if (it != graph_inputs.end()) { + if (node->InputDefs().size() > 1) { + const auto& start_arg = node->InputDefs()[1]; + const auto& end_arg = node->InputDefs()[2]; + cond_for_slice |= initializers.find(start_arg->Name()) == initializers.end(); + cond_for_slice |= initializers.find(end_arg->Name()) == initializers.end(); + } + if (node->InputDefs().size() > 3) { + const auto& axes_arg = node->InputDefs()[3]; + cond_for_slice |= initializers.find(axes_arg->Name()) == initializers.end(); + } + } - return cond_for_slice; - } - }; + return cond_for_slice; + }}; op_list_.insert({"Slice", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - //Shape can't have empty axes attribute - const auto& attributes = node->GetAttributes(); - if (attributes.count("axes") == 0) - return true; - return false; - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + //Shape can't have empty axes attribute + const auto& attributes = node->GetAttributes(); + if (attributes.count("axes") == 0) + return true; + return false; + }}; op_list_.insert({"Squeeze", obj}); } { - UnsupportedOpMode obj = {{V_2020_4}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - return node->InputDefs().size() > 1; - } - }; + UnsupportedOpMode obj = {{V_2020_4}, + [this](const Node* node, const InitializedTensorSet&) { + return node->InputDefs().size() > 1; + }}; op_list_.insert({"TopK", obj}); } { - UnsupportedOpMode obj = {{V_2020_4,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet&) { - return (!this->dimension_unsupported(node)); - } - }; + UnsupportedOpMode obj = {{V_2020_4, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + return (!this->dimension_unsupported(node)); + }}; op_list_.insert({"Unsqueeze", obj}); } { - UnsupportedOpMode obj = {{V_2021_1,V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& ) { - //check for attributes - auto& upsample_attr = node->GetAttributes(); - if (upsample_attr.count("scales") > 0) { - auto& upsample_arg = upsample_attr.at("scales"); - auto float_size = upsample_arg.floats_size(); - if (float_size > 2 && (upsample_arg.floats(0) != 1.f || upsample_arg.floats(1) != 1.f)) { - return true; - } - } + UnsupportedOpMode obj = {{V_2021_1, V_2021_2}, + [this](const Node* node, const InitializedTensorSet&) { + //check for attributes + auto& upsample_attr = node->GetAttributes(); + if (upsample_attr.count("scales") > 0) { + auto& upsample_arg = upsample_attr.at("scales"); + auto float_size = upsample_arg.floats_size(); + if (float_size > 2 && (upsample_arg.floats(0) != 1.f || upsample_arg.floats(1) != 1.f)) { + return true; + } + } - //check for input dimensions - const auto& x_arg = node->InputDefs()[0]; - auto shape = x_arg->Shape(); - if (shape != nullptr) { - //input tensor rank cannot be of one dimension - if (shape->dim_size() == 1) { - return true; - } - } - // x_arg supports only float, int8 and float16 type - if ((x_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) || - (x_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8) || - (x_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)) { - return false; - } else { - return true; - } - } - }; + //check for input dimensions + const auto& x_arg = node->InputDefs()[0]; + auto shape = x_arg->Shape(); + if (shape != nullptr) { + //input tensor rank cannot be of one dimension + if (shape->dim_size() == 1) { + return true; + } + } + // x_arg supports only float, int8 and float16 type + if ((x_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) || + (x_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8) || + (x_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)) { + return false; + } else { + return true; + } + }}; op_list_.insert({"Upsample", obj}); } { UnsupportedOpMode obj = {{V_2021_2}, - [this](const Node* node, const Provider_InitializedTensorSet& ) { - //float data type is not supported - const bool data_is_float = node->InputDefs()[1]->Type()->find("float") != std::string::npos; - return data_is_float; - } }; + [this](const Node* node, const InitializedTensorSet&) { + //float data type is not supported + const bool data_is_float = node->InputDefs()[1]->Type()->find("float") != std::string::npos; + return data_is_float; + }}; op_list_.insert({"Where", obj}); } -} +} bool DataOps::op_is_supported(std::string name, std::vector& op_list) { - for (size_t i=0; i < op_list.size(); i++) { + for (size_t i = 0; i < op_list.size(); i++) { if (op_list[i].optype == name) { if (op_list[i].version <= version_id_) { auto it = op_list[i].device_type.begin(); while (it != op_list[i].device_type.end()) { - - //if device supported is all then we support it - if(*it == "All") { + //if device supported is all then we support it + if (*it == "All") { return true; - } - + } + //check for device supported if (device_id_.find(*it) != std::string::npos) { return true; } - + it++; - } + } } } } - + return false; } bool DataOps::type_is_supported(const NodeArg* node_arg, bool is_initializer) { - const auto* type_proto = node_arg->TypeAsProto(); if (!type_proto) { return false; @@ -779,13 +744,13 @@ bool DataOps::type_is_supported(const NodeArg* node_arg, bool is_initializer) { if (is_initializer) { auto dtype = type_proto->tensor_type().elem_type(); - for (auto const &var : supported_types_initializer_) { - if ((var.first <= version_id_) && + for (auto const& var : supported_types_initializer_) { + if ((var.first <= version_id_) && (var.second == dtype)) { return true; } } - + #ifndef NDEBUG if (openvino_ep::backend_utils::IsDebugEnabled()) { std::cout << "Initializer Data Type is not supported" << std::endl; @@ -793,55 +758,53 @@ bool DataOps::type_is_supported(const NodeArg* node_arg, bool is_initializer) { #endif return false; } else { - auto dtype = type_proto->tensor_type().elem_type(); if (device_id_ == "MYRIAD" || device_id_ == "HDDL" || device_id_.find("HETERO") != std::string::npos || device_id_.find("MULTI") != std::string::npos) { - for (auto const &var : supported_types_vpu_) { - if ((var.first <= version_id_) && + for (auto const& var : supported_types_vpu_) { + if ((var.first <= version_id_) && (var.second == dtype)) { return true; } } - + #ifndef NDEBUG - if (openvino_ep::backend_utils::IsDebugEnabled()) { - std::cout << "I/O data type is not supported" << std::endl; - } + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "I/O data type is not supported" << std::endl; + } #endif - return false; - + return false; + } else if (device_id_ == "CPU") { - for (auto const &var : supported_types_cpu_) { - if ((var.first <= version_id_) && + for (auto const& var : supported_types_cpu_) { + if ((var.first <= version_id_) && (var.second == dtype)) { return true; } } - #ifndef NDEBUG - if (openvino_ep::backend_utils::IsDebugEnabled()) { - std::cout << "I/O data type is not supported" << std::endl; - } - #endif - return false; - +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "I/O data type is not supported" << std::endl; + } +#endif + return false; + } else if (device_id_ == "GPU") { auto prec_str = openvino_ep::BackendManager::GetGlobalContext().precision_str; if (prec_str == "FP32" && dtype == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) return false; - for (auto const &var : supported_types_gpu_) { - if ((var.first <= version_id_) && + for (auto const& var : supported_types_gpu_) { + if ((var.first <= version_id_) && (var.second == dtype)) { return true; } } - #ifndef NDEBUG - if (openvino_ep::backend_utils::IsDebugEnabled()) { - std::cout << "I/O data type is not supported" << std::endl; - } - #endif - return false; - +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "I/O data type is not supported" << std::endl; + } +#endif + return false; } return true; } @@ -853,7 +816,7 @@ bool DataOps::unsupported_op_mode(const Node* node) { const auto& initializers = graph_viewer_.GetAllInitializedTensors(); auto iter = op_list_.equal_range(optype); - for (auto it=iter.first; it!=iter.second; ++it) { + for (auto it = iter.first; it != iter.second; ++it) { auto ob = it->second; if (std::find(ob.ver.begin(), ob.ver.end(), version_id_) != ob.ver.end()) { return ob.func(node, initializers); @@ -886,7 +849,6 @@ bool DataOps::dimension_unsupported(const Node* node) { bool DataOps::node_is_supported(const std::map>& op_map, const NodeIndex node_idx) { - const auto& node = graph_viewer_.GetNode(node_idx); const auto& optype = node->OpType(); @@ -991,27 +953,25 @@ bool DataOps::node_is_supported(const std::map DataOps::GetUnsupportedNodeIndices(std::unordered_set& ng_required_initializers) { +std::vector DataOps::GetUnsupportedNodeIndices(std::unordered_set& ng_required_initializers) { + const auto ng_supported_ops = GetNgSupportedOps(GetOnnxOpSet(graph_viewer_)); - const auto ng_supported_ops = GetNgSupportedOps(GetOnnxOpSet(graph_viewer_)); + std::vector unsupported_nodes_idx; - std::vector unsupported_nodes_idx; - - for (const auto& node_idx : graph_viewer_.GetNodesInTopologicalOrder()) { - if (node_is_supported(ng_supported_ops, node_idx)) { - // Collect inputs that are initializers - graph_viewer_.GetNode(node_idx)->ForEachDef([&ng_required_initializers, this](const NodeArg& node_arg, bool is_input) { + for (const auto& node_idx : graph_viewer_.GetNodesInTopologicalOrder()) { + if (node_is_supported(ng_supported_ops, node_idx)) { + // Collect inputs that are initializers + graph_viewer_.GetNode(node_idx)->ForEachDef([&ng_required_initializers, this](const NodeArg& node_arg, bool is_input) { if(is_input && this->graph_viewer_.GetAllInitializedTensors().count(node_arg.Name())) { ng_required_initializers.insert(node_arg.Name()); } }, true); - } else { - unsupported_nodes_idx.push_back(node_idx); - } + } else { + unsupported_nodes_idx.push_back(node_idx); } - return unsupported_nodes_idx; + } + return unsupported_nodes_idx; } bool DataOps::IsOpSupportedOnlyInModel(std::string name) { @@ -1019,60 +979,60 @@ bool DataOps::IsOpSupportedOnlyInModel(std::string name) { } bool DataOps::SpecialConditionForClusterSizeOne(std::unordered_set& ng_required_initializers, const Node* node) { - if (node->OpType() == "Reshape") { - const auto& shape_arg = node->InputDefs()[1]; - if (ng_required_initializers.find(shape_arg->Name()) == ng_required_initializers.end()) { + if (node->OpType() == "Reshape") { + const auto& shape_arg = node->InputDefs()[1]; + if (ng_required_initializers.find(shape_arg->Name()) == ng_required_initializers.end()) { + return true; + } + } else if (node->OpType() == "Expand") { + // nGraph only supports constant shape input values + const auto& output = node->OutputDefs()[0]; + if (output->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) + return true; + } else if (node->OpType() == "RoiAlign") { + using onnx_dtype = ONNX_NAMESPACE::TensorProto_DataType; + + onnx_dtype input_0_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + onnx_dtype input_1_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); + onnx_dtype input_2_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[2]->TypeAsProto()->tensor_type().elem_type(); + onnx_dtype output_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + + if ((input_0_data_type != onnx_dtype::TensorProto_DataType_FLOAT16) || + (input_1_data_type != onnx_dtype::TensorProto_DataType_FLOAT16) || + (input_2_data_type != onnx_dtype::TensorProto_DataType_FLOAT) || + (output_data_type != onnx_dtype::TensorProto_DataType_FLOAT16)) + return true; + } else if ((node->OpType() == "Greater") || (node->OpType() == "Less")) { + if (device_id_.find("MYRIAD") != std::string::npos) { + auto input_0_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + auto input_1_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); + auto output_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + + if (!((output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) || + (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16))) { return true; } - } else if (node->OpType() == "Expand") { - // nGraph only supports constant shape input values - const auto& output = node->OutputDefs()[0]; - if (output->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) - return true; - } else if (node->OpType() == "RoiAlign") { - using onnx_dtype = ONNX_NAMESPACE::TensorProto_DataType; - onnx_dtype input_0_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - onnx_dtype input_1_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); - onnx_dtype input_2_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[2]->TypeAsProto()->tensor_type().elem_type(); - onnx_dtype output_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - - if ((input_0_data_type != onnx_dtype::TensorProto_DataType_FLOAT16) || - (input_1_data_type != onnx_dtype::TensorProto_DataType_FLOAT16) || - (input_2_data_type != onnx_dtype::TensorProto_DataType_FLOAT) || - (output_data_type != onnx_dtype::TensorProto_DataType_FLOAT16)) - return true; - } else if ((node->OpType() == "Greater") || (node->OpType() == "Less")) { - if (device_id_.find("MYRIAD") != std::string::npos) { - auto input_0_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - auto input_1_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); - auto output_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - - if (!((output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) || - (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16))) { - return true; - } - - if ((input_0_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) || - (input_1_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)) { - return true; - } - } - } else if (node->OpType() == "MaxPool" && device_id_.find("MYRIAD") != std::string::npos) { - auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - if (output_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT || - output_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) { + if ((input_0_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) || + (input_1_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)) { return true; } } - return false; + } else if (node->OpType() == "MaxPool" && device_id_.find("MYRIAD") != std::string::npos) { + auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + if (output_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT || + output_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) { + return true; + } + } + return false; } bool DataOps::DoNotOmitSubGraph(const std::string& name) { return op_is_supported(name, subgraph_supported_); } -bool DataOps::InsertNode(const Node *node, const std::string& optype) { +bool DataOps::InsertNode(const Node* node, const std::string& optype) { if (optype == "TopK" || optype == "NonZero") { return true; } @@ -1088,4 +1048,4 @@ bool DataOps::InsertNode(const Node *node, const std::string& optype) { } } // namespace openvino_ep -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h index 06546b9aa9..2fe382e7e0 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h @@ -4,9 +4,9 @@ namespace onnxruntime { namespace openvino_ep { -using VarianceFunc = std::function; +using VarianceFunc = std::function; -enum versionNum{ +enum versionNum { V_2020_4, V_2021_1, V_2021_2 @@ -17,46 +17,44 @@ using VersionNum = enum versionNum; struct supportedOp { std::string optype; VersionNum version; - std::vector device_type; + std::vector device_type; }; -struct unsupportedOpMode{ +struct unsupportedOpMode { std::vector ver; - VarianceFunc func; + VarianceFunc func; }; using SupportedOp = struct supportedOp; using UnsupportedOpMode = struct unsupportedOpMode; -using Pairs = std::pair; +using Pairs = std::pair; -class DataOps{ +class DataOps { + private: + const GraphViewer& graph_viewer_; + VersionNum version_id_; + std::string device_id_; + std::multimap op_list_; + std::vector subgraph_supported_; + std::vector no_dimension_supported_; + std::set supported_types_vpu_; + std::set supported_types_cpu_; + std::set supported_types_gpu_; + std::set supported_types_initializer_; -private: -const GraphViewer& graph_viewer_; -VersionNum version_id_; -std::string device_id_; -std::multimap op_list_; -std::vector subgraph_supported_; -std::vector no_dimension_supported_; -std::set supported_types_vpu_; -std::set supported_types_cpu_; -std::set supported_types_gpu_; -std::set supported_types_initializer_; - -protected: + protected: virtual void populate_op_mode_supported(); virtual void populate_types_supported(); bool op_is_supported(std::string name, std::vector& list); bool dimension_unsupported(const Node* node); bool unsupported_op_mode(const Node* node); bool type_is_supported(const NodeArg* node_arg, bool is_initializer); - bool node_is_supported(const std::map>& op_map, + bool node_is_supported(const std::map>& op_map, const NodeIndex node_idx); - -public: - DataOps(const GraphViewer& graph_viewer_param, VersionNum ver, std::string dev_id): - graph_viewer_(graph_viewer_param), version_id_(ver), device_id_(dev_id) { + + public: + DataOps(const GraphViewer& graph_viewer_param, VersionNum ver, std::string dev_id) : graph_viewer_(graph_viewer_param), version_id_(ver), device_id_(dev_id) { populate_op_mode_supported(); populate_types_supported(); } @@ -67,7 +65,6 @@ public: virtual bool DoNotOmitSubGraph(const std::string& name); virtual bool InsertNode(const Node* node, const std::string& name); VersionNum GetVersion() const { return version_id_; } - }; } //namespace openvino_ep diff --git a/onnxruntime/core/providers/openvino/ov_versions/utils.cc b/onnxruntime/core/providers/openvino/ov_versions/utils.cc index bc252db08f..2ca9f03817 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/utils.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/utils.cc @@ -21,7 +21,7 @@ namespace onnxruntime { namespace openvino_ep { //Gets the input count of given node -int GetInputCount(const Node* node, const Provider_InitializedTensorSet& initializer_set) { +int GetInputCount(const Node* node, const InitializedTensorSet& initializer_set) { int count = 0; for (const auto& input : node->InputDefs()) { auto name = input->Name(); @@ -175,7 +175,7 @@ void GetInputsOutputsOfCluster(const GraphViewer& graph_viewer, // Collect all inputs and outputs node->ForEachDef( [&input_args, &ordered_input_args, &output_args](const NodeArg& node_arg, bool is_input) { - if(node_arg.Name() != ""){ + if (node_arg.Name() != "") { if (is_input) { if (!input_args.count(node_arg.Name())) { ordered_input_args.push_back(node_arg.Name()); diff --git a/onnxruntime/core/providers/openvino/ov_versions/utils.h b/onnxruntime/core/providers/openvino/ov_versions/utils.h index 51b957aed7..0d923fe39b 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/utils.h +++ b/onnxruntime/core/providers/openvino/ov_versions/utils.h @@ -4,7 +4,7 @@ namespace onnxruntime { namespace openvino_ep { -int GetInputCount(const Node* node, const Provider_InitializedTensorSet& initializer_set); +int GetInputCount(const Node* node, const InitializedTensorSet& initializer_set); bool IsOpSupportedOnlyInModel(std::string name); diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 2b778360e8..421f14cbcd 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include "onnx/common/stl_backports.h" @@ -92,12 +93,14 @@ struct AttributeProto; struct GraphProto; struct ModelProto; struct NodeProto; +struct SparseTensorProto; struct TensorProto; struct TensorProtos; // RepeatedPtrField struct TensorShapeProto_Dimension; struct TensorShapeProto_Dimensions; // RepeatedPtrField struct TensorShapeProto; struct TypeProto_Tensor; +struct TypeProto_SparseTensor; struct TypeProto; struct ValueInfoProto; struct ValueInfoProtos; // RepeatedPtrField @@ -113,13 +116,6 @@ enum class DataType { } // namespace logging -enum class AutoPadType { - NOTSET = 0, - VALID = 1, - SAME_UPPER = 2, - SAME_LOWER = 3, -}; - // OnnxRuntime Types (these are the internal types) struct CPUIDInfo; namespace logging { @@ -128,7 +124,6 @@ struct Capture; } // namespace logging struct ComputeCapability; struct DataTransferManager; -struct IDataTransfer; struct IndexedSubGraph; struct IndexedSubGraph_MetaDef; struct KernelCreateInfo; @@ -143,6 +138,7 @@ struct Path; struct Node; struct NodeArg; struct NodeAttributes; +class OpKernel; struct OpKernelContext; struct OpKernelInfo; struct PrimitiveDataTypeBase; @@ -156,16 +152,29 @@ class ScatterNDBase; enum class Mode : int; class GatherBase; class ConcatBase; +template +class Scan; +struct EinsumComputePreprocessor; +template +struct EinsumTypedComputeProcessor; class DataTypeImpl; using MLDataType = const DataTypeImpl*; +// be used with class MLValue +using DeleteFunc = void (*)(void*); using NodeArgInfo = ONNX_NAMESPACE::ValueInfoProto; } // namespace onnxruntime +#include "core/platform/threadpool.h" +#include "core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.h" +#include "core/framework/data_transfer.h" #include "core/framework/execution_provider.h" #include "provider_interfaces.h" #include "core/framework/op_kernel.h" #include "core/framework/data_types_internal.h" +#include "core/framework/tensorprotoutils.h" +#include "core/providers/common.h" +#include "core/providers/op_kernel_type_control_utils.h" namespace onnxruntime { @@ -201,6 +210,8 @@ constexpr const char* kTensorrtExecutionProvider = "TensorrtExecutionProvider"; template using IAllocatorUniquePtr = std::unique_ptr>; +inline OrtStatus* CreateStatus(OrtErrorCode code, _In_ const char* msg) noexcept { return g_host->CreateStatus(code, msg); } + std::unique_ptr CreateCPUAllocator(const OrtMemoryInfo& memory_info); std::unique_ptr CreateCUDAAllocator(int16_t device_id, const char* name); std::unique_ptr CreateCUDAPinnedAllocator(int16_t device_id, const char* name); @@ -214,15 +225,6 @@ std::unordered_set GetCpuPreferredNodes(const onnxruntime::GraphViewe std::string GetEnvironmentVar(const std::string& var_name); -inline AutoPadType StringToAutoPadType(const std::string& str) { return g_host->StringToAutoPadType(str); } - -void AllocatorManager__InsertAllocator(AllocatorManager* p, AllocatorPtr allocator); -AllocatorPtr AllocatorManager__GetAllocator(AllocatorManager* p, int id, OrtMemType mem_type); - -inline int64_t HandleNegativeAxis(int64_t axis, int64_t tensor_rank) { return g_host->HandleNegativeAxis(axis, tensor_rank); } -//inline bool IsScalarOr1ElementVector(const Tensor* input) { return g_host->IsScalarOr1ElementVector(input); } -bool IsScalarOr1ElementVector(const Tensor* input); - namespace logging { struct Category { @@ -242,9 +244,47 @@ constexpr T roundUpPow2(T a) { } uint16_t floatToHalf(float f); +float halfToFloat(uint16_t h); } // namespace math +namespace utils { + +template +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; } +template <> +constexpr ONNXTensorElementDataType GetONNXTensorElementDataType() { return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; } + +} // namespace utils + +} // namespace onnxruntime + #define CREATE_MESSAGE(logger, severity, category, datatype) \ ::onnxruntime::logging::Capture::Create(logger, ::onnxruntime::logging::Severity::k##severity, category, datatype, ORT_WHERE) @@ -258,14 +298,3 @@ uint16_t floatToHalf(float f); #define LOGS_DEFAULT(severity) \ LOGS_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime) - -// Use within macro definitions to create a custom vector of constraints. -// Example: #define REG_KERNEL(OP, VERSION, KERNEL_CLASS, Type, ...) -// .TypeConstraint("T", BuildKernelDefConstraints()) -template -inline std::vector BuildKernelDefConstraints() { - return {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()...}; -} - -} // namespace onnxruntime - diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc index 609a619214..7f5ceee439 100644 --- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc +++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc @@ -9,7 +9,6 @@ #include "core/providers/shared/common.h" namespace onnxruntime { - ProviderHost* g_host = Provider_GetHost(); } @@ -23,8 +22,6 @@ void operator delete(void* p, size_t /*size*/) { return Provider_GetHost()->Heap namespace onnxruntime { -//ProviderHost* g_host = Provider_GetHost(); - static std::unique_ptr>> s_run_on_unload_; void RunOnUnload(std::function function) { @@ -49,16 +46,19 @@ struct OnUnload { } g_on_unload; +void* CPUAllocator::Alloc(size_t size) { return g_host->CPUAllocator__Alloc(this, size); } +void CPUAllocator::Free(void* p) { g_host->CPUAllocator__Free(this, p); } + AllocatorPtr CreateAllocator(const AllocatorCreationInfo& info) { return g_host->CreateAllocator(info); } -void AllocatorManager__InsertAllocator(AllocatorManager* p, AllocatorPtr allocator) { - return g_host->AllocatorManager__InsertAllocator(p, allocator); +void AllocatorManager::InsertAllocator(AllocatorPtr allocator) { + return g_host->AllocatorManager__InsertAllocator(this, allocator); } -AllocatorPtr AllocatorManager__GetAllocator(AllocatorManager* p, int id, OrtMemType mem_type) { - return g_host->AllocatorManager__GetAllocator(p, id, mem_type); +AllocatorPtr AllocatorManager::GetAllocator(int id, OrtMemType mem_type) const { + return g_host->AllocatorManager__GetAllocator(this, id, mem_type); } template <> @@ -66,6 +66,11 @@ MLDataType DataTypeImpl::GetType() { return g_host->DataTypeImpl__GetType_float(); } +template <> +MLDataType DataTypeImpl::GetType() { + return g_host->DataTypeImpl__GetType_Tensor(); +} + template <> MLDataType DataTypeImpl::GetTensorType() { return g_host->DataTypeImpl__GetTensorType_bool(); @@ -121,6 +126,11 @@ MLDataType DataTypeImpl::GetTensorType() { return g_host->DataTypeImpl__GetTensorType_double(); } +template <> +MLDataType DataTypeImpl::GetTensorType() { + return Provider_GetHost()->DataTypeImpl__GetTensorType_BFloat16(); +} + template <> MLDataType DataTypeImpl::GetTensorType() { return Provider_GetHost()->DataTypeImpl__GetTensorType_MLFloat16(); @@ -297,10 +307,13 @@ const std::string& Status::EmptyString() noexcept { namespace math { uint16_t floatToHalf(float f) { return g_host->math__floatToHalf(f); } +float halfToFloat(uint16_t h) { return g_host->math__halfToFloat(h); } } // namespace math -bool IsScalarOr1ElementVector(const Tensor* input) { return g_host->IsScalarOr1ElementVector(input); } +float MLFloat16::ToFloat() const { + return math::halfToFloat(val); +} std::vector GetStackTrace() { return g_host->GetStackTrace(); } @@ -323,8 +336,33 @@ std::unique_ptr CopyOpKernelInfo(const OpKernelInfo& info) { #include "core/providers/cpu/tensor/padbase.h" #include "core/providers/cpu/tensor/concatbase.h" #include "core/providers/cpu/tensor/gatherbase.h" +#include "core/providers/cpu/controlflow/scan.h" +#include "core/providers/cpu/controlflow/loop.h" +#include "core/providers/cpu/tensor/tile.h" +#include "core/providers/cpu/object_detection/non_max_suppression.h" +#include "core/framework/random_generator.h" +#include "core/providers/cpu/math/einsum.h" + +#ifndef DISABLE_CONTRIB_OPS +#include "contrib_ops/cpu/bert/bias_gelu_helper.h" +#include "contrib_ops/cpu/bert/embed_layer_norm_helper.h" +#include "contrib_ops/cpu/bert/longformer_attention_base.h" +#include "contrib_ops/cpu/bert/attention_base.h" +#endif + +#ifdef ENABLE_TRAINING +#include "orttraining/training_ops/cpu/controlflow/group.h" +#endif namespace onnxruntime { + +bool TileOp::IsTileMemcpy(const TensorShape& input_shape, const int64_t* repeats, size_t rank, bool& is_batched_memcpy, size_t& num_of_elements_per_batch, size_t& num_of_copies_per_batch, size_t& num_of_batch_copies) { + return g_host->TileOp__IsTileMemcpy(input_shape, repeats, rank, is_batched_memcpy, num_of_elements_per_batch, num_of_copies_per_batch, num_of_batch_copies); +} + +Status NonMaxSuppressionBase::PrepareCompute(OpKernelContext* ctx, PrepareContext& pc) { return g_host->NonMaxSuppressionBase__PrepareCompute(ctx, pc); } +Status NonMaxSuppressionBase::GetThresholdsFromInputs(const PrepareContext& pc, int64_t& max_output_boxes_per_class, float& iou_threshold, float& score_threshold) { return g_host->NonMaxSuppressionBase__GetThresholdsFromInputs(pc, max_output_boxes_per_class, iou_threshold, score_threshold); } + Status UnsqueezeBase::PrepareCompute(OpKernelContext* ctx, UnsqueezeBase::Prepare& p) const { return g_host->UnsqueezeBase__PrepareCompute(this, ctx, reinterpret_cast(p)); } Status SliceBase::PrepareForCompute(const std::vector& raw_starts, @@ -364,4 +402,69 @@ Status ConcatBase::PrepareForCompute(OpKernelContext* ctx, const std::vectorGatherBase__PrepareForCompute(this, context, reinterpret_cast(p)); } +PhiloxGenerator& PhiloxGenerator::Default() { return g_host->PhiloxGenerator__Default(); } + +Status Einsum::Compute(OpKernelContext* context) const { return g_host->Einsum__Compute(this, context); } + +std::unique_ptr> EinsumTypedComputeProcessor::Create(OpKernelContext* context, AllocatorPtr allocator, concurrency::ThreadPool* tp, EinsumComputePreprocessor& einsum_compute_preprocessor, void* einsum_cuda_assets) { return g_host->EinsumTypedComputeProcessor_float__Create(context, allocator, tp, einsum_compute_preprocessor, einsum_cuda_assets); } +std::unique_ptr> EinsumTypedComputeProcessor::Create(OpKernelContext* context, AllocatorPtr allocator, concurrency::ThreadPool* tp, EinsumComputePreprocessor& einsum_compute_preprocessor, void* einsum_cuda_assets) { return g_host->EinsumTypedComputeProcessor_double__Create(context, allocator, tp, einsum_compute_preprocessor, einsum_cuda_assets); } +std::unique_ptr> EinsumTypedComputeProcessor::Create(OpKernelContext* context, AllocatorPtr allocator, concurrency::ThreadPool* tp, EinsumComputePreprocessor& einsum_compute_preprocessor, void* einsum_cuda_assets) { return g_host->EinsumTypedComputeProcessor_MLFloat16__Create(context, allocator, tp, einsum_compute_preprocessor, einsum_cuda_assets); } + +namespace utils { +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ bool* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ float* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ double* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ MLFloat16* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int8_t* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint8_t* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int16_t* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint16_t* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int32_t* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint32_t* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int64_t* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } +template <> +Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint64_t* p_data, size_t expected_size) { return g_host->UnpackTensor(tensor, raw_data, raw_data_len, p_data, expected_size); } + +} // namespace utils + +#ifndef DISABLE_CONTRIB_OPS +namespace contrib { +Status embed_layer_norm::CheckInputs(const OpKernelContext* context) { return g_host->embed_layer_norm__CheckInputs(context); } +Status bias_gelu_helper::CheckInputs(const OpKernelContext* context) { return g_host->bias_gelu_helper__CheckInputs(context); } +Status LongformerAttentionBase::CheckInputs(const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, const TensorShape& mask_shape, const TensorShape& global_weights_shape, const TensorShape& global_bias_shape, const TensorShape& global_shape) const { + return g_host->LongformerAttentionBase__CheckInputs(this, input_shape, weights_shape, bias_shape, mask_shape, global_weights_shape, global_bias_shape, global_shape); +} + +Status AttentionBase::CheckInputs(const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, const Tensor*& mask_index, const Tensor* past, const int max_threads_per_block) const { + return g_host->AttentionBase__CheckInputs(this, input_shape, weights_shape, bias_shape, mask_index, past, max_threads_per_block); +} +Tensor* AttentionBase::GetPresent(OpKernelContext* context, const Tensor* past, int batch_size, int head_size, int sequence_length, int& past_sequence_length) const { + return g_host->AttentionBase__GetPresent(this, context, past, batch_size, head_size, sequence_length, past_sequence_length); +} + +} // namespace contrib +#endif + +std::unique_ptr Loop::Create(const OpKernelInfo& info, const Loop::ConcatOutput& concat_output_func, void* stream) { + return g_host->CreateOpKernel_CPU_Loop(info, &concat_output_func, stream); +} + +#ifdef ENABLE_TRAINING +namespace contrib { +Status Group::Compute(OpKernelContext* context) const { return g_host->contrib__Group__Compute(this, context); } +Status PassThrough::Compute(OpKernelContext* context) const { return g_host->contrib__PassThrough__Compute(this, context); } +} // namespace contrib +#endif + } // namespace onnxruntime diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 94abbf7b9a..6f9bf685c4 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -32,7 +32,18 @@ struct ProviderHost; class UnsqueezeBase__Prepare; // Directly maps to UnsqueezeBase::Prepare class SliceOp__PrepareForComputeMetadata; // Directly maps to SliceOp::PrepareForComputeMetadata struct Prepare; // ConcatBase, TODO: Scope to ConcatBase +struct PrepareContext; class GatherBase__Prepare; +class PhiloxGenerator; +class Einsum; + +namespace contrib { +class LongformerAttentionBase; +class AttentionBase; +class Group; +class PassThrough; +} // namespace contrib + template struct IteratorHolder { IteratorHolder(std::unique_ptr&& p) : p_{std::move(p)} {} @@ -55,8 +66,8 @@ struct NodeAttributes_Iterator { virtual void operator++() = 0; virtual const std::pair& operator*() const = 0; - // virtual const std::string& first() const = 0; - // virtual const Provider_AttributeProto& second() const = 0; + virtual const std::string& first() const = 0; + virtual const ONNX_NAMESPACE::AttributeProto& second() const = 0; }; struct TensorShapeProto_Dimension_Iterator { @@ -72,7 +83,7 @@ using NodeIndex = size_t; // We can't just reinterpret_cast this one, since it's an unordered_map of object BY VALUE (can't do anything by value on the real types) // using NodeAttributes = std::unordered_map; -using Provider_InitializedTensorSet = std::unordered_map; +using InitializedTensorSet = std::unordered_map; struct Node__NodeIterator { virtual ~Node__NodeIterator() {} @@ -100,7 +111,7 @@ struct Provider { // Old simple device_id API to create provider factories, currently used by DNNL And TensorRT virtual std::shared_ptr CreateExecutionProviderFactory(int /*device_id*/) { return nullptr; } - virtual const void* GetInfo() { return nullptr; } // Returns a provider specific information interface if it exists + virtual void* GetInfo() { return nullptr; } // Returns a provider specific information interface if it exists virtual void Shutdown() = 0; }; @@ -114,9 +125,14 @@ struct ProviderHost { virtual logging::Logger* LoggingManager_GetDefaultLogger() = 0; + virtual OrtStatus* CreateStatus(OrtErrorCode code, _In_ const char* msg) noexcept = 0; + virtual std::unique_ptr CreateCPUAllocator(const OrtMemoryInfo& memory_info) = 0; -#ifdef USE_TENSORRT + virtual void* CPUAllocator__Alloc(CPUAllocator* p, size_t size) = 0; + virtual void CPUAllocator__Free(CPUAllocator* p, void* allocation) = 0; + +#ifdef USE_CUDA virtual std::unique_ptr CreateCUDAAllocator(int16_t device_id, const char* name) = 0; virtual std::unique_ptr CreateCUDAPinnedAllocator(int16_t device_id, const char* name) = 0; virtual std::unique_ptr CreateGPUDataTransfer(void* stream) = 0; @@ -152,27 +168,43 @@ struct ProviderHost { MLDataType (*DataTypeImpl__GetTensorType_uint64)(); MLDataType (*DataTypeImpl__GetTensorType_float)(); MLDataType (*DataTypeImpl__GetTensorType_double)(); + MLDataType (*DataTypeImpl__GetTensorType_BFloat16)(); MLDataType (*DataTypeImpl__GetTensorType_MLFloat16)(); virtual const char* DataTypeImpl__ToString(MLDataType type) = 0; + virtual bool DataTypeImpl__IsTensorType(const DataTypeImpl* p) = 0; + virtual bool DataTypeImpl__IsTensorSequenceType(const DataTypeImpl* p) = 0; + virtual bool DataTypeImpl__IsSparseTensorType(const DataTypeImpl* p) = 0; + virtual DeleteFunc DataTypeImpl__GetDeleteFunc(const DataTypeImpl* p) = 0; virtual const std::vector& DataTypeImpl__AllFixedSizeTensorTypes() = 0; virtual const std::vector& DataTypeImpl__AllTensorTypes() = 0; virtual const std::vector& DataTypeImpl__AllIEEEFloatTensorTypes() = 0; virtual size_t DataTypeImpl__Size(const DataTypeImpl* p) = 0; virtual const PrimitiveDataTypeBase* DataTypeImpl__AsPrimitiveDataType(const DataTypeImpl* p) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ bool* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ float* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ double* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ MLFloat16* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int8_t* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint8_t* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int16_t* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint16_t* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int32_t* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint32_t* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ int64_t* p_data, size_t expected_size) = 0; + virtual Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, /*out*/ uint64_t* p_data, size_t expected_size) = 0; + virtual void* HeapAllocate(size_t size) = 0; virtual void HeapFree(void*) = 0; - virtual AutoPadType StringToAutoPadType(const std::string& str) = 0; - virtual int64_t HandleNegativeAxis(int64_t axis, int64_t tensor_rank) = 0; - virtual void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file, const char* function, uint32_t line) = 0; virtual std::vector GetStackTrace() = 0; virtual uint16_t math__floatToHalf(float f) = 0; + virtual float math__halfToFloat(uint16_t h) = 0; // IAllocator virtual bool IAllocator__CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t alignment, size_t* out) = 0; @@ -224,6 +256,7 @@ struct ProviderHost { virtual const int64_t& int64s__Get(const ONNX_NAMESPACE::int64s* p, int index) = 0; // TypeProto_Tensor + virtual bool TypeProto_Tensor__has_shape(const ONNX_NAMESPACE::TypeProto_Tensor* p) = 0; virtual const ONNX_NAMESPACE::TensorShapeProto& TypeProto_Tensor__shape(const ONNX_NAMESPACE::TypeProto_Tensor* p) = 0; virtual ONNX_NAMESPACE::TensorShapeProto* TypeProto_Tensor__mutable_shape(ONNX_NAMESPACE::TypeProto_Tensor* p) = 0; virtual int32_t TypeProto_Tensor__elem_type(const ONNX_NAMESPACE::TypeProto_Tensor* p) = 0; @@ -231,6 +264,7 @@ struct ProviderHost { // TypeProto virtual const ONNX_NAMESPACE::TypeProto_Tensor& TypeProto__tensor_type(const ONNX_NAMESPACE::TypeProto* p) = 0; virtual ONNX_NAMESPACE::TypeProto_Tensor* TypeProto__mutable_tensor_type(ONNX_NAMESPACE::TypeProto* p) = 0; + virtual int TypeProto__value_case(const ONNX_NAMESPACE::TypeProto* p) = 0; // AttributeProto virtual std::unique_ptr AttributeProto__construct() = 0; @@ -285,10 +319,19 @@ struct ProviderHost { virtual void ModelProto__set_ir_version(ONNX_NAMESPACE::ModelProto* p, int64_t value) = 0; // TensorProto + virtual std::unique_ptr TensorProto__construct() = 0; virtual void TensorProto__operator_delete(ONNX_NAMESPACE::TensorProto* p) = 0; virtual void TensorProto__operator_assign(ONNX_NAMESPACE::TensorProto* p, const ONNX_NAMESPACE::TensorProto& v) = 0; + virtual bool TensorProto__has_name(const ONNX_NAMESPACE::TensorProto* p) = 0; + virtual int TensorProto__dims_size(const ONNX_NAMESPACE::TensorProto* p) = 0; + virtual const ONNX_NAMESPACE::int64s& TensorProto__dims(const ONNX_NAMESPACE::TensorProto* p) = 0; virtual bool TensorProto__has_data_location(const ONNX_NAMESPACE::TensorProto* p) = 0; virtual int TensorProto__data_location(const ONNX_NAMESPACE::TensorProto* p) = 0; + virtual bool TensorProto__has_raw_data(const ONNX_NAMESPACE::TensorProto* p) = 0; + virtual const std::string& TensorProto__raw_data(const ONNX_NAMESPACE::TensorProto* p) = 0; + virtual int32_t TensorProto__data_type(const ONNX_NAMESPACE::TensorProto* p) = 0; + + virtual bool TensorProto_DataType_IsValid(int value) = 0; // TensorProtos virtual ONNX_NAMESPACE::TensorProto* TensorProtos__Add(ONNX_NAMESPACE::TensorProtos* p) = 0; @@ -378,10 +421,13 @@ struct ProviderHost { virtual void KernelDefBuilder__TypeConstraint(KernelDefBuilder* p, const char* arg_name, MLDataType supported_type) = 0; virtual void KernelDefBuilder__TypeConstraint(KernelDefBuilder* p, const char* arg_name, const std::vector& supported_types) = 0; virtual void KernelDefBuilder__InputMemoryType(KernelDefBuilder* p, OrtMemType type, int input_index) = 0; + virtual void KernelDefBuilder__InputMemoryType(KernelDefBuilder* p, OrtMemType type, const std::vector& input_indexes) = 0; virtual void KernelDefBuilder__OutputMemoryType(KernelDefBuilder* p, OrtMemType type, int input_index) = 0; virtual void KernelDefBuilder__ExecQueueId(KernelDefBuilder* p, int queue_id) = 0; virtual void KernelDefBuilder__MayInplace(KernelDefBuilder* p, int input_index, int output_index) = 0; virtual void KernelDefBuilder__Alias(KernelDefBuilder* p, int input_index, int output_index) = 0; + virtual void KernelDefBuilder__Alias(KernelDefBuilder* p, const std::vector>& aliases) = 0; + virtual void KernelDefBuilder__VariadicAlias(KernelDefBuilder* p, int input_offset, int output_offset) = 0; virtual std::unique_ptr KernelDefBuilder__Build(KernelDefBuilder* p) = 0; @@ -491,7 +537,7 @@ struct ProviderHost { virtual const std::vector& GraphViewer__GetOutputs(const GraphViewer* p) noexcept = 0; virtual const std::vector& GraphViewer__GetValueInfo(const GraphViewer* p) noexcept = 0; - virtual const Provider_InitializedTensorSet& GraphViewer__GetAllInitializedTensors(const GraphViewer* p) = 0; + virtual const InitializedTensorSet& GraphViewer__GetAllInitializedTensors(const GraphViewer* p) = 0; virtual bool GraphViewer__GetInitializedTensor(const GraphViewer* p, const std::string& tensor_name, const ONNX_NAMESPACE::TensorProto*& value) = 0; virtual const std::unordered_map& GraphViewer__DomainToVersionMap(const GraphViewer* p) = 0; @@ -506,9 +552,13 @@ struct ProviderHost { // OpKernelContext virtual const Tensor* OpKernelContext__Input_Tensor(const OpKernelContext* p, int index) = 0; + virtual const Tensor& OpKernelContext__RequiredInput_Tensor(const OpKernelContext* p, int index) = 0; virtual Tensor* OpKernelContext__Output(OpKernelContext* p, int index, const TensorShape& shape) = 0; + virtual Tensor& OpKernelContext__RequiredOutput(OpKernelContext* p, int index, const TensorShape& shape) = 0; virtual int OpKernelContext__InputCount(const OpKernelContext* p) = 0; virtual int OpKernelContext__OutputCount(const OpKernelContext* p) = 0; + virtual Status OpKernelContext__GetTempSpaceAllocator(const OpKernelContext* p, AllocatorPtr* output) = 0; + virtual bool OpKernelContext__GetUseDeterministicCompute(const OpKernelContext* p) = 0; // OpKernelInfo virtual std::unique_ptr CopyOpKernelInfo(const OpKernelInfo& info) = 0; @@ -517,6 +567,7 @@ struct ProviderHost { virtual Status OpKernelInfo__GetAttr_int64(const OpKernelInfo* p, const std::string& name, int64_t* value) = 0; virtual Status OpKernelInfo__GetAttr_float(const OpKernelInfo* p, const std::string& name, float* value) = 0; virtual Status OpKernelInfo__GetAttr_string(const OpKernelInfo* p, const std::string& name, std::string* value) = 0; + virtual Status OpKernelInfo__GetAttr_TensorProto(const OpKernelInfo* p, const std::string& name, ONNX_NAMESPACE::TensorProto* value) = 0; virtual Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) = 0; virtual Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) = 0; virtual Status OpKernelInfo__GetAttrs(const OpKernelInfo* p, const std::string& name, std::vector& values) = 0; @@ -533,6 +584,7 @@ struct ProviderHost { // Tensor virtual std::unique_ptr Tensor__construct(MLDataType p_type, const TensorShape& shape, std::shared_ptr allocator) = 0; + virtual std::unique_ptr Tensor__construct(MLDataType p_type, const TensorShape& shape, void* p_data, const OrtMemoryInfo& alloc, ptrdiff_t offset) = 0; virtual void Tensor__operator_delete(Tensor* p) = 0; virtual bool* Tensor__MutableData_bool(Tensor* p) = 0; @@ -546,6 +598,7 @@ struct ProviderHost { virtual uint64_t* Tensor__MutableData_uint64(Tensor* p) = 0; virtual float* Tensor__MutableData_float(Tensor* p) = 0; virtual double* Tensor__MutableData_double(Tensor* p) = 0; + virtual BFloat16* Tensor__MutableData_BFloat16(Tensor* p) = 0; virtual MLFloat16* Tensor__MutableData_MLFloat16(Tensor* p) = 0; virtual const bool* Tensor__Data_bool(const Tensor* p) = 0; @@ -559,8 +612,11 @@ struct ProviderHost { virtual const uint64_t* Tensor__Data_uint64(const Tensor* p) = 0; virtual const float* Tensor__Data_float(const Tensor* p) = 0; virtual const double* Tensor__Data_double(const Tensor* p) = 0; + virtual const BFloat16* Tensor__Data_BFloat16(const Tensor* p) = 0; virtual const MLFloat16* Tensor__Data_MLFloat16(const Tensor* p) = 0; + virtual gsl::span Tensor__DataAsSpan_int64(const Tensor* p) = 0; + virtual void* Tensor__MutableDataRaw(Tensor* p, MLDataType type) = 0; virtual const void* Tensor__DataRaw(const Tensor* p, MLDataType type) = 0; virtual void* Tensor__MutableDataRaw(Tensor* p) noexcept = 0; @@ -581,6 +637,9 @@ struct ProviderHost { virtual bool Tensor__IsDataTypeString(const Tensor* p) noexcept = 0; virtual const TensorShape& Tensor__Shape(const Tensor* p) = 0; + virtual void Tensor__Reshape(Tensor* p, const TensorShape& new_shape) = 0; + virtual void Tensor__SetByteOffset(Tensor* p, ptrdiff_t byte_offset) = 0; + virtual ptrdiff_t Tensor__ByteOffset(const Tensor* p) = 0; virtual size_t Tensor__SizeInBytes(const Tensor* p) = 0; virtual const OrtMemoryInfo& Tensor__Location(const Tensor* p) = 0; virtual int32_t Tensor__GetElementType(const Tensor* p) = 0; @@ -589,13 +648,22 @@ struct ProviderHost { // GatherElements virtual Status GatherElements__ValidateInputShapes(const TensorShape& input_data_shape, const TensorShape& indices_shape, int64_t axis) = 0; + // cumsum.cc + virtual Status cumsum_op__GetAxis(const Tensor* axis_tensor, int64_t input_rank, int64_t& axis_out) = 0; + + // TileOp + virtual bool TileOp__IsTileMemcpy(const TensorShape& input_shape, const int64_t* repeats, size_t rank, bool& is_batched_memcpy, size_t& num_of_elements_per_batch, size_t& num_of_copies_per_batch, size_t& num_of_batch_copies) = 0; + + // ROI + virtual Status CheckROIAlignValidInput(const Tensor* X_ptr, const Tensor* rois_ptr, const Tensor* batch_indices_ptr) = 0; + + virtual Status NonMaxSuppressionBase__PrepareCompute(OpKernelContext* ctx, PrepareContext& pc) = 0; + virtual Status NonMaxSuppressionBase__GetThresholdsFromInputs(const PrepareContext& pc, int64_t& max_output_boxes_per_class, float& iou_threshold, float& score_threshold) = 0; + // From onehot.h virtual Status ValidateInputs(const Tensor* depth, const Tensor* values) = 0; virtual Status PrepareOutputShape(const Tensor* indices, const int64_t depth_val, const int64_t axis, int64_t& prefix_dim_size, int64_t& suffix_dim_size, std::vector& output_shape) = 0; - // From Providers/Common.h - virtual bool IsScalarOr1ElementVector(const Tensor* input) = 0; - // From cpu/tensor/unsqueeze.h virtual Status UnsqueezeBase__PrepareCompute(const UnsqueezeBase* p, OpKernelContext* ctx, UnsqueezeBase__Prepare& prepare) = 0; // From cpu/tensor/slice.h @@ -634,9 +702,63 @@ struct ProviderHost { // From cpu/tensor/gatherbase.h virtual Status GatherBase__PrepareForCompute(const GatherBase* p, OpKernelContext* context, GatherBase__Prepare& prepare) = 0; + virtual PhiloxGenerator& PhiloxGenerator__Default() = 0; + + virtual Status Einsum__Compute(const Einsum* p, OpKernelContext* context) = 0; + + // EinsumComputePreprocessor + virtual void EinsumComputePreprocessor__operator_delete(EinsumComputePreprocessor* p) = 0; + virtual std::unique_ptr EinsumComputePreprocessor__Create(EinsumEquationPreprocessor& equation_preprocessor, + const std::vector& inputs, + AllocatorPtr allocator, + void* einsum_cuda_assets) = 0; + + virtual Status EinsumComputePreprocessor__Run(EinsumComputePreprocessor* p) = 0; + virtual void EinsumComputePreprocessor__SetDeviceHelpers(EinsumComputePreprocessor* p, const EinsumOp::DeviceHelpers::Diagonal& diagonal_func, const EinsumOp::DeviceHelpers::Transpose& transpose_func) = 0; + + // EinsumTypedComputeProcessor + virtual void EinsumTypedComputeProcessor__operator_delete(EinsumTypedComputeProcessor* p) = 0; + virtual void EinsumTypedComputeProcessor__operator_delete(EinsumTypedComputeProcessor* p) = 0; + virtual void EinsumTypedComputeProcessor__operator_delete(EinsumTypedComputeProcessor* p) = 0; + virtual std::unique_ptr> EinsumTypedComputeProcessor_float__Create(OpKernelContext* context, AllocatorPtr allocator, concurrency::ThreadPool* tp, EinsumComputePreprocessor& einsum_compute_preprocessor, void* einsum_cuda_assets) = 0; + virtual std::unique_ptr> EinsumTypedComputeProcessor_double__Create(OpKernelContext* context, AllocatorPtr allocator, concurrency::ThreadPool* tp, EinsumComputePreprocessor& einsum_compute_preprocessor, void* einsum_cuda_assets) = 0; + virtual std::unique_ptr> EinsumTypedComputeProcessor_MLFloat16__Create(OpKernelContext* context, AllocatorPtr allocator, concurrency::ThreadPool* tp, EinsumComputePreprocessor& einsum_compute_preprocessor, void* einsum_cuda_assets) = 0; + virtual void EinsumTypedComputeProcessor__SetDeviceHelpers(EinsumTypedComputeProcessor* p, const EinsumOp::DeviceHelpers::Transpose& device_transpose_func, const EinsumOp::DeviceHelpers::MatMul& device_matmul_func, const EinsumOp::DeviceHelpers::ReduceSum& device_reduce_sum_func, const EinsumOp::DeviceHelpers::DataCopy& device_data_copy_func) = 0; + virtual void EinsumTypedComputeProcessor__SetDeviceHelpers(EinsumTypedComputeProcessor* p, const EinsumOp::DeviceHelpers::Transpose& device_transpose_func, const EinsumOp::DeviceHelpers::MatMul& device_matmul_func, const EinsumOp::DeviceHelpers::ReduceSum& device_reduce_sum_func, const EinsumOp::DeviceHelpers::DataCopy& device_data_copy_func) = 0; + virtual void EinsumTypedComputeProcessor__SetDeviceHelpers(EinsumTypedComputeProcessor* p, const EinsumOp::DeviceHelpers::Transpose& device_transpose_func, const EinsumOp::DeviceHelpers::MatMul& device_matmul_func, const EinsumOp::DeviceHelpers::ReduceSum& device_reduce_sum_func, const EinsumOp::DeviceHelpers::DataCopy& device_data_copy_func) = 0; + virtual Status EinsumTypedComputeProcessor__Run(EinsumTypedComputeProcessor* p) = 0; + virtual Status EinsumTypedComputeProcessor__Run(EinsumTypedComputeProcessor* p) = 0; + virtual Status EinsumTypedComputeProcessor__Run(EinsumTypedComputeProcessor* p) = 0; + // AllocatorManager virtual void AllocatorManager__InsertAllocator(AllocatorManager* p, AllocatorPtr allocator) = 0; - virtual AllocatorPtr AllocatorManager__GetAllocator(AllocatorManager* p, int id, OrtMemType mem_type) = 0; + virtual AllocatorPtr AllocatorManager__GetAllocator(const AllocatorManager* p, int id, OrtMemType mem_type) = 0; + + virtual std::unique_ptr CreateOpKernel_CPU_If(const OpKernelInfo& info) = 0; + virtual std::unique_ptr CreateOpKernel_CPU_Loop(const OpKernelInfo& info, const void* concat_output_func, void* stream) = 0; + virtual std::unique_ptr CreateOpKernel_CPU_Scan_8(const OpKernelInfo& info) = 0; + virtual std::unique_ptr CreateOpKernel_CPU_Scan_9(const OpKernelInfo& info) = 0; + + // ContribOps +#ifndef DISABLE_CONTRIB_OPS + virtual Status embed_layer_norm__CheckInputs(const OpKernelContext* context) = 0; + virtual Status bias_gelu_helper__CheckInputs(const OpKernelContext* context) = 0; + virtual Status LongformerAttentionBase__CheckInputs(const contrib::LongformerAttentionBase* p, const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, const TensorShape& mask_shape, const TensorShape& global_weights_shape, const TensorShape& global_bias_shape, const TensorShape& global_shape) = 0; + + virtual Status AttentionBase__CheckInputs(const contrib::AttentionBase* p, const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, const Tensor*& mask_index, const Tensor* past, const int max_threads_per_block) = 0; + virtual Tensor* AttentionBase__GetPresent(const contrib::AttentionBase* p, OpKernelContext* context, const Tensor* past, int batch_size, int head_size, int sequence_length, int& past_sequence_length) = 0; +#endif + +#ifdef ENABLE_TRAINING + virtual void contrib__record_event_in_tensor(const Tensor& event_id_tensor) = 0; + virtual void contrib__wait_event_in_tensor(const Tensor& event_id_tensor) = 0; + virtual Status contrib__Group__Compute(const contrib::Group* p, OpKernelContext* context) = 0; + virtual Status contrib__PassThrough__Compute(const contrib::PassThrough* p, OpKernelContext* context) = 0; + virtual void contrib__VerifyLogitWeightAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, const TensorShape* weight_shape) = 0; + virtual void contrib__GetNDCFromLogitAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, int64_t& N_D, int64_t& C) = 0; + virtual void contrib__GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, std::vector& new_shape, std::vector& permutations) = 0; + virtual Status contrib__PrepareForTrainingCompute(const TensorShape& input_shape, int num_outputs, int64_t& axis, int& before_dims, int& after_dims_including_split_axis, int& after_dims_excluding_split, std::vector& split_sizes) = 0; +#endif }; extern ProviderHost* g_host; @@ -678,16 +800,6 @@ struct Capture { void operator=(const Capture&) = delete; }; } // namespace logging - -namespace Utils { - -struct DataTypeUtils { - static const std::string* ToType(const ONNX_NAMESPACE::TypeProto& type_proto) { return g_host->Utils__DataTypeUtils__ToType(type_proto); } - - PROVIDER_DISALLOW_ALL(DataTypeUtils) -}; - -} // namespace Utils } namespace ONNX_NAMESPACE { @@ -695,6 +807,7 @@ namespace ONNX_NAMESPACE { struct int64s { int size() const { return g_host->int64s__size(this); } const int64_t& Get(int index) const { return g_host->int64s__Get(this, index); } + const int64_t& operator[](int index) const { return Get(index); } PROVIDER_DISALLOW_ALL(int64s) }; @@ -720,6 +833,21 @@ struct AttributeProto { void set_type(AttributeProto_AttributeType value) { return g_host->AttributeProto__set_type(this, value); } TensorProto* add_tensors() { return g_host->AttributeProto__add_tensors(this); } + typedef AttributeProto_AttributeType AttributeType; + static constexpr AttributeType UNDEFINED = AttributeProto_AttributeType_UNDEFINED; + static constexpr AttributeType FLOAT = AttributeProto_AttributeType_FLOAT; + static constexpr AttributeType INT = AttributeProto_AttributeType_INT; + static constexpr AttributeType STRING = AttributeProto_AttributeType_STRING; + static constexpr AttributeType TENSOR = AttributeProto_AttributeType_TENSOR; + static constexpr AttributeType GRAPH = AttributeProto_AttributeType_GRAPH; + static constexpr AttributeType SPARSE_TENSOR = AttributeProto_AttributeType_SPARSE_TENSOR; + static constexpr AttributeType FLOATS = AttributeProto_AttributeType_FLOATS; + static constexpr AttributeType INTS = AttributeProto_AttributeType_INTS; + static constexpr AttributeType STRINGS = AttributeProto_AttributeType_STRINGS; + static constexpr AttributeType TENSORS = AttributeProto_AttributeType_TENSORS; + static constexpr AttributeType GRAPHS = AttributeProto_AttributeType_GRAPHS; + static constexpr AttributeType SPARSE_TENSORS = AttributeProto_AttributeType_SPARSE_TENSORS; + AttributeProto() = delete; AttributeProto(const AttributeProto&) = delete; }; @@ -765,12 +893,28 @@ struct ModelProto { }; struct TensorProto { + static std::unique_ptr Create() { return g_host->TensorProto__construct(); } static void operator delete(void* p) { g_host->TensorProto__operator_delete(reinterpret_cast(p)); } void operator=(const TensorProto& v) { g_host->TensorProto__operator_assign(this, v); } + bool has_name() const { return g_host->TensorProto__has_name(this); } + + int dims_size() const { return g_host->TensorProto__dims_size(this); } + const int64s& dims() const { return g_host->TensorProto__dims(this); } + bool has_data_location() const { return g_host->TensorProto__has_data_location(this); } TensorProto_DataLocation data_location() const { return TensorProto_DataLocation(g_host->TensorProto__data_location(this)); } + bool has_raw_data() const { return g_host->TensorProto__has_raw_data(this); } + const std::string& raw_data() const { return g_host->TensorProto__raw_data(this); } + + int32_t data_type() const { return g_host->TensorProto__data_type(this); } + + typedef TensorProto_DataType DataType; + static constexpr DataType UNDEFINED = TensorProto_DataType_UNDEFINED; + + static bool DataType_IsValid(int value) { return g_host->TensorProto_DataType_IsValid(value); } + TensorProto() = delete; TensorProto(const TensorProto&) = delete; }; @@ -816,6 +960,7 @@ struct TensorShapeProto { }; struct TypeProto_Tensor { + bool has_shape() const { return g_host->TypeProto_Tensor__has_shape(this); } const TensorShapeProto& shape() const { return g_host->TypeProto_Tensor__shape(this); } TensorShapeProto* mutable_shape() { return g_host->TypeProto_Tensor__mutable_shape(this); } int32_t elem_type() const { return g_host->TypeProto_Tensor__elem_type(this); } @@ -827,6 +972,17 @@ struct TypeProto { const TypeProto_Tensor& tensor_type() const { return g_host->TypeProto__tensor_type(this); } TypeProto_Tensor* mutable_tensor_type() { return g_host->TypeProto__mutable_tensor_type(this); } + enum ValueCase { + kTensorType = 1, + kSequenceType = 4, + kMapType = 5, + kSparseTensorType = 8, + kOpaqueType = 7, + VALUE_NOT_SET = 0, + }; + + ValueCase value_case() const { return ValueCase(g_host->TypeProto__value_case(this)); } + PROVIDER_DISALLOW_ALL(TypeProto) }; @@ -852,6 +1008,16 @@ struct ValueInfoProtos { namespace onnxruntime { +namespace Utils { + +struct DataTypeUtils { + static const std::string* ToType(const ONNX_NAMESPACE::TypeProto& type_proto) { return g_host->Utils__DataTypeUtils__ToType(type_proto); } + + PROVIDER_DISALLOW_ALL(DataTypeUtils) +}; + +} // namespace Utils + struct ComputeCapability { static std::unique_ptr Create(std::unique_ptr t_sub_graph) { return g_host->ComputeCapability__construct(std::move(t_sub_graph)); } static void operator delete(void* p) { g_host->ComputeCapability__operator_delete(reinterpret_cast(p)); } @@ -872,16 +1038,6 @@ struct DataTransferManager { PROVIDER_DISALLOW_ALL(DataTransferManager) }; -#if 0 -struct IDataTransfer { - static void operator delete(void* p) { g_host->IDataTransfer__operator_delete(reinterpret_cast(p)); } - - IDataTransfer() = delete; - IDataTransfer(const IDataTransfer&) = delete; - void operator=(const IDataTransfer&) = delete; -}; -#endif - struct IndexedSubGraph_MetaDef { static std::unique_ptr Create() { return g_host->IndexedSubGraph_MetaDef__construct(); } static void operator delete(void* p) { g_host->IndexedSubGraph_MetaDef__operator_delete(reinterpret_cast(p)); } @@ -938,7 +1094,6 @@ struct KernelDef { using BuildKernelCreateInfoFn = KernelCreateInfo (*)(); -#ifdef SHARED_PROVIDER struct KernelDefBuilder { static std::unique_ptr Create() { return g_host->KernelDefBuilder__construct(); } static void operator delete(void* p) { g_host->KernelDefBuilder__operator_delete(reinterpret_cast(p)); } @@ -975,6 +1130,10 @@ struct KernelDefBuilder { g_host->KernelDefBuilder__InputMemoryType(this, type, input_index); return *this; } + KernelDefBuilder& InputMemoryType(OrtMemType type, const std::vector& input_indexes) { + g_host->KernelDefBuilder__InputMemoryType(this, type, input_indexes); + return *this; + } KernelDefBuilder& OutputMemoryType(OrtMemType type, int input_index) { g_host->KernelDefBuilder__OutputMemoryType(this, type, input_index); return *this; @@ -987,10 +1146,18 @@ struct KernelDefBuilder { g_host->KernelDefBuilder__MayInplace(this, input_index, output_index); return *this; } + KernelDefBuilder& KernelDefBuilder::Alias(const std::vector>& aliases) { + g_host->KernelDefBuilder__Alias(this, aliases); + return *this; + } KernelDefBuilder& Alias(int input_index, int output_index) { g_host->KernelDefBuilder__Alias(this, input_index, output_index); return *this; } + KernelDefBuilder& VariadicAlias(int input_offset, int output_offset) { + g_host->KernelDefBuilder__VariadicAlias(this, input_offset, output_offset); + return *this; + } std::unique_ptr Build() { return g_host->KernelDefBuilder__Build(this); @@ -1029,6 +1196,11 @@ class DataTypeImpl { template static MLDataType GetTensorType(); + bool IsTensorType() const { return g_host->DataTypeImpl__IsTensorType(this); } + bool IsTensorSequenceType() const { return g_host->DataTypeImpl__IsTensorSequenceType(this); } + bool IsSparseTensorType() const { return g_host->DataTypeImpl__IsSparseTensorType(this); } + DeleteFunc GetDeleteFunc() const { return g_host->DataTypeImpl__GetDeleteFunc(this); } + static const std::vector& AllFixedSizeTensorTypes() { return g_host->DataTypeImpl__AllFixedSizeTensorTypes(); } static const std::vector& AllTensorTypes() { return g_host->DataTypeImpl__AllTensorTypes(); } static const std::vector& AllIEEEFloatTensorTypes() { return g_host->DataTypeImpl__AllIEEEFloatTensorTypes(); } @@ -1195,7 +1367,7 @@ struct GraphViewer { const std::vector& GetOutputs() const noexcept { return g_host->GraphViewer__GetOutputs(this); } const std::vector& GetValueInfo() const noexcept { return g_host->GraphViewer__GetValueInfo(this); } - const Provider_InitializedTensorSet& GetAllInitializedTensors() const noexcept { return g_host->GraphViewer__GetAllInitializedTensors(this); } + const InitializedTensorSet& GetAllInitializedTensors() const noexcept { return g_host->GraphViewer__GetAllInitializedTensors(this); } bool GetInitializedTensor(const std::string& tensor_name, const ONNX_NAMESPACE::TensorProto*& value) const { return g_host->GraphViewer__GetInitializedTensor(this, tensor_name, value); } const std::unordered_map& DomainToVersionMap() const noexcept { return g_host->GraphViewer__DomainToVersionMap(this); } @@ -1213,13 +1385,11 @@ struct Path { PROVIDER_DISALLOW_ALL(Path) }; -#endif -#endif - -#ifdef SHARED_PROVIDER struct OpKernelContext { - const Tensor* Input_Tensor(int index) const { return g_host->OpKernelContext__Input_Tensor(this, index); } + template + const T& RequiredInput(int index) const; + Tensor& RequiredOutput(int index, const TensorShape& shape) { return g_host->OpKernelContext__RequiredOutput(this, index, shape); } template const T* Input(int index) const; @@ -1228,12 +1398,21 @@ struct OpKernelContext { Tensor* Output(int index, const TensorShape& shape) { return g_host->OpKernelContext__Output(this, index, shape); } int OutputCount() const { return g_host->OpKernelContext__OutputCount(this); } + Status GetTempSpaceAllocator(AllocatorPtr* output) const { return g_host->OpKernelContext__GetTempSpaceAllocator(this, output); } + + bool GetUseDeterministicCompute() const { return g_host->OpKernelContext__GetUseDeterministicCompute(this); } + PROVIDER_DISALLOW_ALL(OpKernelContext) }; template <> inline const Tensor* OpKernelContext::Input(int index) const { - return Input_Tensor(index); + return g_host->OpKernelContext__Input_Tensor(this, index); +} + +template <> +inline const Tensor& OpKernelContext::RequiredInput(int index) const { + return g_host->OpKernelContext__RequiredInput_Tensor(this, index); } struct OpKernelInfo { @@ -1286,6 +1465,8 @@ inline Status OpKernelInfo::GetAttr(const std::string& name, float* value template <> inline Status OpKernelInfo::GetAttr(const std::string& name, std::string* value) const { return g_host->OpKernelInfo__GetAttr_string(this, name, value); } template <> +inline Status OpKernelInfo::GetAttr(const std::string& name, ONNX_NAMESPACE::TensorProto* value) const { return g_host->OpKernelInfo__GetAttr_TensorProto(this, name, value); } +template <> inline Status OpKernelInfo::GetAttrs(const std::string& name, std::vector& values) const { return g_host->OpKernelInfo__GetAttrs(this, name, values); } template <> inline Status OpKernelInfo::GetAttrs(const std::string& name, std::vector& values) const { return g_host->OpKernelInfo__GetAttrs(this, name, values); } @@ -1301,6 +1482,8 @@ class SessionState { struct Tensor { static std::unique_ptr Create(MLDataType p_type, const TensorShape& shape, std::shared_ptr allocator) { return g_host->Tensor__construct(p_type, shape, allocator); } + static std::unique_ptr Create(MLDataType p_type, const TensorShape& shape, void* p_data, const OrtMemoryInfo& alloc, ptrdiff_t offset = 0) { return g_host->Tensor__construct(p_type, shape, p_data, alloc, offset); } + static void operator delete(void* p) { g_host->Tensor__operator_delete(reinterpret_cast(p)); } template @@ -1309,6 +1492,9 @@ struct Tensor { template const T* Data() const; + template + gsl::span DataAsSpan() const; + void* MutableDataRaw(MLDataType type) { return g_host->Tensor__MutableDataRaw(this, type); } const void* DataRaw(MLDataType type) const { return g_host->Tensor__DataRaw(this, type); } @@ -1316,6 +1502,9 @@ struct Tensor { const void* DataRaw() const noexcept { return g_host->Tensor__DataRaw(this); } const TensorShape& Shape() const { return g_host->Tensor__Shape(this); } + void Reshape(const TensorShape& new_shape) { g_host->Tensor__Reshape(this, new_shape); } + void SetByteOffset(ptrdiff_t byte_offset) { return g_host->Tensor__SetByteOffset(this, byte_offset); } + ptrdiff_t ByteOffset() const { return g_host->Tensor__ByteOffset(this); } size_t SizeInBytes() const { return g_host->Tensor__SizeInBytes(this); } const OrtMemoryInfo& Location() const { return g_host->Tensor__Location(this); } @@ -1379,6 +1568,8 @@ inline float* Tensor::MutableData() { return g_host->Tensor__MutableData_ template <> inline double* Tensor::MutableData() { return g_host->Tensor__MutableData_double(this); } template <> +inline BFloat16* Tensor::MutableData() { return g_host->Tensor__MutableData_BFloat16(this); } +template <> inline MLFloat16* Tensor::MutableData() { return g_host->Tensor__MutableData_MLFloat16(this); } template <> @@ -1404,31 +1595,82 @@ inline const float* Tensor::Data() const { return g_host->Tensor__Data_fl template <> inline const double* Tensor::Data() const { return g_host->Tensor__Data_double(this); } template <> +inline const BFloat16* Tensor::Data() const { return g_host->Tensor__Data_BFloat16(this); } +template <> inline const MLFloat16* Tensor::Data() const { return g_host->Tensor__Data_MLFloat16(this); } +template <> +inline gsl::span Tensor::DataAsSpan() const { return g_host->Tensor__DataAsSpan_int64(this); } + namespace utils { - -inline bool HasDimValue(const ONNX_NAMESPACE::TensorShapeProto_Dimension& dim) { - return dim.value_case() == ONNX_NAMESPACE::TensorShapeProto_Dimension::kDimValue; -} - bool IsDataTypeString(MLDataType dt_type); } // namespace utils +#ifdef ENABLE_TRAINING +namespace contrib { +inline void record_event_in_tensor(const Tensor& event_id_tensor) { return g_host->contrib__record_event_in_tensor(event_id_tensor); } +inline void wait_event_in_tensor(const Tensor& event_id_tensor) { return g_host->contrib__wait_event_in_tensor(event_id_tensor); } + +inline void VerifyLogitWeightAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, const TensorShape* weight_shape) { g_host->contrib__VerifyLogitWeightAndLabelShape(logit_shape, label_shape, weight_shape); } +inline void GetNDCFromLogitAndLabelShape(const TensorShape& logit_shape, const TensorShape& label_shape, int64_t& N_D, int64_t& C) { g_host->contrib__GetNDCFromLogitAndLabelShape(logit_shape, label_shape, N_D, C); } +inline void GetPermutationAndShape(bool ncd_to_ndc, const TensorShape& tensor_shape, std::vector& new_shape, std::vector& permutations) { g_host->contrib__GetPermutationAndShape(ncd_to_ndc, tensor_shape, new_shape, permutations); } +inline Status PrepareForTrainingCompute(const TensorShape& input_shape, int num_outputs, int64_t& axis, int& before_dims, int& after_dims_including_split_axis, int& after_dims_excluding_split, std::vector& split_sizes) { return g_host->contrib__PrepareForTrainingCompute(input_shape, num_outputs, axis, before_dims, after_dims_including_split_axis, after_dims_excluding_split, split_sizes); } +} // namespace contrib +#endif + namespace GatherElements { inline Status ValidateInputShapes(const TensorShape& input_data_shape, const TensorShape& indices_shape, int64_t axis) { return g_host->GatherElements__ValidateInputShapes(input_data_shape, indices_shape, axis); } } // namespace GatherElements +namespace cumsum_op { +inline Status GetAxis(const Tensor* axis_tensor, int64_t input_rank, int64_t& axis_out) { return g_host->cumsum_op__GetAxis(axis_tensor, input_rank, axis_out); } +} // namespace cumsum_op + +inline Status CheckROIAlignValidInput(const Tensor* X_ptr, const Tensor* rois_ptr, const Tensor* batch_indices_ptr) { return g_host->CheckROIAlignValidInput(X_ptr, rois_ptr, batch_indices_ptr); } + // From onehot.h inline Status ValidateInputs(const Tensor* depth, const Tensor* values) { return g_host->ValidateInputs(depth, values); } inline Status PrepareOutputShape(const Tensor* indices, const int64_t depth_val, const int64_t axis, int64_t& prefix_dim_size, int64_t& suffix_dim_size, std::vector& output_shape) { return g_host->PrepareOutputShape(indices, depth_val, axis, prefix_dim_size, suffix_dim_size, output_shape); } +struct EinsumComputePreprocessor { + static void operator delete(void* p) { g_host->EinsumComputePreprocessor__operator_delete(reinterpret_cast(p)); } + static std::unique_ptr Create(EinsumEquationPreprocessor& equation_preprocessor, + const std::vector& inputs, + AllocatorPtr allocator, + void* einsum_cuda_assets) { return g_host->EinsumComputePreprocessor__Create(equation_preprocessor, inputs, allocator, einsum_cuda_assets); } + + Status Run() { return g_host->EinsumComputePreprocessor__Run(this); } + + void SetDeviceHelpers(const EinsumOp::DeviceHelpers::Diagonal& diagonal_func, const EinsumOp::DeviceHelpers::Transpose& transpose_func) { return g_host->EinsumComputePreprocessor__SetDeviceHelpers(this, diagonal_func, transpose_func); } +}; + +template +struct EinsumTypedComputeProcessor { + static void operator delete(void* p) { g_host->EinsumTypedComputeProcessor__operator_delete(reinterpret_cast(p)); } + static std::unique_ptr Create(OpKernelContext* context, AllocatorPtr allocator, + concurrency::ThreadPool* tp, + EinsumComputePreprocessor& einsum_compute_preprocessor, + void* einsum_cuda_assets); + + void SetDeviceHelpers(const EinsumOp::DeviceHelpers::Transpose& device_transpose_func, + const EinsumOp::DeviceHelpers::MatMul& device_matmul_func, + const EinsumOp::DeviceHelpers::ReduceSum& device_reduce_sum_func, + const EinsumOp::DeviceHelpers::DataCopy& device_data_copy_func) { + g_host->EinsumTypedComputeProcessor__SetDeviceHelpers(this, device_transpose_func, device_matmul_func, device_reduce_sum_func, device_data_copy_func); + } + + Status Run() { return g_host->EinsumTypedComputeProcessor__Run(this); } +}; + +inline std::unique_ptr CreateOpKernel_CPU_If(const OpKernelInfo& info) { return g_host->CreateOpKernel_CPU_If(info); } +inline std::unique_ptr CreateOpKernel_CPU_Scan_8(const OpKernelInfo& info) { return g_host->CreateOpKernel_CPU_Scan_8(info); } +inline std::unique_ptr CreateOpKernel_CPU_Scan_9(const OpKernelInfo& info) { return g_host->CreateOpKernel_CPU_Scan_9(info); } + #endif } // namespace onnxruntime - diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 1eabe1561e..62c0a24567 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -10,6 +10,7 @@ #include "tensorrt_execution_provider.h" #include "core/providers/cuda/shared_inc/cuda_call.h" #include "core/providers/cuda/math/unary_elementwise_ops_impl.h" +#include "core/providers/cuda/gpu_data_transfer.h" #include "cuda_runtime_api.h" #include "gsl/gsl" #include @@ -507,16 +508,16 @@ AllocatorPtr TensorrtExecutionProvider::GetAllocator(int id, OrtMemType mem_type } void TensorrtExecutionProvider::RegisterAllocator(std::shared_ptr allocator_manager) { - allocator_ = AllocatorManager__GetAllocator(allocator_manager.get(), device_id_, OrtMemTypeDefault); + allocator_ = allocator_manager->GetAllocator(device_id_, OrtMemTypeDefault); if (nullptr == allocator_) { AllocatorCreationInfo default_memory_info( [](OrtDevice::DeviceId device_id) { return CreateCUDAAllocator(device_id, onnxruntime::CUDA); }, device_id_); allocator_ = CreateAllocator(default_memory_info); - AllocatorManager__InsertAllocator(allocator_manager.get(), allocator_); + allocator_manager->InsertAllocator(allocator_); } TryInsertAllocator(allocator_); - auto cuda_pinned_alloc = AllocatorManager__GetAllocator(allocator_manager.get(), DEFAULT_CPU_ALLOCATOR_DEVICE_ID, OrtMemTypeCPUOutput); + auto cuda_pinned_alloc = allocator_manager->GetAllocator(DEFAULT_CPU_ALLOCATOR_DEVICE_ID, OrtMemTypeCPUOutput); if (nullptr == cuda_pinned_alloc) { AllocatorCreationInfo pinned_allocator_info( [](OrtDevice::DeviceId device_id) { @@ -524,7 +525,7 @@ void TensorrtExecutionProvider::RegisterAllocator(std::shared_ptrInsertAllocator(cuda_pinned_alloc); } TryInsertAllocator(cuda_pinned_alloc); } diff --git a/onnxruntime/test/framework/cuda/allocator_cuda_test.cc b/onnxruntime/test/framework/cuda/allocator_cuda_test.cc index c6b32db17f..0cfba44d4a 100644 --- a/onnxruntime/test/framework/cuda/allocator_cuda_test.cc +++ b/onnxruntime/test/framework/cuda/allocator_cuda_test.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if 0 +#if 0 // TODO: Can't call these directly from external code as Cuda is now a shared library #include "core/framework/allocatormgr.h" #include "test/framework/test_utils.h" diff --git a/onnxruntime/test/framework/cuda/fence_cuda_test.cc b/onnxruntime/test/framework/cuda/fence_cuda_test.cc index cb0e849911..6e9e699d80 100644 --- a/onnxruntime/test/framework/cuda/fence_cuda_test.cc +++ b/onnxruntime/test/framework/cuda/fence_cuda_test.cc @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if 0 +#if 0 // TODO: Can't call these directly from external code as Cuda is now a shared library #include "core/graph/onnx_protobuf.h" #include "core/session/inference_session.h" diff --git a/onnxruntime/test/framework/test_utils.cc b/onnxruntime/test/framework/test_utils.cc index 15b1e219c9..a1da10cc5f 100644 --- a/onnxruntime/test/framework/test_utils.cc +++ b/onnxruntime/test/framework/test_utils.cc @@ -1,7 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#undef USE_CUDA - #include "test_utils.h" #include "core/graph/graph.h" @@ -13,34 +11,6 @@ IExecutionProvider* TestCPUExecutionProvider() { return &cpu_provider; } -#ifdef USE_CUDA -IExecutionProvider* TestCudaExecutionProvider() { - static CUDAExecutionProviderInfo info; - static CUDAExecutionProvider cuda_provider(info); - return &cuda_provider; -} -#endif - -#ifdef USE_TENSORRT -#if 0 // TODO: TensorRT is shared, can't access these directly anymore -IExecutionProvider* TestTensorrtExecutionProvider() { - static TensorrtExecutionProviderInfo info; - static TensorrtExecutionProvider trt_provider(info); - return &trt_provider; -} -#endif -#endif - -#ifdef USE_OPENVINO -#if 0 // TODO: OpenVINO is shared, can't access these directly anymore -IExecutionProvider* TestOpenVINOExecutionProvider() { - static OpenVINOExecutionProviderInfo info; - static OpenVINOExecutionProvider openvino_provider(info); - return &openvino_provider; -} -#endif -#endif - #ifdef USE_NNAPI IExecutionProvider* TestNnapiExecutionProvider() { static NnapiExecutionProvider nnapi_provider(0); diff --git a/onnxruntime/test/framework/test_utils.h b/onnxruntime/test/framework/test_utils.h index 8d8fd8294d..68cff00ada 100644 --- a/onnxruntime/test/framework/test_utils.h +++ b/onnxruntime/test/framework/test_utils.h @@ -13,7 +13,7 @@ #include "gsl/gsl" #ifdef USE_CUDA -#include "core/providers/cuda/cuda_execution_provider.h" +#include "core/providers/providers.h" #endif #ifdef USE_NNAPI #include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h" @@ -32,20 +32,6 @@ namespace test { // Doesn't work with ExecutionProviders class and KernelRegistryManager IExecutionProvider* TestCPUExecutionProvider(); -#ifdef USE_CUDA -// Doesn't work with ExecutionProviders class and KernelRegistryManager -IExecutionProvider* TestCudaExecutionProvider(); -#endif - -#ifdef USE_TENSORRT -// Doesn't work with ExecutionProviders class and KernelRegistryManager -IExecutionProvider* TestTensorrtExecutionProvider(); -#endif - -#ifdef USE_OPENVINO -IExecutionProvider* TestOpenVINOExecutionProvider(); -#endif - #ifdef USE_NNAPI IExecutionProvider* TestNnapiExecutionProvider(); #endif diff --git a/onnxruntime/test/providers/cuda/reduction_functions_test.cc b/onnxruntime/test/providers/cuda/reduction_functions_test.cc index 8b46d94b89..7a677b6ed2 100644 --- a/onnxruntime/test/providers/cuda/reduction_functions_test.cc +++ b/onnxruntime/test/providers/cuda/reduction_functions_test.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if 0 +#if 0 // TODO: Can't call these directly from external code as Cuda is now a shared library //#ifdef USE_CUDA #include diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h index f15c0cb4a3..6a43ca9b43 100644 --- a/onnxruntime/test/providers/provider_test_utils.h +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -19,6 +19,7 @@ #include "test/framework/TestAllocatorManager.h" #include "core/framework/TensorSeq.h" #include "core/framework/session_options.h" +#include "core/providers/providers.h" #include "test/util/include/asserts.h" #include "gmock/gmock.h" diff --git a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc index b605428cab..c24401a1b8 100644 --- a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc +++ b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc @@ -27,7 +27,9 @@ void VerifyOutputs(const std::vector& fetches, const std::vector inputs; @@ -68,7 +70,7 @@ TEST(TensorrtExecutionProviderTest, EngineCachingTest) { run_options.run_tag = so.session_logid; InferenceSession session_object{so, GetEnvironment()}; auto allocator_manager = session_object.GetAllocatorManager(); - auto cuda_provider = TestCudaExecutionProvider(); + auto cuda_provider = DefaultCudaExecutionProvider(); cuda_provider->RegisterAllocator(allocator_manager); auto cpu_allocator = cuda_provider->GetAllocator(0, OrtMemTypeCPU); // First run with input shape {1, 3, 2} @@ -180,7 +182,7 @@ TEST(TensorrtExecutionProviderTest, FunctionTest) { InferenceSession session_object{so, GetEnvironment()}; auto allocator_manager = session_object.GetAllocatorManager(); - auto cuda_provider = TestCudaExecutionProvider(); + auto cuda_provider = DefaultCudaExecutionProvider(); cuda_provider->RegisterAllocator(allocator_manager); auto cpu_allocator = cuda_provider->GetAllocator(0, OrtMemTypeCPU); @@ -292,7 +294,7 @@ TEST(TensorrtExecutionProviderTest, NodeIndexMappingTest) { InferenceSession session_object{so, GetEnvironment()}; auto allocator_manager = session_object.GetAllocatorManager(); - auto cuda_provider = TestCudaExecutionProvider(); + auto cuda_provider = DefaultCudaExecutionProvider(); cuda_provider->RegisterAllocator(allocator_manager); auto cpu_allocator = cuda_provider->GetAllocator(0, OrtMemTypeCPU); @@ -413,7 +415,7 @@ TEST(TensorrtExecutionProviderTest, RemoveCycleTest) { InferenceSession session_object{so, GetEnvironment()}; auto allocator_manager = session_object.GetAllocatorManager(); - auto cuda_provider = TestCudaExecutionProvider(); + auto cuda_provider = DefaultCudaExecutionProvider(); cuda_provider->RegisterAllocator(allocator_manager); auto cpu_allocator = cuda_provider->GetAllocator(0, OrtMemTypeCPU); diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index afdd32c1ac..a30ac0bc6e 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -71,10 +71,10 @@ std::unique_ptr DefaultOpenVINOExecutionProvider() { std::unique_ptr DefaultCudaExecutionProvider() { #ifdef USE_CUDA OrtCUDAProviderOptions provider_options{}; - return CreateExecutionProviderFactory_Cuda(&provider_options)->CreateProvider(); -#else - return nullptr; + if (auto factory = CreateExecutionProviderFactory_Cuda(&provider_options)) + return factory->CreateProvider(); #endif + return nullptr; } std::unique_ptr DefaultDnnlExecutionProvider(bool enable_arena) { diff --git a/orttraining/orttraining/core/session/tensor_helper.cc b/orttraining/orttraining/core/session/tensor_helper.cc index 2060ff34a5..a8d084426b 100644 --- a/orttraining/orttraining/core/session/tensor_helper.cc +++ b/orttraining/orttraining/core/session/tensor_helper.cc @@ -3,11 +3,11 @@ #include "orttraining/core/session/tensor_helper.h" -#ifdef USE_CUDA -#include "core/providers/cuda/cuda_common.h" -#endif - namespace onnxruntime { + +// Copy a chunk of memory to CPU from GPU. +void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const OrtMemoryInfo& dst_location, const OrtMemoryInfo& src_location); + namespace training { // Return the shape of a tensor slice. @@ -55,41 +55,6 @@ OrtValue CreateCpuTensorValue( return cpu_value; } -// Copy a chunk of memory to CPU from GPU. -void CopyGpuToCpu( - void* dst_ptr, - const void* src_ptr, - const size_t size, - const OrtMemoryInfo& dst_location, - const OrtMemoryInfo& src_location) { - ORT_ENFORCE(dst_location.device.Type() == OrtDevice::CPU); - -#ifdef USE_CUDA - // Current CUDA device. - int device; - CUDA_CALL(cudaGetDevice(&device)); - - if (device != src_location.id) { - // Need to switch to the allocating device. - CUDA_CALL(cudaSetDevice(src_location.id)); - // Copy from GPU to CPU. - CUDA_CALL(cudaMemcpy(dst_ptr, src_ptr, size, cudaMemcpyDeviceToHost)); - // Switch back to current device. - CUDA_CALL(cudaSetDevice(device)); - } else { - // Copy from GPU to CPU. - CUDA_CALL(cudaMemcpy(dst_ptr, src_ptr, size, cudaMemcpyDeviceToHost)); - } -#else - ORT_UNUSED_PARAMETER(dst_ptr); - ORT_UNUSED_PARAMETER(src_ptr); - ORT_UNUSED_PARAMETER(size); - ORT_UNUSED_PARAMETER(dst_location); - ORT_UNUSED_PARAMETER(src_location); - ORT_THROW("CPU-to-CPU copy is not implemented."); -#endif -} - // Copy a chunk of memory to CPU from CPU. void CopyCpuToCpu( void* dst_ptr, @@ -270,7 +235,7 @@ void CopyConcat( auto chunk_size = src_shape[concat_axis] * segment_size; // Bias of the i_seg-th segment in input tensor. Its unit is the number of tensor elements. - // chunk_size * + // chunk_size * auto src_bias = UpdateLinearIndex(i_seg, 0, src_shape[concat_axis]) * segment_size; auto dst_bias = UpdateLinearIndex(i_seg, anchor_bias, dst_shape[concat_axis]) * segment_size; @@ -333,4 +298,4 @@ OrtValue ConcatenateTensors( return result_value; } } // namespace training -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/orttraining/orttraining/core/session/training_session.cc b/orttraining/orttraining/core/session/training_session.cc index 2f3d15d8e6..a02ed353db 100644 --- a/orttraining/orttraining/core/session/training_session.cc +++ b/orttraining/orttraining/core/session/training_session.cc @@ -28,11 +28,6 @@ //Gist Encoding #include "orttraining/core/optimizer/gist_encode_decode.h" -#ifdef USE_CUDA -#include "core/providers/cuda/cuda_common.h" -#include "core/providers/cuda/cuda_allocator.h" -#endif - #include "orttraining/training_ops/cpu/controlflow/event_pool.h" #if defined(USE_CUDA) && defined(ORT_USE_NCCL) && defined(USE_NCCL_P2P) #include "orttraining/training_ops/cuda/communication/nccl_service.h" @@ -1002,8 +997,8 @@ static Status UpdateWeightsBeforeSaving( return Status::OK(); } -Status TrainingSession::SaveWithExternalInitializers(const PathString& model_uri, - const std::string& external_file_name, +Status TrainingSession::SaveWithExternalInitializers(const PathString& model_uri, + const std::string& external_file_name, size_t initializer_size_threshold) { // Delete the old files before saving. std::remove(ToMBString(model_uri).c_str()); diff --git a/orttraining/orttraining/models/bert/main.cc b/orttraining/orttraining/models/bert/main.cc index 49d43554d1..ece51245e9 100644 --- a/orttraining/orttraining/models/bert/main.cc +++ b/orttraining/orttraining/models/bert/main.cc @@ -12,8 +12,10 @@ #include "core/framework/random_seed.h" #include "core/providers/cpu/cpu_provider_factory_creator.h" #ifdef USE_CUDA -#include "core/providers/cuda/cuda_allocator.h" -#include "core/providers/cuda/cuda_provider_factory_creator.h" +namespace onnxruntime { +std::shared_ptr CreateExecutionProviderFactory_Cuda(const OrtCUDAProviderOptions* provider_options); +std::unique_ptr CreateCUDAPinnedAllocator(int16_t device_id, const char* name); +} // namespace onnxruntime #endif #ifdef USE_ROCM #include "core/providers/rocm/rocm_allocator.h" @@ -54,7 +56,7 @@ static SessionOptions session_options = { false, //use_deterministic_compute {}, //session_configurations {}, // initializers_to_share_map -}; +}; struct BertParameters : public TrainingRunner::Parameters { int max_sequence_length = 512; @@ -599,15 +601,22 @@ void setup_training_params(BertParameters& params) { #ifdef USE_CUDA { - CUDAExecutionProviderInfo info{}; - info.device_id = gsl::narrow(MPIContext::GetInstance().GetLocalRank()); + OrtCUDAProviderOptions info{ + gsl::narrow(MPIContext::GetInstance().GetLocalRank()), + OrtCudnnConvAlgoSearch::EXHAUSTIVE, + std::numeric_limits::max(), + 0, + true, + 0, + nullptr}; + if (params.cuda_mem_limit_in_gb > 0) { info.cuda_mem_limit = gsl::narrow(params.cuda_mem_limit_in_gb * 1024 * 1024 * 1024); } info.cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE; - params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(info)); - params.input_allocator = std::make_shared(info.device_id, CUDA_PINNED); + params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_Cuda(&info)); + params.input_allocator = CreateCUDAPinnedAllocator(info.device_id, CUDA_PINNED); } #endif @@ -828,7 +837,8 @@ int main(int argc, char* argv[]) { OrtParameters ort_params{}; RETURN_IF_FAIL(ParseArguments(argc, argv, params, ort_params)); bool keep_looping = params.debug_break; - while(keep_looping); + while (keep_looping) + ; // setup logger, be noted: LOGS_DEFAULT must be after logging manager initialization. string default_logger_id{"Default"}; diff --git a/orttraining/orttraining/models/gpt2/main.cc b/orttraining/orttraining/models/gpt2/main.cc index 09ee3bd7f7..4a342069d6 100644 --- a/orttraining/orttraining/models/gpt2/main.cc +++ b/orttraining/orttraining/models/gpt2/main.cc @@ -9,9 +9,12 @@ #include "core/session/environment.h" #include "core/framework/random_seed.h" #include "core/framework/bfc_arena.h" +#include "core/providers/providers.h" #ifdef USE_CUDA -#include "core/providers/cuda/cuda_allocator.h" -#include "core/providers/cuda/cuda_provider_factory_creator.h" +namespace onnxruntime { +std::shared_ptr CreateExecutionProviderFactory_Cuda(const OrtCUDAProviderOptions* provider_options); +std::unique_ptr CreateCUDAPinnedAllocator(int16_t device_id, const char* name); +} // namespace onnxruntime #endif #include "orttraining/core/framework/communication/mpi/mpi_context.h" #include "orttraining/core/framework/tensorboard/event_writer.h" @@ -237,7 +240,7 @@ Status ParseArguments(int argc, char* argv[], GPT2Parameters& params, OrtParamet int64_t seed = flags["seed"].as(); if (params.horizontal_parallel_size > 1 && seed <= 0) { - seed = 8211; // Megatron needs a random seed. + seed = 8211; // Megatron needs a random seed. } if (seed > 0) { utils::SetRandomSeed(seed); @@ -277,7 +280,7 @@ float GetLossValue(const Tensor& loss_tensor) { // mapping to define what to be stored in mapped_dimensions // see GetTensorDimensionsFromInputs() in training_util.h and training_runner.cc for more details const std::map> input_to_dimension_mapping = { - {"input_ids", {"SeqLen", 0}}, // int64[batch,seqlen] "seqlen" -> "SeqLen", 0 + {"input_ids", {"SeqLen", 0}}, // int64[batch,seqlen] "seqlen" -> "SeqLen", 0 }; // generic properties for storing perf metrics @@ -342,11 +345,17 @@ void setup_training_params(GPT2Parameters& params) { #ifdef USE_CUDA { - CUDAExecutionProviderInfo info{}; - info.device_id = gsl::narrow(MPIContext::GetInstance().GetLocalRank()); + OrtCUDAProviderOptions info{ + gsl::narrow(MPIContext::GetInstance().GetLocalRank()), + OrtCudnnConvAlgoSearch::EXHAUSTIVE, + std::numeric_limits::max(), + 0, + true, + 0, + nullptr}; - params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(info)); - params.input_allocator = std::make_shared(info.device_id, CUDA_PINNED); + params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_Cuda(&info)); + params.input_allocator = CreateCUDAPinnedAllocator(info.device_id, CUDA_PINNED); } #endif diff --git a/orttraining/orttraining/models/mnist/main.cc b/orttraining/orttraining/models/mnist/main.cc index 3e42692b19..b15cf1de66 100644 --- a/orttraining/orttraining/models/mnist/main.cc +++ b/orttraining/orttraining/models/mnist/main.cc @@ -6,8 +6,11 @@ #include "core/common/logging/sinks/clog_sink.h" #include "core/framework/bfc_arena.h" #include "core/platform/env.h" +#include "core/providers/providers.h" #ifdef USE_CUDA -#include "core/providers/cuda/cuda_provider_factory_creator.h" +namespace onnxruntime { +std::shared_ptr CreateExecutionProviderFactory_Cuda(const OrtCUDAProviderOptions* provider_options); +} // namespace onnxruntime #endif #include "core/session/environment.h" #include "orttraining/core/session/training_session.h" @@ -22,11 +25,6 @@ #include namespace onnxruntime { -std::shared_ptr CreateExecutionProviderFactory_CUDA(OrtDevice::DeviceId device_id, - OrtCudnnConvAlgoSearch cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::EXHAUSTIVE, - size_t cuda_mem_limit = std::numeric_limits::max(), - onnxruntime::ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo, - bool do_copy_in_default_stream = true); std::shared_ptr CreateExecutionProviderFactory_Dnnl(int use_arena); } @@ -167,7 +165,16 @@ Status ParseArguments(int argc, char* argv[], MnistParameters& params) { #ifdef USE_CUDA bool use_cuda = flags.count("use_cuda") > 0; if (use_cuda) { - params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_CUDA(CUDAExecutionProviderInfo{})); + OrtCUDAProviderOptions info{ + 0, + OrtCudnnConvAlgoSearch::EXHAUSTIVE, + std::numeric_limits::max(), + 0, + true, + 0, + nullptr}; + + params.providers.emplace(kCudaExecutionProvider, CreateExecutionProviderFactory_Cuda(&info)); } #endif diff --git a/orttraining/orttraining/test/framework/slice_concatenate_test.cc b/orttraining/orttraining/test/framework/slice_concatenate_test.cc index 5e8998ea92..3f4cf9905e 100644 --- a/orttraining/orttraining/test/framework/slice_concatenate_test.cc +++ b/orttraining/orttraining/test/framework/slice_concatenate_test.cc @@ -7,10 +7,15 @@ #include "core/session/inference_session.h" #include "gtest/gtest.h" #include "orttraining/core/session/tensor_helper.h" +#include "test/util/include/default_providers.h" #include "test/framework/test_utils.h" #include "test/test_environment.h" namespace onnxruntime { +#ifdef USE_CUDA +void cudaMemcpy_HostToDevice(void* dst, const void* src, size_t count); +#endif + namespace test { typedef std::vector ArgMap; @@ -18,7 +23,7 @@ typedef std::vector ArgMap; // Create ML value. OrtValue CreateTensorValue(const std::vector& shape, const std::vector& initializer, const bool allocate_on_gpu) { #ifdef USE_CUDA - auto cpu_allocator = allocate_on_gpu ? TestCudaExecutionProvider()->GetAllocator(0, OrtMemTypeDefault) : TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault); + auto cpu_allocator = allocate_on_gpu ? DefaultCudaExecutionProvider()->GetAllocator(0, OrtMemTypeDefault) : TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault); #else ORT_ENFORCE(allocate_on_gpu != true); auto cpu_allocator = TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault); @@ -34,7 +39,7 @@ OrtValue CreateTensorValue(const std::vector& shape, const std::vector< memcpy(p_tensor->MutableData(), initializer.data(), initializer.size() * sizeof(float)); } else { #ifdef USE_CUDA - cudaMemcpy(p_tensor->MutableData(), initializer.data(), initializer.size() * sizeof(float), cudaMemcpyHostToDevice); + cudaMemcpy_HostToDevice(p_tensor->MutableData(), initializer.data(), initializer.size() * sizeof(float)); #else ORT_THROW("Cannot use CUDA function when ORT is not built with CUDA."); #endif @@ -79,8 +84,7 @@ void InitializeSession(onnxruntime::InferenceSession& session, onnxruntime::Mode // Initialize the session. #if defined(USE_CUDA) - onnxruntime::CUDAExecutionProviderInfo xp_info; - ASSERT_STATUS_OK(session.RegisterExecutionProvider(onnxruntime::make_unique(xp_info))); + ASSERT_STATUS_OK(session.RegisterExecutionProvider(DefaultCudaExecutionProvider())); #endif ASSERT_STATUS_OK(session.Initialize()); } @@ -378,4 +382,4 @@ TEST(PipelineParallel, FloatTensorConcat1dGpu) { #endif } // namespace test -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc b/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc index fdb2cb036f..6aed6bf115 100644 --- a/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc +++ b/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc @@ -6,6 +6,7 @@ #include "orttraining/core/optimizer/gist_encode_decode.h" #include "test/providers/provider_test_utils.h" #include "test/framework/test_utils.h" +#include "test/util/include/default_providers.h" #include "core/common/path_utils.h" #include "core/providers/cpu/cpu_execution_provider.h" #include "core/session/environment.h" @@ -17,7 +18,6 @@ #ifdef USE_CUDA #include "bert_toy_fetches.h" -#include "core/providers/cuda/cuda_execution_provider.h" #endif using namespace onnxruntime::logging; @@ -316,8 +316,7 @@ static void RunBertTrainingWithChecks( auto model_metadata = res.second; std::cout << "Loaded " << model_metadata->graph_name << '\n'; - CUDAExecutionProviderInfo xp_info; - ASSERT_STATUS_OK(training_session->RegisterExecutionProvider(onnxruntime::make_unique(xp_info))); + ASSERT_STATUS_OK(training_session->RegisterExecutionProvider(DefaultCudaExecutionProvider())); ASSERT_STATUS_OK(training_session->Initialize()); @@ -1002,21 +1001,21 @@ class PipelineBatchPlanner { }; void RetrieveEventOperators( - Graph& graph, - const int stage_index, - const int num_stages, - Node** forward_recv_wait, - Node** forward_recv_record, - Node** forward_compute_wait, - Node** forward_compute_record, - Node** forward_send_wait, - Node** forward_send_record, - Node** backward_recv_wait, - Node** backward_recv_record, - Node** backward_compute_wait, - Node** backward_compute_record, - Node** backward_send_wait, - Node** backward_send_record) { + Graph& graph, + const int stage_index, + const int num_stages, + Node** forward_recv_wait, + Node** forward_recv_record, + Node** forward_compute_wait, + Node** forward_compute_record, + Node** forward_send_wait, + Node** forward_send_record, + Node** backward_recv_wait, + Node** backward_recv_record, + Node** backward_compute_wait, + Node** backward_compute_record, + Node** backward_send_wait, + Node** backward_send_record) { // Initialize retrieved nodes. // Non-existing nodes may hold NULL forever. // Existing nodes may get valid pointers below. @@ -1102,16 +1101,16 @@ void RetrieveEventOperators( *backward_send_record = records[2]; } else { ORT_THROW("Wrong number of WaitEvent operators: ", - waits.size(), " allowed value range is [0, ", num_stages - 1, ")."); + waits.size(), " allowed value range is [0, ", num_stages - 1, ")."); } } void RetrieveSendRecvOperators( - Graph& graph, - Node** forward_recv, - Node** forward_send, - Node** backward_recv, - Node** backward_send) { + Graph& graph, + Node** forward_recv, + Node** forward_send, + Node** backward_recv, + Node** backward_send) { // Initialize retrieved nodes. // Non-existing nodes may hold NULL forever. // Existing nodes may get valid pointers below. @@ -1163,32 +1162,30 @@ PathString GenerateFileNameWithIndex(const std::string& base_str, int index, con // DistributedRunTestContext provides a method to override existing DistributedRunTestContext instance. // This is for test purpose only. Please don't use it for other scenarios. -class DistributedRunTestContext : public DistributedRunContext -{ -public: - DistributedRunTestContext(const TrainingSession::TrainingConfiguration &config) - : DistributedRunContext(config.distributed_config.world_rank, - config.distributed_config.world_size, - config.distributed_config.local_rank, - config.distributed_config.local_size, - config.distributed_config.data_parallel_size, - config.distributed_config.horizontal_parallel_size, - config.distributed_config.pipeline_parallel_size) - { - } +class DistributedRunTestContext : public DistributedRunContext { + public: + DistributedRunTestContext(const TrainingSession::TrainingConfiguration& config) + : DistributedRunContext(config.distributed_config.world_rank, + config.distributed_config.world_size, + config.distributed_config.local_rank, + config.distributed_config.local_size, + config.distributed_config.data_parallel_size, + config.distributed_config.horizontal_parallel_size, + config.distributed_config.pipeline_parallel_size) { + } - // Reset the static DistributedRunContext object with new value. - void ResetDistributedRunContext(){ - DistributedRunContext::GetRunConfig() = params_; - auto& dp_group = DistributedRunContext::GetWorkerGroup(WorkerGroupType::DataParallel); - dp_group = groups_[WorkerGroupType::DataParallel]; + // Reset the static DistributedRunContext object with new value. + void ResetDistributedRunContext() { + DistributedRunContext::GetRunConfig() = params_; + auto& dp_group = DistributedRunContext::GetWorkerGroup(WorkerGroupType::DataParallel); + dp_group = groups_[WorkerGroupType::DataParallel]; - auto& hp_group = DistributedRunContext::GetWorkerGroup(WorkerGroupType::HorizontalParallel); - hp_group = groups_[WorkerGroupType::HorizontalParallel]; + auto& hp_group = DistributedRunContext::GetWorkerGroup(WorkerGroupType::HorizontalParallel); + hp_group = groups_[WorkerGroupType::HorizontalParallel]; - auto& mp_group = DistributedRunContext::GetInstance().GetWorkerGroup(WorkerGroupType::PipelineParallel); - mp_group = groups_[WorkerGroupType::PipelineParallel]; - } + auto& mp_group = DistributedRunContext::GetInstance().GetWorkerGroup(WorkerGroupType::PipelineParallel); + mp_group = groups_[WorkerGroupType::PipelineParallel]; + } }; void OverwritePipelineRank(const TrainingSession::TrainingConfiguration& config, const int pipeline_rank) { @@ -1331,7 +1328,7 @@ TEST(GradientGraphBuilderTest, PipelineOnlinePartition_MLP) { // 2 test variations - full precision and mixed precision const std::vector test_with_fp32{true, false}; - for(auto is_fp32 : test_with_fp32) { + for (auto is_fp32 : test_with_fp32) { // graph is partitioned into 3 parts. for (int i = 0; i < 3; ++i) { PathString output_file = GenerateFileNameWithIndex("pipeline_partition_", i, "_back.onnx"); @@ -1356,7 +1353,7 @@ TEST(GradientGraphBuilderTest, PipelineOnlinePartition_MLP) { PathString backprop_model_file; Status status = BuildBackPropGraph(model_uri, config, backprop_model_file); - ASSERT_TRUE(status.IsOK()) << status<<" (is_fp32 = " << is_fp32 << ", stage = " << i << ").\n"; + ASSERT_TRUE(status.IsOK()) << status << " (is_fp32 = " << is_fp32 << ", stage = " << i << ").\n"; // Skip the re-load for mixed-precision case. This model contains grad op that has function body, // which takes a const tensor input. Const cast for input in function body won't be saved in the output @@ -1367,7 +1364,7 @@ TEST(GradientGraphBuilderTest, PipelineOnlinePartition_MLP) { std::shared_ptr model; // Ensure the partitioned model load. status = Model::Load(backprop_model_file, model, nullptr, DefaultLoggingManager().DefaultLogger()); - ASSERT_TRUE(status.IsOK()) << status<<" (is_fp32 = " << is_fp32 << ", stage = " << i << ").\n"; + ASSERT_TRUE(status.IsOK()) << status << " (is_fp32 = " << is_fp32 << ", stage = " << i << ").\n"; } } } @@ -1426,7 +1423,7 @@ TEST(GradientGraphBuilderTest, PipelineOnlinePartition_Invalid_Input) { // verify pipeline config can load and gradient graph can construct. TEST(GradientGraphBuilderTest, TrainingSession_PipelineTransform_base) { - std::string filename_base = "testdata/test_training_model_"; + std::string filename_base = "testdata/test_training_model_"; auto load_and_check_gradient_graph = [](int stageIdx, PathString& input_file, PathString& output_file) { auto config = MakeBasicTrainingConfig(); @@ -1462,21 +1459,21 @@ TEST(GradientGraphBuilderTest, TrainingSession_PipelineTransform_base) { // Find event nodes. RetrieveEventOperators( - graph, - stageIdx, - 3, - &forward_recv_wait, - &forward_recv_record, - &forward_compute_wait, - &forward_compute_record, - &forward_send_wait, - &forward_send_record, - &backward_recv_wait, - &backward_recv_record, - &backward_compute_wait, - &backward_compute_record, - &backward_send_wait, - &backward_send_record); + graph, + stageIdx, + 3, + &forward_recv_wait, + &forward_recv_record, + &forward_compute_wait, + &forward_compute_record, + &forward_send_wait, + &forward_send_record, + &backward_recv_wait, + &backward_recv_record, + &backward_compute_wait, + &backward_compute_record, + &backward_send_wait, + &backward_send_record); // Check event nodes. if (stageIdx == 2) { @@ -1538,11 +1535,11 @@ TEST(GradientGraphBuilderTest, TrainingSession_PipelineTransform_base) { Node* backward_send{nullptr}; RetrieveSendRecvOperators( - graph, - &forward_recv, - &forward_send, - &backward_recv, - &backward_send); + graph, + &forward_recv, + &forward_send, + &backward_recv, + &backward_send); // Except the last partion, each partition should have send forward and recv backward. if (stageIdx == 0 || stageIdx == 1) { @@ -1607,23 +1604,21 @@ TEST(GradientGraphBuilderTest, TrainingSession_WithPipeline) { {}, {}, {}}, - {{ - "MeanSquaredError_reduce_mean_Grad/Sized_X", - "MeanSquaredError_reduce_mean_Grad/Sized_Grad", - "MeanSquaredError_reduce_mean_Grad/Scale", - "MeanSquaredError_reduce_mean_Grad/Scaled_Grad", - "MeanSquaredError_reduce_mean_Grad/Shaped_X", - "MeanSquaredError_diff_square_grad", - "MeanSquaredError_pow_Grad/Sub_I1", - "MeanSquaredError_pow_Grad/Pow_I0", - "MeanSquaredError_pow_Grad/Mul_Pow_I0_I1", - "MeanSquaredError_diff_grad", - "predictions_grad", - "B3_grad", - "T7_grad", - "W3_grad", - "T6_grad" - }, + {{"MeanSquaredError_reduce_mean_Grad/Sized_X", + "MeanSquaredError_reduce_mean_Grad/Sized_Grad", + "MeanSquaredError_reduce_mean_Grad/Scale", + "MeanSquaredError_reduce_mean_Grad/Scaled_Grad", + "MeanSquaredError_reduce_mean_Grad/Shaped_X", + "MeanSquaredError_diff_square_grad", + "MeanSquaredError_pow_Grad/Sub_I1", + "MeanSquaredError_pow_Grad/Pow_I0", + "MeanSquaredError_pow_Grad/Mul_Pow_I0_I1", + "MeanSquaredError_diff_grad", + "predictions_grad", + "B3_grad", + "T7_grad", + "W3_grad", + "T6_grad"}, {}, {"T6_grad"}, {}, diff --git a/orttraining/orttraining/test/model/training_runner_test.cc b/orttraining/orttraining/test/model/training_runner_test.cc index d3dd253e44..6e257b9943 100644 --- a/orttraining/orttraining/test/model/training_runner_test.cc +++ b/orttraining/orttraining/test/model/training_runner_test.cc @@ -14,6 +14,7 @@ #include "core/providers/cuda/cuda_execution_provider.h" #include "orttraining/models/runner/data_loader.h" #include "orttraining/models/runner/training_util.h" +#include "test/util/include/default_providers.h" namespace onnxruntime { namespace training { @@ -45,8 +46,7 @@ TEST(TrainingRunnerTest, Basic) { TrainingRunner runner{params, *env}; - CUDAExecutionProviderInfo xp_info; - ASSERT_TRUE(runner.GetSession().RegisterExecutionProvider(onnxruntime::make_unique(xp_info)).IsOK()); + ASSERT_TRUE(runner.GetSession().RegisterExecutionProvider(onnxruntime::test::DefaultCudaExecutionProvider()).IsOK()); auto status = runner.Initialize(); ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); @@ -122,4 +122,4 @@ TEST(TrainingRunnerTest, DISABLED_PipelineRun) { } // namespace training } // namespace onnxruntime -#endif \ No newline at end of file +#endif diff --git a/orttraining/orttraining/test/session/training_session_test_utils.cc b/orttraining/orttraining/test/session/training_session_test_utils.cc index bb2dd5d14d..a2703c473e 100644 --- a/orttraining/orttraining/test/session/training_session_test_utils.cc +++ b/orttraining/orttraining/test/session/training_session_test_utils.cc @@ -3,6 +3,7 @@ #include "orttraining/test/session/training_session_test_utils.h" #include "orttraining/core/graph/optimizer_builder.h" +#include "test/util/include/default_providers.h" using namespace onnxruntime::logging; using namespace onnxruntime::training; @@ -180,8 +181,7 @@ std::unique_ptr BuildAndRunTrainingSessionWithChecks( std::cout << "Loaded " << model_metadata->graph_name << '\n'; #ifdef USE_CUDA - CUDAExecutionProviderInfo xp_info; - ORT_THROW_IF_ERROR(training_session->RegisterExecutionProvider(onnxruntime::make_unique(xp_info))); + ORT_THROW_IF_ERROR(training_session->RegisterExecutionProvider(DefaultCudaExecutionProvider())); #endif ORT_THROW_IF_ERROR(training_session->Initialize()); diff --git a/orttraining/orttraining/test/training_ops/cpu/reduction/reduction_ops_test.cc b/orttraining/orttraining/test/training_ops/cpu/reduction/reduction_ops_test.cc index 18218b76f2..9e2aad84ce 100644 --- a/orttraining/orttraining/test/training_ops/cpu/reduction/reduction_ops_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/reduction/reduction_ops_test.cc @@ -6,10 +6,6 @@ #include #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" -//#include "test/providers/cpu/reduction/reduction_test_cases.h" -#ifdef USE_CUDA -#include "core/providers/cuda/reduction/reduction_functions.h" -#endif namespace onnxruntime { namespace test { @@ -87,8 +83,8 @@ TEST(AllOpTest, All_1d_large) { #endif class ReductionOpTest : public ::testing::TestWithParam { - protected: - bool use_determinism; + protected: + bool use_determinism; }; TEST_P(ReductionOpTest, ReduceAllL2) { diff --git a/orttraining/orttraining/test/training_ops/cuda/cuda_utils_test.cc b/orttraining/orttraining/test/training_ops/cuda/cuda_utils_test.cc index 51141b3eb4..b58ff0bc14 100644 --- a/orttraining/orttraining/test/training_ops/cuda/cuda_utils_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/cuda_utils_test.cc @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifdef USE_CUDA +#if 0 // TODO: Can't call these directly from external code as Cuda is now a shared library +//#ifdef USE_CUDA #include #include diff --git a/orttraining/orttraining/training_ops/cpu/loss/reduction_type.h b/orttraining/orttraining/training_ops/cpu/loss/reduction_type.h index 97c4a8362c..96681fffad 100644 --- a/orttraining/orttraining/training_ops/cpu/loss/reduction_type.h +++ b/orttraining/orttraining/training_ops/cpu/loss/reduction_type.h @@ -3,8 +3,10 @@ #pragma once +#ifndef SHARED_PROVIDER #include "core/common/common.h" #include "core/common/exceptions.h" +#endif namespace onnxruntime { diff --git a/orttraining/orttraining/training_ops/cuda/activation/activations_grad.cc b/orttraining/orttraining/training_ops/cuda/activation/activations_grad.cc index 1049079082..8490d6d6b7 100644 --- a/orttraining/orttraining/training_ops/cuda/activation/activations_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/activation/activations_grad.cc @@ -14,7 +14,7 @@ namespace cuda { ver, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .MayInplace(0, 0), \ x); diff --git a/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad.cc b/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad.cc index 703623f16b..30c174dd77 100644 --- a/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad.cc @@ -21,7 +21,7 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("T", BuildKernelDefConstraints()) .MayInplace(0, 0), BiasGeluGrad_dX); @@ -31,7 +31,7 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("T", BuildKernelDefConstraints()) .MayInplace(0, 0), BiasGeluGrad_dX); diff --git a/orttraining/orttraining/training_ops/cuda/controlflow/group.cc b/orttraining/orttraining/training_ops/cuda/controlflow/group.cc index 38456b4866..128407c97c 100644 --- a/orttraining/orttraining/training_ops/cuda/controlflow/group.cc +++ b/orttraining/orttraining/training_ops/cuda/controlflow/group.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/providers/shared_library/provider_api.h" #include "orttraining/training_ops/cpu/controlflow/group.h" #include "core/providers/cuda/cuda_fwd.h" @@ -12,8 +13,8 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() - .OutputMemoryType(0) + (*KernelDefBuilder::Create()) + .OutputMemoryType(OrtMemTypeCPUOutput, 0) .TypeConstraint("B", DataTypeImpl::GetTensorType()) .TypeConstraint("T", DataTypeImpl::AllTensorTypes()), onnxruntime::contrib::Group); @@ -23,7 +24,7 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("T", DataTypeImpl::AllTensorTypes()) .VariadicAlias(0, 0), // outputs and inputs are mapped one to one onnxruntime::contrib::PassThrough); diff --git a/orttraining/orttraining/training_ops/cuda/controlflow/record.cc b/orttraining/orttraining/training_ops/cuda/controlflow/record.cc index ef47938785..91a98966bd 100644 --- a/orttraining/orttraining/training_ops/cuda/controlflow/record.cc +++ b/orttraining/orttraining/training_ops/cuda/controlflow/record.cc @@ -17,8 +17,8 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) /* Keep EventIdentifier in CPU */ + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) /* Keep EventIdentifier in CPU */ .TypeConstraint("TInt64", DataTypeImpl::GetTensorType()) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()) .VariadicAlias(1, 0), // outputs and inputs are mapped one to one, with input offset by 1 @@ -32,7 +32,7 @@ Status RecordEvent::ComputeInternal(OpKernelContext* ctx) const { auto& profile_context = profile::Context::GetInstance(); const auto tag = profile_context.GetThreadTagOrDefault(std::this_thread::get_id()); profile::NvtxRangeCreator range( - "Batch-" + tag + " Record-" + std::to_string(event_id), profile::Color::Magenta); + "Batch-" + tag + " Record-" + std::to_string(event_id), profile::Color::Magenta); range.Begin(); #endif diff --git a/orttraining/orttraining/training_ops/cuda/controlflow/wait.cc b/orttraining/orttraining/training_ops/cuda/controlflow/wait.cc index 47dc0c15b7..13b32fa7af 100644 --- a/orttraining/orttraining/training_ops/cuda/controlflow/wait.cc +++ b/orttraining/orttraining/training_ops/cuda/controlflow/wait.cc @@ -17,8 +17,8 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) /* CPU variable */ + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) /* CPU variable */ .TypeConstraint("TInt64", DataTypeImpl::GetTensorType()) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()) .VariadicAlias(1, 0), // outputs and inputs are mapped one to one, with input offset by 1 @@ -32,7 +32,7 @@ Status WaitEvent::ComputeInternal(OpKernelContext* ctx) const { auto& profile_context = profile::Context::GetInstance(); const auto tag = profile_context.GetThreadTagOrDefault(std::this_thread::get_id()); profile::NvtxRangeCreator range( - "Batch-" + tag + " Wait-" + std::to_string(event_id), profile::Color::Blue); + "Batch-" + tag + " Wait-" + std::to_string(event_id), profile::Color::Blue); range.Begin(); #endif diff --git a/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.cc b/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.cc index 8fd6e9a298..0e70138b8b 100644 --- a/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.cc +++ b/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.cc @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include "core/providers/shared_library/provider_api.h" #include "core/providers/cuda/cuda_fwd.h" #include "core/providers/cuda/cuda_pch.h" -#include "core/framework/kernel_registry.h" using namespace onnxruntime::common; diff --git a/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.h b/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.h index 1304d53e06..87e5a2afdd 100644 --- a/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.h +++ b/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.h @@ -3,13 +3,10 @@ #pragma once -#include "core/common/common.h" -#include "core/framework/kernel_registry.h" - namespace onnxruntime { namespace cuda { Status RegisterCudaTrainingKernels(KernelRegistry& kernel_registry); } // namespace cuda -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cc b/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cc index 5ff75fb54b..ecb4b524f4 100644 --- a/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cc +++ b/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cc @@ -4,13 +4,21 @@ #include "core/providers/cuda/math/softmax.h" #include "core/providers/cuda/reduction/reduction_functions.h" #include "core/providers/cuda/tensor/transpose.h" -#include "core/providers/cpu/controlflow/scan_utils.h" +#include "core/framework/ml_value.h" #include "orttraining/training_ops/cpu/loss/softmax_cross_entropy_loss.h" #include "orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.h" namespace onnxruntime { namespace cuda { +OrtValue AllocateTensorInMLValue(const MLDataType data_type, const TensorShape& shape, AllocatorPtr& allocator) { + auto new_tensor = Tensor::Create(data_type, shape, allocator); + + auto ml_tensor = DataTypeImpl::GetType(); + return OrtValue{new_tensor.release(), ml_tensor, + ml_tensor->GetDeleteFunc()}; +}; + #define REGISTER_KERNEL_VERSIONED_TYPED_TWO_TYPES(Class, T, Tin, domain, startver, endver) \ ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_EX( \ Class, \ @@ -18,7 +26,7 @@ namespace cuda { startver, endver, \ T, Tin, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("Tin", DataTypeImpl::GetTensorType()), \ Class); @@ -30,7 +38,7 @@ namespace cuda { version, \ T, Tin, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("Tin", DataTypeImpl::GetTensorType()), \ Class); @@ -76,7 +84,6 @@ Status SoftmaxCrossEntropyLoss::ComputeInternal(OpKernelContext* ctx) co } OrtValue transpose_output; - Tensor transpose_tensor; std::vector new_shape; std::vector permutations; AllocatorPtr alloc; @@ -86,7 +93,7 @@ Status SoftmaxCrossEntropyLoss::ComputeInternal(OpKernelContext* ctx) co if (logit_shape.NumDimensions() > 2) { ORT_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&alloc)); onnxruntime::contrib::GetPermutationAndShape(true, logit_shape, new_shape, permutations); - transpose_output = scan::detail::AllocateTensorInMLValue(logit.DataType(), new_shape, alloc); + transpose_output = AllocateTensorInMLValue(logit.DataType(), new_shape, alloc); ORT_RETURN_IF_ERROR(cuda::Transpose::DoTranspose(cuda::Transpose(info), permutations, logit, *transpose_output.GetMutable())); logit_data = (*transpose_output.GetMutable()).template Data(); } @@ -200,7 +207,7 @@ Status SoftmaxCrossEntropyLossGrad::ComputeInternal(OpKernelContext* ctx if (probability_shape.NumDimensions() > 2) { ORT_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&alloc)); onnxruntime::contrib::GetPermutationAndShape(true, probability_shape, new_shape, permutations); - transpose_output = scan::detail::AllocateTensorInMLValue(log_prob.DataType(), new_shape, alloc); + transpose_output = AllocateTensorInMLValue(log_prob.DataType(), new_shape, alloc); ORT_RETURN_IF_ERROR(cuda::Transpose::DoTranspose(cuda::Transpose(info), permutations, log_prob, *transpose_output.GetMutable())); log_prob_data = (*transpose_output.GetMutable()).template Data(); } diff --git a/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cu b/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cu index 02bd7ebf25..5e384909ee 100644 --- a/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cu +++ b/orttraining/orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.cu @@ -1,9 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/providers/cuda/cu_inc/common.cuh" -#include "orttraining/training_ops/cuda/loss/softmax_cross_entropy_loss_impl.h" #include "core/providers/cuda/cuda_common.h" +#include "core/providers/cuda/cu_inc/common.cuh" namespace onnxruntime { namespace cuda { diff --git a/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cc b/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cc index 441a39d21d..5d2a27be56 100644 --- a/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cc +++ b/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cc @@ -7,14 +7,14 @@ namespace onnxruntime { namespace cuda { -#define REGISTER_KERNEL_TYPED(Class, T, domain, version) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - Class, \ - domain, \ - version, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define REGISTER_KERNEL_TYPED(Class, T, domain, version) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Class, \ + domain, \ + version, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ Class); #define REGISTER_KERNEL_TYPED_TWO_TYPES(Class, T, Tin, domain, version) \ @@ -24,7 +24,7 @@ namespace cuda { version, \ T, Tin, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("Tin", DataTypeImpl::GetTensorType()), \ Class); diff --git a/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cu b/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cu index a9165fd9ed..9fd01720b5 100644 --- a/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cu +++ b/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.cu @@ -1,9 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/providers/cuda/cu_inc/common.cuh" -#include "softmaxcrossentropy_impl.h" #include "core/providers/cuda/cuda_common.h" +#include "core/providers/cuda/cu_inc/common.cuh" namespace onnxruntime { namespace cuda { diff --git a/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.h b/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.h index d41718d276..094f61bd01 100644 --- a/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.h +++ b/orttraining/orttraining/training_ops/cuda/loss/softmaxcrossentropy_impl.h @@ -108,6 +108,5 @@ class SparseSoftmaxCrossEntropyGrad final : public LossBase { Status ComputeInternal(OpKernelContext* context) const override; }; - } // namespace cuda } // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/cuda/math/div_grad.cc b/orttraining/orttraining/training_ops/cuda/math/div_grad.cc index 21b3dd9c8d..a9ad00c199 100644 --- a/orttraining/orttraining/training_ops/cuda/math/div_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/math/div_grad.cc @@ -9,14 +9,14 @@ using namespace onnxruntime::common; namespace onnxruntime { namespace cuda { -#define DIVGRAD_REGISTER_KERNEL_TYPED(T) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - DivGrad, \ - kMSDomain, \ - 1, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define DIVGRAD_REGISTER_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + DivGrad, \ + kMSDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ DivGrad); DIVGRAD_REGISTER_KERNEL_TYPED(MLFloat16) diff --git a/orttraining/orttraining/training_ops/cuda/math/isfinite.cc b/orttraining/orttraining/training_ops/cuda/math/isfinite.cc index 07ba929cc2..9550dcef30 100644 --- a/orttraining/orttraining/training_ops/cuda/math/isfinite.cc +++ b/orttraining/orttraining/training_ops/cuda/math/isfinite.cc @@ -15,7 +15,7 @@ namespace cuda { 1, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("T1", DataTypeImpl::GetTensorType()), \ IsFiniteOp); @@ -44,7 +44,7 @@ REGISTER_ISFINITE_KERNEL_TYPED(double) 1, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("V", DataTypeImpl::GetTensorType()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ IsAllFiniteOp); diff --git a/orttraining/orttraining/training_ops/cuda/math/mixed_precision_scale.cc b/orttraining/orttraining/training_ops/cuda/math/mixed_precision_scale.cc index 1ab0a00307..af9dd95ed1 100644 --- a/orttraining/orttraining/training_ops/cuda/math/mixed_precision_scale.cc +++ b/orttraining/orttraining/training_ops/cuda/math/mixed_precision_scale.cc @@ -9,25 +9,26 @@ namespace onnxruntime { namespace cuda { #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 -#define ALL_IEEE_FLOAT_TENSOR_TYPES {DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType()} +#define ALL_IEEE_FLOAT_TENSOR_TYPES \ + { DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType() } #else #define ALL_IEEE_FLOAT_TENSOR_TYPES DataTypeImpl::AllIEEEFloatTensorTypes() #endif -#define REGISTER_MIXEDPRECISIONSCALE_KERNEL_TYPED(SrcT) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - MixedPrecisionScale, \ - kMSDomain, \ - 1, \ - SrcT, \ - kCudaExecutionProvider, \ - KernelDefBuilder() \ - .TypeConstraint("SrcT", DataTypeImpl::GetTensorType()) \ - .TypeConstraint("ScaleT", DataTypeImpl::GetTensorType()) \ - .TypeConstraint("DstT", ALL_IEEE_FLOAT_TENSOR_TYPES), \ +#define REGISTER_MIXEDPRECISIONSCALE_KERNEL_TYPED(SrcT) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + MixedPrecisionScale, \ + kMSDomain, \ + 1, \ + SrcT, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()) \ + .TypeConstraint("SrcT", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("ScaleT", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("DstT", ALL_IEEE_FLOAT_TENSOR_TYPES), \ MixedPrecisionScale); Status BytesPerElement(ONNX_NAMESPACE::TensorProto_DataType to, size_t& bytes_per_elem) { diff --git a/orttraining/orttraining/training_ops/cuda/math/scale.cc b/orttraining/orttraining/training_ops/cuda/math/scale.cc index 2345213d6b..8b2ebb45a3 100644 --- a/orttraining/orttraining/training_ops/cuda/math/scale.cc +++ b/orttraining/orttraining/training_ops/cuda/math/scale.cc @@ -8,21 +8,21 @@ using namespace onnxruntime::common; namespace onnxruntime { namespace cuda { -#define REGISTER_SCALE_KERNEL_TYPED(T) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - Scale, \ - kMSDomain, \ - 1, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder() \ - .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ - .TypeConstraint("ScaleT", {DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType()}) \ - .InputMemoryType(1), \ +#define REGISTER_SCALE_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Scale, \ + kMSDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()) \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("ScaleT", {DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType()}) \ + .InputMemoryType(OrtMemTypeCPUInput, 1), \ Scale); template diff --git a/orttraining/orttraining/training_ops/cuda/math/softmax_grad.cc b/orttraining/orttraining/training_ops/cuda/math/softmax_grad.cc index 74976d0639..9e00f9ede1 100644 --- a/orttraining/orttraining/training_ops/cuda/math/softmax_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/math/softmax_grad.cc @@ -63,52 +63,52 @@ Status SoftMaxGradComputeHelper( #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 // cudnnSoftmaxForward/Backward doesn't support BFloat16. -#define SPECIALIZED_SOFTMAXGRAD_HELPER_IMPL_BFloat16(is_log_softmax) \ - template <> \ - Status SoftMaxGradComputeHelper( \ - cudaStream_t stream, \ - const BFloat16* dY, \ - const TensorShape& input_shape, \ - const BFloat16* Y, \ - BFloat16* dX, \ - cudnnHandle_t, \ - int64_t axis) { \ - typedef typename ToCudaType::MappedType CudaT; \ - const int64_t normalized_axis = HandleNegativeAxis(axis, input_shape.NumDimensions()); \ - int64_t N = input_shape.SizeToDimension(normalized_axis); \ - int64_t D = input_shape.SizeFromDimension(normalized_axis); \ - auto dY_data = reinterpret_cast(dY); \ - auto Y_data = reinterpret_cast(Y); \ - auto dX_data = reinterpret_cast(dX); \ - dispatch_softmax_backward, is_log_softmax>( \ - stream, dX_data, dY_data, Y_data, gsl::narrow_cast(D), gsl::narrow_cast(D), gsl::narrow_cast(N)); \ - return Status::OK(); \ +#define SPECIALIZED_SOFTMAXGRAD_HELPER_IMPL_BFloat16(is_log_softmax) \ + template <> \ + Status SoftMaxGradComputeHelper( \ + cudaStream_t stream, \ + const BFloat16* dY, \ + const TensorShape& input_shape, \ + const BFloat16* Y, \ + BFloat16* dX, \ + cudnnHandle_t, \ + int64_t axis) { \ + typedef typename ToCudaType::MappedType CudaT; \ + const int64_t normalized_axis = HandleNegativeAxis(axis, input_shape.NumDimensions()); \ + int64_t N = input_shape.SizeToDimension(normalized_axis); \ + int64_t D = input_shape.SizeFromDimension(normalized_axis); \ + auto dY_data = reinterpret_cast(dY); \ + auto Y_data = reinterpret_cast(Y); \ + auto dX_data = reinterpret_cast(dX); \ + dispatch_softmax_backward, is_log_softmax>( \ + stream, dX_data, dY_data, Y_data, gsl::narrow_cast(D), gsl::narrow_cast(D), gsl::narrow_cast(N)); \ + return Status::OK(); \ } SPECIALIZED_SOFTMAXGRAD_HELPER_IMPL_BFloat16(true) -SPECIALIZED_SOFTMAXGRAD_HELPER_IMPL_BFloat16(false) + SPECIALIZED_SOFTMAXGRAD_HELPER_IMPL_BFloat16(false) #endif -#define REGISTER_GRADIENT_KERNEL_TYPED(T) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - SoftmaxGrad, \ - kMSDomain, \ - 1, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - SoftmaxGrad); \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - LogSoftmaxGrad, \ - kMSDomain, \ - 1, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define REGISTER_GRADIENT_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + SoftmaxGrad, \ + kMSDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + SoftmaxGrad); \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + LogSoftmaxGrad, \ + kMSDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ SoftmaxGrad); -template -Status SoftmaxGrad::ComputeInternal(OpKernelContext* ctx) const { + template + Status SoftmaxGrad::ComputeInternal(OpKernelContext* ctx) const { const Tensor* dY = ctx->Input(0); const TensorShape& input_shape{dY->Shape()}; const Tensor* Y = ctx->Input(1); diff --git a/orttraining/orttraining/training_ops/cuda/nn/batch_norm_grad.cc b/orttraining/orttraining/training_ops/cuda/nn/batch_norm_grad.cc index 7267c46013..4f59eba9ee 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/batch_norm_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/nn/batch_norm_grad.cc @@ -10,14 +10,14 @@ using namespace std; namespace onnxruntime { namespace cuda { -#define REGISTER_GRADIENT_KERNEL_TYPED(T) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - BatchNormalizationGrad, \ - kMSDomain, \ - 1, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define REGISTER_GRADIENT_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + BatchNormalizationGrad, \ + kMSDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ BatchNormalizationGrad); template diff --git a/orttraining/orttraining/training_ops/cuda/nn/dropout.cc b/orttraining/orttraining/training_ops/cuda/nn/dropout.cc index 8d970bdd90..9f210018a5 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/dropout.cc +++ b/orttraining/orttraining/training_ops/cuda/nn/dropout.cc @@ -10,27 +10,28 @@ namespace onnxruntime { namespace cuda { #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 -#define ALL_IEEE_FLOAT_TENSOR_TYPES {DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType()} +#define ALL_IEEE_FLOAT_TENSOR_TYPES \ + { DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType() } #define ALL_IEEE_FLOAT_DATA_TYPES float, MLFloat16, double, BFloat16 #else #define ALL_IEEE_FLOAT_TENSOR_TYPES DataTypeImpl::AllIEEEFloatTensorTypes() #define ALL_IEEE_FLOAT_DATA_TYPES float, MLFloat16, double #endif -#define REGISTER_GRADIENT_KERNEL(OpName) \ - ONNX_OPERATOR_KERNEL_EX( \ - OpName, \ - kMSDomain, \ - 1, \ - kCudaExecutionProvider, \ - KernelDefBuilder() \ - .TypeConstraint("T", ALL_IEEE_FLOAT_TENSOR_TYPES) \ - .TypeConstraint("T1", ALL_IEEE_FLOAT_TENSOR_TYPES) \ - .TypeConstraint("T2", DataTypeImpl::GetTensorType()) \ - .InputMemoryType(2), \ +#define REGISTER_GRADIENT_KERNEL(OpName) \ + ONNX_OPERATOR_KERNEL_EX( \ + OpName, \ + kMSDomain, \ + 1, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()) \ + .TypeConstraint("T", ALL_IEEE_FLOAT_TENSOR_TYPES) \ + .TypeConstraint("T1", ALL_IEEE_FLOAT_TENSOR_TYPES) \ + .TypeConstraint("T2", DataTypeImpl::GetTensorType()) \ + .InputMemoryType(OrtMemTypeCPUInput, 2), \ DropoutGrad); REGISTER_GRADIENT_KERNEL(DropoutGrad) @@ -90,12 +91,12 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("T", ALL_IEEE_FLOAT_TENSOR_TYPES) .TypeConstraint("T1", ALL_IEEE_FLOAT_TENSOR_TYPES) .TypeConstraint("T2", DataTypeImpl::GetTensorType()) - .InputMemoryType(3) - .InputMemoryType(4), + .InputMemoryType(OrtMemTypeCPUInput, 3) + .InputMemoryType(OrtMemTypeCPUInput, 4), BiasDropout); template diff --git a/orttraining/orttraining/training_ops/cuda/nn/layer_norm.cc b/orttraining/orttraining/training_ops/cuda/nn/layer_norm.cc index 64262a1ed1..5b0bcb684f 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/layer_norm.cc +++ b/orttraining/orttraining/training_ops/cuda/nn/layer_norm.cc @@ -16,7 +16,7 @@ namespace cuda { 1, \ T##_##U, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("U", DataTypeImpl::GetTensorType()), \ LayerNormGrad); \ @@ -26,7 +26,7 @@ namespace cuda { 1, \ T##_##U, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("U", DataTypeImpl::GetTensorType()), \ InvertibleLayerNormGrad); \ @@ -36,11 +36,11 @@ namespace cuda { 1, \ T##_##U, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("U", DataTypeImpl::GetTensorType()), \ LayerNormGrad); - + REGISTER_GRADIENT_KERNEL_TYPED(float, float) REGISTER_GRADIENT_KERNEL_TYPED(double, double) REGISTER_GRADIENT_KERNEL_TYPED(MLFloat16, float) @@ -71,7 +71,7 @@ Status LayerNormGrad::ComputeInternal(OpKernelContext* p_op_ke auto Y_grad_data = reinterpret_cast(Y_grad->template Data()); auto X_data = reinterpret_cast(X->template Data()); auto scale_data = reinterpret_cast(scale->template Data()); - auto mean_data = simplified ? nullptr: reinterpret_cast(mean->template Data()); + auto mean_data = simplified ? nullptr : reinterpret_cast(mean->template Data()); auto inv_std_var_data = reinterpret_cast(inv_std_var->template Data()); const TensorShape& x_shape = X->Shape(); @@ -96,9 +96,9 @@ Status LayerNormGrad::ComputeInternal(OpKernelContext* p_op_ke auto part_grad_beta = GetScratchBuffer(part_size * n2); HostLayerNormGradient(GetDeviceProp(), Stream(), Y_grad_data, X_data, reinterpret_cast(NULL), - scale_data, reinterpret_cast(NULL), mean_data, inv_std_var_data, n1, n2, - X_grad_data, scale_grad_data, bias_grad_data, - part_grad_gamma.get(), part_grad_beta.get(), part_size); + scale_data, reinterpret_cast(NULL), mean_data, inv_std_var_data, n1, n2, + X_grad_data, scale_grad_data, bias_grad_data, + part_grad_gamma.get(), part_grad_beta.get(), part_size); return Status::OK(); } @@ -145,9 +145,9 @@ Status InvertibleLayerNormGrad::ComputeInternal(OpKernelContext* p_op_kern auto part_grad_beta = GetScratchBuffer(part_size * n2); HostLayerNormGradient(GetDeviceProp(), Stream(), Y_grad_data, reinterpret_cast(NULL), Y_data, - scale_data, bias_data, reinterpret_cast(NULL), inv_std_var_data, n1, n2, - X_grad_data, scale_grad_data, bias_grad_data, - part_grad_gamma.get(), part_grad_beta.get(), part_size); + scale_data, bias_data, reinterpret_cast(NULL), inv_std_var_data, n1, n2, + X_grad_data, scale_grad_data, bias_grad_data, + part_grad_gamma.get(), part_grad_beta.get(), part_size); return Status::OK(); } diff --git a/orttraining/orttraining/training_ops/cuda/optimizer/adam.cc b/orttraining/orttraining/training_ops/cuda/optimizer/adam.cc index 03fdcc3685..5dba709815 100644 --- a/orttraining/orttraining/training_ops/cuda/optimizer/adam.cc +++ b/orttraining/orttraining/training_ops/cuda/optimizer/adam.cc @@ -18,16 +18,16 @@ namespace cuda { 1, \ T1##_##T2##_##T3##_##T4##_##T_GRAD##_##T_GRAD_NORM##_##T_MIXED_PRECISION_FP, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .Alias(1, 0) /* Update step count in-place */ \ .Alias(2, 3) /* Update weights in-place */ \ .Alias(3, 4) /* Update gradients in-place */ \ .Alias(4, 1) /* Update moment-1 in-place */ \ .Alias(5, 2) /* Update moment-2 in-place */ \ .Alias(6, 5) /* Update mixed_precision weights in-place */ \ - .InputMemoryType(1) /* Keep step count in CPU */ \ - .InputMemoryType(9) /* Keep do_update in CPU */ \ - .OutputMemoryType(0) /* Keep step count in CPU */ \ + .InputMemoryType(OrtMemTypeCPUInput, 1) /* Keep step count in CPU */ \ + .InputMemoryType(OrtMemTypeCPUInput, 9) /* Keep do_update in CPU */ \ + .OutputMemoryType(OrtMemTypeCPUOutput, 0) /* Keep step count in CPU */ \ .TypeConstraint("T1", DataTypeImpl::GetTensorType()) \ .TypeConstraint("T2", DataTypeImpl::GetTensorType()) \ .TypeConstraint("T3", DataTypeImpl::GetTensorType()) \ diff --git a/orttraining/orttraining/training_ops/cuda/optimizer/gradient_control.cc b/orttraining/orttraining/training_ops/cuda/optimizer/gradient_control.cc index a8c4de6f5f..ab116b20d7 100644 --- a/orttraining/orttraining/training_ops/cuda/optimizer/gradient_control.cc +++ b/orttraining/orttraining/training_ops/cuda/optimizer/gradient_control.cc @@ -17,9 +17,9 @@ namespace cuda { 1, \ T##_##T_GRAD, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .Alias(0, 0) /* Accumulate tensors in-place */ \ - .InputMemoryType(2) /* Keep do_update in CPU */ \ + .InputMemoryType(OrtMemTypeCPUInput, 2) /* Keep do_update in CPU */ \ .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ .TypeConstraint("T_GRAD", DataTypeImpl::GetTensorType()), \ InPlaceAccumulator); @@ -54,7 +54,7 @@ Status ZeroGradient::ComputeInternal(OpKernelContext* ctx) const { 1, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .Alias(0, 0) /* Zero out gradients in-place */ \ .TypeConstraint("T1", DataTypeImpl::GetTensorType()) \ .TypeConstraint("T2", DataTypeImpl::AllTensorTypes()), \ diff --git a/orttraining/orttraining/training_ops/cuda/optimizer/lamb.cc b/orttraining/orttraining/training_ops/cuda/optimizer/lamb.cc index ded60d2373..2ed5d163cd 100644 --- a/orttraining/orttraining/training_ops/cuda/optimizer/lamb.cc +++ b/orttraining/orttraining/training_ops/cuda/optimizer/lamb.cc @@ -53,11 +53,11 @@ std::vector> GenerateLambExtraAliasMapping() { 1, \ T1##_##T2##_##T3##_##T4##_##T_GRAD_NORM##_##T_MIXED_PRECISION_FP, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ + (*KernelDefBuilder::Create()) \ .Alias(GenerateLambExtraAliasMapping()) \ - .InputMemoryType(0) /* Keep do_update in CPU */ \ - .InputMemoryType(4) /* Keep iteration_count in CPU */ \ - .OutputMemoryType(0) /* Keep iteration_count in CPU */ \ + .InputMemoryType(OrtMemTypeCPUInput, 0) /* Keep do_update in CPU */ \ + .InputMemoryType(OrtMemTypeCPUInput, 4) /* Keep iteration_count in CPU */ \ + .OutputMemoryType(OrtMemTypeCPUOutput, 0) /* Keep iteration_count in CPU */ \ .TypeConstraint("T1", DataTypeImpl::GetTensorType()) \ .TypeConstraint("T2", DataTypeImpl::GetTensorType()) \ .TypeConstraint("T3", DataTypeImpl::GetTensorType()) \ diff --git a/orttraining/orttraining/training_ops/cuda/optimizer/sg.cc b/orttraining/orttraining/training_ops/cuda/optimizer/sg.cc index 048f1da8e7..e2133af960 100644 --- a/orttraining/orttraining/training_ops/cuda/optimizer/sg.cc +++ b/orttraining/orttraining/training_ops/cuda/optimizer/sg.cc @@ -14,7 +14,7 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .Alias(1, 0) // Update weights in-place .Alias(2, 1) // Update gradients in-place .TypeConstraint("T", DataTypeImpl::GetTensorType()), diff --git a/orttraining/orttraining/training_ops/cuda/reduction/all.cc b/orttraining/orttraining/training_ops/cuda/reduction/all.cc index 9e1c282667..746c001969 100644 --- a/orttraining/orttraining/training_ops/cuda/reduction/all.cc +++ b/orttraining/orttraining/training_ops/cuda/reduction/all.cc @@ -6,14 +6,14 @@ namespace onnxruntime { namespace cuda { -#define REGISTER_ALL_KERNEL_TYPED(T) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - All, \ - kMSDomain, \ - 1, \ - T, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), \ +#define REGISTER_ALL_KERNEL_TYPED(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + All, \ + kMSDomain, \ + 1, \ + T, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ All); template diff --git a/orttraining/orttraining/training_ops/cuda/reduction/reduction_all.cc b/orttraining/orttraining/training_ops/cuda/reduction/reduction_all.cc index 654b915ffe..71d04393e6 100644 --- a/orttraining/orttraining/training_ops/cuda/reduction/reduction_all.cc +++ b/orttraining/orttraining/training_ops/cuda/reduction/reduction_all.cc @@ -3,21 +3,20 @@ #include "orttraining/training_ops/cuda/reduction/reduction_all.h" -#include "core/framework/op_kernel_context_internal.h" #include "core/providers/cuda/reduction/reduction_functions.h" #include "core/providers/cuda/shared_inc/accumulation_type.h" namespace onnxruntime { namespace cuda { -#define REGISTER_REDUCE_ALL_KERNEL_TYPED(Name, TIn, TOut) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - Name, \ - kMSDomain, \ - 1, \ - TIn##_##TOut, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("TIn", DataTypeImpl::GetTensorType()).TypeConstraint("TOut", DataTypeImpl::GetTensorType()), \ +#define REGISTER_REDUCE_ALL_KERNEL_TYPED(Name, TIn, TOut) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + Name, \ + kMSDomain, \ + 1, \ + TIn##_##TOut, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("TIn", DataTypeImpl::GetTensorType()).TypeConstraint("TOut", DataTypeImpl::GetTensorType()), \ Name); template @@ -55,7 +54,7 @@ Status ReduceAllL2::ComputeInternal(OpKernelContext* ctx) const { // Check if all values are finite and write true to deviceOutput. // Otherwise, false will be written. launch_multi_tensor_functor<1, TFunctor>(Stream(), - 2048 * 32, tensor_sizes, grouped_tensor_pointers, functor, p_output); + 2048 * 32, tensor_sizes, grouped_tensor_pointers, functor, p_output); // *p_output is the squared sum of all elements. // Let's take a sqrt to get the actual L2-norm. @@ -111,4 +110,4 @@ REGISTER_REDUCE_ALL_KERNEL_TYPED(ReduceAllL2, BFloat16, BFloat16) #endif } // namespace cuda -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/cuda/reduction/reduction_ops.cc b/orttraining/orttraining/training_ops/cuda/reduction/reduction_ops.cc index adfaeff30f..a01152140d 100644 --- a/orttraining/orttraining/training_ops/cuda/reduction/reduction_ops.cc +++ b/orttraining/orttraining/training_ops/cuda/reduction/reduction_ops.cc @@ -8,7 +8,6 @@ #include "core/providers/cuda/math/binary_elementwise_ops_impl.h" #include "core/providers/cuda/math/binary_elementwise_ops.h" #include "core/providers/cpu/tensor/utils.h" -#include "core/framework/op_kernel_context_internal.h" using namespace onnxruntime::common; namespace onnxruntime { @@ -21,8 +20,8 @@ namespace cuda { 1, \ T, \ kCudaExecutionProvider, \ - KernelDefBuilder() \ - .InputMemoryType(1) \ + (*KernelDefBuilder::Create()) \ + .InputMemoryType(OrtMemTypeCPUInput, 1) \ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ name); diff --git a/orttraining/orttraining/training_ops/cuda/tensor/concat.cc b/orttraining/orttraining/training_ops/cuda/tensor/concat.cc index 0404c1fa4e..1626ebec23 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/concat.cc +++ b/orttraining/orttraining/training_ops/cuda/tensor/concat.cc @@ -3,14 +3,13 @@ #include "orttraining/training_ops/cuda/tensor/concat.h" #include "core/providers/cuda/tensor/concat_impl.h" - namespace onnxruntime { namespace cuda { ONNX_OPERATOR_KERNEL_EX(ConcatTraining, kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), ConcatTraining); diff --git a/orttraining/orttraining/training_ops/cuda/tensor/concat.h b/orttraining/orttraining/training_ops/cuda/tensor/concat.h index 3d401433b6..702c363b4c 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/concat.h +++ b/orttraining/orttraining/training_ops/cuda/tensor/concat.h @@ -1,9 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #pragma once -#include "core/common/common.h" #include "core/providers/cuda/cuda_kernel.h" -#include "core/providers/cpu/tensor/concat.h" +#include "core/providers/cpu/tensor/concatbase.h" namespace onnxruntime { namespace cuda { diff --git a/orttraining/orttraining/training_ops/cuda/tensor/gather_elements_grad.cc b/orttraining/orttraining/training_ops/cuda/tensor/gather_elements_grad.cc index dae18fc35f..267efda5f0 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/gather_elements_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/tensor/gather_elements_grad.cc @@ -14,8 +14,8 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(1) // 'GatherElements' data shape needs to be on CPU + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 1) // 'GatherElements' data shape needs to be on CPU .TypeConstraint("T", DataTypeImpl::AllIEEEFloatTensorTypes()) .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("Tind", std::vector{DataTypeImpl::GetTensorType(), diff --git a/orttraining/orttraining/training_ops/cuda/tensor/gather_grad.cc b/orttraining/orttraining/training_ops/cuda/tensor/gather_grad.cc index 9a89cda8c1..865443d207 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/gather_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/tensor/gather_grad.cc @@ -11,10 +11,11 @@ namespace onnxruntime { namespace cuda { #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 -#define ALL_IEEE_FLOAT_TENSOR_TYPES {DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType()} +#define ALL_IEEE_FLOAT_TENSOR_TYPES \ + { DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType() } #else #define ALL_IEEE_FLOAT_TENSOR_TYPES DataTypeImpl::AllIEEEFloatTensorTypes() #endif @@ -24,8 +25,8 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(0) + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 0) .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("T", ALL_IEEE_FLOAT_TENSOR_TYPES) .TypeConstraint("Tind", std::vector{ diff --git a/orttraining/orttraining/training_ops/cuda/tensor/gather_nd_grad.cc b/orttraining/orttraining/training_ops/cuda/tensor/gather_nd_grad.cc index 6bfc0eb68f..285c7e66f1 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/gather_nd_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/tensor/gather_nd_grad.cc @@ -9,27 +9,25 @@ namespace onnxruntime { namespace cuda { #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 -#define ALL_IEEE_FLOAT_TENSOR_TYPES {DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType()} +#define ALL_IEEE_FLOAT_TENSOR_TYPES \ + { DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType() } #define ALL_IEEE_FLOAT_DATA_TYPES float, MLFloat16, double, BFloat16 #else #define ALL_IEEE_FLOAT_TENSOR_TYPES DataTypeImpl::AllIEEEFloatTensorTypes() #define ALL_IEEE_FLOAT_DATA_TYPES float, MLFloat16, double #endif -#define REGISTER_KERNEL_TYPED_GATHER_ND_GRAD(TIndex) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - GatherNDGrad, \ - kMSDomain, \ - 1, \ - TIndex, \ - kCudaExecutionProvider, \ - KernelDefBuilder().TypeConstraint("T", ALL_IEEE_FLOAT_TENSOR_TYPES) \ - .TypeConstraint("Tind", DataTypeImpl::GetTensorType()) \ - .TypeConstraint("T1", DataTypeImpl::GetTensorType()) \ - .InputMemoryType(0), \ +#define REGISTER_KERNEL_TYPED_GATHER_ND_GRAD(TIndex) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + GatherNDGrad, \ + kMSDomain, \ + 1, \ + TIndex, \ + kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", ALL_IEEE_FLOAT_TENSOR_TYPES).TypeConstraint("Tind", DataTypeImpl::GetTensorType()).TypeConstraint("T1", DataTypeImpl::GetTensorType()).InputMemoryType(OrtMemTypeCPUInput, 0), \ GatherNDGrad); REGISTER_KERNEL_TYPED_GATHER_ND_GRAD(int64_t) diff --git a/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.cc b/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.cc index 7004cbfb70..f76d968687 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/tensor/slice_grad.cc @@ -13,12 +13,12 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(1) - .InputMemoryType(2) - .InputMemoryType(3) - .InputMemoryType(4) - .InputMemoryType(5) + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 1) + .InputMemoryType(OrtMemTypeCPUInput, 2) + .InputMemoryType(OrtMemTypeCPUInput, 3) + .InputMemoryType(OrtMemTypeCPUInput, 4) + .InputMemoryType(OrtMemTypeCPUInput, 5) .TypeConstraint("I", DataTypeImpl::GetTensorType()) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()) .TypeConstraint("Tind", std::vector{ diff --git a/orttraining/orttraining/training_ops/cuda/tensor/split.cc b/orttraining/orttraining/training_ops/cuda/tensor/split.cc index 37a18c603f..956c423255 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/split.cc +++ b/orttraining/orttraining/training_ops/cuda/tensor/split.cc @@ -12,8 +12,8 @@ ONNX_OPERATOR_KERNEL_EX(SplitTraining, kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() - .InputMemoryType(1) + (*KernelDefBuilder::Create()) + .InputMemoryType(OrtMemTypeCPUInput, 1) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()), SplitTraining); diff --git a/orttraining/orttraining/training_ops/cuda/tensor/view.cc b/orttraining/orttraining/training_ops/cuda/tensor/view.cc index 6d5d9da000..47de07ae1f 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/view.cc +++ b/orttraining/orttraining/training_ops/cuda/tensor/view.cc @@ -32,10 +32,10 @@ ONNX_OPERATOR_KERNEL_EX( kMSDomain, 1, kCudaExecutionProvider, - KernelDefBuilder() + (*KernelDefBuilder::Create()) .TypeConstraint("T", DataTypeImpl::AllFixedSizeTensorTypes()) .TypeConstraint("Shape", DataTypeImpl::GetTensorType()) - .InputMemoryType(GenerateInputMemoryType()) // all shape inputs are in CPU + .InputMemoryType(OrtMemTypeCPUInput, GenerateInputMemoryType()) // all shape inputs are in CPU .Alias(GenerateAliasMapping()), // all output tensors are sharing the same bffer as input[0], // execept that the byte_offset is different View);