From 329fd03bb4458e77a08288457479a284f783bd3c Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Thu, 1 Apr 2021 19:32:34 +1000 Subject: [PATCH] Add int32_t as required type to some operators (#7192) * Updates to some operators to always support int32 and int64 based on testing of Android package build config with a minimal build. If an operator can be used for shape manipulation (int64) it is frequently used for indices manipulation (int32), so we enable both types for that set of ops. - e.g. BERT models take indices as input - Scatter/Gather ops utilize indices Misc. fix to python bindings to exclude call that fails in a minimal build. --- .../core/providers/cpu/generator/range.cc | 4 +-- .../providers/cpu/math/element_wise_ops.cc | 4 +-- .../core/providers/cpu/tensor/cast_op.cc | 8 ++++- .../core/providers/cpu/tensor/gather.cc | 4 +-- onnxruntime/core/providers/cpu/tensor/pad.cc | 3 ++ .../core/providers/cpu/tensor/slice.cc | 4 +-- .../core/providers/cpu/tensor/split.cc | 12 ++++---- .../python/onnxruntime_pybind_state.cc | 4 +++ .../operator_type_usage_processors.py | 29 ++++++++++++------- 9 files changed, 47 insertions(+), 25 deletions(-) diff --git a/onnxruntime/core/providers/cpu/generator/range.cc b/onnxruntime/core/providers/cpu/generator/range.cc index 1e77560271..05ef9f508a 100644 --- a/onnxruntime/core/providers/cpu/generator/range.cc +++ b/onnxruntime/core/providers/cpu/generator/range.cc @@ -15,8 +15,8 @@ ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES_ALL_OPSETS( float, double, int16_t, int32_t, int64_t); ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES_ALL_OPSETS( kCpuExecutionProvider, kOnnxDomain, Range, Input, 0, - int64_t); -} + int32_t, int64_t); +} // namespace op_kernel_type_control using RangeDataTypes = ORT_OP_KERNEL_ARG_DEFAULT_TYPE_LIST_ALL_OPSETS( kCpuExecutionProvider, kOnnxDomain, Range, Input, 0); diff --git a/onnxruntime/core/providers/cpu/math/element_wise_ops.cc b/onnxruntime/core/providers/cpu/math/element_wise_ops.cc index b95b75cd1f..30d6c7c011 100644 --- a/onnxruntime/core/providers/cpu/math/element_wise_ops.cc +++ b/onnxruntime/core/providers/cpu/math/element_wise_ops.cc @@ -20,14 +20,14 @@ ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES(kCpuExecutionProvider, kOnnxDomain, Max, ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES(kCpuExecutionProvider, kOnnxDomain, Max, 12, Input, 0, float, double, MLFloat16, int32_t, uint32_t, int64_t, uint64_t); ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES(kCpuExecutionProvider, kOnnxDomain, Max, 12, Input, 0, - int64_t); + int32_t, int64_t); // Min ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES(kCpuExecutionProvider, kOnnxDomain, Min, 8, Input, 0, float, double); ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES(kCpuExecutionProvider, kOnnxDomain, Min, 12, Input, 0, float, double, MLFloat16, int32_t, uint32_t, int64_t, uint64_t); ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES(kCpuExecutionProvider, kOnnxDomain, Min, 12, Input, 0, - int64_t); + int32_t, int64_t); // Mod ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES_ALL_OPSETS(kCpuExecutionProvider, kOnnxDomain, Mod, Input, 0, diff --git a/onnxruntime/core/providers/cpu/tensor/cast_op.cc b/onnxruntime/core/providers/cpu/tensor/cast_op.cc index 2130ad43ef..1c33536bd9 100644 --- a/onnxruntime/core/providers/cpu/tensor/cast_op.cc +++ b/onnxruntime/core/providers/cpu/tensor/cast_op.cc @@ -33,13 +33,19 @@ namespace op_kernel_type_control { ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPE_LIST_ALL_OPSETS( kCpuExecutionProvider, kOnnxDomain, Cast, Input, 0, element_type_lists::All); + ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES_ALL_OPSETS( kCpuExecutionProvider, kOnnxDomain, Cast, Input, 0, - int64_t); + bool, int32_t, int64_t); + ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPE_LIST_ALL_OPSETS( kCpuExecutionProvider, kOnnxDomain, Cast, Output, 0, element_type_lists::All); + +ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES_ALL_OPSETS( + kCpuExecutionProvider, kOnnxDomain, Cast, Output, 0, + bool, int32_t, int64_t); } // namespace op_kernel_type_control namespace { diff --git a/onnxruntime/core/providers/cpu/tensor/gather.cc b/onnxruntime/core/providers/cpu/tensor/gather.cc index 7ed3cf2f22..fb2cf909c3 100644 --- a/onnxruntime/core/providers/cpu/tensor/gather.cc +++ b/onnxruntime/core/providers/cpu/tensor/gather.cc @@ -14,8 +14,8 @@ namespace op_kernel_type_control { ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES_ALL_OPSETS( kCpuExecutionProvider, kOnnxDomain, Gather, Input, 1, int32_t, int64_t); ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES_ALL_OPSETS( - kCpuExecutionProvider, kOnnxDomain, Gather, Input, 1, int64_t); -} + kCpuExecutionProvider, kOnnxDomain, Gather, Input, 1, int32_t, int64_t); +} // namespace op_kernel_type_control namespace { using IndexTypes = ORT_OP_KERNEL_ARG_DEFAULT_TYPE_LIST_ALL_OPSETS(kCpuExecutionProvider, kOnnxDomain, diff --git a/onnxruntime/core/providers/cpu/tensor/pad.cc b/onnxruntime/core/providers/cpu/tensor/pad.cc index 3e9ba7dbdb..20c72cb590 100644 --- a/onnxruntime/core/providers/cpu/tensor/pad.cc +++ b/onnxruntime/core/providers/cpu/tensor/pad.cc @@ -53,6 +53,9 @@ ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES( uint64_t, int8_t, uint8_t); + +ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES( + kCpuExecutionProvider, kOnnxDomain, Pad, 11, Input, 0, int32_t, int64_t); } // namespace op_kernel_type_control using Pad2Types = ORT_OP_KERNEL_ARG_DEFAULT_TYPE_LIST( diff --git a/onnxruntime/core/providers/cpu/tensor/slice.cc b/onnxruntime/core/providers/cpu/tensor/slice.cc index 490edfdada..0557598732 100644 --- a/onnxruntime/core/providers/cpu/tensor/slice.cc +++ b/onnxruntime/core/providers/cpu/tensor/slice.cc @@ -22,12 +22,12 @@ ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPE_LIST_ALL_OPSETS( kCpuExecutionProvider, kOnnxDomain, Slice, Input, 0, element_type_lists::All); ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES_ALL_OPSETS( - kCpuExecutionProvider, kOnnxDomain, Slice, Input, 0, int64_t); + kCpuExecutionProvider, kOnnxDomain, Slice, Input, 0, int32_t, int64_t); ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES_ALL_OPSETS( kCpuExecutionProvider, kOnnxDomain, Slice, Input, 1, int32_t, int64_t); ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES_ALL_OPSETS( - kCpuExecutionProvider, kOnnxDomain, Slice, Input, 1, int64_t); + kCpuExecutionProvider, kOnnxDomain, Slice, Input, 1, int32_t, int64_t); } // namespace op_kernel_type_control namespace { diff --git a/onnxruntime/core/providers/cpu/tensor/split.cc b/onnxruntime/core/providers/cpu/tensor/split.cc index d895832626..bfa21f4ab2 100644 --- a/onnxruntime/core/providers/cpu/tensor/split.cc +++ b/onnxruntime/core/providers/cpu/tensor/split.cc @@ -17,7 +17,10 @@ namespace op_kernel_type_control { ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES_ALL_OPSETS( kCpuExecutionProvider, kOnnxDomain, Split, Input, 0, float, int32_t, int64_t, uint8_t, std::string); -} +ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES_ALL_OPSETS( + kCpuExecutionProvider, kOnnxDomain, Split, Input, 0, + int32_t, int64_t); +} // namespace op_kernel_type_control using SplitDataTypes = ORT_OP_KERNEL_ARG_DEFAULT_TYPE_LIST_ALL_OPSETS( kCpuExecutionProvider, kOnnxDomain, Split, Input, 0); @@ -77,9 +80,9 @@ Status SplitBase::PrepareForCompute(const TensorShape& input_shape, int num_outp split_sizes = std::vector(static_cast(num_outputs), split_dim_size / num_outputs); } else { int64_t split_size_sum = split_size_sum_; - if (split_size_sum == -1){ + if (split_size_sum == -1) { split_size_sum = std::accumulate(split_sizes.cbegin(), split_sizes.cend(), 0LL); - } + } if (split_sizes.size() != static_cast(num_outputs) || split_size_sum != split_dim_size) return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Cannot split using values in 'split' attribute. Axis=", axis_, @@ -146,8 +149,7 @@ Status Split::ComputeImpl(OpKernelContext& context, const Tensor& input) const { auto nDims = static_cast(split_tensor->Shape()[0]); const auto* data = split_tensor->template Data(); split_sizes.assign(data, data + nDims); - } - else{ + } else { split_sizes.assign(split_sizes_.begin(), split_sizes_.end()); } ORT_RETURN_IF_ERROR(PrepareForCompute(input_shape, diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 0ed51d6346..25bfc7611a 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -744,9 +744,13 @@ void InitializeSession(InferenceSession* sess, const std::vector& p RegisterExecutionProviders(sess, provider_types, provider_options_map); } +#if !defined(ORT_MINIMAL_BUILD) if (!disabled_optimizer_names.empty()) { OrtPybindThrowIfError(sess->FilterEnabledOptimizers(disabled_optimizer_names)); } +#else + ORT_UNUSED_PARAMETER(disabled_optimizer_names); +#endif OrtPybindThrowIfError(sess->Initialize()); } diff --git a/tools/python/util/ort_format_model/operator_type_usage_processors.py b/tools/python/util/ort_format_model/operator_type_usage_processors.py index 0ff00653ea..b6bfbbf54e 100644 --- a/tools/python/util/ort_format_model/operator_type_usage_processors.py +++ b/tools/python/util/ort_format_model/operator_type_usage_processors.py @@ -325,14 +325,13 @@ def _create_operator_type_usage_processors(): default_processor_onnx_ops = ['Abs', 'ArgMax', 'ArgMin', 'AveragePool', 'BatchNormalization', 'BitShift', 'Ceil', 'Clip', 'Conv', 'CumSum', - 'DequantizeLinear', 'Exp', 'Expand', 'Floor', 'Gemm', 'IsNaN', 'Log', 'LogSoftmax', 'LpNormalization', 'MatMul', 'Max', 'MaxPool', 'Mean', 'Min', - 'Neg', 'NonZero', + 'NonZero', 'Pad', 'Range', 'Reciprocal', 'ReduceL1', 'ReduceL2', 'ReduceLogSum', 'ReduceLogSumExp', 'ReduceMax', 'ReduceMean', 'ReduceMin', 'ReduceProd', 'ReduceSum', 'ReduceSumSquare', @@ -343,13 +342,15 @@ def _create_operator_type_usage_processors(): 'Unique', 'Where'] - default_processor_onnx_ops_requiring_int64_for_input_0 = ['Add', - 'Div', - 'Equal', - 'Greater', - 'Less', - 'Mul', - 'Sub'] + # ops that are used to manipulate shapes or indices so require int32_t and int64_t to be available + default_processor_onnx_ops_requiring_ints_for_input_0 = ['Add', + 'Div', + 'Equal', + 'Greater', + 'Less', + 'Mul', + 'Neg', # used in tflite TransposeConv conversion + 'Sub'] internal_ops = ['QLinearAdd', 'QLinearMul'] @@ -365,8 +366,8 @@ def _create_operator_type_usage_processors(): default_processor_onnxml_ops = [] [add(DefaultTypeUsageProcessor('ai.onnx', op)) for op in default_processor_onnx_ops] - [add(DefaultTypeUsageProcessor('ai.onnx', op, required_input_types={0: {"int64_t"}})) - for op in default_processor_onnx_ops_requiring_int64_for_input_0] + [add(DefaultTypeUsageProcessor('ai.onnx', op, required_input_types={0: {"int32_t", "int64_t"}})) + for op in default_processor_onnx_ops_requiring_ints_for_input_0] [add(DefaultTypeUsageProcessor('ai.onnx.ml', op)) for op in default_processor_onnxml_ops] [add(DefaultTypeUsageProcessor('com.microsoft', op)) for op in internal_ops] @@ -395,6 +396,12 @@ def _create_operator_type_usage_processors(): add(Output0TypedRegistrationProcessor('ai.onnx', 'QuantizeLinear')) add(Output0TypedRegistrationProcessor('ai.onnx', 'DynamicQuantizeLinear')) + # make sure all the dequantize types are enabled. we use int32_t for parts of GEMM and Conv so just + # enabling int8 and uint8 is not enough. + # TODO: Only apply required types to the global type list and ignore if it's model based per-op type reduction + add(DefaultTypeUsageProcessor('ai.onnx', 'DequantizeLinear', inputs=[0], + required_input_types={0: {'int8_t', 'uint8_t', 'int32_t'}})) + # OneHot concatenates type strings into a triple in the typed registration # e.g. float_int64_t_int64_t add(OneHotProcessor())