diff --git a/tools/ci_build/github/android/mobile_package.required_operators.config b/tools/ci_build/github/android/mobile_package.required_operators.config index 5ce34a7ba0..6a6ba8c3c9 100644 --- a/tools/ci_build/github/android/mobile_package.required_operators.config +++ b/tools/ci_build/github/android/mobile_package.required_operators.config @@ -9,19 +9,30 @@ # allow float, int8, uint8. operators that manipulate shapes or indices have int32 and int64 enabled internally. !globally_allowed_types;float,int8_t,uint8_t -# ops used by the tf2onnx tflite converter. same list for opsets 12 and 13. -ai.onnx;12;Abs,Add,And,ArgMax,ArgMin,AveragePool,Cast,Ceil,Clip,Concat,ConstantOfShape,Conv,ConvTranspose,Cos,CumSum,DepthToSpace,DequantizeLinear,Div,DynamicQuantizeLinear,Elu,Equal,Exp,Expand,Flatten,Floor,Gather,GatherND,Gemm,Greater,GreaterOrEqual,Identity,If,LRN,LeakyRelu,Less,LessOrEqual,Log,LogSoftmax,Loop,MatMul,Max,MaxPool,Mean,Min,Mul,Neg,NonMaxSuppression,NonZero,Not,Or,PRelu,Pad,Pow,QuantizeLinear,Range,Reciprocal,ReduceMax,ReduceMean,ReduceMin,ReduceProd,ReduceSum,Relu,Reshape,Resize,ReverseSequence,Round,ScatterND,Shape,Sigmoid,Sin,Size,Slice,Softmax,SpaceToDepth,Split,Sqrt,Squeeze,Sub,Sum,Tanh,ThresholdedRelu,Tile,TopK,Transpose,Unique,Unsqueeze,Where -ai.onnx;13;Abs,Add,And,ArgMax,ArgMin,AveragePool,Cast,Ceil,Clip,Concat,ConstantOfShape,Conv,ConvTranspose,Cos,CumSum,DepthToSpace,DequantizeLinear,Div,DynamicQuantizeLinear,Elu,Equal,Exp,Expand,Flatten,Floor,Gather,GatherND,Gemm,Greater,GreaterOrEqual,Identity,If,LRN,LeakyRelu,Less,LessOrEqual,Log,LogSoftmax,Loop,MatMul,Max,MaxPool,Mean,Min,Mul,Neg,NonMaxSuppression,NonZero,Not,Or,PRelu,Pad,Pow,QuantizeLinear,Range,Reciprocal,ReduceMax,ReduceMean,ReduceMin,ReduceProd,ReduceSum,Relu,Reshape,Resize,ReverseSequence,Round,ScatterND,Shape,Sigmoid,Sin,Size,Slice,Softmax,SpaceToDepth,Split,Sqrt,Squeeze,Sub,Sum,Tanh,ThresholdedRelu,Tile,TopK,Transpose,Unique,Unsqueeze,Where +# ops used by the tf2onnx tflite converter. +ai.onnx;12,13,14,15;Abs,Add,And,ArgMax,ArgMin,AveragePool,Cast,Ceil,Clip,Concat,ConstantOfShape,Conv,ConvTranspose,Cos,CumSum,DepthToSpace,DequantizeLinear,Div,DynamicQuantizeLinear,Elu,Equal,Exp,Expand,Flatten,Floor,Gather,GatherND,Gemm,Greater,GreaterOrEqual,Identity,If,LRN,LeakyRelu,Less,LessOrEqual,Log,LogSoftmax,Loop,MatMul,Max,MaxPool,Mean,Min,Mul,Neg,NonMaxSuppression,NonZero,Not,Or,PRelu,Pad,Pow,QuantizeLinear,Range,Reciprocal,ReduceMax,ReduceMean,ReduceMin,ReduceProd,ReduceSum,Relu,Reshape,Resize,ReverseSequence,Round,ScatterND,Shape,Sigmoid,Sin,Size,Slice,Softmax,SpaceToDepth,Split,Sqrt,Squeeze,Sub,Sum,Tanh,ThresholdedRelu,Tile,TopK,Transpose,Unique,Unsqueeze,Where # other ops found in test models -ai.onnx;12;Erf,GlobalAveragePool,InstanceNormalization,HardSigmoid,MatMulInteger,QLinearConv,QLinearMatMul -ai.onnx;13;Erf,GlobalAveragePool,InstanceNormalization,HardSigmoid,MatMulInteger,QLinearConv,QLinearMatMul +ai.onnx;12,13,14,15;Erf,GlobalAveragePool,InstanceNormalization,HardSigmoid,MatMulInteger,QLinearConv,QLinearMatMul # Control flow ops # - If and Loop are covered by the tflite converter list # - Scan tends to be used in speech models (it's more efficient than Loop) so include it for support of those -ai.onnx;12;Scan -ai.onnx;13;Scan +ai.onnx;12,13,14,15;Scan + +# Changed ONNX ops by opset version for the above ops. This list is to provide context as to how much was added +# for each additional opset we support. +# +# opset 13 +# Abs,Add,ArgMax,ArgMin,Cast,Ceil,Clip,Concat,DepthToSpace,DequantizeLinear,Div,Equal,Erf,Exp,Expand,Flatten,Floor, +# Gather,GatherND,Gemm,Greater,Identity,If,LRN,Less,Log,LogSoftmax,Loop,MatMul,Max,Mean,Min,Mul,Neg,NonZero,Pad, +# Pow,QuantizeLinear,Reciprocal,ReduceMax,ReduceMean,ReduceMin,ReduceProd,ReduceSum,Relu,Reshape,Resize, +# ScatterND,Shape,Sigmoid,Size,Slice,Softmax,SpaceToDepth,Split,Sqrt,Squeeze,Sub,Sum,Tanh,Tile,Transpose,Unsqueeze +# opset 14 +# Add,CumSum,Div,Identity,Mul,Relu,Reshape,Sub +# opset 15 +# Pow,Shape + # internal ops added by optimizers # Note: LayerNormalization is an internal op even though it is (incorrectly) registered in the ONNX domain. diff --git a/tools/python/check_model_can_use_ort_mobile_pkg.py b/tools/python/check_model_can_use_ort_mobile_pkg.py new file mode 100644 index 0000000000..5ea3dde24b --- /dev/null +++ b/tools/python/check_model_can_use_ort_mobile_pkg.py @@ -0,0 +1,219 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Helper script that will check if the types and operators used in an ONNX model +# would be supported by the pre-built ORT Mobile package. + +import argparse +import onnx +import os +import pathlib +import sys +from onnx import shape_inference +from util import reduced_build_config_parser + +cpp_to_tensorproto_type = { + 'float': 1, + 'uint8_t': 2, + 'int8_t': 3, + 'uint16_t': 4, + 'int16_t': 5, + 'int32_t': 6, + 'int64_t': 7, + 'std::string': 8, + 'bool': 9, + 'MLFloat16': 10, + 'double': 11, + 'uint32_t': 12, + 'uint64_t': 13, + 'Complex64': 14, # not supported by ORT + 'Complex128': 15, # not supported by ORT + 'BFloat16': 16 +} + +tensorproto_type_to_cpp = {v: k for k, v in cpp_to_tensorproto_type.items()} + + +def check_graph(graph, opsets, required_ops, global_types, special_types, unsupported_ops): + ''' + Check the graph and any subgraphs for usage of types or operators which we know are not supported. + :param graph: Graph to process. + :param opsets: Map of domain to opset version that the model imports. + :param required_ops: Operators that are included in the pre-built package. + :param global_types: Types globally enabled in the pre-built package. + :param special_types: Types that are always enabled for a subset of operators and are _usually_ supported but are + are not guaranteed to be. We would need to add a lot of infrastructure to know for sure so + currently we treat them as supported. + :param unsupported_ops: Set of unsupported operators that is updated as they are found and returned to the caller. + :return: Returns whether the graph uses unsupported operators or types. + ''' + has_unsupported_types = False + value_info_map = {vi.name: vi for vi in graph.value_info} + + def _is_type_supported(value_info, description): + is_supported = True + type_name = i.type.WhichOneof('value') + if type_name == 'tensor_type': + t = i.type.tensor_type.elem_type + if t not in global_types and t not in special_types: + cpp_type = tensorproto_type_to_cpp[t] + print(f'Data type {cpp_type} of {description} is not supported.') + is_supported = False + else: + # we don't support sequences, map, sparse tensors, or optional types in the pre-built package + print(f'Data type {type_name} of {description} is not supported.') + is_supported = False + + return is_supported + + def _input_output_is_supported(value_info, input_output): + return _is_type_supported(value_info, f'graph {input_output} {value_info.name}') + + # node outputs are simpler to check. + # node inputs have a much wider mix of types, some of which come from initializers and most likely are always + # enabled as we generally do type reduction on the user data input to the operator and not the weights/etc. which + # come from initializers. + def _node_output_is_supported(name): + is_supported = True + if name in value_info_map: + vi = value_info_map[name] + is_supported = _is_type_supported(vi, f'node output {name}') + else: + # we don't have type info so ignore + pass + + return is_supported + + for i in graph.input: + if not _input_output_is_supported(i, 'input'): + has_unsupported_types = True + + for i in graph.output: + if not _input_output_is_supported(i, 'output'): + has_unsupported_types = True + + for node in graph.node: + # required_ops are map of [domain][opset] to set of op_type names. '' == ai.onnx + domain = node.domain or 'ai.onnx' + + # special case Constant as we will convert to an initializer during model load + if domain == 'ai.onnx' and node.op_type == 'Constant': + continue + + # some models don't have complete imports. use 1 as a default as that's valid for custom domains and should + # result in an error for any others. not sure why ONNX or ORT validation allows this though. + opset = opsets[domain] if domain in opsets else 1 + if domain not in required_ops or \ + opset not in required_ops[domain] or \ + node.op_type not in required_ops[domain][opset]: + unsupported_ops.add(f'{domain}:{opset}:{node.op_type}') + + for output_name in node.output: + if not _node_output_is_supported(output_name): + has_unsupported_types = True + + # recurse into subgraph for control flow nodes (Scan/Loop/If) + for attr in node.attribute: + if attr.HasField('g'): + check_graph(attr.g, opsets, required_ops, global_types, special_types, unsupported_ops) + + return has_unsupported_types or unsupported_ops + + +def _get_global_tensorproto_types(op_type_impl_filter): + ''' + Map the globally supported types (C++) to onnx.TensorProto.DataType values used in the model + See https://github.com/onnx/onnx/blob/1faae95520649c93ae8d0b403816938a190f4fa7/onnx/onnx.proto#L485 + + Additionally return a set of types we special case as being able to generally be considered as supported. + :param op_type_impl_filter: type filter from reduced build configuration parser + :return: tuple of globally enabled types and special cased types + ''' + global_cpp_types = op_type_impl_filter.global_type_list() + global_onnx_tensorproto_types = set() + + for t in global_cpp_types: + if t in cpp_to_tensorproto_type: + global_onnx_tensorproto_types.add(cpp_to_tensorproto_type[t]) + else: + print(f'Error: Unexpected data type of {t}') + sys.exit(-1) + + # a subset of operators require int32 and int64 to always be enabled, as those types are used for dimensions in + # shapes and indices. + # additionally we have a number of operators (e.g. Not, Where) that always require the use of bool. + # this _may_ mean values involving these types can be processed, but without adding a lot more code we don't know + # for sure. + special_types = [cpp_to_tensorproto_type['int32_t'], + cpp_to_tensorproto_type['int64_t'], + cpp_to_tensorproto_type['bool']] + + return global_onnx_tensorproto_types, special_types + + +def main(): + parser = argparse.ArgumentParser( + description='Check if model is likely to be able to be run using the ONNX Runtime Mobile Pre-Built Package', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + script_dir = os.path.dirname(os.path.realpath(__file__)) + + # config that was used to create the pre-built package. + # TODO: Would be nice to specify ORT release and pull the config for that release. + default_config_path = \ + pathlib.Path(os.path.join(script_dir, '../ci_build/github/android/mobile_package.required_operators.config') + ).resolve() + + parser.add_argument('--config_path', + help='Path to required operators and types configuration used to build ' + 'the pre-built ORT mobile package.', + required=False, + type=pathlib.Path, default=default_config_path) + + parser.add_argument('model', help='Path to ONNX model to check', type=pathlib.Path) + + args = parser.parse_args() + config_file = args.config_path.resolve(strict=True) # must exist so strict=True + model_file = args.model.resolve(strict=True) + model = onnx.load(model_file) + + # we need to run shape inferencing to populate that type info for node outputs. + # we will get warnings if the model uses ORT contrib ops, and type/shape inferencing will be lost downstream + # of those. due to that we should also (or only) support ORT format models in this processing as they will have full + # type/shape info for all ops. + model_with_type_info = shape_inference.infer_shapes(model) + + # get type reduction and required ops from pre-built package config + # we assume type reduction is enabled as that's what we use for our builds + enable_type_reduction = True + required_ops, op_type_impl_filter = reduced_build_config_parser.parse_config(config_file, enable_type_reduction) + global_onnx_tensorproto_types, special_types = _get_global_tensorproto_types(op_type_impl_filter) + + # get the opset imports + opsets = {} + for entry in model.opset_import: + # if empty it's ai.onnx + domain = entry.domain or 'ai.onnx' + opsets[domain] = entry.version + + unsupported_ops = set() + print('Checking if the data types and operators used in the model are supported in the pre-built ORT package...\n') + unsupported = check_graph(model_with_type_info.graph, opsets, required_ops, + global_onnx_tensorproto_types, special_types, + unsupported_ops) + + if unsupported_ops: + print(' Unsupported operators:') + for entry in sorted(unsupported_ops): + print(' ' + entry) + + if unsupported: + print('\nModel is not supported by the pre-built package due to unsupported types or operators.') + print('Please see https://onnxruntime.ai/docs/reference/mobile/prebuilt-package/ for further details.') + else: + print('Model is most likely supported. ' + 'Note that this check is not comprehensive so testing to validate is still required.') + + +if __name__ == '__main__': + main() diff --git a/tools/python/util/reduced_build_config_parser.py b/tools/python/util/reduced_build_config_parser.py index 420111ee01..a55f81489e 100644 --- a/tools/python/util/reduced_build_config_parser.py +++ b/tools/python/util/reduced_build_config_parser.py @@ -24,8 +24,12 @@ def parse_config(config_file: str, enable_type_reduction: bool = False): 1. Specifying required operators - The basic format for specifying required operators is `domain;opset;op1,op2...` - e.g. `ai.onnx;11;Add,Cast,Clip,... + The basic format for specifying required operators is `domain;opset1,opset2;op1,op2...` + e.g. `ai.onnx;11;Add,Cast,Clip,... for a single opset + `ai.onnx;11,12;Add,Cast,Clip,... for multiple opsets + + note: Configuration information is accrued as the file is parsed. If an operator requires support from multiple + opsets that can be done with one entry for each opset, or one entry with multiple opsets in it. If the configuration file is generated from ORT format models it may optionally contain JSON for per-operator type reduction. The required types are generally listed per input and/or output of the operator. @@ -113,7 +117,7 @@ def parse_config(config_file: str, enable_type_reduction: bool = False): continue domain, opset_str, operators_str = [segment.strip() for segment in line.split(';')] - opset = int(opset_str) + opsets = [int(s) for s in opset_str.split(',')] # any type reduction information is serialized json that starts/ends with { and }. # type info is optional for each operator. @@ -169,12 +173,13 @@ def parse_config(config_file: str, enable_type_reduction: bool = False): else: operators = set([op.strip() for op in operators_str.split(',')]) - if domain not in required_ops: - required_ops[domain] = {opset: operators} - elif opset not in required_ops[domain]: - required_ops[domain][opset] = operators - else: - required_ops[domain][opset].update(operators) + for opset in opsets: + if domain not in required_ops: + required_ops[domain] = {opset: operators} + elif opset not in required_ops[domain]: + required_ops[domain][opset] = operators + else: + required_ops[domain][opset].update(operators) if len(required_ops) == 0 and no_ops_specified_means_all_ops_are_required: required_ops = None