mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-24 19:43:35 +00:00
Android package infrastructure (#7430)
* Include ORT format model conversion scripts and infrastructure in ORT python package. - tweak existing script setup so it can be easily run directly and from the ORT python package Add config file and readme for Android minimal build package Update ORT Mobile doco Disable warning if 'all' optimizations are enabled but NCHWc transformer is excluded (device specific optimizations don't apply in this scenario so the warning is moot). * Address PR comments
This commit is contained in:
Scott McKay
GitHub
parent
3d92723d1c
commit
d6df5764d7
9 changed files with 379 additions and 209 deletions
|
|
@ -278,6 +278,14 @@ file(GLOB onnxruntime_python_datasets_data CONFIGURE_DEPENDS
|
|||
"${ONNXRUNTIME_ROOT}/python/datasets/*.onnx"
|
||||
)
|
||||
|
||||
# Files needed to convert ONNX model to ORT format
|
||||
set(onnxruntime_ort_format_model_conversion_srcs
|
||||
${REPO_ROOT}/tools/python/util/convert_onnx_models_to_ort.py
|
||||
${REPO_ROOT}/tools/python/util/logger.py
|
||||
)
|
||||
file(GLOB onnxruntime_ort_format_model_srcs CONFIGURE_DEPENDS
|
||||
${REPO_ROOT}/tools/python/util/ort_format_model/*.py)
|
||||
|
||||
set(build_output_target onnxruntime_common)
|
||||
if(NOT onnxruntime_ENABLE_STATIC_ANALYSIS)
|
||||
add_custom_command(
|
||||
|
|
@ -288,6 +296,8 @@ add_custom_command(
|
|||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/datasets
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/featurizer_ops
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/ort_format_model
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/ort_format_model/ort_flatbuffers_py
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/transformers
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/transformers/longformer
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization
|
||||
|
|
@ -332,6 +342,15 @@ add_custom_command(
|
|||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${onnxruntime_python_tools_featurizers_src}
|
||||
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/featurizer_ops/
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${onnxruntime_ort_format_model_conversion_srcs}
|
||||
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${onnxruntime_ort_format_model_srcs}
|
||||
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/ort_format_model/
|
||||
COMMAND ${CMAKE_COMMAND} -E copy_directory
|
||||
${ONNXRUNTIME_ROOT}/core/flatbuffers/ort_flatbuffers_py
|
||||
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/tools/ort_format_model/ort_flatbuffers_py
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${onnxruntime_python_quantization_src}
|
||||
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/
|
||||
|
|
|
|||
|
|
@ -2,16 +2,18 @@
|
|||
|
||||
## Overview
|
||||
|
||||
<img align="left" width=40% src="images/Mobile.png" alt="Steps to build for mobile platforms."/>
|
||||
|
||||
ONNX Runtime now supports an internal model format to minimize the build size for usage in mobile and embedded scenarios. An ONNX model can be converted to an internal ONNX Runtime format ('ORT format model') using the below instructions.
|
||||
ONNX Runtime now supports an internal model format to minimize the build size for usage in mobile, edge and embedded scenarios. An ONNX model can be converted to an internal ONNX Runtime format ('ORT format model') using the below instructions.
|
||||
|
||||
A minimal build can be used with any ORT format model, provided that the kernels for the operators used in the model were included in the build.
|
||||
I.e., the custom build provides a set of kernels, and if that set satisfies a given ORT format model's needs, the model can be loaded and executed.
|
||||
i.e. the custom build provides a set of kernels, and if that set satisfies a given ORT format model's needs, the model can be loaded and executed.
|
||||
|
||||
<br>
|
||||
<img align="center" src="images/Mobile.png" alt="Steps to build for mobile platforms."/>
|
||||
<br>
|
||||
|
||||
## Steps to create model and minimal build
|
||||
|
||||
You will need a script from the ONNX Runtime repository and to also perform a custom build, so you will need to clone the repository locally. See [here](https://www.onnxruntime.ai/docs/how-to/build.html#prerequisites) for initial steps.
|
||||
As you will need to perform a custom build of ONNX Runtime, you will need to clone the repository locally. See [here](https://www.onnxruntime.ai/docs/how-to/build.html#prerequisites) for initial steps.
|
||||
|
||||
The directory the ONNX Runtime repository was cloned into is referred to as `<ONNX Runtime repository root>` in this documentation.
|
||||
|
||||
|
|
@ -28,26 +30,29 @@ It is also possible (and optional) to further prune the operator kernel implemen
|
|||
This pruning is referred to as "operator type reduction" in this documentation.
|
||||
|
||||
- The helper python script requires the standard ONNX Runtime python package to be installed. Install the ONNX Runtime python package from https://pypi.org/project/onnxruntime/. Version 1.5.2 or later is required.
|
||||
To enable operator type reduction, version 1.7 or later is required.
|
||||
- `pip install onnxruntime`
|
||||
- Ensure that any existing ONNX Runtime python package was uninstalled first, or use `-U` with the above command to upgrade an existing package.
|
||||
|
||||
- Additionally, if you want to enable operator type reduction, the Flatbuffers python package should be installed.
|
||||
- Additionally, if you want to enable operator type reduction, ONNX Runtime version 1.7 or later is required, and the flatbuffers python package must be installed.
|
||||
- `pip install flatbuffers`
|
||||
|
||||
- Copy all the ONNX models you wish to convert and use with the minimal build into a directory.
|
||||
|
||||
- Convert the ONNX models to ORT format
|
||||
- `python <ONNX Runtime repository root>/tools/python/convert_onnx_models_to_ort.py <path to directory containing one or more .onnx models>`
|
||||
- To enable operator type reduction, specify the `--enable_type_reduction` option.
|
||||
- For each ONNX model an ORT format model will be created with '.ort' as the file extension.
|
||||
- Run the helper script to convert the models
|
||||
- For ONNX Runtime version 1.8 or later you can use ONNX Runtime python package:
|
||||
- `python -m onnxruntime.tools.convert_onnx_models_to_ort <path to directory containing one or more .onnx models>`
|
||||
- Alternatively use the conversion script in the ONNX Runtime repository:
|
||||
- `python <ONNX Runtime repository root>/tools/python/convert_onnx_models_to_ort.py <path to directory containing one or more onnx models>`
|
||||
- To enable operator type reduction, specify the `--enable_type_reduction` option
|
||||
- For each ONNX model an ORT format model will be created with '.ort' as the file extension
|
||||
- A configuration file will also be created.
|
||||
If operator type reduction is enabled, the file will be called `required_operators_and_types.config`.
|
||||
Otherwise, the file will be called `required_operators.config`.
|
||||
|
||||
Example:
|
||||
|
||||
Running `'python <ORT repository root>/tools/python/convert_onnx_models_to_ort.py /models'` where the '/models' directory contains ModelA.onnx and ModelB.onnx
|
||||
Running `python -m onnxruntime.tools.convert_onnx_models_to_ort /models` where the '/models' directory contains ModelA.onnx and ModelB.onnx
|
||||
- Will create /models/ModelA.ort and /models/ModelB.ort
|
||||
- Will create /models/required_operators.config
|
||||
|
||||
|
|
@ -144,10 +149,10 @@ For a more in-depth analysis of the performance considerations when using NNAPI
|
|||
|
||||
### Limit ORT format model to ONNX operators
|
||||
|
||||
The NNAPI Execution Provider is only able to execute ONNX operators using NNAPI. When creating the ORT format model it is recommended to limit the optimization level to 'basic' so that custom internal ONNX Runtime operators are not added by the 'extended' optimizations. This will ensure that the maximum number of nodes can be executed using NNAPI. See the [graph optimization](https://www.onnxruntime.ai/docs/resources/graph-optimizations.html) documentation for details on the optimization levels.
|
||||
The NNAPI Execution Provider is only able to execute ONNX operators using NNAPI. When creating the ORT format model it is recommended to limit the optimization level to 'basic' so that custom internal ONNX Runtime operators are not added by the 'extended' or 'all' optimization levels. This will ensure that the maximum number of nodes can be executed using NNAPI. See the [graph optimization](https://www.onnxruntime.ai/docs/resources/graph-optimizations.html) documentation for details on the optimization levels.
|
||||
|
||||
To limit the optimization level when creating the ORT format models using `tools\python\convert_onnx_models_to_ort.py` as per the above [instructions](#1-Create-ORT-format-model-and-configuration-file-with-required-operators), add `--optimization_level basic` to the arguments.
|
||||
- e.g. `python <ORT repository root>/tools/python/convert_onnx_models_to_ort.py --optimization_level basic /models`
|
||||
- e.g. `python -m onnxruntime.tools.convert_onnx_models_to_ort --optimization_level basic /models` or `python <ORT repository root>/tools/python/convert_onnx_models_to_ort.py --optimization_level basic /models`
|
||||
|
||||
### Create a minimal build for Android with NNAPI support
|
||||
|
||||
|
|
|
|||
|
|
@ -1290,11 +1290,13 @@ common::Status InferenceSession::Initialize() {
|
|||
"Please disable any execution providers which generate compiled nodes."));
|
||||
}
|
||||
|
||||
if (session_options_.graph_optimization_level >= TransformerLevel::Level3) {
|
||||
// add a warning if the NchwcTransformer was enabled, as it contains the hardware specific logic
|
||||
if (session_options_.graph_optimization_level >= TransformerLevel::Level3 &&
|
||||
optimizers_to_disable_.find("NchwcTransformer") == optimizers_to_disable_.cend()) {
|
||||
LOGS(*session_logger_, WARNING)
|
||||
<< "Serializing optimized model with Graph Optimization level greater than ORT_ENABLE_EXTENDED. "
|
||||
"The generated model may contain hardware and execution provider specific optimizations, "
|
||||
"and should only be used in the same environment the model was optimized for.";
|
||||
<< "Serializing optimized model with Graph Optimization level greater than ORT_ENABLE_EXTENDED and the "
|
||||
"NchwcTransformer enabled. The generated model may contain hardware specific optimizations, and "
|
||||
"should only be used in the same environment the model was optimized in.";
|
||||
}
|
||||
|
||||
if (saving_ort_format) {
|
||||
|
|
|
|||
4
setup.py
4
setup.py
|
|
@ -232,6 +232,10 @@ packages = [
|
|||
'onnxruntime.capi.training',
|
||||
'onnxruntime.datasets',
|
||||
'onnxruntime.tools',
|
||||
'onnxruntime.tools.ort_format_model',
|
||||
'onnxruntime.tools.ort_format_model.ort_flatbuffers_py',
|
||||
'onnxruntime.tools.ort_format_model.ort_flatbuffers_py.experimental',
|
||||
'onnxruntime.tools.ort_format_model.ort_flatbuffers_py.experimental.fbs',
|
||||
'onnxruntime.quantization',
|
||||
'onnxruntime.quantization.operators',
|
||||
'onnxruntime.quantization.CalTableFlatBuffers',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
# Android package for ORT Mobile operator and type reduction configuration
|
||||
#
|
||||
# The list of operators was generated from:
|
||||
# - the ONNX operators use by the tf2onnx tflite converter
|
||||
# - the operators used in a set of tflite models from tfhub, the tflite examples, and the mlperf mobile models
|
||||
# - models were optimized with optimizations set to 'basic', 'extended' and 'all'
|
||||
# - see the readme file for full details
|
||||
|
||||
# allow float, int8, uint8. operators that manipulate shapes or indices have int32 and int64 enabled internally.
|
||||
!globally_allowed_types;float,int8_t,uint8_t
|
||||
|
||||
# ops used by the tf2onnx tflite converter. same list for opsets 12 and 13.
|
||||
ai.onnx;12;Abs,Add,And,ArgMax,ArgMin,AveragePool,Cast,Ceil,Clip,Concat,ConstantOfShape,Conv,ConvTranspose,Cos,CumSum,DepthToSpace,DequantizeLinear,Div,DynamicQuantizeLinear,Elu,Equal,Exp,Expand,Flatten,Floor,Gather,GatherND,Gemm,Greater,GreaterOrEqual,Identity,If,LRN,LeakyRelu,Less,LessOrEqual,Log,LogSoftmax,Loop,MatMul,Max,MaxPool,Mean,Min,Mul,Neg,NonMaxSuppression,NonZero,Not,Or,PRelu,Pad,Pow,QuantizeLinear,Range,Reciprocal,ReduceMax,ReduceMean,ReduceMin,ReduceProd,ReduceSum,Relu,Reshape,Resize,ReverseSequence,Round,ScatterND,Shape,Sigmoid,Sin,Size,Slice,Softmax,SpaceToDepth,Split,Sqrt,Squeeze,Sub,Sum,Tanh,ThresholdedRelu,Tile,TopK,Transpose,Unique,Unsqueeze,Where
|
||||
ai.onnx;13;Abs,Add,And,ArgMax,ArgMin,AveragePool,Cast,Ceil,Clip,Concat,ConstantOfShape,Conv,ConvTranspose,Cos,CumSum,DepthToSpace,DequantizeLinear,Div,DynamicQuantizeLinear,Elu,Equal,Exp,Expand,Flatten,Floor,Gather,GatherND,Gemm,Greater,GreaterOrEqual,Identity,If,LRN,LeakyRelu,Less,LessOrEqual,Log,LogSoftmax,Loop,MatMul,Max,MaxPool,Mean,Min,Mul,Neg,NonMaxSuppression,NonZero,Not,Or,PRelu,Pad,Pow,QuantizeLinear,Range,Reciprocal,ReduceMax,ReduceMean,ReduceMin,ReduceProd,ReduceSum,Relu,Reshape,Resize,ReverseSequence,Round,ScatterND,Shape,Sigmoid,Sin,Size,Slice,Softmax,SpaceToDepth,Split,Sqrt,Squeeze,Sub,Sum,Tanh,ThresholdedRelu,Tile,TopK,Transpose,Unique,Unsqueeze,Where
|
||||
|
||||
# other ops found in test models
|
||||
ai.onnx;12;GlobalAveragePool,MatMulInteger,QLinearConv,QLinearMatMul
|
||||
ai.onnx;13;GlobalAveragePool,MatMulInteger,QLinearConv,QLinearMatMul
|
||||
|
||||
# Control flow ops
|
||||
# - If and Loop are covered by the tflite converter list
|
||||
# - Scan tends to be used in speech models (it's more efficient than Loop) so include it for support of those
|
||||
ai.onnx;12;Scan
|
||||
ai.onnx;13;Scan
|
||||
|
||||
# internal ops added by optimizers
|
||||
com.microsoft;1;DynamicQuantizeMatMul,FusedConv,FusedGemm,FusedMatMul,MatMulIntegerToFloat,NhwcMaxPool,QLinearAdd,QLinearAveragePool,QLinearConv,QLinearGlobalAveragePool,QLinearMul,QLinearSigmoid
|
||||
# NHWC transformer also uses this, so assuming it's valuable enough to include
|
||||
com.microsoft;1;QLinearLeakyRelu
|
||||
|
||||
# Quantized contrib ops that are registered but no usage was found. Excluding for now.
|
||||
# com.microsoft;1;DynamicQuantizeLSTM,QAttention
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
The required operators config file was generated from a number of models (details below), with optimizations run using 'all', 'extended' and 'basic'.
|
||||
Following that, some additional operators were added, as per the comments in the config file.
|
||||
|
||||
The global types to support were selected to support quantized and float32 models
|
||||
Additionally there is internal 'required' type support for int32 and int64_t in selected operators that work with the dimensions in a shape or indices so that we don't need to enable those types at a global level.
|
||||
|
||||
Models used as input (Converted using tf2onnx in early March 2021):
|
||||
Models from TF Lite Examples https://www.tensorflow.org/lite/examples
|
||||
- lite-model_deeplabv3_1_metadata_2.tflite.onnx
|
||||
- lite-model_esrgan-tf2_1.tflite.onnx
|
||||
- lite-model_mobilebert_1_metadata_1.tflite.onnx
|
||||
- mnist.tflite.onnx
|
||||
- mobilenet_v1_1.0_224_quant.tflite.onnx
|
||||
- model_history10_top100.tflite.onnx
|
||||
- posenet_mobilenet_float_075_1_default_1.tflite.onnx
|
||||
- posenet_mobilenet_v1_100_257x257_multi_kpt_stripped.tflite.onnx
|
||||
- ssd_mobilenet_v1_1_metadata_1.tflite.onnx
|
||||
- text_classification_v2.tflite.onnx
|
||||
|
||||
Assorted models from TF Hub that were able to be converted with tf2onnx
|
||||
TFLite v1 https://tfhub.dev/s?deployment-format=lite&tf-version=tf1
|
||||
- efficientnet_lite1_fp32_2.tflite.onnx
|
||||
- efficientnet_lite1_int8_2.tflite.onnx
|
||||
- efficientnet_lite4_fp32_2.tflite.onnx
|
||||
- efficientnet_lite4_int8_2.tflite.onnx
|
||||
- lite-model_aiy_vision_classifier_birds_V1_3.tflite.onnx
|
||||
- lite-model_aiy_vision_classifier_food_V1_1.tflite.onnx
|
||||
- lite-model_aiy_vision_classifier_plants_V1_3.tflite.onnx
|
||||
- lite-model_midas_v2_1_small_1_lite_1.tflite.onnx
|
||||
- lite-model_object_detection_mobile_object_labeler_v1_1.tflite.onnx
|
||||
- magenta_arbitrary-image-stylization-v1-256_int8_prediction_1.tflite.onnx
|
||||
- magenta_arbitrary-image-stylization-v1-256_int8_transfer_1.tflite.onnx
|
||||
- object_detection_mobile_object_localizer_v1_1_default_1.tflite.onnx
|
||||
|
||||
TFLite v2 https://tfhub.dev/s?deployment-format=lite&tf-version=tf2
|
||||
- tf2\albert_lite_base_squadv1_1.tflite.onnx
|
||||
- tf2\lite-model_disease-classification_1.tflite.onnx
|
||||
- tf2\lite-model_efficientdet_lite0_detection_default_1.tflite.onnx
|
||||
- tf2\lite-model_efficientdet_lite0_int8_1.tflite.onnx
|
||||
- tf2\lite-model_efficientdet_lite1_detection_default_1.tflite.onnx
|
||||
- tf2\lite-model_efficientdet_lite2_detection_default_1.tflite.onnx
|
||||
- tf2\lite-model_efficientdet_lite3_detection_default_1.tflite.onnx
|
||||
- tf2\lite-model_efficientdet_lite4_detection_default_1.tflite.onnx
|
||||
- tf2\lite-model_esrgan-tf2_1.tflite.onnx
|
||||
- tf2\lite-model_german-mbmelgan_lite_1.tflite.onnx
|
||||
- tf2\lite-model_nonsemantic-speech-benchmark_trill-distilled_1.tflite.onnx
|
||||
- tf2\lite-model_yamnet_tflite_1.tflite.onnx
|
||||
|
||||
Models from MLPerf Mobile
|
||||
(mainly models converted from TFLite and quantized in different ways, but some from TF for completeness as those also have batch handling)
|
||||
- deeplabv3_mnv2_ade20k_float-int8.onnx
|
||||
- deeplabv3_mnv2_ade20k_float.onnx
|
||||
- deeplabv3_mnv2_ade20k-qdq.onnx
|
||||
- mobilebert-int8.onnx
|
||||
- mobilebert-qdq.onnx
|
||||
- mobilebert.onnx
|
||||
- mobiledet-int8.onnx
|
||||
- mobiledet-qdq.onnx
|
||||
- mobiledet.onnx
|
||||
- mobilenet_edgetpu_224_1.0_float-int8.onnx
|
||||
- mobilenet_edgetpu_224_1.0_float.onnx
|
||||
- mobilenet_edgetpu_224_1.0-qdq.onnx
|
||||
- mobilenet_v1_1.0_224.opset12.onnx
|
||||
- resnet50_v1-int8.onnx
|
||||
- resnet50_v1.onnx
|
||||
- ssd_mobilenet_v2_300_float-int8.onnx
|
||||
- ssd_mobilenet_v2_300_float.onnx
|
||||
- ssd_mobilenet_v2_300-qdq.onnx
|
||||
|
||||
Other
|
||||
Mobilenet v2 from pytortch
|
||||
- pytorch.mobilenet_v2_float.onnx
|
||||
- pytorch.mobilenet_v2_uint8.onnx
|
||||
|
|
@ -2,193 +2,11 @@
|
|||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
import onnxruntime as ort
|
||||
|
||||
|
||||
def _path_match_suffix_ignore_case(path: typing.Union[pathlib.Path, str], suffix: str):
|
||||
if not isinstance(path, str):
|
||||
path = str(path)
|
||||
return path.casefold().endswith(suffix.casefold())
|
||||
|
||||
|
||||
def _onnx_model_path_to_ort_model_path(onnx_model_path: pathlib.Path):
|
||||
assert onnx_model_path.is_file() and _path_match_suffix_ignore_case(onnx_model_path, ".onnx")
|
||||
return onnx_model_path.with_suffix(".ort")
|
||||
|
||||
|
||||
def _create_config_file_from_ort_models(onnx_model_path_or_dir: pathlib.Path, enable_type_reduction: bool):
|
||||
if onnx_model_path_or_dir.is_dir():
|
||||
# model directory
|
||||
model_path_or_dir = onnx_model_path_or_dir
|
||||
config_path = None # default path in model directory
|
||||
else:
|
||||
# single model
|
||||
model_path_or_dir = _onnx_model_path_to_ort_model_path(onnx_model_path_or_dir)
|
||||
config_suffix = ".{}".format(
|
||||
'required_operators_and_types.config' if enable_type_reduction else 'required_operators.config')
|
||||
config_path = model_path_or_dir.with_suffix(config_suffix)
|
||||
|
||||
from util.ort_format_model import create_config_from_models
|
||||
create_config_from_models(model_path_or_dir=str(model_path_or_dir),
|
||||
output_file=str(config_path) if config_path is not None else None,
|
||||
enable_type_reduction=enable_type_reduction)
|
||||
|
||||
|
||||
def _create_session_options(optimization_level: ort.GraphOptimizationLevel,
|
||||
output_model_path: pathlib.Path,
|
||||
custom_op_library: pathlib.Path):
|
||||
so = ort.SessionOptions()
|
||||
so.optimized_model_filepath = str(output_model_path)
|
||||
so.graph_optimization_level = optimization_level
|
||||
|
||||
if custom_op_library:
|
||||
so.register_custom_ops_library(str(custom_op_library))
|
||||
|
||||
return so
|
||||
|
||||
|
||||
def _convert(model_path_or_dir: pathlib.Path, optimization_level: ort.GraphOptimizationLevel, use_nnapi: bool,
|
||||
custom_op_library: pathlib.Path, create_optimized_onnx_model: bool):
|
||||
models = []
|
||||
if model_path_or_dir.is_file() and _path_match_suffix_ignore_case(model_path_or_dir, ".onnx"):
|
||||
models.append(model_path_or_dir)
|
||||
elif model_path_or_dir.is_dir():
|
||||
for root, _, files in os.walk(model_path_or_dir):
|
||||
for file in files:
|
||||
if _path_match_suffix_ignore_case(file, ".onnx"):
|
||||
models.append(pathlib.Path(root, file))
|
||||
|
||||
if len(models) == 0:
|
||||
raise ValueError("No .onnx files were found in '{}'".format(model_path_or_dir))
|
||||
|
||||
providers = ['CPUExecutionProvider']
|
||||
if use_nnapi:
|
||||
# providers are priority based, so register NNAPI first
|
||||
providers.insert(0, 'NnapiExecutionProvider')
|
||||
|
||||
# if the optimization level is 'all' we manually exclude the NCHWc transformer. It's not applicable to ARM
|
||||
# devices, and creates a device specific model which won't run on all hardware.
|
||||
# If someone really really really wants to run it they could manually create an optimized onnx model first,
|
||||
# or they could comment out this code.
|
||||
optimizer_filter = None
|
||||
if optimization_level == ort.GraphOptimizationLevel.ORT_ENABLE_ALL:
|
||||
optimizer_filter = ['NchwcTransformer']
|
||||
|
||||
for model in models:
|
||||
# ignore any files with an extension of .optimized.onnx which are presumably from previous executions
|
||||
# of this script
|
||||
if _path_match_suffix_ignore_case(model, ".optimized.onnx"):
|
||||
print("Ignoring '{}'".format(model))
|
||||
continue
|
||||
|
||||
# create .ort file in same dir as original onnx model
|
||||
ort_target_path = _onnx_model_path_to_ort_model_path(model)
|
||||
|
||||
if create_optimized_onnx_model:
|
||||
# Create an ONNX file with the same optimizations that will be used for the ORT format file.
|
||||
# This allows the ONNX equivalent of the ORT format model to be easily viewed in Netron.
|
||||
optimized_target_path = model.with_suffix(".optimized.onnx")
|
||||
so = _create_session_options(optimization_level, optimized_target_path, custom_op_library)
|
||||
|
||||
print("Saving optimized ONNX model {} to {}".format(model, optimized_target_path))
|
||||
_ = ort.InferenceSession(str(model), sess_options=so, providers=providers,
|
||||
disabled_optimizers=optimizer_filter)
|
||||
|
||||
# Load ONNX model, optimize, and save to ORT format
|
||||
so = _create_session_options(optimization_level, ort_target_path, custom_op_library)
|
||||
so.add_session_config_entry('session.save_model_format', 'ORT')
|
||||
|
||||
print("Converting optimized ONNX model to ORT format model {}".format(ort_target_path))
|
||||
_ = ort.InferenceSession(str(model), sess_options=so, providers=providers,
|
||||
disabled_optimizers=optimizer_filter)
|
||||
|
||||
# orig_size = os.path.getsize(onnx_target_path)
|
||||
# new_size = os.path.getsize(ort_target_path)
|
||||
# print("Serialized {} to {}. Sizes: orig={} new={} diff={} new:old={:.4f}:1.0".format(
|
||||
# onnx_target_path, ort_target_path, orig_size, new_size, new_size - orig_size, new_size / orig_size))
|
||||
|
||||
|
||||
def _get_optimization_level(level):
|
||||
if level == 'disable':
|
||||
return ort.GraphOptimizationLevel.ORT_DISABLE_ALL
|
||||
if level == 'basic':
|
||||
# Constant folding and other optimizations that only use ONNX operators
|
||||
return ort.GraphOptimizationLevel.ORT_ENABLE_BASIC
|
||||
if level == 'extended':
|
||||
# Optimizations using custom operators, excluding NCHWc and NHWC layout optimizers
|
||||
return ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
|
||||
if level == 'all':
|
||||
return ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
|
||||
raise ValueError('Invalid optimization level of ' + level)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
os.path.basename(__file__),
|
||||
description='''Convert the ONNX format model/s in the provided directory to ORT format models.
|
||||
All files with a `.onnx` extension will be processed. For each one, an ORT format model will be created in the
|
||||
same directory. A configuration file will also be created called `required_operators.config`, and will contain
|
||||
the list of required operators for all converted models.
|
||||
This configuration file should be used as input to the minimal build via the `--include_ops_by_config`
|
||||
parameter.
|
||||
'''
|
||||
)
|
||||
|
||||
parser.add_argument('--use_nnapi', action='store_true',
|
||||
help='Enable the NNAPI Execution Provider when creating models and determining required '
|
||||
'operators. Note that this will limit the optimizations possible on nodes that the '
|
||||
'NNAPI execution provider takes, in order to preserve those nodes in the ORT format '
|
||||
'model.')
|
||||
|
||||
parser.add_argument('--optimization_level', default='all',
|
||||
choices=['disable', 'basic', 'extended', 'all'],
|
||||
help="Level to optimize ONNX model with, prior to converting to ORT format model. "
|
||||
"These map to the onnxruntime.GraphOptimizationLevel values. "
|
||||
"If the level is 'all' the NCHWc transformer is manually disabled as it contains device "
|
||||
"specific logic, so the ORT format model must be generated on the device it will run on. "
|
||||
"Additionally, the NCHWc optimizations are not applicable to ARM devices."
|
||||
)
|
||||
|
||||
parser.add_argument('--enable_type_reduction', action='store_true',
|
||||
help='Add operator specific type information to the configuration file to potentially reduce '
|
||||
'the types supported by individual operator implementations.')
|
||||
|
||||
parser.add_argument('--custom_op_library', type=pathlib.Path, default=None,
|
||||
help='Provide path to shared library containing custom operator kernels to register.')
|
||||
|
||||
parser.add_argument('--save_optimized_onnx_model', action='store_true',
|
||||
help='Save the optimized version of each ONNX model. '
|
||||
'This will have the same optimizations applied as the ORT format model.')
|
||||
|
||||
parser.add_argument('model_path_or_dir', type=pathlib.Path,
|
||||
help='Provide path to ONNX model or directory containing ONNX model/s to convert. '
|
||||
'All files with a .onnx extension, including in subdirectories, will be processed.')
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
model_path_or_dir = args.model_path_or_dir.resolve()
|
||||
custom_op_library = args.custom_op_library.resolve() if args.custom_op_library else None
|
||||
|
||||
if not model_path_or_dir.is_dir() and not model_path_or_dir.is_file():
|
||||
raise FileNotFoundError("Model path '{}' is not a file or directory.".format(model_path_or_dir))
|
||||
|
||||
if custom_op_library and not custom_op_library.is_file():
|
||||
raise FileNotFoundError("Unable to find custom operator library '{}'".format(custom_op_library))
|
||||
|
||||
optimization_level = _get_optimization_level(args.optimization_level)
|
||||
_convert(model_path_or_dir, optimization_level, args.use_nnapi, custom_op_library, args.save_optimized_onnx_model)
|
||||
_create_config_file_from_ort_models(model_path_or_dir, args.enable_type_reduction)
|
||||
# This script is a stub that uses the model conversion script from the util subdirectory.
|
||||
# We do it this way so we can use relative imports in that script, which makes it easy to include
|
||||
# in the ORT python package (where it must use relative imports)
|
||||
from util.convert_onnx_models_to_ort import convert_onnx_models_to_ort
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
convert_onnx_models_to_ort()
|
||||
|
|
|
|||
209
tools/python/util/convert_onnx_models_to_ort.py
Normal file
209
tools/python/util/convert_onnx_models_to_ort.py
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
import onnxruntime as ort
|
||||
from .ort_format_model import create_config_from_models
|
||||
|
||||
|
||||
def _path_match_suffix_ignore_case(path: typing.Union[pathlib.Path, str], suffix: str):
|
||||
if not isinstance(path, str):
|
||||
path = str(path)
|
||||
return path.casefold().endswith(suffix.casefold())
|
||||
|
||||
|
||||
def _onnx_model_path_to_ort_model_path(onnx_model_path: pathlib.Path, optimization_level_str: str):
|
||||
assert onnx_model_path.is_file() and _path_match_suffix_ignore_case(onnx_model_path, ".onnx")
|
||||
return onnx_model_path.with_suffix(".{}.ort".format(optimization_level_str))
|
||||
|
||||
|
||||
def _create_config_file_from_ort_models(onnx_model_path_or_dir: pathlib.Path, enable_type_reduction: bool):
|
||||
if onnx_model_path_or_dir.is_dir():
|
||||
# model directory
|
||||
model_path_or_dir = onnx_model_path_or_dir
|
||||
config_path = None # default path in model directory
|
||||
else:
|
||||
# single model
|
||||
model_path_or_dir = _onnx_model_path_to_ort_model_path(onnx_model_path_or_dir)
|
||||
config_suffix = ".{}".format(
|
||||
'required_operators_and_types.config' if enable_type_reduction else 'required_operators.config')
|
||||
config_path = model_path_or_dir.with_suffix(config_suffix)
|
||||
|
||||
create_config_from_models(model_path_or_dir=str(model_path_or_dir),
|
||||
output_file=str(config_path) if config_path is not None else None,
|
||||
enable_type_reduction=enable_type_reduction)
|
||||
|
||||
|
||||
def _create_session_options(optimization_level: ort.GraphOptimizationLevel,
|
||||
output_model_path: pathlib.Path,
|
||||
custom_op_library: pathlib.Path):
|
||||
so = ort.SessionOptions()
|
||||
so.optimized_model_filepath = str(output_model_path)
|
||||
so.graph_optimization_level = optimization_level
|
||||
|
||||
if custom_op_library:
|
||||
so.register_custom_ops_library(str(custom_op_library))
|
||||
|
||||
return so
|
||||
|
||||
|
||||
def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_nnapi: bool,
|
||||
custom_op_library: pathlib.Path, create_optimized_onnx_model: bool):
|
||||
|
||||
optimization_level = _get_optimization_level(optimization_level_str)
|
||||
|
||||
models = []
|
||||
if model_path_or_dir.is_file() and _path_match_suffix_ignore_case(model_path_or_dir, ".onnx"):
|
||||
models.append(model_path_or_dir)
|
||||
elif model_path_or_dir.is_dir():
|
||||
for root, _, files in os.walk(model_path_or_dir):
|
||||
for file in files:
|
||||
if _path_match_suffix_ignore_case(file, ".onnx"):
|
||||
models.append(pathlib.Path(root, file))
|
||||
|
||||
if len(models) == 0:
|
||||
raise ValueError("No .onnx files were found in '{}'".format(model_path_or_dir))
|
||||
|
||||
providers = ['CPUExecutionProvider']
|
||||
if use_nnapi:
|
||||
# providers are priority based, so register NNAPI first
|
||||
providers.insert(0, 'NnapiExecutionProvider')
|
||||
|
||||
# if the optimization level is 'all' we manually exclude the NCHWc transformer. It's not applicable to ARM
|
||||
# devices, and creates a device specific model which won't run on all hardware.
|
||||
# If someone really really really wants to run it they could manually create an optimized onnx model first,
|
||||
# or they could comment out this code.
|
||||
optimizer_filter = None
|
||||
if optimization_level == ort.GraphOptimizationLevel.ORT_ENABLE_ALL:
|
||||
optimizer_filter = ['NchwcTransformer']
|
||||
|
||||
num_failures = 0
|
||||
|
||||
for model in models:
|
||||
try:
|
||||
|
||||
# ignore any files with an extension of .optimized.onnx which are presumably from previous executions
|
||||
# of this script
|
||||
if _path_match_suffix_ignore_case(model, ".optimized.onnx"):
|
||||
print("Ignoring '{}'".format(model))
|
||||
continue
|
||||
|
||||
# create .ort file in same dir as original onnx model
|
||||
ort_target_path = _onnx_model_path_to_ort_model_path(model, optimization_level_str)
|
||||
|
||||
if create_optimized_onnx_model:
|
||||
# Create an ONNX file with the same optimizations that will be used for the ORT format file.
|
||||
# This allows the ONNX equivalent of the ORT format model to be easily viewed in Netron.
|
||||
optimized_target_path = model.with_suffix(".{}.optimized.onnx".format(optimization_level_str))
|
||||
so = _create_session_options(optimization_level, optimized_target_path, custom_op_library)
|
||||
|
||||
print("Saving optimized ONNX model {} to {}".format(model, optimized_target_path))
|
||||
_ = ort.InferenceSession(str(model), sess_options=so, providers=providers,
|
||||
disabled_optimizers=optimizer_filter)
|
||||
|
||||
# Load ONNX model, optimize, and save to ORT format
|
||||
so = _create_session_options(optimization_level, ort_target_path, custom_op_library)
|
||||
so.add_session_config_entry('session.save_model_format', 'ORT')
|
||||
|
||||
print("Converting optimized ONNX model {} to ORT format model {}".format(model, ort_target_path))
|
||||
_ = ort.InferenceSession(str(model), sess_options=so, providers=providers,
|
||||
disabled_optimizers=optimizer_filter)
|
||||
|
||||
# orig_size = os.path.getsize(onnx_target_path)
|
||||
# new_size = os.path.getsize(ort_target_path)
|
||||
# print("Serialized {} to {}. Sizes: orig={} new={} diff={} new:old={:.4f}:1.0".format(
|
||||
# onnx_target_path, ort_target_path, orig_size, new_size, new_size - orig_size, new_size / orig_size))
|
||||
except Exception as e:
|
||||
print("Error converting {}: {}".format(model, e))
|
||||
num_failures += 1
|
||||
|
||||
print("Converted {} models. {} failures.".format(len(models), num_failures))
|
||||
|
||||
|
||||
def _get_optimization_level(level):
|
||||
if level == 'disable':
|
||||
return ort.GraphOptimizationLevel.ORT_DISABLE_ALL
|
||||
if level == 'basic':
|
||||
# Constant folding and other optimizations that only use ONNX operators
|
||||
return ort.GraphOptimizationLevel.ORT_ENABLE_BASIC
|
||||
if level == 'extended':
|
||||
# Optimizations using custom operators, excluding NCHWc and NHWC layout optimizers
|
||||
return ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
|
||||
if level == 'all':
|
||||
return ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
|
||||
raise ValueError('Invalid optimization level of ' + level)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
os.path.basename(__file__),
|
||||
description='''Convert the ONNX format model/s in the provided directory to ORT format models.
|
||||
All files with a `.onnx` extension will be processed. For each one, an ORT format model will be created in the
|
||||
same directory. A configuration file will also be created called `required_operators.config`, and will contain
|
||||
the list of required operators for all converted models.
|
||||
This configuration file should be used as input to the minimal build via the `--include_ops_by_config`
|
||||
parameter.
|
||||
'''
|
||||
)
|
||||
|
||||
parser.add_argument('--use_nnapi', action='store_true',
|
||||
help='Enable the NNAPI Execution Provider when creating models and determining required '
|
||||
'operators. Note that this will limit the optimizations possible on nodes that the '
|
||||
'NNAPI execution provider takes, in order to preserve those nodes in the ORT format '
|
||||
'model.')
|
||||
|
||||
parser.add_argument('--optimization_level', default='all',
|
||||
choices=['disable', 'basic', 'extended', 'all'],
|
||||
help="Level to optimize ONNX model with, prior to converting to ORT format model. "
|
||||
"These map to the onnxruntime.GraphOptimizationLevel values. "
|
||||
"If the level is 'all' the NCHWc transformer is manually disabled as it contains device "
|
||||
"specific logic, so the ORT format model must be generated on the device it will run on. "
|
||||
"Additionally, the NCHWc optimizations are not applicable to ARM devices."
|
||||
)
|
||||
|
||||
parser.add_argument('--enable_type_reduction', action='store_true',
|
||||
help='Add operator specific type information to the configuration file to potentially reduce '
|
||||
'the types supported by individual operator implementations.')
|
||||
|
||||
parser.add_argument('--custom_op_library', type=pathlib.Path, default=None,
|
||||
help='Provide path to shared library containing custom operator kernels to register.')
|
||||
|
||||
parser.add_argument('--save_optimized_onnx_model', action='store_true',
|
||||
help='Save the optimized version of each ONNX model. '
|
||||
'This will have the same optimizations applied as the ORT format model.')
|
||||
|
||||
parser.add_argument('model_path_or_dir', type=pathlib.Path,
|
||||
help='Provide path to ONNX model or directory containing ONNX model/s to convert. '
|
||||
'All files with a .onnx extension, including in subdirectories, will be processed.')
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def convert_onnx_models_to_ort():
|
||||
args = parse_args()
|
||||
|
||||
model_path_or_dir = args.model_path_or_dir.resolve()
|
||||
custom_op_library = args.custom_op_library.resolve() if args.custom_op_library else None
|
||||
|
||||
if not model_path_or_dir.is_dir() and not model_path_or_dir.is_file():
|
||||
raise FileNotFoundError("Model path '{}' is not a file or directory.".format(model_path_or_dir))
|
||||
|
||||
if custom_op_library and not custom_op_library.is_file():
|
||||
raise FileNotFoundError("Unable to find custom operator library '{}'".format(custom_op_library))
|
||||
|
||||
if args.use_nnapi and 'NnapiExecutionProvider' not in ort.get_available_providers():
|
||||
raise ValueError('The NNAPI Execution Provider was not included in this build of ONNX Runtime.')
|
||||
|
||||
_convert(model_path_or_dir, args.optimization_level, args.use_nnapi, custom_op_library,
|
||||
args.save_optimized_onnx_model)
|
||||
|
||||
_create_config_file_from_ort_models(model_path_or_dir, args.enable_type_reduction)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
convert_onnx_models_to_ort()
|
||||
|
|
@ -4,11 +4,19 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
# need to add the path to the ORT flatbuffers python module before we import anything else here
|
||||
script_path = os.path.dirname(os.path.realpath(__file__))
|
||||
ort_root = os.path.join(script_path, '..', '..', '..', '..')
|
||||
ort_fbs_py_path = os.path.abspath(os.path.join(ort_root, 'onnxruntime', 'core', 'flatbuffers'))
|
||||
sys.path.append(ort_fbs_py_path)
|
||||
# need to add the path to the ORT flatbuffers python module before we import anything else here.
|
||||
# we also auto-magically adjust to whether we're running from the ORT repo, or from within the ORT python package
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
fbs_py_schema_dirname = 'ort_flatbuffers_py'
|
||||
if os.path.isdir(os.path.join(script_dir, fbs_py_schema_dirname)):
|
||||
# fbs bindings are in this directory, so we're running in the ORT python package
|
||||
ort_fbs_py_parent_dir = script_dir
|
||||
else:
|
||||
# running directly from ORT repo, so fbs bindings are under onnxruntime/core/flatbuffers
|
||||
ort_root = os.path.abspath(os.path.join(script_dir, '..', '..', '..', '..'))
|
||||
ort_fbs_py_parent_dir = os.path.join(ort_root, 'onnxruntime', 'core', 'flatbuffers')
|
||||
|
||||
sys.path.append(ort_fbs_py_parent_dir)
|
||||
|
||||
from .utils import create_config_from_models # noqa
|
||||
from .ort_model_processor import OrtFormatModelProcessor # noqa
|
||||
|
|
|
|||
Loading…
Reference in a new issue