From 0cf2ed029be11a4c63eae71b5548c4c2c87cb7cf Mon Sep 17 00:00:00 2001 From: Rachel Guo <35738743+YUNQIUGUO@users.noreply.github.com> Date: Thu, 29 Jul 2021 10:06:47 -0700 Subject: [PATCH] Add python binding for CoreML EP (#8472) * add pybind binding for coreml ep * update merged files * address comments * format * remove lines for non-macOS platform Co-authored-by: rachguo --- cmake/onnxruntime_python.cmake | 10 ++++++++ .../core/providers/get_execution_providers.cc | 8 ++++++ .../python/onnxruntime_pybind_schema.cc | 3 +++ .../python/onnxruntime_pybind_state.cc | 4 +++ .../python/onnxruntime_pybind_state_common.h | 1 + .../python/util/convert_onnx_models_to_ort.py | 25 +++++++++++++++++-- 6 files changed, 49 insertions(+), 2 deletions(-) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 0760533fe6..535b407a22 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -82,6 +82,7 @@ target_link_libraries(onnxruntime_pybind11_state PRIVATE ${PROVIDERS_NUPHAR} ${PROVIDERS_VITISAI} ${PROVIDERS_NNAPI} + ${PROVIDERS_COREML} ${PROVIDERS_RKNPU} ${PROVIDERS_DML} ${PROVIDERS_ACL} @@ -522,6 +523,15 @@ if (onnxruntime_USE_NNAPI_BUILTIN) $/onnxruntime/capi/ ) endif() + +if (onnxruntime_USE_COREML) + add_custom_command( + TARGET onnxruntime_pybind11_state POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + $ + $/onnxruntime/capi/ + ) +endif() endif() if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) include(onnxruntime_language_interop_ops.cmake) diff --git a/onnxruntime/core/providers/get_execution_providers.cc b/onnxruntime/core/providers/get_execution_providers.cc index 866d71feef..78239625b4 100644 --- a/onnxruntime/core/providers/get_execution_providers.cc +++ b/onnxruntime/core/providers/get_execution_providers.cc @@ -87,6 +87,14 @@ constexpr ProviderInfo kProvidersInPriorityOrder[] = true, #else false, +#endif + }, + { + kCoreMLExecutionProvider, +#ifdef USE_COREML + true, +#else + false, #endif }, { diff --git a/onnxruntime/python/onnxruntime_pybind_schema.cc b/onnxruntime/python/onnxruntime_pybind_schema.cc index db8083ab7d..9caddaf6fb 100644 --- a/onnxruntime/python/onnxruntime_pybind_schema.cc +++ b/onnxruntime/python/onnxruntime_pybind_schema.cc @@ -73,6 +73,9 @@ void addGlobalSchemaFunctions(pybind11::module& m) { #endif #ifdef USE_RKNPU onnxruntime::CreateExecutionProviderFactory_Rknpu(), +#endif +#ifdef USE_COREML + onnxruntime::CreateExecutionProviderFactory_CoreML(0), #endif }; diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 4b926c4ae7..da983aacea 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -667,6 +667,10 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector } else if (type == kRknpuExecutionProvider) { #ifdef USE_RKNPU RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_Rknpu()); +#endif + } else if (type == kCoreMLExecutionProvider) { +#if defined(USE_COREML) + RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_CoreML(0)); #endif } else { // check whether it is a dynamic load EP: diff --git a/onnxruntime/python/onnxruntime_pybind_state_common.h b/onnxruntime/python/onnxruntime_pybind_state_common.h index d5ca023e63..47645642ed 100644 --- a/onnxruntime/python/onnxruntime_pybind_state_common.h +++ b/onnxruntime/python/onnxruntime_pybind_state_common.h @@ -452,5 +452,6 @@ std::shared_ptr CreateExecutionProviderFactory_DML(in std::shared_ptr CreateExecutionProviderFactory_Nnapi( uint32_t flags, const optional& partitioning_stop_ops_list); std::shared_ptr CreateExecutionProviderFactory_Rknpu(); +std::shared_ptr CreateExecutionProviderFactory_CoreML(uint32_t flags); } // namespace onnxruntime diff --git a/tools/python/util/convert_onnx_models_to_ort.py b/tools/python/util/convert_onnx_models_to_ort.py index dd2449ff08..8b515b95b7 100644 --- a/tools/python/util/convert_onnx_models_to_ort.py +++ b/tools/python/util/convert_onnx_models_to_ort.py @@ -5,6 +5,7 @@ import argparse import os import pathlib import typing +import sys import onnxruntime as ort from .ort_format_model import create_config_from_models @@ -52,7 +53,7 @@ def _create_session_options(optimization_level: ort.GraphOptimizationLevel, return so -def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_nnapi: bool, +def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_nnapi: bool, use_coreml: bool, custom_op_library: pathlib.Path, create_optimized_onnx_model: bool): optimization_level = _get_optimization_level(optimization_level_str) @@ -73,6 +74,9 @@ def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_n if use_nnapi: # providers are priority based, so register NNAPI first providers.insert(0, 'NnapiExecutionProvider') + if use_coreml: + # providers are priority based, so register CoreML first + providers.insert(0, 'CoreMLExecutionProvider') # if the optimization level is 'all' we manually exclude the NCHWc transformer. It's not applicable to ARM # devices, and creates a device specific model which won't run on all hardware. @@ -140,6 +144,10 @@ def _get_optimization_level(level): raise ValueError('Invalid optimization level of ' + level) +def is_macOS(): + return sys.platform.startswith("darwin") + + def parse_args(): parser = argparse.ArgumentParser( os.path.basename(__file__), @@ -158,6 +166,12 @@ def parse_args(): 'NNAPI execution provider takes, in order to preserve those nodes in the ORT format ' 'model.') + parser.add_argument('--use_coreml', action='store_true', + help='Enable the CoreML Execution Provider when creating models and determining required ' + 'operators. Note that this will limit the optimizations possible on nodes that the ' + 'CoreML execution provider takes, in order to preserve those nodes in the ORT format ' + 'model.') + parser.add_argument('--optimization_level', default='all', choices=['disable', 'basic', 'extended', 'all'], help="Level to optimize ONNX model with, prior to converting to ORT format model. " @@ -200,7 +214,14 @@ def convert_onnx_models_to_ort(): if args.use_nnapi and 'NnapiExecutionProvider' not in ort.get_available_providers(): raise ValueError('The NNAPI Execution Provider was not included in this build of ONNX Runtime.') - _convert(model_path_or_dir, args.optimization_level, args.use_nnapi, custom_op_library, + if args.use_coreml: + if not is_macOS(): + # Check if the script is run on a Mac Device in this case + raise ValueError('--use_coreml option requires a MacOS environment.') + if 'CoreMLExecutionProvider' not in ort.get_available_providers(): + raise ValueError('The CoreML Execution Provider was not included in this build of ONNX Runtime.') + + _convert(model_path_or_dir, args.optimization_level, args.use_nnapi, args.use_coreml, custom_op_library, args.save_optimized_onnx_model) _create_config_file_from_ort_models(model_path_or_dir, args.optimization_level, args.enable_type_reduction)