Add python binding for CoreML EP (#8472)

* add pybind binding for coreml ep

* update merged files

* address comments

* format

* remove lines for non-macOS platform

Co-authored-by: rachguo <rachguo@rachguos-Mini.attlocal.net>

cmake/onnxruntime_python.cmake

@@ -82,6 +82,7 @@ target_link_libraries(onnxruntime_pybind11_state PRIVATE
     ${PROVIDERS_NUPHAR}
     ${PROVIDERS_VITISAI}
     ${PROVIDERS_NNAPI}
+    ${PROVIDERS_COREML}
     ${PROVIDERS_RKNPU}
     ${PROVIDERS_DML}
     ${PROVIDERS_ACL}
@@ -522,6 +523,15 @@ if (onnxruntime_USE_NNAPI_BUILTIN)
         $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/capi/
   )
 endif()
+
+if (onnxruntime_USE_COREML)
+  add_custom_command(
+    TARGET onnxruntime_pybind11_state POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy
+        $<TARGET_FILE:onnxruntime_providers_coreml>
+        $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/capi/
+  )
+endif()
 endif()
 if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
   include(onnxruntime_language_interop_ops.cmake)

onnxruntime/core/providers/get_execution_providers.cc

@@ -87,6 +87,14 @@ constexpr ProviderInfo kProvidersInPriorityOrder[] =
             true,
 #else
             false,
+#endif
+        },
+        {
+            kCoreMLExecutionProvider,
+#ifdef USE_COREML
+            true,
+#else
+            false,
 #endif
         },
         {

onnxruntime/python/onnxruntime_pybind_schema.cc

@@ -73,6 +73,9 @@ void addGlobalSchemaFunctions(pybind11::module& m) {
 #endif
 #ifdef USE_RKNPU
             onnxruntime::CreateExecutionProviderFactory_Rknpu(),
+#endif
+#ifdef USE_COREML
+            onnxruntime::CreateExecutionProviderFactory_CoreML(0),
 #endif
         };
 

onnxruntime/python/onnxruntime_pybind_state.cc

@@ -667,6 +667,10 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
     } else if (type == kRknpuExecutionProvider) {
 #ifdef USE_RKNPU
       RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_Rknpu());
+#endif
+    } else if (type == kCoreMLExecutionProvider) {
+#if defined(USE_COREML)
+      RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_CoreML(0));
 #endif
     } else {
       // check whether it is a dynamic load EP:

onnxruntime/python/onnxruntime_pybind_state_common.h

@@ -452,5 +452,6 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_DML(in
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nnapi(
     uint32_t flags, const optional<std::string>& partitioning_stop_ops_list);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Rknpu();
+std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CoreML(uint32_t flags);
 
 }  // namespace onnxruntime

tools/python/util/convert_onnx_models_to_ort.py

@@ -5,6 +5,7 @@ import argparse
 import os
 import pathlib
 import typing
+import sys
 
 import onnxruntime as ort
 from .ort_format_model import create_config_from_models
@@ -52,7 +53,7 @@ def _create_session_options(optimization_level: ort.GraphOptimizationLevel,
     return so
 
 
-def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_nnapi: bool,
+def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_nnapi: bool, use_coreml: bool,
              custom_op_library: pathlib.Path, create_optimized_onnx_model: bool):
 
     optimization_level = _get_optimization_level(optimization_level_str)
@@ -73,6 +74,9 @@ def _convert(model_path_or_dir: pathlib.Path, optimization_level_str: str, use_n
     if use_nnapi:
         # providers are priority based, so register NNAPI first
         providers.insert(0, 'NnapiExecutionProvider')
+    if use_coreml:
+        # providers are priority based, so register CoreML first
+        providers.insert(0, 'CoreMLExecutionProvider')
 
     # if the optimization level is 'all' we manually exclude the NCHWc transformer. It's not applicable to ARM
     # devices, and creates a device specific model which won't run on all hardware.
@@ -140,6 +144,10 @@ def _get_optimization_level(level):
     raise ValueError('Invalid optimization level of ' + level)
 
 
+def is_macOS():
+    return sys.platform.startswith("darwin")
+
+
 def parse_args():
     parser = argparse.ArgumentParser(
         os.path.basename(__file__),
@@ -158,6 +166,12 @@ def parse_args():
                              'NNAPI execution provider takes, in order to preserve those nodes in the ORT format '
                              'model.')
 
+    parser.add_argument('--use_coreml', action='store_true',
+                        help='Enable the CoreML Execution Provider when creating models and determining required '
+                             'operators. Note that this will limit the optimizations possible on nodes that the '
+                             'CoreML execution provider takes, in order to preserve those nodes in the ORT format '
+                             'model.')
+
     parser.add_argument('--optimization_level', default='all',
                         choices=['disable', 'basic', 'extended', 'all'],
                         help="Level to optimize ONNX model with, prior to converting to ORT format model. "
@@ -200,7 +214,14 @@ def convert_onnx_models_to_ort():
     if args.use_nnapi and 'NnapiExecutionProvider' not in ort.get_available_providers():
         raise ValueError('The NNAPI Execution Provider was not included in this build of ONNX Runtime.')
 
-    _convert(model_path_or_dir, args.optimization_level, args.use_nnapi, custom_op_library,
+    if args.use_coreml:
+        if not is_macOS():
+            # Check if the script is run on a Mac Device in this case
+            raise ValueError('--use_coreml option requires a MacOS environment.')
+        if 'CoreMLExecutionProvider' not in ort.get_available_providers():
+            raise ValueError('The CoreML Execution Provider was not included in this build of ONNX Runtime.')
+
+    _convert(model_path_or_dir, args.optimization_level, args.use_nnapi, args.use_coreml, custom_op_library,
              args.save_optimized_onnx_model)
 
     _create_config_file_from_ort_models(model_path_or_dir, args.optimization_level, args.enable_type_reduction)