[CoreML EP] Add CI for CoreML EP (macOS) and add coreml_flags for EP options (#6481)

* Add macos coreml CI and coreml_flags

* Move save debuggubg model to use environment var

* Move pipeline off from macos CI template

* Fix an issue building using unix make, add parallel to build script

* Fixed build break for shared_lib and cmpile warning

* Fix a compile warning

* test

* Revert the accidental push from another branch

This reverts commit 472029ba25d50f9508474c9eeceb3454cead7877.

cmake/onnxruntime_providers.cmake

@@ -656,6 +656,7 @@ if (onnxruntime_USE_COREML)
   target_include_directories(onnxruntime_coreml_proto PUBLIC $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_INCLUDE_DIRECTORIES> "${CMAKE_CURRENT_BINARY_DIR}")
   target_compile_definitions(onnxruntime_coreml_proto PUBLIC $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_COMPILE_DEFINITIONS>)
   set_target_properties(onnxruntime_coreml_proto PROPERTIES COMPILE_FLAGS "-fvisibility=hidden")
+  set_target_properties(onnxruntime_coreml_proto PROPERTIES COMPILE_FLAGS "-fvisibility-inlines-hidden")
   set(_src_sub_dir "coreml/")
   onnxruntime_protobuf_generate(
     APPEND_PATH
@@ -694,7 +695,7 @@ if (onnxruntime_USE_COREML)
   source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_coreml_cc_srcs})
   add_library(onnxruntime_providers_coreml ${onnxruntime_providers_coreml_cc_srcs} ${onnxruntime_providers_coreml_objcc_srcs})
   onnxruntime_add_include_to_target(onnxruntime_providers_coreml onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf-lite flatbuffers onnxruntime_coreml_proto)
-  target_link_libraries(onnxruntime_providers_coreml "-framework Foundation" "-framework CoreML")
+  target_link_libraries(onnxruntime_providers_coreml PRIVATE onnxruntime_coreml_proto "-framework Foundation" "-framework CoreML")
   add_dependencies(onnxruntime_providers_coreml onnx onnxruntime_coreml_proto ${onnxruntime_EXTERNAL_DEPENDENCIES})
   set_target_properties(onnxruntime_providers_coreml PROPERTIES CXX_STANDARD_REQUIRED ON)
   set_target_properties(onnxruntime_providers_coreml PROPERTIES FOLDER "ONNXRuntime")

include/onnxruntime/core/providers/coreml/coreml_provider_factory.h

@@ -4,11 +4,32 @@
 
 #include "onnxruntime_c_api.h"
 
+// COREMLFlags are bool options we want to set for CoreML EP
+// This enum is defined as bit flats, and cannot have negative value
+// To generate an uint32_t coreml_flags for using with OrtSessionOptionsAppendExecutionProvider_CoreML below,
+//   uint32_t coreml_flags = 0;
+//   coreml_flags |= COREML_FLAG_USE_CPU_ONLY;
+enum COREMLFlags {
+  COREML_FLAG_USE_NONE = 0x000,
+
+  // Using CPU only in CoreML EP, this may decrease the perf but will provide
+  // reference output value without precision loss, which is useful for validation
+  COREML_FLAG_USE_CPU_ONLY = 0x001,
+
+  // Enable CoreML EP on subgraph
+  COREML_FLAG_ENABLE_ON_SUBGRAPH = 0x002,
+
+  // Keep COREML_FLAG_MAX at the end of the enum definition
+  // And assign the last COREMLFlag to it
+  COREML_FLAG_LAST = COREML_FLAG_ENABLE_ON_SUBGRAPH,
+};
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CoreML, _In_ OrtSessionOptions* options);
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CoreML,
+               _In_ OrtSessionOptions* options, uint32_t coreml_flags);
 
 #ifdef __cplusplus
 }

onnxruntime/core/providers/coreml/builders/helper.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <stdint.h>
+#include <functional>
 
 namespace onnxruntime {
 

onnxruntime/core/providers/coreml/builders/model_builder.cc

@@ -15,9 +15,10 @@
 namespace onnxruntime {
 namespace coreml {
 
-ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const logging::Logger& logger)
+ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const logging::Logger& logger, uint32_t coreml_flags)
     : graph_viewer_(graph_viewer),
-      logger_(logger) {
+      logger_(logger),
+      coreml_flags_(coreml_flags) {
 }
 
 Status ModelBuilder::Initialize() {
@@ -191,7 +192,7 @@ Status ModelBuilder::RegisterModelOutputs() {
 
 Status ModelBuilder::Compile(std::unique_ptr<Model>& model, const std::string& path) {
   ORT_RETURN_IF_ERROR(SaveCoreMLModel(path));
-  model.reset(new Model(path, logger_));
+  model.reset(new Model(path, logger_, coreml_flags_));
   model->SetScalarOutputs(std::move(scalar_outputs_));
   model->SetInputOutputInfo(std::move(input_output_info_));
   return model->LoadModel();
@@ -202,9 +203,11 @@ Status ModelBuilder::SaveCoreMLModel(const std::string& path) {
   std::ofstream stream(path, std::ofstream::out | std::ofstream::binary);
   ORT_RETURN_IF_NOT(coreml_model_->SerializeToOstream(&stream), "Save the CoreML model failed");
 
-  // Delete, debug only
-  std::ofstream temp_stream("/Users/gwang/temp/aaa.mlmodel", std::ofstream::out | std::ofstream::binary);
-  ORT_RETURN_IF_NOT(coreml_model_->SerializeToOstream(&temp_stream), "Save the CoreML model failed");
+  // TODO, Delete, debug only
+  if (const char* path = std::getenv("ORT_COREML_EP_CONVERTED_MODEL_PATH")) {
+    std::ofstream temp_stream(path, std::ofstream::out | std::ofstream::binary);
+    ORT_RETURN_IF_NOT(coreml_model_->SerializeToOstream(&temp_stream), "Save the CoreML model failed");
+  }
 
   return Status::OK();
 }

onnxruntime/core/providers/coreml/builders/model_builder.h

@@ -17,7 +17,7 @@ struct OnnxTensorInfo;
 
 class ModelBuilder {
  public:
-  ModelBuilder(const GraphViewer& graph_viewer, const logging::Logger& logger);
+  ModelBuilder(const GraphViewer& graph_viewer, const logging::Logger& logger, uint32_t coreml_flags);
   ~ModelBuilder() = default;
 
   Status Compile(std::unique_ptr<Model>& model, const std::string& path) ORT_MUST_USE_RESULT;
@@ -37,6 +37,7 @@ class ModelBuilder {
  private:
   const GraphViewer& graph_viewer_;
   const logging::Logger& logger_;
+  uint32_t coreml_flags_;
 
   std::unique_ptr<CoreML::Specification::Model> coreml_model_;
   std::unordered_set<std::string> scalar_outputs_;

onnxruntime/core/providers/coreml/coreml_execution_provider.cc

@@ -17,8 +17,9 @@ namespace onnxruntime {
 
 constexpr const char* COREML = "CoreML";
 
-CoreMLExecutionProvider::CoreMLExecutionProvider()
-    : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider, true} {
+CoreMLExecutionProvider::CoreMLExecutionProvider(uint32_t coreml_flags)
+    : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider, true},
+      coreml_flags_(coreml_flags) {
   AllocatorCreationInfo device_info(
       [](int) {
         return onnxruntime::make_unique<CPUAllocator>(OrtMemoryInfo(COREML, OrtAllocatorType::OrtDeviceAllocator));
@@ -44,7 +45,7 @@ CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie
 
   // We do not run CoreML EP on subgraph, instead we cover this in the control flow nodes
   // TODO investigate whether we want to support subgraph using CoreML EP
-  if (graph_viewer.IsSubgraph()) {
+  if (graph_viewer.IsSubgraph() && !(coreml_flags_ & COREML_FLAG_ENABLE_ON_SUBGRAPH)) {
     return result;
   }
 
@@ -169,7 +170,7 @@ common::Status CoreMLExecutionProvider::Compile(const std::vector<FusedNodeAndGr
     Node& fused_node = fused_node_and_graph.fused_node;
     const onnxruntime::GraphViewer& graph_viewer(fused_node_and_graph.filtered_graph);
 
-    coreml::ModelBuilder builder(graph_viewer, *GetLogger());
+    coreml::ModelBuilder builder(graph_viewer, *GetLogger(), coreml_flags_);
     std::unique_ptr<coreml::Model> coreml_model;
     const std::string coreml_model_file_path = coreml::util::GetTemporaryFilePath();
     ORT_RETURN_IF_ERROR(builder.Compile(coreml_model, coreml_model_file_path));

onnxruntime/core/providers/coreml/coreml_execution_provider.h

@@ -13,7 +13,7 @@ class Model;
 
 class CoreMLExecutionProvider : public IExecutionProvider {
  public:
-  CoreMLExecutionProvider();
+  CoreMLExecutionProvider(uint32_t coreml_flags);
   virtual ~CoreMLExecutionProvider();
 
   std::vector<std::unique_ptr<ComputeCapability>>
@@ -28,6 +28,10 @@ class CoreMLExecutionProvider : public IExecutionProvider {
                          std::vector<NodeComputeInfo>& node_compute_funcs) override;
 #endif
 
+  // The bit flags which define bool options for COREML EP, bits are defined as
+  // COREMLFlags in include/onnxruntime/core/providers/coreml/coreml_provider_factory.h
+  const uint32_t coreml_flags_;
+
  private:
   // <fused_node_name, <coreml_model_file_path, compiled_coreml_model>>
   std::unordered_map<std::string, std::unique_ptr<onnxruntime::coreml::Model>> coreml_models_;

onnxruntime/core/providers/coreml/coreml_provider_factory.cc

@@ -9,22 +9,25 @@ using namespace onnxruntime;
 
 namespace onnxruntime {
 struct CoreMLProviderFactory : IExecutionProviderFactory {
-  CoreMLProviderFactory() {}
+  CoreMLProviderFactory(uint32_t coreml_flags)
+      : coreml_flags_(coreml_flags) {}
   ~CoreMLProviderFactory() override {}
 
   std::unique_ptr<IExecutionProvider> CreateProvider() override;
+  uint32_t coreml_flags_;
 };
 
 std::unique_ptr<IExecutionProvider> CoreMLProviderFactory::CreateProvider() {
-  return onnxruntime::make_unique<CoreMLExecutionProvider>();
+  return onnxruntime::make_unique<CoreMLExecutionProvider>(coreml_flags_);
 }
 
-std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CoreML() {
-  return std::make_shared<onnxruntime::CoreMLProviderFactory>();
+std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CoreML(uint32_t coreml_flags) {
+  return std::make_shared<onnxruntime::CoreMLProviderFactory>(coreml_flags);
 }
 }  // namespace onnxruntime
 
-ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CoreML, _In_ OrtSessionOptions* options) {
-  options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CoreML());
+ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CoreML,
+                    _In_ OrtSessionOptions* options, uint32_t coreml_flags) {
+  options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CoreML(coreml_flags));
   return nullptr;
 }

onnxruntime/core/providers/coreml/model/model.h

@@ -56,7 +56,7 @@ class Model {
 
   OrtMutex mutex_;
 
-  Model(const std::string& path, const logging::Logger& logger);
+  Model(const std::string& path, const logging::Logger& logger, uint32_t coreml_flags);
   onnxruntime::common::Status LoadModel();
 
   void SetInputOutputInfo(std::unordered_map<std::string, OnnxTensorInfo>&& input_output_info) {

onnxruntime/core/providers/coreml/model/model.mm

@@ -9,6 +9,7 @@
 #include "core/common/logging/logging.h"
 #include "core/graph/onnx_protobuf.h"
 #include "core/providers/coreml/builders/helper.h"
+#include "core/providers/coreml/coreml_provider_factory.h"
 #include "host_utils.h"
 #include "model.h"
 
@@ -38,10 +39,12 @@
   NSString* coreml_model_path_;
   NSString* compiled_model_path_;
   const onnxruntime::logging::Logger* logger_;
+  uint32_t coreml_flags_;
 }
 
 - (instancetype)initWithPath:(const std::string&)path
-                      logger:(const onnxruntime::logging::Logger&)logger;
+                      logger:(const onnxruntime::logging::Logger&)logger
+                coreml_flags:(uint32_t)coreml_flags;
 - (void)cleanup;
 - (void)dealloc;
 - (onnxruntime::common::Status)loadModel API_AVAILABLE_OS_VERSIONS;
@@ -129,10 +132,12 @@
 @implementation CoreMLExecution
 
 - (instancetype)initWithPath:(const std::string&)path
-                      logger:(const onnxruntime::logging::Logger&)logger {
+                      logger:(const onnxruntime::logging::Logger&)logger
+                coreml_flags:(uint32_t)coreml_flags {
   if (self = [super init]) {
     coreml_model_path_ = [NSString stringWithUTF8String:path.c_str()];
     logger_ = &logger;
+    coreml_flags_ = coreml_flags;
   }
   return self;
 }
@@ -202,8 +207,7 @@
   }
 
   MLPredictionOptions* options = [[MLPredictionOptions alloc] init];
-  // TODO add options
-  // options.usesCPUOnly = YES;
+  options.usesCPUOnly = coreml_flags_ & COREML_FLAG_USE_CPU_ONLY;
   NSError* error = nil;
   id<MLFeatureProvider> output_feature = [_model predictionFromFeatures:input_feature
                                                                 options:options
@@ -262,7 +266,7 @@ namespace coreml {
 // This class will bridge Model (c++) with CoreMLExecution (objective c++)
 class Execution {
  public:
-  Execution(const std::string& path, const logging::Logger& logger);
+  Execution(const std::string& path, const logging::Logger& logger, uint32_t coreml_flags);
   ~Execution(){};
 
   Status LoadModel();
@@ -274,8 +278,10 @@ class Execution {
   CoreMLExecution* execution_;
 };
 
-Execution::Execution(const std::string& path, const logging::Logger& logger) {
-  execution_ = [[CoreMLExecution alloc] initWithPath:path logger:logger];
+Execution::Execution(const std::string& path, const logging::Logger& logger, uint32_t coreml_flags) {
+  execution_ = [[CoreMLExecution alloc] initWithPath:path
+                                              logger:logger
+                                        coreml_flags:coreml_flags];
 }
 
 Status Execution::LoadModel() {
@@ -303,8 +309,8 @@ Status Execution::Predict(const std::unordered_map<std::string, OnnxTensorData>&
   return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Execution::LoadModel requires macos 10.15+ or ios 13+ ");
 }
 
-Model::Model(const std::string& path, const logging::Logger& logger)
-    : execution_(onnxruntime::make_unique<Execution>(path, logger)) {
+Model::Model(const std::string& path, const logging::Logger& logger, uint32_t coreml_flags)
+    : execution_(onnxruntime::make_unique<Execution>(path, logger, coreml_flags)) {
 }
 
 Model::~Model() {}

onnxruntime/test/framework/test_utils.cc

@@ -55,8 +55,8 @@ IExecutionProvider* TestRknpuExecutionProvider() {
 #endif
 
 #ifdef USE_COREML
-IExecutionProvider* TestCoreMLExecutionProvider() {
-  static CoreMLExecutionProvider coreml_provider;
+IExecutionProvider* TestCoreMLExecutionProvider(uint32_t coreml_flags) {
+  static CoreMLExecutionProvider coreml_provider(coreml_flags);
   return &coreml_provider;
 }
 #endif

onnxruntime/test/framework/test_utils.h

@@ -55,7 +55,7 @@ IExecutionProvider* TestRknpuExecutionProvider();
 #endif
 
 #ifdef USE_COREML
-IExecutionProvider* TestCoreMLExecutionProvider();
+IExecutionProvider* TestCoreMLExecutionProvider(uint32_t coreml_flags);
 #endif
 
 template <typename T>

onnxruntime/test/providers/coreml/coreml_basic_test.cc

@@ -3,6 +3,7 @@
 
 #include "core/common/logging/logging.h"
 #include "core/providers/coreml/coreml_execution_provider.h"
+#include "core/providers/coreml/coreml_provider_factory.h"
 #include "core/session/inference_session.h"
 #include "test/common/tensor_op_test_utils.h"
 #include "test/framework/test_utils.h"
@@ -65,13 +66,18 @@ TEST(CoreMLExecutionProviderTest, FunctionTest) {
   std::vector<int64_t> dims_mul_x = {1, 1, 3, 2};
   std::vector<float> values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
   OrtValue ml_value_x;
-  CreateMLValue<float>(TestCoreMLExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x,
+
+  // We want to run UT on CPU only to get output value without losing precision
+  uint32_t coreml_flags = 0;
+  coreml_flags |= COREML_FLAG_USE_CPU_ONLY;
+
+  CreateMLValue<float>(TestCoreMLExecutionProvider(coreml_flags)->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x,
                        &ml_value_x);
   OrtValue ml_value_y;
-  CreateMLValue<float>(TestCoreMLExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x,
+  CreateMLValue<float>(TestCoreMLExecutionProvider(coreml_flags)->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x,
                        &ml_value_y);
   OrtValue ml_value_z;
-  CreateMLValue<float>(TestCoreMLExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x,
+  CreateMLValue<float>(TestCoreMLExecutionProvider(coreml_flags)->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x,
                        &ml_value_z);
   NameMLValMap feeds;
   feeds.insert(std::make_pair("X", ml_value_x));
@@ -79,7 +85,7 @@ TEST(CoreMLExecutionProviderTest, FunctionTest) {
   feeds.insert(std::make_pair("Z", ml_value_z));
 
   RunAndVerifyOutputsWithEP(model_file_name, "CoreMLExecutionProviderTest.FunctionTest",
-                            onnxruntime::make_unique<CoreMLExecutionProvider>(),
+                            onnxruntime::make_unique<CoreMLExecutionProvider>(coreml_flags),
                             feeds);
 }
 

onnxruntime/test/util/default_providers.cc

@@ -10,6 +10,9 @@
 #ifdef USE_ROCM
 #include "core/providers/rocm/rocm_provider_factory_creator.h"
 #endif
+#ifdef USE_COREML
+#include "core/providers/coreml/coreml_provider_factory.h"
+#endif
 #include "core/session/onnxruntime_cxx_api.h"
 
 namespace onnxruntime {
@@ -26,7 +29,7 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tensor
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_MIGraphX(int device_id);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ACL(int use_arena);
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ArmNN(int use_arena);
-std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CoreML();
+std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CoreML(uint32_t);
 
 // EP for internal testing
 std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_InternalTesting(
@@ -136,7 +139,10 @@ std::unique_ptr<IExecutionProvider> DefaultRocmExecutionProvider() {
 
 std::unique_ptr<IExecutionProvider> DefaultCoreMLExecutionProvider() {
 #if defined(USE_COREML)
-  return CreateExecutionProviderFactory_CoreML()->CreateProvider();
+  // We want to run UT on CPU only to get output value without losing precision
+  uint32_t coreml_flags = 0;
+  coreml_flags |= COREML_FLAG_USE_CPU_ONLY;
+  return CreateExecutionProviderFactory_CoreML(coreml_flags)->CreateProvider();
 #else
   return nullptr;
 #endif

tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml

@@ -0,0 +1,19 @@
+jobs:
+- job: CoreML_CI
+  pool:
+    vmImage: 'macOS-10.15'
+  timeoutInMinutes: 120
+  steps:
+  - script: brew install coreutils ninja
+    displayName: Install coreutils and ninja
+
+  - script: |
+      python3 tools/ci_build/build.py \
+      --build_dir build \
+      --skip_submodule_sync \
+      --cmake_generator=Ninja \
+      --parallel \
+      --build_shared_lib \
+      --config Debug \
+      --use_coreml
+    displayName: CoreML EP, Build and Test on macOS