diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index f89ce5b4ec..716a4e2073 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -656,6 +656,7 @@ if (onnxruntime_USE_COREML) target_include_directories(onnxruntime_coreml_proto PUBLIC $ "${CMAKE_CURRENT_BINARY_DIR}") target_compile_definitions(onnxruntime_coreml_proto PUBLIC $) set_target_properties(onnxruntime_coreml_proto PROPERTIES COMPILE_FLAGS "-fvisibility=hidden") + set_target_properties(onnxruntime_coreml_proto PROPERTIES COMPILE_FLAGS "-fvisibility-inlines-hidden") set(_src_sub_dir "coreml/") onnxruntime_protobuf_generate( APPEND_PATH @@ -694,7 +695,7 @@ if (onnxruntime_USE_COREML) source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_coreml_cc_srcs}) add_library(onnxruntime_providers_coreml ${onnxruntime_providers_coreml_cc_srcs} ${onnxruntime_providers_coreml_objcc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_coreml onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf-lite flatbuffers onnxruntime_coreml_proto) - target_link_libraries(onnxruntime_providers_coreml "-framework Foundation" "-framework CoreML") + target_link_libraries(onnxruntime_providers_coreml PRIVATE onnxruntime_coreml_proto "-framework Foundation" "-framework CoreML") add_dependencies(onnxruntime_providers_coreml onnx onnxruntime_coreml_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) set_target_properties(onnxruntime_providers_coreml PROPERTIES CXX_STANDARD_REQUIRED ON) set_target_properties(onnxruntime_providers_coreml PROPERTIES FOLDER "ONNXRuntime") diff --git a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h index c9eddd6244..5acedfc12d 100644 --- a/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h +++ b/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h @@ -4,11 +4,32 @@ #include "onnxruntime_c_api.h" +// COREMLFlags are bool options we want to set for CoreML EP +// This enum is defined as bit flats, and cannot have negative value +// To generate an uint32_t coreml_flags for using with OrtSessionOptionsAppendExecutionProvider_CoreML below, +// uint32_t coreml_flags = 0; +// coreml_flags |= COREML_FLAG_USE_CPU_ONLY; +enum COREMLFlags { + COREML_FLAG_USE_NONE = 0x000, + + // Using CPU only in CoreML EP, this may decrease the perf but will provide + // reference output value without precision loss, which is useful for validation + COREML_FLAG_USE_CPU_ONLY = 0x001, + + // Enable CoreML EP on subgraph + COREML_FLAG_ENABLE_ON_SUBGRAPH = 0x002, + + // Keep COREML_FLAG_MAX at the end of the enum definition + // And assign the last COREMLFlag to it + COREML_FLAG_LAST = COREML_FLAG_ENABLE_ON_SUBGRAPH, +}; + #ifdef __cplusplus extern "C" { #endif -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CoreML, _In_ OrtSessionOptions* options); +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CoreML, + _In_ OrtSessionOptions* options, uint32_t coreml_flags); #ifdef __cplusplus } diff --git a/onnxruntime/core/providers/coreml/builders/helper.h b/onnxruntime/core/providers/coreml/builders/helper.h index 3ec8e010c0..9d9977d512 100644 --- a/onnxruntime/core/providers/coreml/builders/helper.h +++ b/onnxruntime/core/providers/coreml/builders/helper.h @@ -4,6 +4,7 @@ #pragma once #include +#include namespace onnxruntime { diff --git a/onnxruntime/core/providers/coreml/builders/model_builder.cc b/onnxruntime/core/providers/coreml/builders/model_builder.cc index 7dfe6fb66c..16e35ced4c 100644 --- a/onnxruntime/core/providers/coreml/builders/model_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/model_builder.cc @@ -15,9 +15,10 @@ namespace onnxruntime { namespace coreml { -ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const logging::Logger& logger) +ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const logging::Logger& logger, uint32_t coreml_flags) : graph_viewer_(graph_viewer), - logger_(logger) { + logger_(logger), + coreml_flags_(coreml_flags) { } Status ModelBuilder::Initialize() { @@ -191,7 +192,7 @@ Status ModelBuilder::RegisterModelOutputs() { Status ModelBuilder::Compile(std::unique_ptr& model, const std::string& path) { ORT_RETURN_IF_ERROR(SaveCoreMLModel(path)); - model.reset(new Model(path, logger_)); + model.reset(new Model(path, logger_, coreml_flags_)); model->SetScalarOutputs(std::move(scalar_outputs_)); model->SetInputOutputInfo(std::move(input_output_info_)); return model->LoadModel(); @@ -202,9 +203,11 @@ Status ModelBuilder::SaveCoreMLModel(const std::string& path) { std::ofstream stream(path, std::ofstream::out | std::ofstream::binary); ORT_RETURN_IF_NOT(coreml_model_->SerializeToOstream(&stream), "Save the CoreML model failed"); - // Delete, debug only - std::ofstream temp_stream("/Users/gwang/temp/aaa.mlmodel", std::ofstream::out | std::ofstream::binary); - ORT_RETURN_IF_NOT(coreml_model_->SerializeToOstream(&temp_stream), "Save the CoreML model failed"); + // TODO, Delete, debug only + if (const char* path = std::getenv("ORT_COREML_EP_CONVERTED_MODEL_PATH")) { + std::ofstream temp_stream(path, std::ofstream::out | std::ofstream::binary); + ORT_RETURN_IF_NOT(coreml_model_->SerializeToOstream(&temp_stream), "Save the CoreML model failed"); + } return Status::OK(); } diff --git a/onnxruntime/core/providers/coreml/builders/model_builder.h b/onnxruntime/core/providers/coreml/builders/model_builder.h index 9b7fee113f..6be3a0bb74 100644 --- a/onnxruntime/core/providers/coreml/builders/model_builder.h +++ b/onnxruntime/core/providers/coreml/builders/model_builder.h @@ -17,7 +17,7 @@ struct OnnxTensorInfo; class ModelBuilder { public: - ModelBuilder(const GraphViewer& graph_viewer, const logging::Logger& logger); + ModelBuilder(const GraphViewer& graph_viewer, const logging::Logger& logger, uint32_t coreml_flags); ~ModelBuilder() = default; Status Compile(std::unique_ptr& model, const std::string& path) ORT_MUST_USE_RESULT; @@ -37,6 +37,7 @@ class ModelBuilder { private: const GraphViewer& graph_viewer_; const logging::Logger& logger_; + uint32_t coreml_flags_; std::unique_ptr coreml_model_; std::unordered_set scalar_outputs_; diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc index 5f5222b7d8..0cd2a114ce 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.cc +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.cc @@ -17,8 +17,9 @@ namespace onnxruntime { constexpr const char* COREML = "CoreML"; -CoreMLExecutionProvider::CoreMLExecutionProvider() - : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider, true} { +CoreMLExecutionProvider::CoreMLExecutionProvider(uint32_t coreml_flags) + : IExecutionProvider{onnxruntime::kCoreMLExecutionProvider, true}, + coreml_flags_(coreml_flags) { AllocatorCreationInfo device_info( [](int) { return onnxruntime::make_unique(OrtMemoryInfo(COREML, OrtAllocatorType::OrtDeviceAllocator)); @@ -44,7 +45,7 @@ CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie // We do not run CoreML EP on subgraph, instead we cover this in the control flow nodes // TODO investigate whether we want to support subgraph using CoreML EP - if (graph_viewer.IsSubgraph()) { + if (graph_viewer.IsSubgraph() && !(coreml_flags_ & COREML_FLAG_ENABLE_ON_SUBGRAPH)) { return result; } @@ -169,7 +170,7 @@ common::Status CoreMLExecutionProvider::Compile(const std::vector coreml_model; const std::string coreml_model_file_path = coreml::util::GetTemporaryFilePath(); ORT_RETURN_IF_ERROR(builder.Compile(coreml_model, coreml_model_file_path)); diff --git a/onnxruntime/core/providers/coreml/coreml_execution_provider.h b/onnxruntime/core/providers/coreml/coreml_execution_provider.h index f07af1215e..6e26e1e84c 100644 --- a/onnxruntime/core/providers/coreml/coreml_execution_provider.h +++ b/onnxruntime/core/providers/coreml/coreml_execution_provider.h @@ -13,7 +13,7 @@ class Model; class CoreMLExecutionProvider : public IExecutionProvider { public: - CoreMLExecutionProvider(); + CoreMLExecutionProvider(uint32_t coreml_flags); virtual ~CoreMLExecutionProvider(); std::vector> @@ -28,6 +28,10 @@ class CoreMLExecutionProvider : public IExecutionProvider { std::vector& node_compute_funcs) override; #endif + // The bit flags which define bool options for COREML EP, bits are defined as + // COREMLFlags in include/onnxruntime/core/providers/coreml/coreml_provider_factory.h + const uint32_t coreml_flags_; + private: // > std::unordered_map> coreml_models_; diff --git a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc index 5a6b23b1b6..6e74dd54a6 100644 --- a/onnxruntime/core/providers/coreml/coreml_provider_factory.cc +++ b/onnxruntime/core/providers/coreml/coreml_provider_factory.cc @@ -9,22 +9,25 @@ using namespace onnxruntime; namespace onnxruntime { struct CoreMLProviderFactory : IExecutionProviderFactory { - CoreMLProviderFactory() {} + CoreMLProviderFactory(uint32_t coreml_flags) + : coreml_flags_(coreml_flags) {} ~CoreMLProviderFactory() override {} std::unique_ptr CreateProvider() override; + uint32_t coreml_flags_; }; std::unique_ptr CoreMLProviderFactory::CreateProvider() { - return onnxruntime::make_unique(); + return onnxruntime::make_unique(coreml_flags_); } -std::shared_ptr CreateExecutionProviderFactory_CoreML() { - return std::make_shared(); +std::shared_ptr CreateExecutionProviderFactory_CoreML(uint32_t coreml_flags) { + return std::make_shared(coreml_flags); } } // namespace onnxruntime -ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CoreML, _In_ OrtSessionOptions* options) { - options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CoreML()); +ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_CoreML, + _In_ OrtSessionOptions* options, uint32_t coreml_flags) { + options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_CoreML(coreml_flags)); return nullptr; } diff --git a/onnxruntime/core/providers/coreml/model/model.h b/onnxruntime/core/providers/coreml/model/model.h index 3d3a6cb16f..e5bb246679 100644 --- a/onnxruntime/core/providers/coreml/model/model.h +++ b/onnxruntime/core/providers/coreml/model/model.h @@ -56,7 +56,7 @@ class Model { OrtMutex mutex_; - Model(const std::string& path, const logging::Logger& logger); + Model(const std::string& path, const logging::Logger& logger, uint32_t coreml_flags); onnxruntime::common::Status LoadModel(); void SetInputOutputInfo(std::unordered_map&& input_output_info) { diff --git a/onnxruntime/core/providers/coreml/model/model.mm b/onnxruntime/core/providers/coreml/model/model.mm index 9b1eb30d28..781814f89e 100644 --- a/onnxruntime/core/providers/coreml/model/model.mm +++ b/onnxruntime/core/providers/coreml/model/model.mm @@ -9,6 +9,7 @@ #include "core/common/logging/logging.h" #include "core/graph/onnx_protobuf.h" #include "core/providers/coreml/builders/helper.h" +#include "core/providers/coreml/coreml_provider_factory.h" #include "host_utils.h" #include "model.h" @@ -38,10 +39,12 @@ NSString* coreml_model_path_; NSString* compiled_model_path_; const onnxruntime::logging::Logger* logger_; + uint32_t coreml_flags_; } - (instancetype)initWithPath:(const std::string&)path - logger:(const onnxruntime::logging::Logger&)logger; + logger:(const onnxruntime::logging::Logger&)logger + coreml_flags:(uint32_t)coreml_flags; - (void)cleanup; - (void)dealloc; - (onnxruntime::common::Status)loadModel API_AVAILABLE_OS_VERSIONS; @@ -129,10 +132,12 @@ @implementation CoreMLExecution - (instancetype)initWithPath:(const std::string&)path - logger:(const onnxruntime::logging::Logger&)logger { + logger:(const onnxruntime::logging::Logger&)logger + coreml_flags:(uint32_t)coreml_flags { if (self = [super init]) { coreml_model_path_ = [NSString stringWithUTF8String:path.c_str()]; logger_ = &logger; + coreml_flags_ = coreml_flags; } return self; } @@ -202,8 +207,7 @@ } MLPredictionOptions* options = [[MLPredictionOptions alloc] init]; - // TODO add options - // options.usesCPUOnly = YES; + options.usesCPUOnly = coreml_flags_ & COREML_FLAG_USE_CPU_ONLY; NSError* error = nil; id output_feature = [_model predictionFromFeatures:input_feature options:options @@ -262,7 +266,7 @@ namespace coreml { // This class will bridge Model (c++) with CoreMLExecution (objective c++) class Execution { public: - Execution(const std::string& path, const logging::Logger& logger); + Execution(const std::string& path, const logging::Logger& logger, uint32_t coreml_flags); ~Execution(){}; Status LoadModel(); @@ -274,8 +278,10 @@ class Execution { CoreMLExecution* execution_; }; -Execution::Execution(const std::string& path, const logging::Logger& logger) { - execution_ = [[CoreMLExecution alloc] initWithPath:path logger:logger]; +Execution::Execution(const std::string& path, const logging::Logger& logger, uint32_t coreml_flags) { + execution_ = [[CoreMLExecution alloc] initWithPath:path + logger:logger + coreml_flags:coreml_flags]; } Status Execution::LoadModel() { @@ -303,8 +309,8 @@ Status Execution::Predict(const std::unordered_map& return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Execution::LoadModel requires macos 10.15+ or ios 13+ "); } -Model::Model(const std::string& path, const logging::Logger& logger) - : execution_(onnxruntime::make_unique(path, logger)) { +Model::Model(const std::string& path, const logging::Logger& logger, uint32_t coreml_flags) + : execution_(onnxruntime::make_unique(path, logger, coreml_flags)) { } Model::~Model() {} diff --git a/onnxruntime/test/framework/test_utils.cc b/onnxruntime/test/framework/test_utils.cc index 8fa6110837..b1da9ec940 100644 --- a/onnxruntime/test/framework/test_utils.cc +++ b/onnxruntime/test/framework/test_utils.cc @@ -55,8 +55,8 @@ IExecutionProvider* TestRknpuExecutionProvider() { #endif #ifdef USE_COREML -IExecutionProvider* TestCoreMLExecutionProvider() { - static CoreMLExecutionProvider coreml_provider; +IExecutionProvider* TestCoreMLExecutionProvider(uint32_t coreml_flags) { + static CoreMLExecutionProvider coreml_provider(coreml_flags); return &coreml_provider; } #endif diff --git a/onnxruntime/test/framework/test_utils.h b/onnxruntime/test/framework/test_utils.h index a465edce37..8d8fd8294d 100644 --- a/onnxruntime/test/framework/test_utils.h +++ b/onnxruntime/test/framework/test_utils.h @@ -55,7 +55,7 @@ IExecutionProvider* TestRknpuExecutionProvider(); #endif #ifdef USE_COREML -IExecutionProvider* TestCoreMLExecutionProvider(); +IExecutionProvider* TestCoreMLExecutionProvider(uint32_t coreml_flags); #endif template diff --git a/onnxruntime/test/providers/coreml/coreml_basic_test.cc b/onnxruntime/test/providers/coreml/coreml_basic_test.cc index 6869aa175d..600f62a24f 100644 --- a/onnxruntime/test/providers/coreml/coreml_basic_test.cc +++ b/onnxruntime/test/providers/coreml/coreml_basic_test.cc @@ -3,6 +3,7 @@ #include "core/common/logging/logging.h" #include "core/providers/coreml/coreml_execution_provider.h" +#include "core/providers/coreml/coreml_provider_factory.h" #include "core/session/inference_session.h" #include "test/common/tensor_op_test_utils.h" #include "test/framework/test_utils.h" @@ -65,13 +66,18 @@ TEST(CoreMLExecutionProviderTest, FunctionTest) { std::vector dims_mul_x = {1, 1, 3, 2}; std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; OrtValue ml_value_x; - CreateMLValue(TestCoreMLExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, + + // We want to run UT on CPU only to get output value without losing precision + uint32_t coreml_flags = 0; + coreml_flags |= COREML_FLAG_USE_CPU_ONLY; + + CreateMLValue(TestCoreMLExecutionProvider(coreml_flags)->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_x); OrtValue ml_value_y; - CreateMLValue(TestCoreMLExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, + CreateMLValue(TestCoreMLExecutionProvider(coreml_flags)->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_y); OrtValue ml_value_z; - CreateMLValue(TestCoreMLExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, + CreateMLValue(TestCoreMLExecutionProvider(coreml_flags)->GetAllocator(0, OrtMemTypeDefault), dims_mul_x, values_mul_x, &ml_value_z); NameMLValMap feeds; feeds.insert(std::make_pair("X", ml_value_x)); @@ -79,7 +85,7 @@ TEST(CoreMLExecutionProviderTest, FunctionTest) { feeds.insert(std::make_pair("Z", ml_value_z)); RunAndVerifyOutputsWithEP(model_file_name, "CoreMLExecutionProviderTest.FunctionTest", - onnxruntime::make_unique(), + onnxruntime::make_unique(coreml_flags), feeds); } diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 035c03fbe7..a0d69269a2 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -10,6 +10,9 @@ #ifdef USE_ROCM #include "core/providers/rocm/rocm_provider_factory_creator.h" #endif +#ifdef USE_COREML +#include "core/providers/coreml/coreml_provider_factory.h" +#endif #include "core/session/onnxruntime_cxx_api.h" namespace onnxruntime { @@ -26,7 +29,7 @@ std::shared_ptr CreateExecutionProviderFactory_Tensor std::shared_ptr CreateExecutionProviderFactory_MIGraphX(int device_id); std::shared_ptr CreateExecutionProviderFactory_ACL(int use_arena); std::shared_ptr CreateExecutionProviderFactory_ArmNN(int use_arena); -std::shared_ptr CreateExecutionProviderFactory_CoreML(); +std::shared_ptr CreateExecutionProviderFactory_CoreML(uint32_t); // EP for internal testing std::shared_ptr CreateExecutionProviderFactory_InternalTesting( @@ -136,7 +139,10 @@ std::unique_ptr DefaultRocmExecutionProvider() { std::unique_ptr DefaultCoreMLExecutionProvider() { #if defined(USE_COREML) - return CreateExecutionProviderFactory_CoreML()->CreateProvider(); + // We want to run UT on CPU only to get output value without losing precision + uint32_t coreml_flags = 0; + coreml_flags |= COREML_FLAG_USE_CPU_ONLY; + return CreateExecutionProviderFactory_CoreML(coreml_flags)->CreateProvider(); #else return nullptr; #endif diff --git a/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml new file mode 100644 index 0000000000..03a05f15b6 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml @@ -0,0 +1,19 @@ +jobs: +- job: CoreML_CI + pool: + vmImage: 'macOS-10.15' + timeoutInMinutes: 120 + steps: + - script: brew install coreutils ninja + displayName: Install coreutils and ninja + + - script: | + python3 tools/ci_build/build.py \ + --build_dir build \ + --skip_submodule_sync \ + --cmake_generator=Ninja \ + --parallel \ + --build_shared_lib \ + --config Debug \ + --use_coreml + displayName: CoreML EP, Build and Test on macOS \ No newline at end of file