diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 0fc5ee4e5d..f7383a9436 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -239,8 +239,6 @@ ORT_API(int, OrtSetSessionThreadPoolSize, _In_ OrtSessionOptions* options, int s * If none are called Ort will use its internal CPU execution provider. */ -ORT_API(void, OrtAppendCustomOpLibPath, _In_ OrtSessionOptions* options, const char* lib_path); - ORT_API_STATUS(OrtSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out); ORT_API_STATUS(OrtSessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out); @@ -564,7 +562,7 @@ typedef struct OrtCustomOp OrtCustomOp; /* * Create a custom op domain. After all sessions using it are released, call OrtReleaseCustomOpDomain */ -ORT_API(OrtCustomOpDomain*, OrtCreateCustomOpDomain, _In_ const char* domain, _In_ int op_version_start, _In_ int op_version_end); +ORT_API(OrtCustomOpDomain*, OrtCreateCustomOpDomain, _In_ const char* domain); /* * Add custom ops to the OrtCustomOpDomain diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index c4e86b32e9..d6bb4d2912 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -125,9 +125,6 @@ class SessionOptionsWrapper { return ret; } #endif - void AppendCustomOpLibPath(_In_ const char* lib_path) { - OrtAppendCustomOpLibPath(value.get(), lib_path); - } }; inline OrtValue* OrtCreateTensorAsOrtValue(_Inout_ OrtAllocator* env, const std::vector& shape, ONNXTensorElementDataType type) { OrtValue* ret; diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt index f79af6fc79..89ecf30612 100644 --- a/onnxruntime/core/providers/cpu/symbols.txt +++ b/onnxruntime/core/providers/cpu/symbols.txt @@ -6,7 +6,6 @@ OrtAllocatorInfoGetId OrtAllocatorInfoGetMemType OrtAllocatorInfoGetName OrtAllocatorInfoGetType -OrtAppendCustomOpLibPath OrtCastTypeInfoToTensorInfo OrtCloneSessionOptions OrtCompareAllocatorInfo diff --git a/onnxruntime/core/session/CustomOpsLoader.cc b/onnxruntime/core/session/CustomOpsLoader.cc deleted file mode 100644 index 7bd237206d..0000000000 --- a/onnxruntime/core/session/CustomOpsLoader.cc +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/session/CustomOpsLoader.h" - -#include "core/framework/custom_ops_author.h" -#include "core/platform/env.h" -#include "core/common/logging/logging.h" -#include "core/common/logging/severity.h" -#include - -using namespace ::onnxruntime::common; -using namespace ::onnxruntime::logging; - -namespace onnxruntime { -void CustomOpsLoader::PreUnloadLibrary(void* handle) { - using FreeKernelsContainerFn = void (*)(KernelsContainer*); - using FreeSchemasContainerFn = void (*)(SchemasContainer*); - auto it = dso_handle_data_map_.find(handle); - if (it == dso_handle_data_map_.end()) - return; - // free memory - if (!handle) - return; - - // free the kernels container - if (it->second.kernels_container) { - void* free_all_kernels_symbol_handle = nullptr; - Env::Default().GetSymbolFromLibrary(handle, - kFreeKernelsContainerSymbol, - &free_all_kernels_symbol_handle); - if (!free_all_kernels_symbol_handle) { - LOGS_DEFAULT(WARNING) << "Got nullptr for " + kFreeKernelsContainerSymbol; - } else { - FreeKernelsContainerFn free_all_kernels_fn = reinterpret_cast(free_all_kernels_symbol_handle); - free_all_kernels_fn(it->second.kernels_container); - } - } - - // free the schemas container - if (it->second.schemas_container) { - void* free_all_schemas_symbol_handle = nullptr; - Env::Default().GetSymbolFromLibrary(handle, - kFreeSchemasContainerSymbol, - &free_all_schemas_symbol_handle); - - if (!free_all_schemas_symbol_handle) { - LOGS_DEFAULT(WARNING) << "Got nullptr for " + kFreeSchemasContainerSymbol; - } else { - FreeSchemasContainerFn free_all_schemas_fn = reinterpret_cast(free_all_schemas_symbol_handle); - free_all_schemas_fn(it->second.schemas_container); - } - } -} - -Status CustomOpsLoader::LoadCustomOps(const std::string& dso_file_path, - std::shared_ptr& custom_registry) { - void* lib_handle = nullptr; - ORT_RETURN_IF_ERROR(LoadExternalLib(dso_file_path, &lib_handle)); - try { - using GetAllKernelsFn = KernelsContainer* (*)(); - using GetAllSchemasFn = SchemasContainer* (*)(); - - // get symbol for GetAllKernels - void* get_all_kernels_symbol_handle = nullptr; - ORT_RETURN_IF_ERROR(Env::Default().GetSymbolFromLibrary(lib_handle, - kGetAllKernelsSymbol, - &get_all_kernels_symbol_handle)); - if (!get_all_kernels_symbol_handle) { - return Status(ONNXRUNTIME, INVALID_ARGUMENT, - "Got null handle for " + kGetAllKernelsSymbol + " for DSO " + dso_file_path); - } - - GetAllKernelsFn get_all_kernels_fn = reinterpret_cast(get_all_kernels_symbol_handle); - KernelsContainer* kernels_container = get_all_kernels_fn(); - if (!kernels_container) { - LOGS_DEFAULT(WARNING) << "Got nullptr for KernelsContainer from the custom op library " << dso_file_path; - return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Got nullptr for KernelsContainer from the custom op library " + dso_file_path); - } - dso_handle_data_map_[lib_handle].kernels_container = kernels_container; - - // register the kernels - custom_registry.reset(); - custom_registry = std::make_shared(); - - for (auto& i : kernels_container->kernels_list) { - ORT_RETURN_IF_ERROR(custom_registry->RegisterCustomKernel(i)); - } - - // get symbol for GetAllSchemas - void* get_all_schemas_symbol_handle = nullptr; - ORT_RETURN_IF_ERROR(Env::Default().GetSymbolFromLibrary(lib_handle, - kGetAllSchemasSymbol, - &get_all_schemas_symbol_handle)); - - if (!get_all_schemas_symbol_handle) { // a custom schema may not be registered - return Status::OK(); - } - - GetAllSchemasFn get_all_schemas_fn = reinterpret_cast(get_all_schemas_symbol_handle); - SchemasContainer* schemas_container = get_all_schemas_fn(); - if (!schemas_container) { - LOGS_DEFAULT(WARNING) << "Got nullptr for SchemasContainer from the custom op library " << dso_file_path; - return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Got nullptr for SchemasContainer from the custom op library " + dso_file_path); - } - dso_handle_data_map_[lib_handle].schemas_container = schemas_container; - - // register the schemas if present - ORT_RETURN_IF_ERROR(custom_registry->RegisterOpSet(schemas_container->schemas_list, - schemas_container->domain, - schemas_container->baseline_opset_version, - schemas_container->opset_version)); - return Status::OK(); - } catch (const std::exception& ex) { - return Status(ONNXRUNTIME, FAIL, "Caught exception while loading custom ops with message: " + std::string(ex.what())); - } -} -} // namespace onnxruntime diff --git a/onnxruntime/core/session/CustomOpsLoader.h b/onnxruntime/core/session/CustomOpsLoader.h deleted file mode 100644 index 66666fb3c5..0000000000 --- a/onnxruntime/core/session/CustomOpsLoader.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include -#include - -#include "core/common/common.h" -#include "core/common/status.h" -#include "core/framework/customregistry.h" -#include "core/framework/custom_ops_author.h" -#include "core/framework/ex_lib_loader.h" - -namespace onnxruntime { -class CustomOpsLoader final : public ExLibLoader { - public: - CustomOpsLoader() = default; - common::Status LoadCustomOps(const std::string& dso_file_path, - std::shared_ptr& custom_registry); - - protected: - virtual void PreUnloadLibrary(void* handle) override; - - private: - const std::string kGetAllKernelsSymbol = "GetAllKernels"; - const std::string kGetAllSchemasSymbol = "GetAllSchemas"; - const std::string kFreeKernelsContainerSymbol = "FreeKernelsContainer"; - const std::string kFreeSchemasContainerSymbol = "FreeSchemasContainer"; - - struct DsoData { - KernelsContainer* kernels_container = nullptr; - SchemasContainer* schemas_container = nullptr; - }; - std::map dso_handle_data_map_; - - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CustomOpsLoader); -}; -} // namespace onnxruntime diff --git a/onnxruntime/core/session/abi_session_options.cc b/onnxruntime/core/session/abi_session_options.cc index e4f7bb350f..b12c63bb02 100644 --- a/onnxruntime/core/session/abi_session_options.cc +++ b/onnxruntime/core/session/abi_session_options.cc @@ -14,7 +14,7 @@ OrtSessionOptions& OrtSessionOptions::operator=(const OrtSessionOptions&) { throw std::runtime_error("not implemented"); } OrtSessionOptions::OrtSessionOptions(const OrtSessionOptions& other) - : value(other.value), custom_op_paths(other.custom_op_paths), provider_factories(other.provider_factories) { + : value(other.value), provider_factories(other.provider_factories) { } ORT_API(OrtSessionOptions*, OrtCreateSessionOptions) { @@ -81,7 +81,3 @@ ORT_API(int, OrtSetSessionThreadPoolSize, _In_ OrtSessionOptions* options, int s options->value.session_thread_pool_size = session_thread_pool_size; return 0; } - -ORT_API(void, OrtAppendCustomOpLibPath, _In_ OrtSessionOptions* options, const char* lib_path) { - options->custom_op_paths.emplace_back(lib_path); -} diff --git a/onnxruntime/core/session/abi_session_options_impl.h b/onnxruntime/core/session/abi_session_options_impl.h index ca57a01b85..2e87ea7e70 100644 --- a/onnxruntime/core/session/abi_session_options_impl.h +++ b/onnxruntime/core/session/abi_session_options_impl.h @@ -12,7 +12,6 @@ struct OrtSessionOptions { onnxruntime::SessionOptions value; - std::vector custom_op_paths; std::vector custom_op_domains_; std::vector> provider_factories; OrtSessionOptions() = default; diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 3d99e23324..d042139534 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -46,7 +46,7 @@ #include "core/optimizer/insert_cast_transformer.h" #include "core/optimizer/transformer_memcpy.h" #include "core/providers/cpu/cpu_execution_provider.h" -#include "core/session/CustomOpsLoader.h" +#include "core/framework/custom_ops_author.h" #include "core/session/IOBinding.h" #include "core/optimizer/rule_based_graph_transformer.h" #include "core/optimizer/graph_transformer_utils.h" @@ -206,21 +206,6 @@ class InferenceSession::Impl { return Status::OK(); } - common::Status LoadCustomOps(const std::vector& dso_list) { - if (dso_list.empty()) { - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Empty list of shared libraries in the input."); - } - for (auto& dso_file_path : dso_list) { - std::shared_ptr custom_registry; - ORT_RETURN_IF_ERROR(custom_ops_loader_.LoadCustomOps(dso_file_path, custom_registry)); - if (!custom_registry) { - return Status(common::ONNXRUNTIME, common::FAIL, "Null custom_registry after loading custom ops."); - } - ORT_RETURN_IF_ERROR(RegisterCustomRegistry(custom_registry)); - } - return Status::OK(); - } - common::Status AddCustomOpDomains(const std::vector& op_domains) { auto custom_registry = std::make_shared(); @@ -228,8 +213,8 @@ class InferenceSession::Impl { SchemasContainer schemas_container; schemas_container.domain = domain->domain_; - schemas_container.baseline_opset_version = domain->op_version_start_; - schemas_container.opset_version = domain->op_version_end_; + schemas_container.baseline_opset_version = 1; + schemas_container.opset_version = 1000; for (auto& op : domain->custom_ops_) { ONNX_NAMESPACE::OpSchema schema(op->GetName(op), "unknown", 0); @@ -250,7 +235,7 @@ class InferenceSession::Impl { DataTypeImpl::ToString(onnxruntime::DataTypeImpl::TensorTypeFromONNXEnum(type))); } - schema.SinceVersion(domain->op_version_start_); + schema.SinceVersion(1); schema.AllowUncheckedAttributes(); schemas_container.schemas_list.push_back(schema); @@ -258,7 +243,7 @@ class InferenceSession::Impl { KernelDefBuilder def_builder; def_builder.SetName(op->GetName(op)) .SetDomain(onnxruntime::kOnnxDomain) - .SinceVersion(domain->op_version_start_) + .SinceVersion(1) .Provider(onnxruntime::kCpuExecutionProvider); KernelCreateFn kernel_create_fn = [&op](const OpKernelInfo& info) -> OpKernel* { return new CustomOpKernel(info, *op); }; KernelCreateInfo create_info(def_builder.Build(), kernel_create_fn); @@ -1012,8 +997,6 @@ class InferenceSession::Impl { return Status::OK(); } - CustomOpsLoader custom_ops_loader_; - const SessionOptions session_options_; onnxruntime::GraphTransformerManager graph_transformation_mgr_; @@ -1217,10 +1200,6 @@ common::Status InferenceSession::Run(IOBinding& io_binding) { return impl_->Run(io_binding); } -common::Status InferenceSession::LoadCustomOps(const std::vector& dso_list) { - return impl_->LoadCustomOps(dso_list); -} - common::Status InferenceSession::AddCustomOpDomains(const std::vector& ops) { return impl_->AddCustomOpDomains(ops); } diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h index 0e52de4cc4..9d21d6de36 100644 --- a/onnxruntime/core/session/inference_session.h +++ b/onnxruntime/core/session/inference_session.h @@ -22,8 +22,6 @@ class ModelProto; struct OrtCustomOpDomain { std::string domain_; - int op_version_start_{}; - int op_version_end_{}; std::vector custom_ops_; }; @@ -150,15 +148,6 @@ class InferenceSession { */ common::Status AddCustomTransformerList(const std::vector& transformers_to_enable); - /** - * Load custom ops implemented in a dynamically linked shared library. - * @param dso_list list of library file paths containing the custom ops implementation. - * In order to implement a custom op please see file: custom_ops_author.h - * TODO add sample code - * @return OK if success - */ - common::Status LoadCustomOps(const std::vector& dso_list); - common::Status AddCustomOpDomains(const std::vector& ops); /** diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 7e7623f392..01264fd010 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -337,11 +337,9 @@ ORT_API_STATUS_IMPL(OrtCreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, API_IMPL_END } -ORT_API(OrtCustomOpDomain*, OrtCreateCustomOpDomain, _In_ const char* domain, int op_version_start, int op_version_end) { +ORT_API(OrtCustomOpDomain*, OrtCreateCustomOpDomain, _In_ const char* domain) { auto custom_op_domain = std::make_unique(); custom_op_domain->domain_ = domain; - custom_op_domain->op_version_start_ = op_version_start; - custom_op_domain->op_version_end_ = op_version_end; return custom_op_domain.release(); } @@ -370,11 +368,6 @@ ORT_API_STATUS_IMPL(OrtCreateSession, _In_ OrtEnv* env, _In_ const ORTCHAR_T* mo options == nullptr ? onnxruntime::SessionOptions() : options->value, env->loggingManager); Status status; if (options != nullptr) { - if (!options->custom_op_paths.empty()) { - status = sess->LoadCustomOps(options->custom_op_paths); - if (!status.IsOK()) - return ToOrtStatus(status); - } if (!options->custom_op_domains_.empty()) { status = sess->AddCustomOpDomains(options->custom_op_domains_); if (!status.IsOK()) @@ -559,7 +552,6 @@ ORT_API_STATUS_IMPL(OrtGetTensorMemSizeInBytesFromTensorProto, _In_ const void* delete reinterpret_cast(value); \ } - ORT_API_STATUS_IMPL(OrtSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index 0a8e840b6c..13f57b852c 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -63,7 +63,7 @@ void TestInference(OrtEnv* env, T model_uri, const std::vector& values_x, const std::vector& expected_dims_y, const std::vector& expected_values_y, - int provider_type, bool custom_op, OrtCustomOpDomain* custom_op_domain_ptr = nullptr) { + int provider_type, OrtCustomOpDomain* custom_op_domain_ptr) { SessionOptionsWrapper sf(env); if (provider_type == 1) { @@ -90,9 +90,6 @@ void TestInference(OrtEnv* env, T model_uri, } else { std::cout << "Running simple inference with default provider" << std::endl; } - if (custom_op) { - sf.AppendCustomOpLibPath("libonnxruntime_custom_op_shared_lib_test.so"); - } if (custom_op_domain_ptr) { ORT_THROW_ON_ERROR(OrtAddCustomOpDomain(sf, custom_op_domain_ptr)); } @@ -151,28 +148,13 @@ TEST_P(CApiTestWithProvider, simple) { std::vector expected_dims_y = {3, 2}; std::vector expected_values_y = {1.0f, 4.0f, 9.0f, 16.0f, 25.0f, 36.0f}; - TestInference(env, MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, GetParam(), false); + TestInference(env, MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, GetParam(), nullptr); } INSTANTIATE_TEST_CASE_P(CApiTestWithProviders, CApiTestWithProvider, ::testing::Values(0, 1, 2, 3, 4)); -#ifndef _WIN32 -//doesn't work, failed in type comparison -TEST_F(CApiTest, DISABLED_custom_op) { - std::cout << "Running custom op inference" << std::endl; - std::vector dims_x = {3, 2}; - std::vector values_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; - - // prepare expected inputs and outputs - std::vector expected_dims_y = {3, 2}; - std::vector expected_values_y = {2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f}; - - TestInference(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, true); -} -#endif - struct OrtTensorDimensions : std::vector { OrtTensorDimensions(OrtValue* value) { OrtTensorTypeAndShapeInfo* info; @@ -249,11 +231,11 @@ TEST_F(CApiTest, custom_op_handler) { std::vector expected_values_y = {2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f}; MyCustomOp custom_op; - OrtCustomOpDomain* custom_op_domain = OrtCreateCustomOpDomain("", 5, 7); + OrtCustomOpDomain* custom_op_domain = OrtCreateCustomOpDomain(""); ORT_THROW_ON_ERROR(OrtCustomOpDomain_Add(custom_op_domain, &custom_op)); - TestInference(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, false, - custom_op_domain); + TestInference(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, 0, custom_op_domain); + OrtReleaseCustomOpDomain(custom_op_domain); } #ifdef ORT_RUN_EXTERNAL_ONNX_TESTS