diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 137e71a7f6..1660f64b1a 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1459,6 +1459,31 @@ if (NOT onnxruntime_BUILD_WEBASSEMBLY AND (NOT onnxruntime_MINIMAL_BUILD OR onnx ${ONNXRUNTIME_CUSTOM_OP_INVALID_LIB_LINK_FLAG}) endif() +if (NOT onnxruntime_BUILD_WEBASSEMBLY AND onnxruntime_USE_OPENVINO AND (NOT onnxruntime_MINIMAL_BUILD OR + onnxruntime_MINIMAL_BUILD_CUSTOM_OPS)) + onnxruntime_add_shared_library_module(custom_op_openvino_wrapper_library + ${TEST_SRC_DIR}/testdata/custom_op_openvino_wrapper_library/custom_op_lib.cc + ${TEST_SRC_DIR}/testdata/custom_op_openvino_wrapper_library/openvino_wrapper.cc) + target_include_directories(custom_op_openvino_wrapper_library PRIVATE ${REPO_ROOT}/include/onnxruntime/core/session) + target_link_libraries(custom_op_openvino_wrapper_library PRIVATE openvino::runtime) + + if(UNIX) + if (APPLE) + set(ONNXRUNTIME_CUSTOM_OP_OPENVINO_WRAPPER_LIB_LINK_FLAG "-Xlinker -dead_strip") + else() + string(CONCAT ONNXRUNTIME_CUSTOM_OP_OPENVINO_WRAPPER_LIB_LINK_FLAG + "-Xlinker --version-script=${TEST_SRC_DIR}/testdata/custom_op_openvino_wrapper_library/custom_op_lib.lds " + "-Xlinker --no-undefined -Xlinker --gc-sections -z noexecstack") + endif() + else() + set(ONNXRUNTIME_CUSTOM_OP_OPENVINO_WRAPPER_LIB_LINK_FLAG + "-DEF:${TEST_SRC_DIR}/testdata/custom_op_openvino_wrapper_library/custom_op_lib.def") + endif() + + set_property(TARGET custom_op_openvino_wrapper_library APPEND_STRING PROPERTY LINK_FLAGS + ${ONNXRUNTIME_CUSTOM_OP_OPENVINO_WRAPPER_LIB_LINK_FLAG}) +endif() + # limit to only test on windows first, due to a runtime path issue on linux if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS" diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index e66c33d2fc..210cd2fa88 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -1678,6 +1678,7 @@ struct OrtApi { /// @} /// \name OrtKernelInfo + /// Custom operator APIs. /// @{ /** \brief Get a float stored as an attribute in the graph node @@ -1727,6 +1728,7 @@ struct OrtApi { /// @} /// \name OrtKernelContext + /// Custom operator APIs. /// @{ /** \brief Used for custom operators, get the input count of a kernel @@ -2598,6 +2600,7 @@ struct OrtApi { /// @} /// \name OrtKernelInfo + /// Custom operator APIs. /// @{ /** \brief Fetch an array of int64_t values stored as an attribute in the graph node @@ -3141,9 +3144,12 @@ struct OrtApi { * \snippet{doc} snippets.dox OrtStatus Return Value */ ORT_API2_STATUS(HasValue, _In_ const OrtValue* value, _Out_ int* out); + /// @} /// \name OrtKernelContext + /// Custom operator APIs. /// @{ + /** \brief Used for custom operators, gets the GPU compute stream to use to launch the custom a GPU kernel * \see ::OrtCustomOp * \param[in] context OrtKernelContext instance @@ -3687,6 +3693,191 @@ struct OrtApi { ORT_API2_STATUS(RegisterCustomOpsUsingFunction, _Inout_ OrtSessionOptions* options, _In_ const char* registration_func_name); + /// @} + /// \name OrtKernelInfo + /// Custom operator APIs. + /// @{ + + /** \brief Get the number of inputs from ::OrtKernelInfo. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the number of inputs + * during kernel/session creation. + * + * \param[in] info Instance of ::OrtKernelInfo. + * \param[out] out Pointer to variable assigned with the result on success. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetInputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); + + /** \brief Get the number of outputs from ::OrtKernelInfo. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the number of outputs + * during kernel/session creation. + * + * \param[in] info Instance of ::OrtKernelInfo. + * \param[out] out Pointer to variable assigned with the result on success. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetOutputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); + + /** \brief Get the name of a ::OrtKernelInfo's input. + * + * Used in the CreateKernel callback of an OrtCustomOp to query an input's name + * during kernel/session creation. + * + * If `out` is nullptr, the value of `size` is set to the size of the name + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the name string's size, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status is returned. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[in] index The index of the input name to get. Returns a failure status if out-of-bounds. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the input's name. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetInputName, _In_ const OrtKernelInfo* info, size_t index, _Out_ char* out, + _Inout_ size_t* size); + + /** \brief Get the name of a ::OrtKernelInfo's output. + * + * Used in the CreateKernel callback of an OrtCustomOp to query an output's name + * during kernel/session creation. + * + * If `out` is nullptr, the value of `size` is set to the size of the name + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the name string's size, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status is returned. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[in] index The index of the output name to get. Returns a failure status if out-of-bounds. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the output's + * name. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetOutputName, _In_ const OrtKernelInfo* info, size_t index, _Out_ char* out, + _Inout_ size_t* size); + + /** \brief Get the type information for a ::OrtKernelInfo's input. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the shape and type information + * of an input during kernel/session creation. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] type_info Pointer set to the resulting ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetInputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get the type information for a ::OrtKernelInfo's output. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the shape and type information + * of an output during kernel/session creation. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] type_info Pointer set to the resulting ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetOutputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get a ::OrtValue tensor stored as an attribute in the graph node. + * + * Used in the CreateKernel callback of an OrtCustomOp to get a tensor attribute. + * + * \param[in] info ::OrtKernelInfo instance. + * \param[in] name UTF-8 null-terminated string representing the attribute's name. + * \param[in] allocator Allocator used to allocate the internal tensor state. + * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue, + * which will also free internal tensor state allocated with the provided allocator. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttribute_tensor, _In_ const OrtKernelInfo* info, _In_z_ const char* name, + _Inout_ OrtAllocator* allocator, _Outptr_ OrtValue** out); + + /// @} + /// \name OrtSessionOptions + /// Custom operator APIs + /// @{ + + /** \brief Checks if the given session configuration entry exists. + * + * The config_key formats are defined in onnxruntime_session_options_config_keys.h + * + * Can be used in a custom operator library to check for session configuration entries + * that target one or more custom operators in the library. Example: The config entry + * custom_op.myop.some_key targets a custom op named "myop". + * + * \param[in] options The ::OrtSessionOptions instance. + * \param[in] config_key A null-terminated UTF-8 string representation of the configuration key. + * \param[out] out Pointer set to 1 if the entry exists and 0 otherwise. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(HasSessionConfigEntry, _In_ const OrtSessionOptions* options, + _In_z_ const char* config_key, _Out_ int* out); + + /** \brief Get a session configuration value. + * + * Returns a failure status if the configuration key does not exist. + * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h + * + * If `config_value` is nullptr, the value of `size` is set to the true size of the string + * value (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the actual string value's size, + * the value of `size` is set to the true size of the string value, the provided memory + * is filled with the value's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string value's size and `config_value` + * is not nullptr, the value of `size` is set to the true size of the string value + * and a failure status is returned. + * + * Can be used in a custom operator library to get session configuration entries + * that target one or more custom operators in the library. Example: The config entry + * custom_op.myop.some_key targets a custom op named "myop". + * + * \param[in] options The session options. + * \param[in] config_key A null-terminated UTF-8 string representation of the config key. + * \param[in] config_value Pointer to memory where the null-terminated UTF-8 string value will be stored. + * \param[in,out] size Pointer to the size of the `config_value` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(GetSessionConfigEntry, _In_ const OrtSessionOptions* options, + _In_z_ const char* config_key, _Out_ char* config_value, _Inout_ size_t* size); + + /// @} + #ifdef __cplusplus OrtApi(const OrtApi&) = delete; // Prevent users from accidentally copying the API structure, it should always be passed as a pointer #endif diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index 8f220693a7..97b2aa4849 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -88,6 +88,22 @@ template #ifdef ORT_API_MANUAL_INIT const OrtApi* Global::api_{}; inline void InitApi() { Global::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); } + +// Used by custom operator libraries that are not linked to onnxruntime. Sets the global API object, which is +// required by C++ APIs. +// +// Example mycustomop.cc: +// +// #define ORT_API_MANUAL_INIT +// #include +// #undef ORT_API_MANUAL_INIT +// +// OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api_base) { +// Ort::InitApi(api_base->GetApi(ORT_API_VERSION)); +// // ... +// } +// +inline void InitApi(const OrtApi* api) { Global::api_ = api; } #else #if defined(_MSC_VER) && !defined(__clang__) #pragma warning(push) @@ -448,6 +464,54 @@ struct RunOptions : detail::Base { RunOptions& UnsetTerminate(); }; + +namespace detail { +// Utility function that returns a SessionOption config entry key for a specific custom operator. +// Ex: custom_op.[custom_op_name].[config] +std::string MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config); +} // namespace detail + +/// +/// Class that represents session configuration entries for one or more custom operators. +/// +/// Example: +/// Ort::CustomOpConfigs op_configs; +/// op_configs.AddConfig("my_custom_op", "device_type", "CPU"); +/// +/// Passed to Ort::SessionOptions::RegisterCustomOpsLibrary. +/// +struct CustomOpConfigs { + CustomOpConfigs() = default; + ~CustomOpConfigs() = default; + CustomOpConfigs(const CustomOpConfigs&) = default; + CustomOpConfigs& operator=(const CustomOpConfigs&) = default; + CustomOpConfigs(CustomOpConfigs&& o) = default; + CustomOpConfigs& operator=(CustomOpConfigs&& o) = default; + + /** \brief Adds a session configuration entry/value for a specific custom operator. + * + * \param custom_op_name The name of the custom operator for which to add a configuration entry. + * Must match the name returned by the CustomOp's GetName() method. + * \param config_key The name of the configuration entry. + * \param config_value The value of the configuration entry. + * \return A reference to this object to enable call chaining. + */ + CustomOpConfigs& AddConfig(const char* custom_op_name, const char* config_key, const char* config_value); + + /** \brief Returns a flattened map of custom operator configuration entries and their values. + * + * The keys has been flattened to include both the custom operator name and the configuration entry key name. + * For example, a prior call to AddConfig("my_op", "key", "value") corresponds to the flattened key/value pair + * {"my_op.key", "value"}. + * + * \return An unordered map of flattened configurations. + */ + const std::unordered_map& GetFlattenedConfigs() const; + + private: + std::unordered_map flat_configs_; +}; + /** \brief Options object used when creating a new Session object * * Wraps ::OrtSessionOptions object and methods @@ -456,12 +520,24 @@ struct RunOptions : detail::Base { struct SessionOptions; namespace detail { +// we separate const-only methods because passing const ptr to non-const methods +// is only discovered when inline methods are compiled which is counter-intuitive template -struct SessionOptionsImpl : Base { +struct ConstSessionOptionsImpl : Base { using B = Base; using B::B; - Ort::SessionOptions Clone() const; ///< Creates and returns a copy of this SessionOptions object. Wraps OrtApi::CloneSessionOptions + SessionOptions Clone() const; ///< Creates and returns a copy of this SessionOptions object. Wraps OrtApi::CloneSessionOptions + + std::string GetConfigEntry(const char* config_key) const; ///< Wraps OrtApi::GetSessionConfigEntry + bool HasConfigEntry(const char* config_key) const; ///< Wraps OrtApi::HasSessionConfigEntry + std::string GetConfigEntryOrDefault(const char* config_key, const std::string& def); +}; + +template +struct SessionOptionsImpl : ConstSessionOptionsImpl { + using B = ConstSessionOptionsImpl; + using B::B; SessionOptionsImpl& SetIntraOpNumThreads(int intra_op_num_threads); ///< Wraps OrtApi::SetIntraOpNumThreads SessionOptionsImpl& SetInterOpNumThreads(int inter_op_num_threads); ///< Wraps OrtApi::SetInterOpNumThreads @@ -489,7 +565,8 @@ struct SessionOptionsImpl : Base { SessionOptionsImpl& DisablePerSessionThreads(); ///< Wraps OrtApi::DisablePerSessionThreads - SessionOptionsImpl& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddSessionConfigEntry + SessionOptionsImpl& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddSessionConfigEntry + SessionOptionsImpl& AddInitializer(const char* name, const OrtValue* ort_val); ///< Wraps OrtApi::AddInitializer SessionOptionsImpl& AddExternalInitializers(const std::vector& names, const std::vector& ort_values); ///< Wraps OrtApi::AddExternalInitializers @@ -510,13 +587,17 @@ struct SessionOptionsImpl : Base { SessionOptionsImpl& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options); ///< Wraps OrtApi::SessionOptionsSetCustomThreadCreationOptions SessionOptionsImpl& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomJoinThreadFn - SessionOptionsImpl& RegisterCustomOpsLibrary(const ORTCHAR_T* library_name); ///< Wraps OrtApi::RegisterCustomOpsLibrary_V2 + ///< Registers the custom operator from the specified shared library via OrtApi::RegisterCustomOpsLibrary_V2. + ///< The custom operator configurations are optional. If provided, custom operator configs are set via + ///< OrtApi::AddSessionConfigEntry. + SessionOptionsImpl& RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, const CustomOpConfigs& custom_op_configs = {}); + SessionOptionsImpl& RegisterCustomOpsUsingFunction(const char* function_name); ///< Wraps OrtApi::RegisterCustomOpsUsingFunction }; } // namespace detail -// No const version required using UnownedSessionOptions = detail::SessionOptionsImpl>; +using ConstSessionOptions = detail::ConstSessionOptionsImpl>; /** \brief Wrapper around ::OrtSessionOptions * @@ -526,6 +607,7 @@ struct SessionOptions : detail::SessionOptionsImpl { SessionOptions(); ///< Wraps OrtApi::CreateSessionOptions explicit SessionOptions(OrtSessionOptions* p) : SessionOptionsImpl{p} {} ///< Used for interop with the C API UnownedSessionOptions GetUnowned() const { return UnownedSessionOptions{this->p_}; } + ConstSessionOptions GetConst() const { return ConstSessionOptions{this->p_}; } }; /** \brief Wrapper around ::OrtModelMetadata @@ -819,13 +901,11 @@ struct MapTypeInfo : detail::MapTypeInfoImpl { ConstMapTypeInfo GetConst() const { return ConstMapTypeInfo{this->p_}; } }; -/// -/// Type information that may contain either TensorTypeAndShapeInfo or -/// the information about contained sequence or map depending on the ONNXType. -/// -struct TypeInfo : detail::Base { - explicit TypeInfo(std::nullptr_t) {} ///< Create an empty TypeInfo object, must be assigned a valid one to be used - explicit TypeInfo(OrtTypeInfo* p) : Base{p} {} ///< C API Interop +namespace detail { +template +struct TypeInfoImpl : detail::Base { + using B = Base; + using B::B; ConstTensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; ///< Wraps OrtApi::CastTypeInfoToTensorInfo ConstSequenceTypeInfo GetSequenceTypeInfo() const; ///< Wraps OrtApi::CastTypeInfoToSequenceTypeInfo @@ -833,6 +913,24 @@ struct TypeInfo : detail::Base { ONNXType GetONNXType() const; }; +} // namespace detail + +/// +/// Contains a constant, unowned OrtTypeInfo that can be copied and passed around by value. +/// Provides access to const OrtTypeInfo APIs. +/// +using ConstTypeInfo = detail::TypeInfoImpl>; + +/// +/// Type information that may contain either TensorTypeAndShapeInfo or +/// the information about contained sequence or map depending on the ONNXType. +/// +struct TypeInfo : detail::TypeInfoImpl { + explicit TypeInfo(std::nullptr_t) {} ///< Create an empty TypeInfo object, must be assigned a valid one to be used + explicit TypeInfo(OrtTypeInfo* p) : TypeInfoImpl{p} {} ///< C API Interop + + ConstTypeInfo GetConst() const { return ConstTypeInfo{this->p_}; } +}; namespace detail { // This structure is used to feed sparse tensor values @@ -1413,7 +1511,6 @@ struct KernelContext { struct KernelInfo; namespace detail { - namespace attr_utils { void GetAttr(const OrtKernelInfo* p, const char* name, float&); void GetAttr(const OrtKernelInfo* p, const char* name, int64_t&); @@ -1442,6 +1539,17 @@ struct KernelInfoImpl : Base { attr_utils::GetAttrs(this->p_, name, result); return result; } + + Value GetTensorAttribute(const char* name, OrtAllocator* allocator) const; + + size_t GetInputCount() const; + size_t GetOutputCount() const; + + std::string GetInputName(size_t index) const; + std::string GetOutputName(size_t index) const; + + TypeInfo GetInputTypeInfo(size_t index) const; + TypeInfo GetOutputTypeInfo(size_t index) const; }; } // namespace detail @@ -1750,6 +1858,17 @@ struct CustomOpBase : OrtCustomOp { bool GetVariadicOutputHomogeneity() const { return true; } + + // Declare list of session config entries used by this Custom Op. + // Implement this function in order to get configs from CustomOpBase::GetSessionConfigs(). + // This default implementation returns an empty vector of config entries. + std::vector GetSessionConfigKeys() const { + return std::vector{}; + } + + protected: + // Helper function that returns a map of session config entries specified by CustomOpBase::GetSessionConfigKeys. + void GetSessionConfigs(std::unordered_map& out, ConstSessionOptions options) const; }; } // namespace Ort diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index 1f7974f38b..6d391ad0f7 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -527,12 +527,42 @@ inline RunOptions& RunOptions::UnsetTerminate() { namespace detail { template -inline Ort::SessionOptions SessionOptionsImpl::Clone() const { +inline Ort::SessionOptions ConstSessionOptionsImpl::Clone() const { OrtSessionOptions* out; ThrowOnError(GetApi().CloneSessionOptions(this->p_, &out)); return SessionOptions{out}; } +template +inline std::string ConstSessionOptionsImpl::GetConfigEntry(const char* config_key) const { + size_t size = 0; + // Feed nullptr for the data buffer to query the true size of the string value + Ort::ThrowOnError(GetApi().GetSessionConfigEntry(this->p_, config_key, nullptr, &size)); + + std::string out; + out.resize(size); + Ort::ThrowOnError(GetApi().GetSessionConfigEntry(this->p_, config_key, &out[0], &size)); + out.resize(size - 1); // remove the terminating character '\0' + + return out; +} + +template +inline bool ConstSessionOptionsImpl::HasConfigEntry(const char* config_key) const { + int out = 0; + Ort::ThrowOnError(GetApi().HasSessionConfigEntry(this->p_, config_key, &out)); + return static_cast(out); +} + +template +inline std::string ConstSessionOptionsImpl::GetConfigEntryOrDefault(const char* config_key, const std::string& def) { + if (!this->HasConfigEntry(config_key)) { + return def; + } + + return this->GetConfigEntry(config_key); +} + template inline SessionOptionsImpl& SessionOptionsImpl::SetIntraOpNumThreads(int intra_op_num_threads) { ThrowOnError(GetApi().SetIntraOpNumThreads(this->p_, intra_op_num_threads)); @@ -749,7 +779,14 @@ inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_Ope } template -inline SessionOptionsImpl& SessionOptionsImpl::RegisterCustomOpsLibrary(const ORTCHAR_T* library_name) { +inline SessionOptionsImpl& SessionOptionsImpl::RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, + const CustomOpConfigs& custom_op_configs) { + // Add custom op config entries before registering the custom op library. Otherwise, the config entries _may_ be ignored by + // the custom op library. + for (const auto& config_iter : custom_op_configs.GetFlattenedConfigs()) { + AddConfigEntry(config_iter.first.c_str(), config_iter.second.c_str()); + } + ThrowOnError(GetApi().RegisterCustomOpsLibrary_V2(this->p_, library_name)); return *this; } @@ -876,6 +913,27 @@ inline SessionOptions::SessionOptions() { ThrowOnError(GetApi().CreateSessionOptions(&this->p_)); } +/// CustomOpConfigs +inline std::string detail::MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config) { + std::string config_key = "custom_op."; + + config_key += custom_op_name; + config_key += "."; + config_key += config; + + return config_key; +} + +inline CustomOpConfigs& CustomOpConfigs::AddConfig(const char* custom_op_name, const char* config_key, const char* config_value) { + const std::string full_flat_key = detail::MakeCustomOpConfigEntryKey(custom_op_name, config_key); + flat_configs_[full_flat_key] = config_value; + return *this; +} + +inline const std::unordered_map& CustomOpConfigs::GetFlattenedConfigs() const { + return flat_configs_; +} + inline Session::Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options) { ThrowOnError(GetApi().CreateSession(env, model_path, options, &this->p_)); } @@ -1004,18 +1062,37 @@ inline std::vector TensorTypeAndShapeInfoImpl::GetShape() const { } // namespace detail -inline ConstTensorTypeAndShapeInfo TypeInfo::GetTensorTypeAndShapeInfo() const { +namespace detail { +template +inline ConstTensorTypeAndShapeInfo TypeInfoImpl::GetTensorTypeAndShapeInfo() const { const OrtTensorTypeAndShapeInfo* out; ThrowOnError(GetApi().CastTypeInfoToTensorInfo(this->p_, &out)); return ConstTensorTypeAndShapeInfo{out}; } -inline ConstSequenceTypeInfo TypeInfo::GetSequenceTypeInfo() const { +template +inline ConstSequenceTypeInfo TypeInfoImpl::GetSequenceTypeInfo() const { const OrtSequenceTypeInfo* out; ThrowOnError(GetApi().CastTypeInfoToSequenceTypeInfo(this->p_, &out)); return ConstSequenceTypeInfo{out}; } +template +inline ConstMapTypeInfo TypeInfoImpl::GetMapTypeInfo() const { + const OrtMapTypeInfo* out; + ThrowOnError(GetApi().CastTypeInfoToMapTypeInfo(this->p_, &out)); + return ConstMapTypeInfo{out}; +} + +template +inline ONNXType TypeInfoImpl::GetONNXType() const { + ONNXType out; + ThrowOnError(GetApi().GetOnnxTypeFromTypeInfo(this->p_, &out)); + return out; +} + +} // namespace detail + namespace detail { template inline TypeInfo SequenceTypeInfoImpl::GetSequenceElementType() const { @@ -1026,12 +1103,6 @@ inline TypeInfo SequenceTypeInfoImpl::GetSequenceElementType() const { } // namespace detail -inline ConstMapTypeInfo TypeInfo::GetMapTypeInfo() const { - const OrtMapTypeInfo* out; - ThrowOnError(GetApi().CastTypeInfoToMapTypeInfo(p_, &out)); - return ConstMapTypeInfo{out}; -} - namespace detail { template inline ONNXTensorElementDataType MapTypeInfoImpl::GetMapKeyType() const { @@ -1048,12 +1119,6 @@ inline TypeInfo MapTypeInfoImpl::GetMapValueType() const { } } // namespace detail -inline ONNXType TypeInfo::GetONNXType() const { - ONNXType out; - ThrowOnError(GetApi().GetOnnxTypeFromTypeInfo(this->p_, &out)); - return out; -} - namespace detail { template @@ -1359,37 +1424,37 @@ inline KernelContext::KernelContext(OrtKernelContext* context) : ctx_(context) { } inline size_t KernelContext::GetInputCount() const { - size_t out; + size_t out = 0; Ort::ThrowOnError(GetApi().KernelContext_GetInputCount(ctx_, &out)); return out; } inline size_t KernelContext::GetOutputCount() const { - size_t out; + size_t out = 0; Ort::ThrowOnError(GetApi().KernelContext_GetOutputCount(ctx_, &out)); return out; } inline ConstValue KernelContext::GetInput(size_t index) const { - const OrtValue* out; + const OrtValue* out = nullptr; Ort::ThrowOnError(GetApi().KernelContext_GetInput(ctx_, index, &out)); return ConstValue{out}; } inline UnownedValue KernelContext::GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const { - OrtValue* out; + OrtValue* out = nullptr; Ort::ThrowOnError(GetApi().KernelContext_GetOutput(ctx_, index, dim_values, dim_count, &out)); return UnownedValue(out); } inline UnownedValue KernelContext::GetOutput(size_t index, const std::vector& dims) const { - OrtValue* out; + OrtValue* out = nullptr; Ort::ThrowOnError(GetApi().KernelContext_GetOutput(ctx_, index, dims.data(), dims.size(), &out)); return UnownedValue(out); } inline void* KernelContext::GetGPUComputeStream() const { - void* out; + void* out = nullptr; Ort::ThrowOnError(GetApi().KernelContext_GetGPUComputeStream(ctx_, &out)); return out; } @@ -1401,11 +1466,76 @@ inline OpAttr::OpAttr(const char* name, const void* data, int len, OrtOpAttrType namespace detail { template inline KernelInfo KernelInfoImpl::Copy() const { - OrtKernelInfo* info_copy; + OrtKernelInfo* info_copy = nullptr; Ort::ThrowOnError(GetApi().CopyKernelInfo(this->p_, &info_copy)); return KernelInfo{info_copy}; } +template +inline size_t KernelInfoImpl::GetInputCount() const { + size_t out = 0; + ThrowOnError(GetApi().KernelInfo_GetInputCount(this->p_, &out)); + return out; +} + +template +inline size_t KernelInfoImpl::GetOutputCount() const { + size_t out = 0; + ThrowOnError(GetApi().KernelInfo_GetOutputCount(this->p_, &out)); + return out; +} + +template +inline std::string KernelInfoImpl::GetInputName(size_t index) const { + size_t size = 0; + + // Feed nullptr for the data buffer to query the true size of the string value + Ort::ThrowOnError(GetApi().KernelInfo_GetInputName(this->p_, index, nullptr, &size)); + + std::string out; + out.resize(size); + Ort::ThrowOnError(GetApi().KernelInfo_GetInputName(this->p_, index, &out[0], &size)); + out.resize(size - 1); // remove the terminating character '\0' + + return out; +} + +template +inline std::string KernelInfoImpl::GetOutputName(size_t index) const { + size_t size = 0; + + // Feed nullptr for the data buffer to query the true size of the string value + Ort::ThrowOnError(GetApi().KernelInfo_GetOutputName(this->p_, index, nullptr, &size)); + + std::string out; + out.resize(size); + Ort::ThrowOnError(GetApi().KernelInfo_GetOutputName(this->p_, index, &out[0], &size)); + out.resize(size - 1); // remove the terminating character '\0' + + return out; +} + +template +inline TypeInfo KernelInfoImpl::GetInputTypeInfo(size_t index) const { + OrtTypeInfo* out = nullptr; + ThrowOnError(GetApi().KernelInfo_GetInputTypeInfo(this->p_, index, &out)); + return TypeInfo{out}; +} + +template +inline TypeInfo KernelInfoImpl::GetOutputTypeInfo(size_t index) const { + OrtTypeInfo* out = nullptr; + ThrowOnError(GetApi().KernelInfo_GetOutputTypeInfo(this->p_, index, &out)); + return TypeInfo{out}; +} + +template +inline Value KernelInfoImpl::GetTensorAttribute(const char* name, OrtAllocator* allocator) const { + OrtValue* out = nullptr; + ThrowOnError(GetApi().KernelInfoGetAttribute_tensor(this->p_, name, allocator, &out)); + return Value{out}; +} + inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, float& out) { Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_float(p, name, &out)); } @@ -1723,4 +1853,22 @@ inline std::vector GetAvailableProviders() { SessionOptions& AddInitializer(const char* name, const OrtValue* ort_val); +template +void CustomOpBase::GetSessionConfigs(std::unordered_map& out, + ConstSessionOptions options) const { + const TOp* derived = static_cast(this); + std::vector keys = derived->GetSessionConfigKeys(); + + out.reserve(keys.size()); + + std::string config_entry_key = detail::MakeCustomOpConfigEntryKey(derived->GetName(), ""); + const size_t prefix_size = config_entry_key.length(); + + for (const auto& key : keys) { + config_entry_key.resize(prefix_size); + config_entry_key.append(key); + out[key] = options.GetConfigEntryOrDefault(config_entry_key.c_str(), ""); + } +} + } // namespace Ort diff --git a/onnxruntime/core/session/abi_session_options.cc b/onnxruntime/core/session/abi_session_options.cc index 636d3585ad..d4c9db344c 100644 --- a/onnxruntime/core/session/abi_session_options.cc +++ b/onnxruntime/core/session/abi_session_options.cc @@ -8,8 +8,10 @@ #include "core/framework/error_code_helper.h" #include #include +#include #include "core/session/inference_session.h" #include "abi_session_options_impl.h" +#include "api_utils.h" OrtSessionOptions::~OrtSessionOptions() = default; @@ -213,6 +215,33 @@ ORT_API_STATUS_IMPL(OrtApis::AddSessionConfigEntry, _Inout_ OrtSessionOptions* o return onnxruntime::ToOrtStatus(options->value.config_options.AddConfigEntry(config_key, config_value)); } +ORT_API_STATUS_IMPL(OrtApis::HasSessionConfigEntry, _In_ const OrtSessionOptions* options, + _In_z_ const char* config_key, _Out_ int* out) { + API_IMPL_BEGIN + auto value_opt = options->value.config_options.GetConfigEntry(config_key); + *out = static_cast(value_opt.has_value()); + return nullptr; + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::GetSessionConfigEntry, _In_ const OrtSessionOptions* options, + _In_z_ const char* config_key, _Out_ char* config_value, _Inout_ size_t* size) { + API_IMPL_BEGIN + auto value_opt = options->value.config_options.GetConfigEntry(config_key); + + if (!value_opt) { + std::ostringstream err_msg; + err_msg << "Session config entry '" << config_key << "' was not found."; + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, err_msg.str().c_str()); + } + + auto status = CopyStringToOutputArg(*value_opt, "Output buffer is not large enough for session config entry", config_value, + size); + + return onnxruntime::ToOrtStatus(status); + API_IMPL_END +} + ORT_API_STATUS_IMPL(OrtApis::AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name, _In_ const OrtValue* val) { API_IMPL_BEGIN diff --git a/onnxruntime/core/session/api_utils.cc b/onnxruntime/core/session/api_utils.cc new file mode 100644 index 0000000000..f7cb8520b1 --- /dev/null +++ b/onnxruntime/core/session/api_utils.cc @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "api_utils.h" + +onnxruntime::common::Status CopyStringToOutputArg(std::string_view str, const char* err_msg, char* out, size_t* size) { + const size_t str_len = str.size(); + const size_t req_size = str_len + 1; + + if (out == nullptr) { // User is querying the total output buffer size + *size = req_size; + return onnxruntime::common::Status::OK(); + } + + if (*size >= req_size) { // User provided a buffer of sufficient size + std::memcpy(out, str.data(), str_len); + out[str_len] = '\0'; + *size = req_size; + return onnxruntime::common::Status::OK(); + } + + // User has provided a buffer that is not large enough + *size = req_size; + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, err_msg); +} diff --git a/onnxruntime/core/session/api_utils.h b/onnxruntime/core/session/api_utils.h new file mode 100644 index 0000000000..27c2bbd66f --- /dev/null +++ b/onnxruntime/core/session/api_utils.h @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include + +onnxruntime::common::Status CopyStringToOutputArg(std::string_view str, const char* err_msg, char* out, size_t* size); diff --git a/onnxruntime/core/session/custom_ops.cc b/onnxruntime/core/session/custom_ops.cc index 489066a65d..dae00b96ea 100644 --- a/onnxruntime/core/session/custom_ops.cc +++ b/onnxruntime/core/session/custom_ops.cc @@ -10,47 +10,64 @@ #include "core/framework/op_kernel_context_internal.h" #include "core/framework/error_code_helper.h" #include "core/framework/tensor_type_and_shape.h" +#include "core/framework/onnxruntime_typeinfo.h" +#include "core/framework/tensorprotoutils.h" #include "core/graph/onnx_protobuf.h" +#include "core/session/allocator_adapters.h" #include "core/session/inference_session.h" #include "core/session/ort_apis.h" #include +#include "api_utils.h" ORT_API_STATUS_IMPL(OrtApis::KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ float* out) { + API_IMPL_BEGIN auto status = reinterpret_cast(info)->GetAttr(name, out); if (status.IsOK()) return nullptr; return onnxruntime::ToOrtStatus(status); + API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ int64_t* out) { + API_IMPL_BEGIN auto status = reinterpret_cast(info)->GetAttr(name, out); if (status.IsOK()) return nullptr; return onnxruntime::ToOrtStatus(status); + API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out) { + API_IMPL_BEGIN *out = reinterpret_cast(context)->InputCount(); return nullptr; + API_IMPL_END }; ORT_API_STATUS_IMPL(OrtApis::KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out) { + API_IMPL_BEGIN *out = reinterpret_cast(context)->OutputCount(); return nullptr; + API_IMPL_END }; ORT_API_STATUS_IMPL(OrtApis::KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index, _Out_ const OrtValue** out) { + API_IMPL_BEGIN *out = reinterpret_cast(reinterpret_cast(context)->GetInputMLValue(index)); return nullptr; + API_IMPL_END }; ORT_API_STATUS_IMPL(OrtApis::KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index, _In_ const int64_t* dim_values, size_t dim_count, _Out_ OrtValue** out) { + API_IMPL_BEGIN onnxruntime::TensorShape shape(dim_values, dim_count); *out = reinterpret_cast(reinterpret_cast(context)->OutputMLValue(index, shape)); return nullptr; + API_IMPL_END }; ORT_API_STATUS_IMPL(OrtApis::KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out, _Inout_ size_t* size) { + API_IMPL_BEGIN std::string value; auto status = reinterpret_cast(info)->GetAttr(name, &value); if (status.IsOK()) { @@ -68,6 +85,7 @@ ORT_API_STATUS_IMPL(OrtApis::KernelInfoGetAttribute_string, _In_ const OrtKernel } } return onnxruntime::ToOrtStatus(status); + API_IMPL_END } #ifdef _WIN32 @@ -76,12 +94,14 @@ ORT_API_STATUS_IMPL(OrtApis::KernelInfoGetAttribute_string, _In_ const OrtKernel #endif ORT_API_STATUS_IMPL(OrtApis::KernelContext_GetGPUComputeStream, _In_ const OrtKernelContext* context, _Outptr_ void** out) { + API_IMPL_BEGIN auto* stream = reinterpret_cast(context)->GetComputeStream(); if (stream) *out = stream->GetHandle(); else *out = nullptr; return nullptr; + API_IMPL_END }; #ifdef _WIN32 @@ -106,22 +126,157 @@ static Status CopyDataFromVectorToMemory(const std::vector& values, T* out, s ORT_API_STATUS_IMPL(OrtApis::KernelInfoGetAttributeArray_float, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ float* out, _Inout_ size_t* size) { + API_IMPL_BEGIN std::vector values; auto status = reinterpret_cast(info)->GetAttrs(name, values); if (status.IsOK()) { status = CopyDataFromVectorToMemory(values, out, size); } return onnxruntime::ToOrtStatus(status); + API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::KernelInfoGetAttributeArray_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ int64_t* out, _Inout_ size_t* size) { + API_IMPL_BEGIN std::vector values; auto status = reinterpret_cast(info)->GetAttrs(name, values); if (status.IsOK()) { status = CopyDataFromVectorToMemory(values, out, size); } return onnxruntime::ToOrtStatus(status); + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::KernelInfoGetAttribute_tensor, _In_ const OrtKernelInfo* info, _In_z_ const char* name, + _Inout_ OrtAllocator* allocator, _Outptr_ OrtValue** out) { + API_IMPL_BEGIN + const auto* op_kinfo = reinterpret_cast(info); + + // Get TensorProto attribute + onnx::TensorProto tensor_proto; + auto status = op_kinfo->GetAttr(name, &tensor_proto); + if (!status.IsOK()) { + return onnxruntime::ToOrtStatus(status); + } + + // Determine the tensor's size in bytes. + size_t req_size = 0; + status = onnxruntime::utils::GetSizeInBytesFromTensorProto<0>(tensor_proto, &req_size); + if (!status.IsOK()) { + return onnxruntime::ToOrtStatus(status); + } + + // Create Tensor that owns buffer memory that will be allocated with the provided OrtAllocator. + onnxruntime::TensorShape tensor_shape = onnxruntime::utils::GetTensorShapeFromTensorProto(tensor_proto); + const auto* const type = onnxruntime::DataTypeImpl::TensorTypeFromONNXEnum(tensor_proto.data_type())->GetElementType(); + onnxruntime::AllocatorPtr alloc_ptr = std::make_shared(allocator); + auto tensorp = std::make_unique(type, tensor_shape, std::move(alloc_ptr)); + + // Deserialize TensorProto into pre-allocated, empty Tensor. + status = onnxruntime::utils::TensorProtoToTensor(onnxruntime::Env::Default(), nullptr, tensor_proto, *tensorp); + if (!status.IsOK()) { + return onnxruntime::ToOrtStatus(status); + } + + // Initialize OrtValue from Tensor. + auto ml_tensor = onnxruntime::DataTypeImpl::GetType(); + auto value = std::make_unique(); + value->Init(tensorp.release(), ml_tensor, ml_tensor->GetDeleteFunc()); + + *out = value.release(); + return nullptr; + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::KernelInfo_GetInputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out) { + API_IMPL_BEGIN + *out = reinterpret_cast(info)->GetInputCount(); + return nullptr; + API_IMPL_END +}; + +ORT_API_STATUS_IMPL(OrtApis::KernelInfo_GetOutputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out) { + API_IMPL_BEGIN + *out = reinterpret_cast(info)->GetOutputCount(); + return nullptr; + API_IMPL_END +}; + +ORT_API_STATUS_IMPL(OrtApis::KernelInfo_GetInputName, _In_ const OrtKernelInfo* info, size_t index, _Out_ char* out, + _Inout_ size_t* size) { + API_IMPL_BEGIN + const auto* op_info = reinterpret_cast(info); + const auto input_defs = op_info->node().InputDefs(); + + if (index >= input_defs.size()) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "::OrtKernelInfo input index is out of bounds"); + } + + auto status = CopyStringToOutputArg(input_defs[index]->Name(), + "Output buffer is not large enough for ::OrtKernelInfo input name", out, size); + + return onnxruntime::ToOrtStatus(status); + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::KernelInfo_GetOutputName, _In_ const OrtKernelInfo* info, size_t index, _Out_ char* out, + _Inout_ size_t* size) { + API_IMPL_BEGIN + const auto* op_info = reinterpret_cast(info); + const auto output_defs = op_info->node().OutputDefs(); + + if (index >= output_defs.size()) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "::OrtKernelInfo output index is out of bounds"); + } + + auto status = CopyStringToOutputArg(output_defs[index]->Name(), + "Output buffer is not large enough for ::OrtKernelInfo output name", out, size); + + return onnxruntime::ToOrtStatus(status); + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::KernelInfo_GetInputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, + _Outptr_ OrtTypeInfo** type_info) { + API_IMPL_BEGIN + const auto* op_info = reinterpret_cast(info); + const auto input_defs = op_info->node().InputDefs(); + + if (index >= input_defs.size()) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "::OrtKernelInfo input index is out of bounds"); + } + + const onnxruntime::NodeArg* node_arg = input_defs[index]; + const ONNX_NAMESPACE::TypeProto* type_proto = node_arg->TypeAsProto(); + + if (type_proto == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_GRAPH, "::OrtKernelInfo input does not have a type"); + } + + return OrtTypeInfo::FromTypeProto(type_proto, type_info); + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::KernelInfo_GetOutputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, + _Outptr_ OrtTypeInfo** type_info) { + API_IMPL_BEGIN + const auto* op_info = reinterpret_cast(info); + const auto output_defs = op_info->node().OutputDefs(); + + if (index >= output_defs.size()) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "::OrtKernelInfo output index is out of bounds"); + } + + const onnxruntime::NodeArg* node_arg = output_defs[index]; + const ONNX_NAMESPACE::TypeProto* type_proto = node_arg->TypeAsProto(); + + if (type_proto == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_GRAPH, "::OrtKernelInfo output does not have a type"); + } + + return OrtTypeInfo::FromTypeProto(type_proto, type_info); + API_IMPL_END } #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS) diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 591f8076c5..a1a5bc7893 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -2651,6 +2651,15 @@ static constexpr OrtApi ort_api_1_to_14 = { &OrtApis::SetGlobalIntraOpThreadAffinity, &OrtApis::RegisterCustomOpsLibrary_V2, &OrtApis::RegisterCustomOpsUsingFunction, + &OrtApis::KernelInfo_GetInputCount, + &OrtApis::KernelInfo_GetOutputCount, + &OrtApis::KernelInfo_GetInputName, + &OrtApis::KernelInfo_GetOutputName, + &OrtApis::KernelInfo_GetInputTypeInfo, + &OrtApis::KernelInfo_GetOutputTypeInfo, + &OrtApis::KernelInfoGetAttribute_tensor, + &OrtApis::HasSessionConfigEntry, + &OrtApis::GetSessionConfigEntry, }; // Asserts to do a some checks to ensure older Versions of the OrtApi never change (will detect an addition or deletion but not if they cancel out each other) diff --git a/onnxruntime/core/session/ort_apis.h b/onnxruntime/core/session/ort_apis.h index b54b350883..a1ce3bcb37 100644 --- a/onnxruntime/core/session/ort_apis.h +++ b/onnxruntime/core/session/ort_apis.h @@ -410,7 +410,25 @@ ORT_API_STATUS_IMPL(SetGlobalIntraOpThreadAffinity, _Inout_ OrtThreadingOptions* ORT_API_STATUS_IMPL(RegisterCustomOpsLibrary_V2, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* library_name); - ORT_API_STATUS_IMPL(RegisterCustomOpsUsingFunction, _Inout_ OrtSessionOptions* options, _In_ const char* registration_func_name); + +ORT_API_STATUS_IMPL(KernelInfo_GetInputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); +ORT_API_STATUS_IMPL(KernelInfo_GetOutputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); +ORT_API_STATUS_IMPL(KernelInfo_GetInputName, _In_ const OrtKernelInfo* info, size_t index, _Out_ char* out, + _Inout_ size_t* size); +ORT_API_STATUS_IMPL(KernelInfo_GetOutputName, _In_ const OrtKernelInfo* info, size_t index, _Out_ char* out, + _Inout_ size_t* size); +ORT_API_STATUS_IMPL(KernelInfo_GetInputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, + _Outptr_ OrtTypeInfo** type_info); +ORT_API_STATUS_IMPL(KernelInfo_GetOutputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, + _Outptr_ OrtTypeInfo** type_info); +ORT_API_STATUS_IMPL(KernelInfoGetAttribute_tensor, _In_ const OrtKernelInfo* info, _In_z_ const char* name, + _Inout_ OrtAllocator* allocator, _Outptr_ OrtValue** out); + +ORT_API_STATUS_IMPL(HasSessionConfigEntry, _In_ const OrtSessionOptions* options, + _In_z_ const char* config_key, _Out_ int* out); +ORT_API_STATUS_IMPL(GetSessionConfigEntry, _In_ const OrtSessionOptions* options, + _In_z_ const char* config_key, _Out_ char* config_value, _Inout_ size_t* size); } // namespace OrtApis + diff --git a/onnxruntime/python/tools/custom_op_wrapper/create_custom_op_wrapper.py b/onnxruntime/python/tools/custom_op_wrapper/create_custom_op_wrapper.py new file mode 100644 index 0000000000..33702d3fb5 --- /dev/null +++ b/onnxruntime/python/tools/custom_op_wrapper/create_custom_op_wrapper.py @@ -0,0 +1,273 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +""" +Creates an ONNX model with a single custom operator node that wraps an opaque model blob +meant to be executed by an external runtime. The model blob is serialized into the custom operator's +attributes. + +Example: + +python3 create_custom_op_wrapper.py --domain "test.domain" + --custom_op_name my_custom_op + --inputs "input0;FLOAT;1,10" "input1;FLOAT;1,10,10" + --outputs "output0;STRING;10" + --attribute_data xml googlenet-v1.xml + --attribute_data bin googlenet-v1.bin + -o test_model.onnx +""" + +import argparse +import os +import sys +from dataclasses import dataclass +from typing import List, Optional, Union + +import onnx +from onnx import TensorProto, helper + +IO_NAME_INDEX = 0 +IO_ELEM_TYPE_INDEX = 1 +IO_SHAPE_INDEX = 2 + +TENSOR_TYPE_MAP = { + "UNDEFINED": TensorProto.UNDEFINED, + "FLOAT": TensorProto.FLOAT, + "UINT8": TensorProto.UINT8, + "INT8": TensorProto.INT8, + "UINT16": TensorProto.UINT16, + "INT16": TensorProto.INT16, + "INT32": TensorProto.INT32, + "INT64": TensorProto.INT64, + "STRING": TensorProto.STRING, + "BOOL": TensorProto.BOOL, + "FLOAT16": TensorProto.FLOAT16, + "DOUBLE": TensorProto.DOUBLE, + "UINT32": TensorProto.UINT32, + "UINT64": TensorProto.UINT64, + "COMPLEX64": TensorProto.COMPLEX64, + "COMPLEX128": TensorProto.COMPLEX128, + "BFLOAT16": TensorProto.BFLOAT16, +} + + +@dataclass +class IOInfo: + """ + Class that represents the index, name, element type, and shape of an input or output. + """ + + index: int + name: str + elem_type: TensorProto.DataType + shape: Optional[List[Union[int, str]]] + + +def str_is_int(string: str) -> bool: + try: + int(string) + return True + except ValueError: + return False + + +def parse_shape(shape_str: str) -> Optional[List[Union[int, str]]]: + try: + shape = [int(s) if str_is_int(s) else s for s in shape_str.split(",")] + except ValueError: + shape = None + + return shape + + +class ParseIOInfoAction(argparse.Action): + def __call__(self, parser, namespace, io_strs, opt_str): + is_input = opt_str == "--inputs" + io_meta_name = "input" if is_input else "output" + ios = [] + + for io_idx, io_str in enumerate(io_strs): + comp_strs = [] + + try: + comp_strs = io_str.split(";") + except ValueError: + parser.error(f"{opt_str}: {io_meta_name} info must be separated by ';'") + + if len(comp_strs) != 3: + parser.error(f"{opt_str}: {io_meta_name} info must have 3 components, but provided {len(comp_strs)}.") + + # Get io name + io_name = comp_strs[IO_NAME_INDEX] + + # Get io element type + io_elem_type = TENSOR_TYPE_MAP.get(comp_strs[IO_ELEM_TYPE_INDEX]) + if io_elem_type is None: + type_options = ",".join(TENSOR_TYPE_MAP.keys()) + parser.error( + f"{opt_str}: invalid {io_meta_name} element type '{comp_strs[IO_ELEM_TYPE_INDEX]}'. " + f"Must be one of {type_options}." + ) + + # Get io shape + io_shape = parse_shape(comp_strs[IO_SHAPE_INDEX]) + if io_shape is None: + parser.error( + f"{opt_str}: invalid {io_meta_name} shape '{comp_strs[IO_SHAPE_INDEX]}'. " + "Expected comma-separated list of integers." + ) + + ios.append( + IOInfo( + index=io_idx, + name=io_name, + elem_type=io_elem_type, + shape=io_shape, + ) + ) + + # Sort ios on index + ios = sorted(ios, key=lambda elem: elem.index) + + setattr(namespace, self.dest, ios) + + +def parse_arguments() -> argparse.Namespace: + io_metavar = '";;"' + parser = argparse.ArgumentParser() + + parser.add_argument( + "-n", + "--custom_op_name", + required=True, + help="The custom operator's name.", + ) + + parser.add_argument( + "-d", + "--domain", + required=True, + help="The ONNX domain name used by the custom operator node.", + ) + + parser.add_argument( + "--inputs", + nargs="+", + required=True, + action=ParseIOInfoAction, + help="List of strings specifying the name, element type, and shape of every input (in order). " + 'Ex: --inputs "input_0;FLOAT;1,10" "input_1;INT8;32"', + metavar=io_metavar, + ) + + parser.add_argument( + "--outputs", + nargs="+", + required=True, + action=ParseIOInfoAction, + help="List of strings specifying the name, element type, and shape of every output (in order). " + 'Ex: --outputs "output_0;FLOAT;1,10" "output_1;INT8;32"', + metavar=io_metavar, + ) + + parser.add_argument( + "-o", + "--output_model", + required=True, + help="The name of the generated output model.", + ) + + parser.add_argument( + "-a", + "--attribute_data", + nargs=2, + required=False, + action="append", + help="The attribute name and file path of the data to serialize as a node attribute. " + "Can be provided multiple times. Ex: -a model_xml ./model.xml -a model_bin ./model.bin", + ) + + parser.add_argument( + "-v", + "--opset_version", + type=int, + required=False, + default=13, + help="The Opset version.", + ) + + return parser.parse_args() + + +def get_attributes(attr_data_info: List[List[str]]): + if not attr_data_info: + return {} + + attrs = {} + + for info in attr_data_info: + filepath = os.path.normpath(info[1]) + + if not os.path.exists(filepath): + print(f"[ERROR] attribute file '{info[1]}' does not exist.", file=sys.stderr) + sys.exit(1) + + data = b"" + with open(filepath, "rb") as file_desc: + data = file_desc.read() + + tensor = helper.make_tensor(name=info[0], data_type=TensorProto.UINT8, dims=[len(data)], vals=data, raw=True) + + attrs[info[0]] = tensor + + return attrs + + +def main(): + args = parse_arguments() + + inputs = [] + input_names = [] + for inp in args.inputs: + inputs.append(helper.make_tensor_value_info(inp.name, inp.elem_type, inp.shape)) + input_names.append(inp.name) + + outputs = [] + output_names = [] + for out in args.outputs: + outputs.append(helper.make_tensor_value_info(out.name, out.elem_type, out.shape)) + output_names.append(out.name) + + attrs = get_attributes(args.attribute_data) + + custom_op_node = helper.make_node( + args.custom_op_name, + name=args.custom_op_name + "_0", + inputs=input_names, + outputs=output_names, + domain=args.domain, + **attrs, + ) + + output_model_path = os.path.normpath(args.output_model) + graph_name, model_ext = os.path.splitext(os.path.basename(output_model_path)) + + if model_ext != ".onnx": + print( + f"[ERROR] Invalid output model name '{output_model_path}'. Must end in '.onnx'", + file=sys.stderr, + ) + sys.exit(1) + + graph_def = helper.make_graph([custom_op_node], graph_name, inputs, outputs) + model_def = helper.make_model(graph_def, opset_imports=[helper.make_opsetid(args.domain, args.opset_version)]) + + onnx.checker.check_model(model_def) + onnx.save(model_def, output_model_path) + + print(f"[INFO] Saved output model to {output_model_path}") + + +if __name__ == "__main__": + main() diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index 2a7b13c78c..a630bcddb7 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -164,6 +164,10 @@ static constexpr PATH_TYPE SEQUENCE_MODEL_URI_2 = TSTR("testdata/optional_sequen #endif static constexpr PATH_TYPE CUSTOM_OP_MODEL_URI = TSTR("testdata/foo_1.onnx"); static constexpr PATH_TYPE CUSTOM_OP_LIBRARY_TEST_MODEL_URI = TSTR("testdata/custom_op_library/custom_op_test.onnx"); +#if defined(USE_OPENVINO) && (!defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS)) +static constexpr PATH_TYPE CUSTOM_OP_OPENVINO_WRAPPER_LIB_TEST_MODEL_URI = TSTR( + "testdata/custom_op_openvino_wrapper_library/custom_op_mnist_ov_wrapper.onnx"); +#endif static constexpr PATH_TYPE OVERRIDABLE_INITIALIZER_MODEL_URI = TSTR("testdata/overridable_initializer.onnx"); static constexpr PATH_TYPE NAMED_AND_ANON_DIM_PARAM_URI = TSTR("testdata/capi_symbolic_dims.onnx"); static constexpr PATH_TYPE MODEL_WITH_CUSTOM_MODEL_METADATA = TSTR("testdata/model_with_valid_ort_config_json.onnx"); @@ -1187,6 +1191,81 @@ TEST(CApiTest, RegisterCustomOpForCPUAndCUDA) { } #endif + +#if defined(USE_OPENVINO) && (!defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS)) +TEST(CApiTest, test_custom_op_openvino_wrapper_library) { + // Tests a custom operator that wraps an OpenVINO MNIST model (.xml and .bin files serialized into node attributes). + // The custom op extracts the serialized .xml/.bin bytes and creates an in-memory OpenVINO model + // during kernel creation. The custom op is passed an image of a hand-drawn "1" as an input during computation, which + // is then inferenced using OpenVINO C++ APIs. + std::vector inputs(1); + inputs[0].name = "Input3"; + inputs[0].dims = {1, 1, 28, 28}; + + // Float image with the digit "1". + inputs[0].values = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.75f, 1.0f, 0.75f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.85f, 0.99f, 0.85f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 1.0f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 0.99f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.98f, 1.0f, 0.98f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.99f, 1.0f, 0.99f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.94f, 0.99f, 0.94f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.1f, 0.75f, 0.75f, 0.75f, 0.1f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + + // prepare expected outputs + std::vector expected_output_dims = {1, 10}; + + // Digit 1 (index 1) has the highest probability (before applying softmax) + std::vector expected_vals = {-5.34957457f, 13.1904755f, -4.79670954f, -3.59232116f, 2.31260920f, + -4.27866220f, -4.31867933f, 0.587718308f, -2.33952785f, -3.88515306f}; + + const ORTCHAR_T* lib_name; +#if defined(_WIN32) + lib_name = ORT_TSTR("custom_op_openvino_wrapper_library.dll"); +#elif defined(__APPLE__) + lib_name = ORT_TSTR("libcustom_op_openvino_wrapper_library.dylib"); +#else + lib_name = ORT_TSTR("./libcustom_op_openvino_wrapper_library.so"); +#endif + + Ort::SessionOptions session_opts; + Ort::CustomOpConfigs custom_op_configs; + + custom_op_configs.AddConfig("OpenVINO_Wrapper", "device_type", "CPU"); + session_opts.RegisterCustomOpsLibrary(lib_name, custom_op_configs); + + Ort::Session session(*ort_env, CUSTOM_OP_OPENVINO_WRAPPER_LIB_TEST_MODEL_URI, session_opts); + auto default_allocator = std::make_unique(); + + RunSession(default_allocator.get(), session, + inputs, + "Plus214_Output_0", + expected_output_dims, + expected_vals, + nullptr); +} +#endif // defined(USE_OPENVINO) && (!defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS)) + // It has memory leak. The OrtCustomOpDomain created in custom_op_library.cc:RegisterCustomOps function was not freed #if defined(__ANDROID__) TEST(CApiTest, DISABLED_test_custom_op_library) { diff --git a/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.cc b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.cc new file mode 100644 index 0000000000..ebdefda2b9 --- /dev/null +++ b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.cc @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include + +#include "custom_op_lib.h" +#include "openvino_wrapper.h" + +static const char* c_OpDomain = "test.customop.ov"; + +static void AddOrtCustomOpDomainToContainer(Ort::CustomOpDomain&& domain) { + static std::vector ort_custom_op_domain_container; + static std::mutex ort_custom_op_domain_mutex; + std::lock_guard lock(ort_custom_op_domain_mutex); + ort_custom_op_domain_container.push_back(std::move(domain)); +} + +OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api_base) { + + // Allow use of Ort::GetApi() in C++ API implementations. + Ort::InitApi(api_base->GetApi(ORT_API_VERSION)); + Ort::UnownedSessionOptions session_options(options); + + static CustomOpOpenVINO c_CustomOpOpenVINO(Ort::ConstSessionOptions{options}); + + OrtStatus* result = nullptr; + + try { + Ort::CustomOpDomain domain{c_OpDomain}; + domain.Add(&c_CustomOpOpenVINO); + + session_options.Add(domain); + AddOrtCustomOpDomainToContainer(std::move(domain)); + + } catch(const std::exception& e) { + Ort::Status status{e}; + result = status.release(); + } + + return result; +} diff --git a/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.def b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.def new file mode 100644 index 0000000000..fb17aac064 --- /dev/null +++ b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.def @@ -0,0 +1,4 @@ +LIBRARY "custom_op_openvino_wrapper_library.dll" +EXPORTS + RegisterCustomOps @1 + diff --git a/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.h b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.h new file mode 100644 index 0000000000..e4f9b34939 --- /dev/null +++ b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.h @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +ORT_EXPORT OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api_base); + +#ifdef __cplusplus +} +#endif diff --git a/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.lds b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.lds new file mode 100644 index 0000000000..bb5d118c7c --- /dev/null +++ b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_lib.lds @@ -0,0 +1,6 @@ +VERS_1.0.0 { + global: + RegisterCustomOps; + local: + *; +}; diff --git a/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_mnist_ov_wrapper.onnx b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_mnist_ov_wrapper.onnx new file mode 100644 index 0000000000..e786f92d51 Binary files /dev/null and b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/custom_op_mnist_ov_wrapper.onnx differ diff --git a/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/openvino_wrapper.cc b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/openvino_wrapper.cc new file mode 100644 index 0000000000..43266848d0 --- /dev/null +++ b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/openvino_wrapper.cc @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "openvino_wrapper.h" + +#include +#include + +static ov::element::Type ConvertONNXToOVType(ONNXTensorElementDataType onnx_type) { + switch (onnx_type) { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: + return ov::element::f32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: + return ov::element::u8; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: + return ov::element::i8; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: + return ov::element::u16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: + return ov::element::i16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: + return ov::element::i32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: + return ov::element::i64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: + return ov::element::boolean; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: + return ov::element::f16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: + return ov::element::f64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: + return ov::element::u32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: + return ov::element::u64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: + return ov::element::bf16; + default: + return ov::element::undefined; + } +} + +static bool AreShapesEqual(const std::vector& ort_shape, const ov::Shape& ov_shape) { + if (ort_shape.size() != ov_shape.size()) { + return false; + } + + const size_t num_dims = ort_shape.size(); + + for (size_t i = 0; i < num_dims; ++i) { + if (static_cast(ort_shape[i]) != ov_shape[i]) { + return false; + } + } + + return true; +} + +static bool AreIOTypesEqual(Ort::ConstTypeInfo type_info, const ov::Output& ov_node) { + Ort::ConstTensorTypeAndShapeInfo type_shape_info = type_info.GetTensorTypeAndShapeInfo(); + + // Check element type. + const ov::element::Type ort_elem_type = ConvertONNXToOVType(type_shape_info.GetElementType()); + const ov::element::Type ov_elem_type = ov_node.get_element_type(); + if (ort_elem_type != ov_elem_type) { + return false; + } + + // Check shape. + const std::vector ort_shape = type_shape_info.GetShape(); + const ov::Shape& ov_shape = ov_node.get_shape(); + if (!AreShapesEqual(ort_shape, ov_shape)) { + return false; + } + + return true; +} + +static bool ValidateInputsAndOutputs(const Ort::ConstKernelInfo& kinfo, const ov::OutputVector& ov_inputs, + const ov::OutputVector& ov_outputs) { + const size_t num_inputs = kinfo.GetInputCount(); + const size_t num_outputs = kinfo.GetOutputCount(); + + // Number of inputs and outputs must match. + if (ov_inputs.size() != num_inputs || ov_outputs.size() != num_outputs) { + return false; + } + + // Check input names, shapes, and element types. + for (size_t i = 0; i < num_inputs; ++i) { + Ort::TypeInfo ort_type = kinfo.GetInputTypeInfo(i); + const auto& ov_input = ov_inputs[i]; + + if (kinfo.GetInputName(i) != ov_input.get_any_name()) { + return false; + } + + if (!AreIOTypesEqual(ort_type.GetConst(), ov_input)) { + return false; + } + } + + // Check output names, shapes, and element types. + for (size_t i = 0; i < num_outputs; ++i) { + Ort::TypeInfo ort_type = kinfo.GetOutputTypeInfo(i); + const auto& ov_output = ov_outputs[i]; + + if (kinfo.GetOutputName(i) != ov_output.get_any_name()) { + return false; + } + + if (!AreIOTypesEqual(ort_type.GetConst(), ov_output)) { + return false; + } + } + + return true; +} + +/// +/// Converts an ORT tensor (as a Ort::Value) into an OpenVINO tensor that uses +/// the same underlying tensor data. +/// +static ov::Tensor OrtToOpenVINOTensor(Ort::UnownedValue ort_tensor) { + Ort::TensorTypeAndShapeInfo type_shape_info = ort_tensor.GetTensorTypeAndShapeInfo(); + std::vector ort_shape = type_shape_info.GetShape(); + + ov::Shape ov_shape(ort_shape.begin(), ort_shape.end()); // Copy shape because ORT uses int64_t, not size_t. + void* raw_data = ort_tensor.GetTensorMutableData(); + auto elem_type = ConvertONNXToOVType(type_shape_info.GetElementType()); + + return ov::Tensor(elem_type, ov_shape, raw_data); +} + +KernelOpenVINO::KernelOpenVINO(const OrtApi& /* api*/, const OrtKernelInfo* info, + const std::unordered_map& session_configs) + : weights_(nullptr) { + Ort::ConstKernelInfo kinfo(info); + Ort::AllocatorWithDefaultOptions allocator; + + // Extract OpenVINO .bin and .xml contents from node attributes. + this->weights_ = kinfo.GetTensorAttribute("BIN", allocator); // Must keep the weights memory alive for inference. + std::string xml_contents = kinfo.GetAttribute("XML"); + + // Create OpenVINO model. + ov::Core core; + const ov::Tensor weights_tensor = OrtToOpenVINOTensor(this->weights_.GetUnowned()); + std::shared_ptr model = core.read_model(xml_contents, weights_tensor); + + // Validate input/output shapes and types. + this->ov_inputs_ = model->inputs(); + this->ov_outputs_ = model->outputs(); + + if (!ValidateInputsAndOutputs(kinfo, this->ov_inputs_, this->ov_outputs_)) { + // A more detailed error message would be better. + ORT_CXX_API_THROW("I/O names, shapes, or element types do not match OpenVINO model.", ORT_INVALID_GRAPH); + } + + // Get OpenVINO device type from provider options. + auto device_type_it = session_configs.find("device_type"); + + if ((device_type_it == session_configs.end()) || device_type_it->second.empty()) { + this->device_type_ = "CPU"; + } else { + this->device_type_ = device_type_it->second; + } + + // Compile OpenVINO model. + this->compiled_model_ = core.compile_model(model, this->device_type_); +} + +void KernelOpenVINO::Compute(OrtKernelContext* context) { + Ort::KernelContext kcontext(context); + + // Create inference request. + ov::InferRequest infer_req = this->compiled_model_.create_infer_request(); + + const size_t num_inputs = kcontext.GetInputCount(); + assert(num_inputs == this->ov_inputs_.size()); + + // Set input tensors. + for (size_t i = 0; i < num_inputs; ++i) { + Ort::ConstValue ort_val = kcontext.GetInput(i); + const auto& input_info = this->ov_inputs_[i]; + + const void* p_input_data = ort_val.GetTensorData(); + + // OpenVINO does not always observe const-correctness. + ov::Tensor input_tensor(input_info.get_element_type(), input_info.get_shape(), const_cast(p_input_data)); + + infer_req.set_input_tensor(i, input_tensor); + } + + const size_t num_outputs = kcontext.GetOutputCount(); + assert(num_outputs == this->ov_outputs_.size()); + + // Set output tensors that are backed by ORT memory. + for (size_t i = 0; i < num_outputs; ++i) { + const auto& output_info = this->ov_outputs_[i]; + const ov::Shape& ov_shape = output_info.get_shape(); + const ov::element::Type ov_elem_type = output_info.get_element_type(); + + std::vector ort_shape(ov_shape.begin(), ov_shape.end()); + Ort::UnownedValue ort_val = kcontext.GetOutput(i, ort_shape); + void* ort_memory = ort_val.GetTensorMutableData(); + + infer_req.set_output_tensor(i, ov::Tensor(ov_elem_type, ov_shape, ort_memory)); + } + + // Run inference. + infer_req.infer(); +} + +// +// CustomOpOpenVINO +// +CustomOpOpenVINO::CustomOpOpenVINO(Ort::ConstSessionOptions session_options) { + GetSessionConfigs(this->session_configs_, session_options); +} + +void* CustomOpOpenVINO::CreateKernel(const OrtApi& api, const OrtKernelInfo* info) const { + return new KernelOpenVINO(api, info, this->session_configs_); +} + +std::vector CustomOpOpenVINO::GetSessionConfigKeys() const { return {"device_type"}; } diff --git a/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/openvino_wrapper.h b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/openvino_wrapper.h new file mode 100644 index 0000000000..738127401b --- /dev/null +++ b/onnxruntime/test/testdata/custom_op_openvino_wrapper_library/openvino_wrapper.h @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#define ORT_API_MANUAL_INIT +#include +#undef ORT_API_MANUAL_INIT + +#ifdef _MSC_VER +#pragma warning(disable: 4100) +#endif +#include +#ifdef _MSC_VER +#pragma warning(default : 4100) +#endif + +#include + +struct KernelOpenVINO { + KernelOpenVINO(const OrtApi& api, const OrtKernelInfo* info, + const std::unordered_map& session_configs); + + KernelOpenVINO(const KernelOpenVINO&) = delete; + KernelOpenVINO& operator=(const KernelOpenVINO&) = delete; + + ~KernelOpenVINO() = default; + + void Compute(OrtKernelContext* context); + + private: + ov::CompiledModel compiled_model_; + ov::OutputVector ov_inputs_; + ov::OutputVector ov_outputs_; + Ort::Value weights_; + std::string device_type_; +}; + +struct CustomOpOpenVINO : Ort::CustomOpBase { + explicit CustomOpOpenVINO(Ort::ConstSessionOptions session_options); + + CustomOpOpenVINO(const CustomOpOpenVINO&) = delete; + CustomOpOpenVINO& operator=(const CustomOpOpenVINO&) = delete; + + void* CreateKernel(const OrtApi& api, const OrtKernelInfo* info) const; + + constexpr const char* GetName() const noexcept { + return "OpenVINO_Wrapper"; + } + + constexpr const char* GetExecutionProviderType() const noexcept { + return "CPUExecutionProvider"; + } + + constexpr size_t GetInputTypeCount() const noexcept { + return 1; + } + + constexpr size_t GetOutputTypeCount() const noexcept { + return 1; + } + + constexpr ONNXTensorElementDataType GetInputType(size_t /* index */) const noexcept { + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + } + + constexpr ONNXTensorElementDataType GetOutputType(size_t /* index */) const noexcept { + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + } + + constexpr OrtCustomOpInputOutputCharacteristic GetInputCharacteristic(size_t /* index */) const noexcept { + return INPUT_OUTPUT_VARIADIC; + } + + constexpr OrtCustomOpInputOutputCharacteristic GetOutputCharacteristic(size_t /* index */) const noexcept { + return INPUT_OUTPUT_VARIADIC; + } + + constexpr bool GetVariadicInputHomogeneity() const noexcept { + return false; // heterogenous + } + + constexpr bool GetVariadicOutputHomogeneity() const noexcept { + return false; // heterogeneous + } + + std::vector GetSessionConfigKeys() const; + + private: + std::unordered_map session_configs_; +};