diff --git a/nodejs/CMakeLists.txt b/nodejs/CMakeLists.txt index e5d23f6840..1eee930bcd 100644 --- a/nodejs/CMakeLists.txt +++ b/nodejs/CMakeLists.txt @@ -26,7 +26,7 @@ endif() # include dirs include_directories(${CMAKE_JS_INC}) -include_directories(${CMAKE_SOURCE_DIR}/../include/onnxruntime) +include_directories(${CMAKE_SOURCE_DIR}/../include/onnxruntime/core/session) include_directories(${CMAKE_SOURCE_DIR}/node_modules/node-addon-api) # source files diff --git a/nodejs/src/onnxruntime_cxx_api.h b/nodejs/src/onnxruntime_cxx_api.h deleted file mode 100644 index 572a9815ba..0000000000 --- a/nodejs/src/onnxruntime_cxx_api.h +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Summary: The Ort C++ API is a header only wrapper around the Ort C API. -// -// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors -// and automatically releasing resources in the destructors. -// -// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers. -// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};). -// -// Only move assignment between objects is allowed, there are no copy constructors. Some objects have explicit 'Clone' -// methods for this purpose. - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace Ort { - -// All C++ methods that can fail will throw an exception of this type -struct Exception : std::exception { - Exception(std::string &&string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {} - - OrtErrorCode GetOrtErrorCode() const { return code_; } - const char *what() const noexcept override { return message_.c_str(); } - -private: - std::string message_; - OrtErrorCode code_; -}; - -// This need to be defined in a .cpp file -extern const OrtApi *g_api_; - -// This returns a reference to the OrtApi interface in use, in case someone wants to use the C API functions -inline const OrtApi &GetApi() { return *g_api_; } - -// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the -// various OrtRelease* functions for every Ort* type This can't be done in the C API since C doesn't have function -// overloading. -#define ORT_DEFINE_RELEASE(NAME) \ - inline void OrtRelease(Ort##NAME *ptr) { GetApi().Release##NAME(ptr); } - -ORT_DEFINE_RELEASE(MemoryInfo); -ORT_DEFINE_RELEASE(CustomOpDomain); -ORT_DEFINE_RELEASE(Env); -ORT_DEFINE_RELEASE(RunOptions); -ORT_DEFINE_RELEASE(Session); -ORT_DEFINE_RELEASE(SessionOptions); -ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo); -ORT_DEFINE_RELEASE(TypeInfo); -ORT_DEFINE_RELEASE(Value); -ORT_DEFINE_RELEASE(ModelMetadata); -ORT_DEFINE_RELEASE(ThreadingOptions); - -// This is used internally by the C++ API. This is the common base class used by the wrapper objects. -template struct Base { - Base() = default; - Base(T *p) : p_{p} { - if (!p) - throw Ort::Exception("Allocation failure", ORT_FAIL); - } - ~Base() { OrtRelease(p_); } - - operator T *() { return p_; } - operator const T *() const { return p_; } - - T *release() { - T *p = p_; - p_ = nullptr; - return p; - } - -protected: - Base(const Base &) = delete; - Base &operator=(const Base &) = delete; - Base(Base &&v) noexcept : p_{v.p_} { v.p_ = nullptr; } - void operator=(Base &&v) noexcept { - if (p_) { - OrtRelease(p_); - } - p_ = v.p_; - v.p_ = nullptr; - } - - T *p_{}; - - template - friend struct Unowned; // This friend line is needed to keep the centos C++ compiler from giving an error -}; - -template struct Unowned : T { - Unowned(decltype(T::p_) p) : T{p} {} - Unowned(Unowned &&v) : T{v.p_} {} - ~Unowned() { this->p_ = nullptr; } -}; - -struct AllocatorWithDefaultOptions; -struct MemoryInfo; -struct Env; -struct TypeInfo; -struct Value; -struct ModelMetadata; - -struct Env : Base { - Env(std::nullptr_t) {} - Env(OrtLoggingLevel default_logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char *logid = ""); - Env(const OrtThreadingOptions *tp_options, OrtLoggingLevel default_logging_level = ORT_LOGGING_LEVEL_WARNING, - _In_ const char *logid = ""); - Env(OrtLoggingLevel default_logging_level, const char *logid, OrtLoggingFunction logging_function, - void *logger_param); - explicit Env(OrtEnv *p) : Base{p} {} - - Env &EnableTelemetryEvents(); - Env &DisableTelemetryEvents(); - - static const OrtApi *s_api; -}; - -struct CustomOpDomain : Base { - explicit CustomOpDomain(std::nullptr_t) {} - explicit CustomOpDomain(const char *domain); - - void Add(OrtCustomOp *op); -}; - -struct RunOptions : Base { - RunOptions(std::nullptr_t) {} - RunOptions(); - - RunOptions &SetRunLogVerbosityLevel(int); - int GetRunLogVerbosityLevel() const; - - RunOptions &SetRunLogSeverityLevel(int); - int GetRunLogSeverityLevel() const; - - RunOptions &SetRunTag(const char *run_tag); - const char *GetRunTag() const; - - // terminate ALL currently executing Session::Run calls that were made using this RunOptions instance - RunOptions &SetTerminate(); - // unset the terminate flag so this RunOptions instance can be used in a new Session::Run call - RunOptions &UnsetTerminate(); -}; - -struct SessionOptions : Base { - explicit SessionOptions(std::nullptr_t) {} - SessionOptions(); - explicit SessionOptions(OrtSessionOptions *p) : Base{p} {} - - SessionOptions Clone() const; - - SessionOptions &SetIntraOpNumThreads(int intra_op_num_threads); - SessionOptions &SetInterOpNumThreads(int inter_op_num_threads); - SessionOptions &SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level); - - SessionOptions &EnableCpuMemArena(); - SessionOptions &DisableCpuMemArena(); - - SessionOptions &SetOptimizedModelFilePath(const ORTCHAR_T *optimized_model_file); - - SessionOptions &EnableProfiling(const ORTCHAR_T *profile_file_prefix); - SessionOptions &DisableProfiling(); - - SessionOptions &EnableMemPattern(); - SessionOptions &DisableMemPattern(); - - SessionOptions &SetExecutionMode(ExecutionMode execution_mode); - - SessionOptions &SetLogId(const char *logid); - SessionOptions &SetLogSeverityLevel(int); - - SessionOptions &Add(OrtCustomOpDomain *custom_op_domain); - - SessionOptions &DisablePerSessionThreads(); -}; - -struct ModelMetadata : Base { - explicit ModelMetadata(std::nullptr_t) {} - explicit ModelMetadata(OrtModelMetadata *p) : Base{p} {} - - char *GetProducerName(OrtAllocator *allocator) const; - char *GetGraphName(OrtAllocator *allocator) const; - char *GetDomain(OrtAllocator *allocator) const; - char *GetDescription(OrtAllocator *allocator) const; - char **GetCustomMetadataMapKeys(OrtAllocator *allocator, _Out_ int64_t &num_keys) const; - char *LookupCustomMetadataMap(const char *key, OrtAllocator *allocator) const; - int64_t GetVersion() const; -}; - -struct Session : Base { - explicit Session(std::nullptr_t) {} - Session(Env &env, const ORTCHAR_T *model_path, const SessionOptions &options); - Session(Env &env, const void *model_data, size_t model_data_length, const SessionOptions &options); - - // Run that will allocate the output values - std::vector Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, - size_t input_count, const char *const *output_names, size_t output_count); - // Run for when there is a list of prealloated outputs - void Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, - const char *const *output_names, Value *output_values, size_t output_count); - - size_t GetInputCount() const; - size_t GetOutputCount() const; - size_t GetOverridableInitializerCount() const; - - char *GetInputName(size_t index, OrtAllocator *allocator) const; - char *GetOutputName(size_t index, OrtAllocator *allocator) const; - char *GetOverridableInitializerName(size_t index, OrtAllocator *allocator) const; - char *EndProfiling(OrtAllocator *allocator) const; - ModelMetadata GetModelMetadata() const; - - TypeInfo GetInputTypeInfo(size_t index) const; - TypeInfo GetOutputTypeInfo(size_t index) const; - TypeInfo GetOverridableInitializerTypeInfo(size_t index) const; -}; - -struct TensorTypeAndShapeInfo : Base { - explicit TensorTypeAndShapeInfo(std::nullptr_t) {} - explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo *p) : Base{p} {} - - ONNXTensorElementDataType GetElementType() const; - size_t GetElementCount() const; - - size_t GetDimensionsCount() const; - void GetDimensions(int64_t *values, size_t values_count) const; - void GetSymbolicDimensions(const char **values, size_t values_count) const; - - std::vector GetShape() const; -}; - -struct TypeInfo : Base { - explicit TypeInfo(std::nullptr_t) {} - explicit TypeInfo(OrtTypeInfo *p) : Base{p} {} - - Unowned GetTensorTypeAndShapeInfo() const; - ONNXType GetONNXType() const; -}; - -struct Value : Base { - template - static Value CreateTensor(const OrtMemoryInfo *info, T *p_data, size_t p_data_element_count, const int64_t *shape, - size_t shape_len); - static Value CreateTensor(const OrtMemoryInfo *info, void *p_data, size_t p_data_byte_count, const int64_t *shape, - size_t shape_len, ONNXTensorElementDataType type); - template static Value CreateTensor(OrtAllocator *allocator, const int64_t *shape, size_t shape_len); - static Value CreateTensor(OrtAllocator *allocator, const int64_t *shape, size_t shape_len, - ONNXTensorElementDataType type); - - static Value CreateMap(Value &keys, Value &values); - static Value CreateSequence(std::vector &values); - - template static Value CreateOpaque(const char *domain, const char *type_name, const T &); - - template void GetOpaqueData(const char *domain, const char *type_name, T &); - - explicit Value(std::nullptr_t) {} - explicit Value(OrtValue *p) : Base{p} {} - Value(Value &&) = default; - Value &operator=(Value &&) = default; - - bool IsTensor() const; - size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements - Value GetValue(int index, OrtAllocator *allocator) const; - - size_t GetStringTensorDataLength() const; - void GetStringTensorContent(void *buffer, size_t buffer_length, size_t *offsets, size_t offsets_count) const; - - template T *GetTensorMutableData(); - - TypeInfo GetTypeInfo() const; - TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; -}; - -struct AllocatorWithDefaultOptions { - AllocatorWithDefaultOptions(); - - operator OrtAllocator *() { return p_; } - operator const OrtAllocator *() const { return p_; } - - void *Alloc(size_t size); - void Free(void *p); - - const OrtMemoryInfo *GetInfo() const; - -private: - OrtAllocator *p_{}; -}; - -struct MemoryInfo : Base { - static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1); - - explicit MemoryInfo(std::nullptr_t) {} - MemoryInfo(const char *name, OrtAllocatorType type, int id, OrtMemType mem_type); - - explicit MemoryInfo(OrtMemoryInfo *p) : Base{p} {} -}; - -// -// Custom OPs (only needed to implement custom OPs) -// - -struct CustomOpApi { - CustomOpApi(const OrtApi &api) : api_(api) {} - - template // T is only implemented for float, int64_t, and string - T KernelInfoGetAttribute(_In_ const OrtKernelInfo *info, _In_ const char *name); - - OrtTensorTypeAndShapeInfo *GetTensorTypeAndShape(_In_ const OrtValue *value); - size_t GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo *info); - ONNXTensorElementDataType GetTensorElementType(const OrtTensorTypeAndShapeInfo *info); - size_t GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo *info); - void GetDimensions(_In_ const OrtTensorTypeAndShapeInfo *info, _Out_ int64_t *dim_values, size_t dim_values_length); - void SetDimensions(OrtTensorTypeAndShapeInfo *info, _In_ const int64_t *dim_values, size_t dim_count); - - template T *GetTensorMutableData(_Inout_ OrtValue *value); - template const T *GetTensorData(_Inout_ const OrtValue *value); - - std::vector GetTensorShape(const OrtTensorTypeAndShapeInfo *info); - void ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo *input); - size_t KernelContext_GetInputCount(const OrtKernelContext *context); - const OrtValue *KernelContext_GetInput(const OrtKernelContext *context, _In_ size_t index); - size_t KernelContext_GetOutputCount(const OrtKernelContext *context); - OrtValue *KernelContext_GetOutput(OrtKernelContext *context, _In_ size_t index, _In_ const int64_t *dim_values, - size_t dim_count); - - void ThrowOnError(OrtStatus *result); - -private: - const OrtApi &api_; -}; - -template struct CustomOpBase : OrtCustomOp { - CustomOpBase() { - OrtCustomOp::version = ORT_API_VERSION; - OrtCustomOp::CreateKernel = [](OrtCustomOp *this_, const OrtApi *api, const OrtKernelInfo *info) { - return static_cast(this_)->CreateKernel(*api, info); - }; - OrtCustomOp::GetName = [](OrtCustomOp *this_) { return static_cast(this_)->GetName(); }; - - OrtCustomOp::GetExecutionProviderType = [](OrtCustomOp *this_) { - return static_cast(this_)->GetExecutionProviderType(); - }; - - OrtCustomOp::GetInputTypeCount = [](OrtCustomOp *this_) { return static_cast(this_)->GetInputTypeCount(); }; - OrtCustomOp::GetInputType = [](OrtCustomOp *this_, size_t index) { - return static_cast(this_)->GetInputType(index); - }; - - OrtCustomOp::GetOutputTypeCount = [](OrtCustomOp *this_) { - return static_cast(this_)->GetOutputTypeCount(); - }; - OrtCustomOp::GetOutputType = [](OrtCustomOp *this_, size_t index) { - return static_cast(this_)->GetOutputType(index); - }; - - OrtCustomOp::KernelCompute = [](void *op_kernel, OrtKernelContext *context) { - static_cast(op_kernel)->Compute(context); - }; - OrtCustomOp::KernelDestroy = [](void *op_kernel) { delete static_cast(op_kernel); }; - } - - // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider - const char *GetExecutionProviderType() const { return nullptr; } -}; - -} // namespace Ort - -#include "onnxruntime_cxx_inline.h" diff --git a/nodejs/src/onnxruntime_cxx_inline.h b/nodejs/src/onnxruntime_cxx_inline.h deleted file mode 100644 index 0fa1e5478f..0000000000 --- a/nodejs/src/onnxruntime_cxx_inline.h +++ /dev/null @@ -1,628 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Don't include this file directly. Please include "onnxruntime_cxx_api.h" instead. -// These are the inline implementations of the C++ header APIs. They're in this separate file as to not clutter -// the main C++ file with implementation details. - -namespace Ort { - -inline void ThrowOnError(const OrtApi &ort, OrtStatus *status) { - if (status) { - std::string error_message = ort.GetErrorMessage(status); - OrtErrorCode error_code = ort.GetErrorCode(status); - ort.ReleaseStatus(status); - throw Ort::Exception(std::move(error_message), error_code); - } -} - -inline void ThrowOnError(OrtStatus *status) { ThrowOnError(GetApi(), status); } - -// This template converts a C++ type into it's ONNXTensorElementDataType -template struct TypeToTensorType; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; -}; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; -}; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; -}; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; -}; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; -}; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; -}; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; -}; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; -}; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; -}; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; -}; -template <> struct TypeToTensorType { - static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; -}; - -inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() { - ThrowOnError(GetApi().GetAllocatorWithDefaultOptions(&p_)); -} - -inline void *AllocatorWithDefaultOptions::Alloc(size_t size) { - void *out; - ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out)); - return out; -} - -inline void AllocatorWithDefaultOptions::Free(void *p) { ThrowOnError(GetApi().AllocatorFree(p_, p)); } - -inline const OrtMemoryInfo *AllocatorWithDefaultOptions::GetInfo() const { - const OrtMemoryInfo *out; - ThrowOnError(GetApi().AllocatorGetInfo(p_, &out)); - return out; -} - -inline MemoryInfo MemoryInfo::CreateCpu(OrtAllocatorType type, OrtMemType mem_type) { - OrtMemoryInfo *p; - ThrowOnError(GetApi().CreateCpuMemoryInfo(type, mem_type, &p)); - return MemoryInfo(p); -} - -inline MemoryInfo::MemoryInfo(const char *name, OrtAllocatorType type, int id, OrtMemType mem_type) { - ThrowOnError(GetApi().CreateMemoryInfo(name, type, id, mem_type, &p_)); -} - -inline Env::Env(OrtLoggingLevel default_warning_level, _In_ const char *logid) { - ThrowOnError(GetApi().CreateEnv(default_warning_level, logid, &p_)); -} - -inline Env::Env(OrtLoggingLevel default_warning_level, const char *logid, OrtLoggingFunction logging_function, - void *logger_param) { - ThrowOnError(GetApi().CreateEnvWithCustomLogger(logging_function, logger_param, default_warning_level, logid, &p_)); -} - -inline Env::Env(const OrtThreadingOptions *tp_options, OrtLoggingLevel default_warning_level, _In_ const char *logid) { - ThrowOnError(GetApi().CreateEnvWithGlobalThreadPools(default_warning_level, logid, tp_options, &p_)); -} - -inline Env &Env::EnableTelemetryEvents() { - ThrowOnError(GetApi().EnableTelemetryEvents(p_)); - return *this; -} - -inline Env &Env::DisableTelemetryEvents() { - ThrowOnError(GetApi().DisableTelemetryEvents(p_)); - return *this; -} - -inline CustomOpDomain::CustomOpDomain(const char *domain) { ThrowOnError(GetApi().CreateCustomOpDomain(domain, &p_)); } - -inline void CustomOpDomain::Add(OrtCustomOp *op) { ThrowOnError(GetApi().CustomOpDomain_Add(p_, op)); } - -inline RunOptions::RunOptions() { ThrowOnError(GetApi().CreateRunOptions(&p_)); } - -inline RunOptions &RunOptions::SetRunLogVerbosityLevel(int level) { - ThrowOnError(GetApi().RunOptionsSetRunLogVerbosityLevel(p_, level)); - return *this; -} - -inline RunOptions &RunOptions::SetRunLogSeverityLevel(int level) { - ThrowOnError(GetApi().RunOptionsSetRunLogSeverityLevel(p_, level)); - return *this; -} - -inline int RunOptions::GetRunLogVerbosityLevel() const { - int out; - ThrowOnError(GetApi().RunOptionsGetRunLogVerbosityLevel(p_, &out)); - return out; -} - -inline RunOptions &RunOptions::SetRunTag(const char *run_tag) { - ThrowOnError(GetApi().RunOptionsSetRunTag(p_, run_tag)); - return *this; -} - -inline const char *RunOptions::GetRunTag() const { - const char *out; - ThrowOnError(GetApi().RunOptionsGetRunTag(p_, &out)); - return out; -} - -inline RunOptions &RunOptions::SetTerminate() { - ThrowOnError(GetApi().RunOptionsSetTerminate(p_)); - return *this; -} - -inline RunOptions &RunOptions::UnsetTerminate() { - ThrowOnError(GetApi().RunOptionsUnsetTerminate(p_)); - return *this; -} - -inline SessionOptions::SessionOptions() { ThrowOnError(GetApi().CreateSessionOptions(&p_)); } - -inline SessionOptions SessionOptions::Clone() const { - OrtSessionOptions *out; - ThrowOnError(GetApi().CloneSessionOptions(p_, &out)); - return SessionOptions{out}; -} - -inline SessionOptions &SessionOptions::SetIntraOpNumThreads(int intra_op_num_threads) { - ThrowOnError(GetApi().SetIntraOpNumThreads(p_, intra_op_num_threads)); - return *this; -} - -inline SessionOptions &SessionOptions::SetInterOpNumThreads(int inter_op_num_threads) { - ThrowOnError(GetApi().SetInterOpNumThreads(p_, inter_op_num_threads)); - return *this; -} - -inline SessionOptions &SessionOptions::SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level) { - ThrowOnError(GetApi().SetSessionGraphOptimizationLevel(p_, graph_optimization_level)); - return *this; -} - -inline SessionOptions &SessionOptions::SetOptimizedModelFilePath(const ORTCHAR_T *optimized_model_filepath) { - ThrowOnError(GetApi().SetOptimizedModelFilePath(p_, optimized_model_filepath)); - return *this; -} - -inline SessionOptions &SessionOptions::EnableProfiling(const ORTCHAR_T *profile_file_prefix) { - ThrowOnError(GetApi().EnableProfiling(p_, profile_file_prefix)); - return *this; -} - -inline SessionOptions &SessionOptions::DisableProfiling() { - ThrowOnError(GetApi().DisableProfiling(p_)); - return *this; -} - -inline SessionOptions &SessionOptions::EnableMemPattern() { - ThrowOnError(GetApi().EnableMemPattern(p_)); - return *this; -} - -inline SessionOptions &SessionOptions::DisableMemPattern() { - ThrowOnError(GetApi().DisableMemPattern(p_)); - return *this; -} - -inline SessionOptions &SessionOptions::EnableCpuMemArena() { - ThrowOnError(GetApi().EnableCpuMemArena(p_)); - return *this; -} - -inline SessionOptions &SessionOptions::DisableCpuMemArena() { - ThrowOnError(GetApi().DisableCpuMemArena(p_)); - return *this; -} - -inline SessionOptions &SessionOptions::SetExecutionMode(ExecutionMode execution_mode) { - ThrowOnError(GetApi().SetSessionExecutionMode(p_, execution_mode)); - return *this; -} - -inline SessionOptions &SessionOptions::SetLogId(const char *logid) { - ThrowOnError(GetApi().SetSessionLogId(p_, logid)); - return *this; -} - -inline SessionOptions &SessionOptions::SetLogSeverityLevel(int level) { - ThrowOnError(GetApi().SetSessionLogSeverityLevel(p_, level)); - return *this; -} - -inline SessionOptions &SessionOptions::Add(OrtCustomOpDomain *custom_op_domain) { - ThrowOnError(GetApi().AddCustomOpDomain(p_, custom_op_domain)); - return *this; -} - -inline Session::Session(Env &env, const ORTCHAR_T *model_path, const SessionOptions &options) { - ThrowOnError(GetApi().CreateSession(env, model_path, options, &p_)); -} - -inline Session::Session(Env &env, const void *model_data, size_t model_data_length, const SessionOptions &options) { - ThrowOnError(GetApi().CreateSessionFromArray(env, model_data, model_data_length, options, &p_)); -} - -inline std::vector Session::Run(const RunOptions &run_options, const char *const *input_names, - const Value *input_values, size_t input_count, const char *const *output_names, - size_t output_names_count) { - std::vector output_values; - for (size_t i = 0; i < output_names_count; i++) - output_values.emplace_back(nullptr); - Run(run_options, input_names, input_values, input_count, output_names, output_values.data(), output_names_count); - return output_values; -} - -inline void Session::Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, - size_t input_count, const char *const *output_names, Value *output_values, - size_t output_count) { - static_assert(sizeof(Value) == sizeof(OrtValue *), - "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely"); - auto ort_input_values = reinterpret_cast(const_cast(input_values)); - auto ort_output_values = reinterpret_cast(output_values); - ThrowOnError(GetApi().Run(p_, run_options, input_names, ort_input_values, input_count, output_names, output_count, - ort_output_values)); -} - -inline size_t Session::GetInputCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetInputCount(p_, &out)); - return out; -} - -inline size_t Session::GetOutputCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetOutputCount(p_, &out)); - return out; -} - -inline size_t Session::GetOverridableInitializerCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetOverridableInitializerCount(p_, &out)); - return out; -} - -inline char *Session::GetInputName(size_t index, OrtAllocator *allocator) const { - char *out; - ThrowOnError(GetApi().SessionGetInputName(p_, index, allocator, &out)); - return out; -} - -inline char *Session::GetOutputName(size_t index, OrtAllocator *allocator) const { - char *out; - ThrowOnError(GetApi().SessionGetOutputName(p_, index, allocator, &out)); - return out; -} - -inline char *Session::GetOverridableInitializerName(size_t index, OrtAllocator *allocator) const { - char *out; - ThrowOnError(GetApi().SessionGetOverridableInitializerName(p_, index, allocator, &out)); - return out; -} - -inline char *Session::EndProfiling(OrtAllocator *allocator) const { - char *out; - ThrowOnError(GetApi().SessionEndProfiling(p_, allocator, &out)); - return out; -} - -inline ModelMetadata Session::GetModelMetadata() const { - OrtModelMetadata *out; - ThrowOnError(GetApi().SessionGetModelMetadata(p_, &out)); - return ModelMetadata{out}; -} - -inline char *ModelMetadata::GetProducerName(OrtAllocator *allocator) const { - char *out; - ThrowOnError(GetApi().ModelMetadataGetProducerName(p_, allocator, &out)); - return out; -} - -inline char *ModelMetadata::GetGraphName(OrtAllocator *allocator) const { - char *out; - ThrowOnError(GetApi().ModelMetadataGetGraphName(p_, allocator, &out)); - return out; -} - -inline char *ModelMetadata::GetDomain(OrtAllocator *allocator) const { - char *out; - ThrowOnError(GetApi().ModelMetadataGetDomain(p_, allocator, &out)); - return out; -} - -inline char *ModelMetadata::GetDescription(OrtAllocator *allocator) const { - char *out; - ThrowOnError(GetApi().ModelMetadataGetDescription(p_, allocator, &out)); - return out; -} - -inline char *ModelMetadata::LookupCustomMetadataMap(const char *key, OrtAllocator *allocator) const { - char *out; - ThrowOnError(GetApi().ModelMetadataLookupCustomMetadataMap(p_, allocator, key, &out)); - return out; -} - -inline char **ModelMetadata::GetCustomMetadataMapKeys(OrtAllocator *allocator, _Out_ int64_t &num_keys) const { - char **out; - ThrowOnError(GetApi().ModelMetadataGetCustomMetadataMapKeys(p_, allocator, &out, &num_keys)); - return out; -} - -inline int64_t ModelMetadata::GetVersion() const { - int64_t out; - ThrowOnError(GetApi().ModelMetadataGetVersion(p_, &out)); - return out; -} - -inline TypeInfo Session::GetInputTypeInfo(size_t index) const { - OrtTypeInfo *out; - ThrowOnError(GetApi().SessionGetInputTypeInfo(p_, index, &out)); - return TypeInfo{out}; -} - -inline TypeInfo Session::GetOutputTypeInfo(size_t index) const { - OrtTypeInfo *out; - ThrowOnError(GetApi().SessionGetOutputTypeInfo(p_, index, &out)); - return TypeInfo{out}; -} - -inline TypeInfo Session::GetOverridableInitializerTypeInfo(size_t index) const { - OrtTypeInfo *out; - ThrowOnError(GetApi().SessionGetOverridableInitializerTypeInfo(p_, index, &out)); - return TypeInfo{out}; -} - -inline ONNXTensorElementDataType TensorTypeAndShapeInfo::GetElementType() const { - ONNXTensorElementDataType out; - ThrowOnError(GetApi().GetTensorElementType(p_, &out)); - return out; -} - -inline size_t TensorTypeAndShapeInfo::GetElementCount() const { - size_t out; - ThrowOnError(GetApi().GetTensorShapeElementCount(p_, &out)); - return static_cast(out); -} - -inline size_t TensorTypeAndShapeInfo::GetDimensionsCount() const { - size_t out; - ThrowOnError(GetApi().GetDimensionsCount(p_, &out)); - return out; -} - -inline void TensorTypeAndShapeInfo::GetDimensions(int64_t *values, size_t values_count) const { - ThrowOnError(GetApi().GetDimensions(p_, values, values_count)); -} - -inline void TensorTypeAndShapeInfo::GetSymbolicDimensions(const char **values, size_t values_count) const { - ThrowOnError(GetApi().GetSymbolicDimensions(p_, values, values_count)); -} - -inline std::vector TensorTypeAndShapeInfo::GetShape() const { - std::vector out(GetDimensionsCount(), 0); - GetDimensions(out.data(), out.size()); - return out; -} - -inline Unowned TypeInfo::GetTensorTypeAndShapeInfo() const { - const OrtTensorTypeAndShapeInfo *out; - ThrowOnError(GetApi().CastTypeInfoToTensorInfo(p_, &out)); - return Unowned{const_cast(out)}; -} - -inline ONNXType TypeInfo::GetONNXType() const { - ONNXType out; - ThrowOnError(GetApi().GetOnnxTypeFromTypeInfo(p_, &out)); - return out; -} - -template -inline Value Value::CreateTensor(const OrtMemoryInfo *info, T *p_data, size_t p_data_element_count, - const int64_t *shape, size_t shape_len) { - return CreateTensor(info, p_data, p_data_element_count * sizeof(T), shape, shape_len, TypeToTensorType::type); -} - -inline Value Value::CreateTensor(const OrtMemoryInfo *info, void *p_data, size_t p_data_byte_count, - const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type) { - OrtValue *out; - ThrowOnError(GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out)); - return Value{out}; -} - -template -inline Value Value::CreateTensor(OrtAllocator *allocator, const int64_t *shape, size_t shape_len) { - return CreateTensor(allocator, shape, shape_len, TypeToTensorType::type); -} - -inline Value Value::CreateTensor(OrtAllocator *allocator, const int64_t *shape, size_t shape_len, - ONNXTensorElementDataType type) { - OrtValue *out; - ThrowOnError(GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out)); - return Value{out}; -} - -inline Value Value::CreateMap(Value &keys, Value &values) { - OrtValue *out; - OrtValue *inputs[2] = {keys, values}; - ThrowOnError(GetApi().CreateValue(inputs, 2, ONNX_TYPE_MAP, &out)); - return Value{out}; -} - -inline Value Value::CreateSequence(std::vector &values) { - OrtValue *out; - std::vector values_ort{values.data(), values.data() + values.size()}; - ThrowOnError(GetApi().CreateValue(values_ort.data(), values_ort.size(), ONNX_TYPE_SEQUENCE, &out)); - return Value{out}; -} - -template -inline Value Value::CreateOpaque(const char *domain, const char *type_name, const T &data_container) { - OrtValue *out; - ThrowOnError(GetApi().CreateOpaqueValue(domain, type_name, &data_container, sizeof(T), &out)); - return Value{out}; -} - -template inline void Value::GetOpaqueData(const char *domain, const char *type_name, T &out) { - ThrowOnError(GetApi().GetOpaqueValue(domain, type_name, p_, &out, sizeof(T))); -} - -inline bool Value::IsTensor() const { - int out; - ThrowOnError(GetApi().IsTensor(p_, &out)); - return out != 0; -} - -inline size_t Value::GetCount() const { - size_t out; - ThrowOnError(GetApi().GetValueCount(p_, &out)); - return out; -} - -inline Value Value::GetValue(int index, OrtAllocator *allocator) const { - OrtValue *out; - ThrowOnError(GetApi().GetValue(p_, index, allocator, &out)); - return Value{out}; -} - -inline size_t Value::GetStringTensorDataLength() const { - size_t out; - ThrowOnError(GetApi().GetStringTensorDataLength(p_, &out)); - return out; -} - -inline void Value::GetStringTensorContent(void *buffer, size_t buffer_length, size_t *offsets, - size_t offsets_count) const { - ThrowOnError(GetApi().GetStringTensorContent(p_, buffer, buffer_length, offsets, offsets_count)); -} - -template T *Value::GetTensorMutableData() { - T *out; - ThrowOnError(GetApi().GetTensorMutableData(p_, (void **)&out)); - return out; -} - -inline TypeInfo Value::GetTypeInfo() const { - OrtTypeInfo *output; - ThrowOnError(GetApi().GetTypeInfo(p_, &output)); - return TypeInfo{output}; -} - -inline TensorTypeAndShapeInfo Value::GetTensorTypeAndShapeInfo() const { - OrtTensorTypeAndShapeInfo *output; - ThrowOnError(GetApi().GetTensorTypeAndShape(p_, &output)); - return TensorTypeAndShapeInfo{output}; -} - -// -// Custom OP API Inlines -// -inline void CustomOpApi::ThrowOnError(OrtStatus *status) { Ort::ThrowOnError(api_, status); } - -template <> -inline float CustomOpApi::KernelInfoGetAttribute(_In_ const OrtKernelInfo *info, _In_ const char *name) { - float out; - ThrowOnError(api_.KernelInfoGetAttribute_float(info, name, &out)); - return out; -} - -template <> -inline int64_t CustomOpApi::KernelInfoGetAttribute(_In_ const OrtKernelInfo *info, _In_ const char *name) { - int64_t out; - ThrowOnError(api_.KernelInfoGetAttribute_int64(info, name, &out)); - return out; -} - -template <> -inline std::string CustomOpApi::KernelInfoGetAttribute(_In_ const OrtKernelInfo *info, - _In_ const char *name) { - size_t size = 0; - std::string out; - OrtStatus *status = api_.KernelInfoGetAttribute_string(info, name, nullptr, &size); - - // The status should be ORT_INVALID_ARGUMENT because the size is insufficient to hold the string - if (api_.GetErrorCode(status) == ORT_INVALID_ARGUMENT) { - api_.ReleaseStatus(status); - out.resize(size); - ThrowOnError(api_.KernelInfoGetAttribute_string(info, name, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - } else { - ThrowOnError(status); - } - return out; -} - -inline OrtTensorTypeAndShapeInfo *CustomOpApi::GetTensorTypeAndShape(_In_ const OrtValue *value) { - OrtTensorTypeAndShapeInfo *out; - ThrowOnError(api_.GetTensorTypeAndShape(value, &out)); - return out; -} - -inline size_t CustomOpApi::GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo *info) { - size_t out; - ThrowOnError(api_.GetTensorShapeElementCount(info, &out)); - return out; -} - -inline ONNXTensorElementDataType CustomOpApi::GetTensorElementType(const OrtTensorTypeAndShapeInfo *info) { - ONNXTensorElementDataType out; - ThrowOnError(api_.GetTensorElementType(info, &out)); - return out; -} - -inline size_t CustomOpApi::GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo *info) { - size_t out; - ThrowOnError(api_.GetDimensionsCount(info, &out)); - return out; -} - -inline void CustomOpApi::GetDimensions(_In_ const OrtTensorTypeAndShapeInfo *info, _Out_ int64_t *dim_values, - size_t dim_values_length) { - ThrowOnError(api_.GetDimensions(info, dim_values, dim_values_length)); -} - -inline void CustomOpApi::SetDimensions(OrtTensorTypeAndShapeInfo *info, _In_ const int64_t *dim_values, - size_t dim_count) { - ThrowOnError(api_.SetDimensions(info, dim_values, dim_count)); -} - -template inline T *CustomOpApi::GetTensorMutableData(_Inout_ OrtValue *value) { - T *data; - ThrowOnError(api_.GetTensorMutableData(value, reinterpret_cast(&data))); - return data; -} - -template inline const T *CustomOpApi::GetTensorData(_Inout_ const OrtValue *value) { - return GetTensorMutableData(const_cast(value)); -} - -inline std::vector CustomOpApi::GetTensorShape(const OrtTensorTypeAndShapeInfo *info) { - std::vector output(GetDimensionsCount(info)); - GetDimensions(info, output.data(), output.size()); - return output; -} - -inline void CustomOpApi::ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo *input) { - api_.ReleaseTensorTypeAndShapeInfo(input); -} - -inline size_t CustomOpApi::KernelContext_GetInputCount(const OrtKernelContext *context) { - size_t out; - ThrowOnError(api_.KernelContext_GetInputCount(context, &out)); - return out; -} - -inline const OrtValue *CustomOpApi::KernelContext_GetInput(const OrtKernelContext *context, _In_ size_t index) { - const OrtValue *out; - ThrowOnError(api_.KernelContext_GetInput(context, index, &out)); - return out; -} - -inline size_t CustomOpApi::KernelContext_GetOutputCount(const OrtKernelContext *context) { - size_t out; - ThrowOnError(api_.KernelContext_GetOutputCount(context, &out)); - return out; -} - -inline OrtValue *CustomOpApi::KernelContext_GetOutput(OrtKernelContext *context, _In_ size_t index, - _In_ const int64_t *dim_values, size_t dim_count) { - OrtValue *out; - ThrowOnError(api_.KernelContext_GetOutput(context, index, dim_values, dim_count, &out)); - return out; -} - -inline SessionOptions &SessionOptions::DisablePerSessionThreads() { - ThrowOnError(GetApi().DisablePerSessionThreads(p_)); - return *this; -} -} // namespace Ort