From 3ef273b84b54cb3327bbc55c2977ab9210d4cf01 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 11 Mar 2019 13:40:25 -0700 Subject: [PATCH] Support memory mapping on Linux --- cmake/onnxruntime_unittests.cmake | 8 +- .../onnxruntime/core/common}/callback.h | 4 +- .../onnxruntime/core/framework/data_types.h | 2 + onnxruntime/core/framework/callback.cc | 3 +- onnxruntime/core/framework/data_types.cc | 120 +++++--- onnxruntime/core/framework/session_state.h | 2 +- .../core/framework/tensor_type_and_shape.cc | 38 +-- .../core/framework/tensorprotoutils.cc | 269 +++++++++--------- onnxruntime/core/framework/tensorprotoutils.h | 1 + .../optimizer/optimizer_execution_frame.cc | 2 +- .../optimizer/optimizer_execution_frame.h | 2 +- onnxruntime/core/platform/env.h | 14 +- onnxruntime/core/platform/posix/env.cc | 103 +++++-- onnxruntime/core/platform/windows/env.cc | 27 +- onnxruntime/core/providers/cpu/tensor/size.cc | 2 +- onnxruntime/core/session/inference_session.cc | 61 +--- onnxruntime/core/session/onnxruntime_c_api.cc | 5 +- onnxruntime/test/shared_lib/test_inference.cc | 3 +- .../test/shared_lib/test_tensor_loader.cc | 23 +- onnxruntime/test/util/compare_mlvalue.cc | 31 +- 20 files changed, 365 insertions(+), 355 deletions(-) rename {onnxruntime/core/framework => include/onnxruntime/core/common}/callback.h (79%) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index a5e98881bb..3f09ff4476 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -498,10 +498,16 @@ if (onnxruntime_BUILD_SHARED_LIB) list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_model_loading.cc) list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_tensor_loader.cc) endif() + set(onnxruntime_shared_lib_test_LIBS onnxruntime_mocked_allocator onnxruntime_test_utils onnxruntime_common + onnx_proto) + if(onnxruntime_USE_NSYNC) + list(APPEND onnxruntime_shared_lib_test_LIBS nsync_cpp) + endif() AddTest(DYN TARGET onnxruntime_shared_lib_test SOURCES ${onnxruntime_shared_lib_test_SRC} - LIBS onnxruntime_mocked_allocator onnxruntime_test_utils onnxruntime_common onnx_proto protobuf::libprotobuf + LIBS ${onnxruntime_shared_lib_test_LIBS} + protobuf::libprotobuf DEPENDS ${all_dependencies} ) #demo diff --git a/onnxruntime/core/framework/callback.h b/include/onnxruntime/core/common/callback.h similarity index 79% rename from onnxruntime/core/framework/callback.h rename to include/onnxruntime/core/common/callback.h index 372dfe08a1..b52288758d 100644 --- a/onnxruntime/core/framework/callback.h +++ b/include/onnxruntime/core/common/callback.h @@ -8,8 +8,8 @@ extern "C" { #endif typedef struct OrtCallback { - void (ORT_API_CALL *f)(void *param) NO_EXCEPTION; - void *param; + void(ORT_API_CALL* f)(void* param) NO_EXCEPTION; + void* param; } OrtDeleter; #ifdef __cplusplus diff --git a/include/onnxruntime/core/framework/data_types.h b/include/onnxruntime/core/framework/data_types.h index 3f191a82b6..3048fc1550 100644 --- a/include/onnxruntime/core/framework/data_types.h +++ b/include/onnxruntime/core/framework/data_types.h @@ -179,6 +179,8 @@ class DataTypeImpl { */ static MLDataType TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto); + static const TensorTypeBase* TensorTypeFromONNXEnum(int type); + static const char* ToString(MLDataType type); // Registers ONNX_NAMESPACE::DataType (internalized string) with // MLDataType. DataType is produced by internalizing an instance of // TypeProto contained within MLDataType diff --git a/onnxruntime/core/framework/callback.cc b/onnxruntime/core/framework/callback.cc index e191596fb4..414b7ad0d2 100644 --- a/onnxruntime/core/framework/callback.cc +++ b/onnxruntime/core/framework/callback.cc @@ -1,8 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/framework/callback.h" - +#include "core/common/callback.h" ORT_API(void, OrtRunCallback, _Frees_ptr_opt_ OrtCallback* f){ if(f == nullptr) return; diff --git a/onnxruntime/core/framework/data_types.cc b/onnxruntime/core/framework/data_types.cc index ad8dbe8154..0323cb6edb 100644 --- a/onnxruntime/core/framework/data_types.cc +++ b/onnxruntime/core/framework/data_types.cc @@ -514,6 +514,93 @@ void DataTypeImpl::RegisterDataType(MLDataType mltype) { data_types_internal::DataTypeRegistry::instance().RegisterDataType(mltype); } +const char* DataTypeImpl::ToString(MLDataType type) { + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(float)"; + } + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(bool)"; + } + + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(int32)"; + } + + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(double)"; + } + + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(string)"; + } + + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(uint8)"; + } + + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(uint16)"; + } + + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(int16)"; + } + + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(int64)"; + } + + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(uint32)"; + } + + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(uint64)"; + } + + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(MLFloat16)"; + } + if (type == DataTypeImpl::GetTensorType()) { + return "tensor(bfloat16)"; + } + return "unknown"; +} +const TensorTypeBase* DataTypeImpl::TensorTypeFromONNXEnum(int type) { + switch (type) { + case TensorProto_DataType_FLOAT: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_BOOL: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_INT32: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_DOUBLE: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_STRING: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_UINT8: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_UINT16: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_INT8: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_INT16: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_INT64: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_UINT32: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_UINT64: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_FLOAT16: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + case TensorProto_DataType_BFLOAT16: + return reinterpret_cast(DataTypeImpl::GetTensorType()); + default: + ORT_NOT_IMPLEMENTED("tensor type ", type, " is not supported"); + } +} + MLDataType DataTypeImpl::TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto) { const auto& registry = data_types_internal::DataTypeRegistry::instance(); @@ -521,38 +608,7 @@ MLDataType DataTypeImpl::TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto) { case TypeProto::ValueCase::kTensorType: { const auto& tensor_type = proto.tensor_type(); ORT_ENFORCE(tensor_type.has_elem_type()); - switch (tensor_type.elem_type()) { - case TensorProto_DataType_FLOAT: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_BOOL: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_INT32: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_DOUBLE: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_STRING: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_UINT8: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_UINT16: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_INT8: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_INT16: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_INT64: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_UINT32: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_UINT64: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_FLOAT16: - return DataTypeImpl::GetTensorType(); - case TensorProto_DataType_BFLOAT16: - return DataTypeImpl::GetTensorType(); - default: - ORT_NOT_IMPLEMENTED("tensor type ", tensor_type.elem_type(), " is not supported"); - } + return TensorTypeFromONNXEnum(tensor_type.elem_type()); } break; // kTensorType case TypeProto::ValueCase::kMapType: { const auto& maptype = proto.map_type(); diff --git a/onnxruntime/core/framework/session_state.h b/onnxruntime/core/framework/session_state.h index 7fff7a6bd7..1628ce49e9 100644 --- a/onnxruntime/core/framework/session_state.h +++ b/onnxruntime/core/framework/session_state.h @@ -19,7 +19,7 @@ #include "core/framework/kernel_registry_manager.h" #include "core/framework/mem_pattern.h" #include "core/framework/ml_value.h" -#include "core/framework/callback.h" +#include "core/common/callback.h" #include "core/framework/mlvalue_name_idx_map.h" #include "core/framework/node_index_info.h" #include "core/graph/graph_viewer.h" diff --git a/onnxruntime/core/framework/tensor_type_and_shape.cc b/onnxruntime/core/framework/tensor_type_and_shape.cc index af3c52c51b..e27adcbf6f 100644 --- a/onnxruntime/core/framework/tensor_type_and_shape.cc +++ b/onnxruntime/core/framework/tensor_type_and_shape.cc @@ -100,42 +100,8 @@ ONNXTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType( return type; } -const onnxruntime::DataTypeImpl* TensorElementDataTypeToMLDataType(ONNXTensorElementDataType type) { - switch (type) { - case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: - return onnxruntime::DataTypeImpl::GetType(); - case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: - return onnxruntime::DataTypeImpl::GetType(); - default: - return nullptr; - } -} - -OrtStatus* GetTensorShapeAndType(const onnxruntime::TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out) { +OrtStatus* GetTensorShapeAndType(const onnxruntime::TensorShape* shape, + const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out) { ONNXTensorElementDataType type = MLDataTypeToOnnxRuntimeTensorElementDataType(tensor_data_type); if (ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED == type) { return OrtCreateStatus(ORT_FAIL, "Not implemented"); diff --git a/onnxruntime/core/framework/tensorprotoutils.cc b/onnxruntime/core/framework/tensorprotoutils.cc index c50249b91f..18fe26dde9 100644 --- a/onnxruntime/core/framework/tensorprotoutils.cc +++ b/onnxruntime/core/framework/tensorprotoutils.cc @@ -14,7 +14,7 @@ #include "core/framework/tensor.h" #include "core/framework/ml_value_patterns_planner.h" #include "core/framework/allocator.h" -#include "core/framework/callback.h" +#include "core/common/callback.h" #include "core/framework/data_types.h" #include "core/framework/path_lib.h" @@ -23,35 +23,6 @@ using namespace ::onnxruntime::common; namespace { - -//TODO: will move OrtBuffer into env.cc and let ReadFileAsString return an OrtBuffer instead of string -//So that, we can put fclose into the destructor of OrtBuffer. -#if 0 -class OrtBuffer { - public: - virtual const void* GetData() = 0; - virtual size_t GetLength() = 0; - virtual ~OrtBuffer() = default; - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OrtBuffer); -}; - -class OrtHeapBuffer { - public: - const void * GetData() { - - } - size_t GetLength() { - - } - static OrtHeapBuffer* Create(){ - - } - private: - void* data_; - size_t length_; -}; -#endif - #ifdef __GNUC__ constexpr inline bool IsLittleEndianOrder() noexcept { return __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; } #else @@ -314,39 +285,6 @@ std::vector GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::Te return tensor_shape_vec; } -template -common::Status GetTensorByTypeFromTensorProto(const TensorProto& tensor_proto, const TensorShape& tensor_shape, - const OrtAllocatorInfo& alloc, - const void* raw_data, size_t raw_data_len, void* preallocated, - size_t preallocated_size, std::unique_ptr& out_tensor) { - int64_t tensor_size = tensor_shape.Size(); - // tensor_size could be zero. see test_slice_start_out_of_bounds\test_data_set_0\output_0.pb - if (tensor_size < 0 || static_cast(tensor_size) > SIZE_MAX) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid shape ", tensor_shape); - } - size_t size_to_allocate; - if (!IAllocator::CalcMemSizeForArrayWithAlignment<0>(static_cast(tensor_size), sizeof(T), - &size_to_allocate)) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow"); - } - - if (preallocated && preallocated_size < size_to_allocate) - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "The buffer planner is not consistent with tensor buffer size, expected ", - size_to_allocate, ", got ", preallocated_size); - - std::unique_ptr t; - t = std::make_unique(DataTypeImpl::GetType(), tensor_shape, preallocated, alloc); - ORT_RETURN_IF_ERROR( - ::onnxruntime::utils::UnpackTensor(tensor_proto, raw_data, raw_data_len, t->MutableData(), tensor_size)); - out_tensor = std::move(t); - return common::Status::OK(); -} - -#define CASE_PROTO(X, Y) \ - case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: \ - ORT_RETURN_IF_ERROR(GetTensorByTypeFromTensorProto(tensor_proto, tensor_shape, allocator, raw_data, \ - raw_data_len, preallocated, preallocated_size, p_tensor)); \ - break; struct UnInitializeParam { void* preallocated; @@ -368,7 +306,7 @@ ORT_API_STATUS(OrtInitializeBufferForTensor, _In_opt_ void* input, size_t input_ */ ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enum ONNXTensorElementDataType type); -static void ORT_API_CALL DeleteHeapBuffer(void* param) noexcept { +static void ORT_API_CALL UnInitTensor(void* param) noexcept { UnInitializeParam* p = reinterpret_cast(param); OrtUninitializeBuffer(p->preallocated, p->preallocated_size, p->ele_type); delete p; @@ -399,35 +337,44 @@ ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enu } } +#define CASE_PROTO(X, Y) \ + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: \ + ORT_RETURN_IF_ERROR( \ + ::onnxruntime::utils::UnpackTensor(tensor_proto, raw_data, raw_data_len, (Y*)preallocated, tensor_size)); \ + break; + +class AutoDelete { + public: + OrtCallback d{nullptr, nullptr}; + AutoDelete() = default; + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(AutoDelete); + ~AutoDelete() { + if (d.f != nullptr) { + d.f(d.param); + } + } +}; + +static void MoveOrtCallback(OrtCallback& from, OrtCallback& to) { + to.f = from.f; + to.param = from.param; + from.f = nullptr; + from.param = nullptr; +} Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_proto_path, const ONNX_NAMESPACE::TensorProto& tensor_proto, const MemBuffer& m, MLValue& value, OrtCallback& deleter) { const OrtAllocatorInfo& allocator = m.GetAllocInfo(); - void* preallocated = m.GetBuffer(); - size_t preallocated_size = m.GetLen(); ONNXTensorElementDataType ele_type = utils::GetTensorElementType(tensor_proto); - if (preallocated != nullptr && ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) { - OrtStatus* status = OrtInitializeBufferForTensor(preallocated, preallocated_size, ele_type); - if (status != nullptr) { - OrtReleaseStatus(status); - return Status(common::ONNXRUNTIME, common::FAIL, "initialize preallocated buffer failed"); - } - - deleter.f = DeleteHeapBuffer; - deleter.param = new UnInitializeParam{preallocated, preallocated_size, ele_type}; - } else { - deleter.f = nullptr; - deleter.param = nullptr; - } - std::unique_ptr p_tensor; - std::string raw_data_from_file; + deleter.f = nullptr; + deleter.param = nullptr; const void* raw_data = nullptr; size_t raw_data_len = 0; + const DataTypeImpl* const type = DataTypeImpl::TensorTypeFromONNXEnum(tensor_proto.data_type())->GetElementType(); + AutoDelete deleter_for_file_data; + void* tensor_data; { - std::vector tensor_shape_vec = GetTensorShapeFromTensorProto(tensor_proto); - // Note: We permit an empty tensor_shape_vec, and treat it as a scalar (a tensor of size 1). - TensorShape tensor_shape{tensor_shape_vec}; if (tensor_proto.data_location() == TensorProto_DataLocation_EXTERNAL) { if (ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) return Status(common::ONNXRUNTIME, common::FAIL, "string tensor can not have raw data"); @@ -446,82 +393,120 @@ Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_proto_path, } // load the file - ORT_RETURN_IF_ERROR(env.ReadFileAsString(full_path.c_str(), &raw_data_from_file)); - raw_data = raw_data_from_file.data(); - raw_data_len = raw_data_from_file.size(); + { + void* file_data; + ORT_RETURN_IF_ERROR(env.ReadFileAsString(full_path.c_str(), file_data, raw_data_len, deleter_for_file_data.d)); + raw_data = file_data; + } } else if (tensor_proto.has_raw_data()) { if (ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) return Status(common::ONNXRUNTIME, common::FAIL, "string tensor can not have raw data"); raw_data = tensor_proto.raw_data().data(); raw_data_len = tensor_proto.raw_data().size(); } - switch (tensor_proto.data_type()) { - CASE_PROTO(FLOAT, float); - CASE_PROTO(DOUBLE, double); - CASE_PROTO(BOOL, bool); - CASE_PROTO(INT8, int8_t); - CASE_PROTO(INT16, int16_t); - CASE_PROTO(INT32, int32_t); - CASE_PROTO(INT64, int64_t); - CASE_PROTO(UINT8, uint8_t); - CASE_PROTO(UINT16, uint16_t); - CASE_PROTO(UINT32, uint32_t); - CASE_PROTO(UINT64, uint64_t); - CASE_PROTO(STRING, std::string); - CASE_PROTO(FLOAT16, MLFloat16); - CASE_PROTO(BFLOAT16, BFloat16); - default: { - std::ostringstream ostr; - ostr << "Initialized tensor with unexpected type: " << tensor_proto.data_type(); - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + if (IsLittleEndianOrder() && raw_data != nullptr && deleter_for_file_data.d.f != nullptr) { + tensor_data = const_cast(raw_data); + MoveOrtCallback(deleter_for_file_data.d, deleter); + } else { + void* preallocated = m.GetBuffer(); + size_t preallocated_size = m.GetLen(); + int64_t tensor_size = 1; + { + for (auto i : tensor_proto.dims()) { + if (i < 0) return Status(common::ONNXRUNTIME, common::FAIL, "tensor can't contain negative dims"); + tensor_size *= i; + } } + // tensor_size could be zero. see test_slice_start_out_of_bounds\test_data_set_0\output_0.pb + if (static_cast(tensor_size) > SIZE_MAX) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow"); + } + size_t size_to_allocate; + if (!IAllocator::CalcMemSizeForArrayWithAlignment<0>(static_cast(tensor_size), type->Size(), + &size_to_allocate)) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow"); + } + + if (preallocated && preallocated_size < size_to_allocate) + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "The buffer planner is not consistent with tensor buffer size, expected ", + size_to_allocate, ", got ", preallocated_size); + switch (tensor_proto.data_type()) { + CASE_PROTO(FLOAT, float); + CASE_PROTO(DOUBLE, double); + CASE_PROTO(BOOL, bool); + CASE_PROTO(INT8, int8_t); + CASE_PROTO(INT16, int16_t); + CASE_PROTO(INT32, int32_t); + CASE_PROTO(INT64, int64_t); + CASE_PROTO(UINT8, uint8_t); + CASE_PROTO(UINT16, uint16_t); + CASE_PROTO(UINT32, uint32_t); + CASE_PROTO(UINT64, uint64_t); + CASE_PROTO(FLOAT16, MLFloat16); + CASE_PROTO(BFLOAT16, BFloat16); + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_STRING: + if (preallocated != nullptr) { + OrtStatus* status = OrtInitializeBufferForTensor(preallocated, preallocated_size, ele_type); + if (status != nullptr) { + OrtReleaseStatus(status); + return Status(common::ONNXRUNTIME, common::FAIL, "initialize preallocated buffer failed"); + } + + deleter.f = UnInitTensor; + deleter.param = new UnInitializeParam{preallocated, preallocated_size, ele_type}; + } + ORT_RETURN_IF_ERROR(::onnxruntime::utils::UnpackTensor(tensor_proto, raw_data, raw_data_len, + (std::string*)preallocated, tensor_size)); + break; + default: { + std::ostringstream ostr; + ostr << "Initialized tensor with unexpected type: " << tensor_proto.data_type(); + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + } + } + tensor_data = preallocated; } } - value.Init(p_tensor.release(), - DataTypeImpl::GetType(), + std::vector tensor_shape_vec = GetTensorShapeFromTensorProto(tensor_proto); + // Note: We permit an empty tensor_shape_vec, and treat it as a scalar (a tensor of size 1). + TensorShape tensor_shape{tensor_shape_vec}; + value.Init(new Tensor(type, tensor_shape, tensor_data, allocator), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); return Status::OK(); } -ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto& tensor_proto) { - switch (tensor_proto.data_type()) { - case TensorProto_DataType_FLOAT: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; - case TensorProto_DataType_UINT8: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; - case TensorProto_DataType_INT8: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; - case TensorProto_DataType_UINT16: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; - case TensorProto_DataType_INT16: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; - case TensorProto_DataType_INT32: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; - case TensorProto_DataType_INT64: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; - case TensorProto_DataType_STRING: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING; - case TensorProto_DataType_BOOL: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; - case TensorProto_DataType_FLOAT16: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; - case TensorProto_DataType_DOUBLE: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; - case TensorProto_DataType_UINT32: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; - case TensorProto_DataType_UINT64: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; - case TensorProto_DataType_COMPLEX64: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64; - case TensorProto_DataType_COMPLEX128: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128; - case TensorProto_DataType_BFLOAT16: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; +#define CASE_TYPE(X) \ + case ONNX_NAMESPACE::TensorProto_DataType_##X: \ + return ONNX_TENSOR_ELEMENT_DATA_TYPE_##X; + +ONNXTensorElementDataType CApiElementTypeFromProtoType(int type) { + switch (type) { + CASE_TYPE(FLOAT) + CASE_TYPE(UINT8) + CASE_TYPE(INT8) + CASE_TYPE(UINT16) + CASE_TYPE(INT16) + CASE_TYPE(INT32) + CASE_TYPE(INT64) + CASE_TYPE(STRING) + CASE_TYPE(BOOL) + CASE_TYPE(FLOAT16) + CASE_TYPE(DOUBLE) + CASE_TYPE(UINT32) + CASE_TYPE(UINT64) + CASE_TYPE(COMPLEX64) + CASE_TYPE(COMPLEX128) + CASE_TYPE(BFLOAT16) default: return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; } } +ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto& tensor_proto) { + return CApiElementTypeFromProtoType(tensor_proto.data_type()); +} + TensorProto::DataType GetTensorProtoType(const Tensor& tensor) { auto tensor_type = tensor.DataType(); TensorProto::DataType dtype = TensorProto_DataType_UNDEFINED; diff --git a/onnxruntime/core/framework/tensorprotoutils.h b/onnxruntime/core/framework/tensorprotoutils.h index c320f2329e..b5fd466b0f 100644 --- a/onnxruntime/core/framework/tensorprotoutils.h +++ b/onnxruntime/core/framework/tensorprotoutils.h @@ -37,6 +37,7 @@ common::Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_prot // This function doesn't support string tensors ONNX_NAMESPACE::TensorProto::DataType GetTensorProtoType(const Tensor& tensor); +ONNXTensorElementDataType CApiElementTypeFromProtoType(int type); ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto& tensor_proto); // How much memory it will need for putting the content of this tensor into a plain array diff --git a/onnxruntime/core/optimizer/optimizer_execution_frame.cc b/onnxruntime/core/optimizer/optimizer_execution_frame.cc index 8dbb8e0cb1..4350cb7073 100644 --- a/onnxruntime/core/optimizer/optimizer_execution_frame.cc +++ b/onnxruntime/core/optimizer/optimizer_execution_frame.cc @@ -8,7 +8,7 @@ #include "core/framework/mldata_type_utils.h" #include "core/framework/kernel_registry.h" #include "core/framework/fuse_nodes_funcs.h" -#include "core/framework/callback.h" +#include "core/common/callback.h" #include "core/optimizer/optimizer_execution_frame.h" namespace onnxruntime { diff --git a/onnxruntime/core/optimizer/optimizer_execution_frame.h b/onnxruntime/core/optimizer/optimizer_execution_frame.h index f49e2d1906..6a65bc36b1 100644 --- a/onnxruntime/core/optimizer/optimizer_execution_frame.h +++ b/onnxruntime/core/optimizer/optimizer_execution_frame.h @@ -10,7 +10,7 @@ #include "core/framework/execution_frame.h" #include "core/framework/mlvalue_name_idx_map.h" #include "core/framework/ml_value.h" -#include "core/framework/callback.h" +#include "core/common/callback.h" namespace onnxruntime { diff --git a/onnxruntime/core/platform/env.h b/onnxruntime/core/platform/env.h index a5f3fdcf94..ba03bc0bce 100644 --- a/onnxruntime/core/platform/env.h +++ b/onnxruntime/core/platform/env.h @@ -24,6 +24,7 @@ limitations under the License. #include #include "core/common/common.h" +#include "core/common/callback.h" #include "core/platform/env_time.h" #ifndef _WIN32 @@ -96,11 +97,18 @@ class Env { virtual Thread* StartThread(const ThreadOptions& thread_options, const std::string& name, std::function fn) const = 0; - /// file_path must point to a regular file, which can't be a pipe/socket/... #ifndef _WIN32 - virtual common::Status ReadFileAsString(const char* file_path, std::string* out) const = 0; + /** + * + * \param file_path file_path must point to a regular file, which can't be a pipe/socket/... + * \param[out] p allocated buffer with the file data + * \param[out] len lenght of p + * @return + */ + virtual common::Status ReadFileAsString(const char* file_path, void*& p, size_t& len, OrtCallback& deleter) const = 0; #else - virtual common::Status ReadFileAsString(const wchar_t* file_path, std::string* out) const = 0; + virtual common::Status ReadFileAsString(const wchar_t* file_path, void*& p, size_t& len, + OrtCallback& deleter) const = 0; #endif #ifdef _WIN32 diff --git a/onnxruntime/core/platform/posix/env.cc b/onnxruntime/core/platform/posix/env.cc index 2a14418d34..21d4d97bf4 100644 --- a/onnxruntime/core/platform/posix/env.cc +++ b/onnxruntime/core/platform/posix/env.cc @@ -17,6 +17,10 @@ limitations under the License. #include #include #include +#include +#include +#include +#include #include #include #include @@ -25,6 +29,7 @@ limitations under the License. #include #include "core/platform/env.h" #include "core/common/common.h" +#include "core/common/logging/logging.h" namespace onnxruntime { @@ -41,6 +46,26 @@ class StdThread : public Thread { std::thread thread_; }; +static void ORT_API_CALL DeleteBuffer(void* param) noexcept { ::free(param); } + +class UnmapFileParam { + public: + void* addr; + size_t len; + int fd; +}; + +static void ORT_API_CALL UnmapFile(void* param) noexcept { + UnmapFileParam* p = reinterpret_cast(param); + int ret = munmap(p->addr, p->len); + if (ret != 0) { + int err = errno; + LOGS_DEFAULT(INFO) << "munmap failed. error code:" << err; + } + (void)close(p->fd); + delete p; +} + class PosixEnv : public Env { public: static PosixEnv& Instance() { @@ -55,16 +80,12 @@ class PosixEnv : public Env { return std::thread::hardware_concurrency(); } - EnvThread* CreateThread(std::function fn) const override { - return new StdThread(fn); - } + EnvThread* CreateThread(std::function fn) const override { return new StdThread(fn); } Task CreateTask(std::function f) const override { return Task{std::move(f)}; } - void ExecuteTask(const Task& t) const override { - t.f(); - } + void ExecuteTask(const Task& t) const override { t.f(); } void SleepForMicroseconds(int64_t micros) const override { while (micros > 0) { @@ -95,13 +116,36 @@ class PosixEnv : public Env { return getpid(); } - common::Status ReadFileAsString(const char* fname, std::string* out) const override { - if (!out) { - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "ReadFileAsString: 'out' cannot be NULL"); - } + static common::Status ReadBinaryFile(int fd, const char* fname, const struct stat& stbuf, void*& p, size_t& len, + OrtCallback& deleter) { + std::unique_ptr buffer(reinterpret_cast(malloc(stbuf.st_size))); + char* wptr = reinterpret_cast(buffer.get()); + auto length_remain = stbuf.st_size; + do { + size_t bytes_to_read = length_remain; + ssize_t bytes_readed = read(fd, wptr, bytes_to_read); + if (bytes_readed <= 0) { + int err = errno; + (void)close(fd); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "read file '", fname, "' fail, error code = ", err); + } + assert(static_cast(bytes_readed) <= bytes_to_read); + wptr += bytes_readed; + length_remain -= bytes_readed; + } while (length_remain > 0); + p = buffer.release(); + len = stbuf.st_size; + deleter.f = DeleteBuffer; + deleter.param = p; + return Status::OK(); + } + + common::Status ReadFileAsString(const char* fname, void*& p, size_t& len, OrtCallback& deleter) const override { if (!fname) { return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "ReadFileAsString: 'fname' cannot be NULL"); } + deleter.f = nullptr; + deleter.param = nullptr; int fd = open(fname, O_RDONLY); if (fd < 0) { int err = errno; @@ -113,25 +157,32 @@ class PosixEnv : public Env { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Get file '", fname, "' size fail"); } if (stbuf.st_size == 0) { - out->clear(); + p = nullptr; + len = 0; } else { - out->resize(stbuf.st_size, '\0'); - char* wptr = const_cast(out->data()); - auto length_remain = stbuf.st_size; - do { - size_t bytes_to_read = length_remain; - ssize_t bytes_readed = read(fd, wptr, bytes_to_read); - if (bytes_readed <= 0) { - int err = errno; - (void)close(fd); - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "read file '", fname, "' fail, error code = ", err); + if (sizeof(fname) <= 4) { + auto st = ReadBinaryFile(fd, fname, stbuf, p, len, deleter); + (void)close(fd); + if (!st.IsOK()) { + return st; } - assert(static_cast(bytes_readed) <= bytes_to_read); - wptr += bytes_readed; - length_remain -= bytes_readed; - } while (length_remain > 0); - (void)close(fd); + } else { + size_t flen = static_cast(stbuf.st_size); + p = mmap(NULL, flen, PROT_READ, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + auto st = ReadBinaryFile(fd, fname, stbuf, p, len, deleter); + (void)close(fd); + if (!st.IsOK()) { + return st; + } + } else { + len = stbuf.st_size; + deleter.f = UnmapFile; + deleter.param = new UnmapFileParam{p, flen, fd}; + } + } } + return common::Status::OK(); } diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc index 605bcb4200..8f1cf920e7 100644 --- a/onnxruntime/core/platform/windows/env.cc +++ b/onnxruntime/core/platform/windows/env.cc @@ -40,6 +40,7 @@ class StdThread : public Thread { private: std::thread thread_; }; +static void ORT_API_CALL DeleteBuffer(void* param) noexcept { ::free(param); } class WindowsEnv : public Env { public: @@ -93,11 +94,12 @@ class WindowsEnv : public Env { void ExecuteTask(const Task& t) const override { t.f(); } - common::Status ReadFileAsString(const wchar_t* fname, std::string* out) const override { - if (!fname) return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "file name is nullptr"); - if (!out) { - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "'out' cannot be NULL"); + common::Status ReadFileAsString(const wchar_t* fname, void*& p, size_t& len, OrtCallback& deleter) const override { + if (!fname) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "ReadFileAsString: 'fname' cannot be NULL"); } + deleter.f = nullptr; + deleter.param = nullptr; HANDLE hFile = CreateFileW(fname, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile == INVALID_HANDLE_VALUE) { int err = GetLastError(); @@ -111,11 +113,12 @@ class WindowsEnv : public Env { } // check the file file for avoiding allocating a zero length buffer if (filesize.QuadPart == 0) { // empty file - out->clear(); + p = nullptr; + len = 0; return Status::OK(); } - out->resize(filesize.QuadPart, '\0'); - char* wptr = const_cast(out->data()); + std::unique_ptr buffer(reinterpret_cast(malloc(filesize.QuadPart))); + char* wptr = reinterpret_cast(buffer.get()); auto length_remain = filesize.QuadPart; DWORD readed = 0; for (; length_remain > 0; wptr += readed, length_remain -= readed) { @@ -128,14 +131,20 @@ class WindowsEnv : public Env { } if (ReadFile(hFile, wptr, bytes_to_read, &readed, nullptr) != TRUE) { int err = GetLastError(); - out->clear(); + p = nullptr; + len = 0; return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ReadFile ", ToMBString(fname), " fail, errcode =", err); } if (readed != bytes_to_read) { - out->clear(); + p = nullptr; + len = 0; return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ReadFile ", ToMBString(fname), " fail: unexpected end"); } } + p = buffer.release(); + len = filesize.QuadPart; + deleter.f = DeleteBuffer; + deleter.param = p; return common::Status::OK(); } diff --git a/onnxruntime/core/providers/cpu/tensor/size.cc b/onnxruntime/core/providers/cpu/tensor/size.cc index ab4af299ae..5000bbd793 100644 --- a/onnxruntime/core/providers/cpu/tensor/size.cc +++ b/onnxruntime/core/providers/cpu/tensor/size.cc @@ -11,8 +11,8 @@ Status Size::Compute(OpKernelContext* ctx) const { if (input_tensor == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch"); TensorShape scalar_shape; Tensor* p_output_tensor = ctx->Output(0, scalar_shape); - assert(p_output_tensor->Size() == sizeof(int64_t)); int64_t* p_output_scalar = p_output_tensor->template MutableData(); + assert(p_output_tensor->Size() == sizeof(int64_t)); *p_output_scalar = input_tensor->Shape().Size(); diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index e70ff06e5c..f8b7b3d27f 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -56,61 +56,6 @@ using namespace ONNX_NAMESPACE; ONNXTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType(const onnxruntime::DataTypeImpl* cpp_type); -const onnxruntime::DataTypeImpl* TensorElementDataTypeToMLDataType(ONNXTensorElementDataType type); - -namespace onnxruntime { -const char* ElementTypeToString(MLDataType type) { - if (type == DataTypeImpl::GetType()) { - return "tensor(float)"; - } else if (type == DataTypeImpl::GetType()) { - return "tensor(bool)"; - } - - else if (type == DataTypeImpl::GetType()) { - return "tensor(int32)"; - } - - else if (type == DataTypeImpl::GetType()) { - return "tensor(double)"; - } - - else if (type == DataTypeImpl::GetType()) { - return "tensor(string)"; - } - - else if (type == DataTypeImpl::GetType()) { - return "tensor(uint8)"; - } - - else if (type == DataTypeImpl::GetType()) { - return "tensor(uint16)"; - } - - else if (type == DataTypeImpl::GetType()) { - return "tensor(int16)"; - } - - else if (type == DataTypeImpl::GetType()) { - return "tensor(int64)"; - } - - else if (type == DataTypeImpl::GetType()) { - return "tensor(uint32)"; - } - - else if (type == DataTypeImpl::GetType()) { - return "tensor(uint64)"; - } - - else if (type == DataTypeImpl::GetType()) { - return "tensor(MLFloat16)"; - } else if (type == DataTypeImpl::GetType()) { - return "tensor(bfloat16)"; - } else { - return "unknown"; - } -} -} // namespace onnxruntime ORT_API_STATUS_IMPL(OrtKernelInfoGetAttribute_float, _In_ OrtKernelInfo* info, _In_ const char* name, _Out_ float* out) { auto status = reinterpret_cast(info)->GetAttr(name, out); @@ -282,14 +227,16 @@ class InferenceSession::Impl { for (size_t i = 0; i < input_count; i++) { auto type = op->GetInputType(op, i); - schema.Input(i, "A", "Description", ElementTypeToString(TensorElementDataTypeToMLDataType(type))); + schema.Input(i, "A", "Description", + DataTypeImpl::ToString(onnxruntime::DataTypeImpl::TensorTypeFromONNXEnum(type))); } auto output_count = op->GetOutputTypeCount(op); for (size_t i = 0; i < output_count; i++) { auto type = op->GetOutputType(op, i); - schema.Output(i, "A", "Description", ElementTypeToString(TensorElementDataTypeToMLDataType(type))); + schema.Output(i, "A", "Description", + DataTypeImpl::ToString(onnxruntime::DataTypeImpl::TensorTypeFromONNXEnum(type))); } schema.SinceVersion(domain->op_version_start_); diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index f359f5b932..7e7623f392 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -17,7 +17,7 @@ #include "core/framework/tensor.h" #include "core/framework/ml_value.h" #include "core/framework/environment.h" -#include "core/framework/callback.h" +#include "core/common/callback.h" #include "core/framework/tensorprotoutils.h" #include "core/framework/onnxruntime_typeinfo.h" #include "core/session/inference_session.h" @@ -366,7 +366,8 @@ ORT_API_STATUS_IMPL(OrtAddCustomOpDomain, _In_ OrtSessionOptions* options, OrtCu ORT_API_STATUS_IMPL(OrtCreateSession, _In_ OrtEnv* env, _In_ const ORTCHAR_T* model_path, _In_ const OrtSessionOptions* options, _Out_ OrtSession** out) { API_IMPL_BEGIN - auto sess = std::make_unique<::onnxruntime::InferenceSession>(options == nullptr ? onnxruntime::SessionOptions() : options->value, env->loggingManager); + auto sess = std::make_unique<::onnxruntime::InferenceSession>( + options == nullptr ? onnxruntime::SessionOptions() : options->value, env->loggingManager); Status status; if (options != nullptr) { if (!options->custom_op_paths.empty()) { diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index ad1129ec2a..0a8e840b6c 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -252,7 +252,8 @@ TEST_F(CApiTest, custom_op_handler) { OrtCustomOpDomain* custom_op_domain = OrtCreateCustomOpDomain("", 5, 7); ORT_THROW_ON_ERROR(OrtCustomOpDomain_Add(custom_op_domain, &custom_op)); - TestInference(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, false, custom_op_domain); + TestInference(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, false, + custom_op_domain); } #ifdef ORT_RUN_EXTERNAL_ONNX_TESTS diff --git a/onnxruntime/test/shared_lib/test_tensor_loader.cc b/onnxruntime/test/shared_lib/test_tensor_loader.cc index 92440d1db7..4f674ec312 100644 --- a/onnxruntime/test/shared_lib/test_tensor_loader.cc +++ b/onnxruntime/test/shared_lib/test_tensor_loader.cc @@ -52,11 +52,14 @@ TEST_F(CApiTest, load_simple_float_tensor) { OrtCallback* deleter; auto st = OrtTensorProtoToOrtValue(s.data(), static_cast(s.size()), nullptr, output.data(), output.size() * sizeof(float), &value, &deleter); - // check the result ASSERT_EQ(st, nullptr) << OrtGetErrorMessage(st); - ASSERT_EQ(output[0], 1.0f); - ASSERT_EQ(output[1], 2.2f); - ASSERT_EQ(output[2], 3.5f); + float* real_output; + st = OrtGetTensorMutableData(value, (void**)&real_output); + ASSERT_EQ(st, nullptr) << OrtGetErrorMessage(st); + // check the result + ASSERT_EQ(real_output[0], 1.0f); + ASSERT_EQ(real_output[1], 2.2f); + ASSERT_EQ(real_output[2], 3.5f); OrtReleaseValue(value); } @@ -104,12 +107,14 @@ static void run_external_data_test() { } auto st = OrtTensorProtoToOrtValue(s.data(), static_cast(s.size()), cwd.empty() ? nullptr : cwd.c_str(), output.data(), output.size() * sizeof(float), &value, &deleter); - - // check the result ASSERT_EQ(st, nullptr) << OrtGetErrorMessage(st); - ASSERT_EQ(output[0], 1.0f); - ASSERT_EQ(output[1], 2.2f); - ASSERT_EQ(output[2], 3.5f); + float* real_output; + st = OrtGetTensorMutableData(value, (void**)&real_output); + ASSERT_EQ(st, nullptr) << OrtGetErrorMessage(st); + // check the result + ASSERT_EQ(real_output[0], 1.0f); + ASSERT_EQ(real_output[1], 2.2f); + ASSERT_EQ(real_output[2], 3.5f); OrtReleaseValue(value); } TEST_F(CApiTest, load_float_tensor_with_external_data) { diff --git a/onnxruntime/test/util/compare_mlvalue.cc b/onnxruntime/test/util/compare_mlvalue.cc index 94b6aaa1d4..0e9a0e917a 100644 --- a/onnxruntime/test/util/compare_mlvalue.cc +++ b/onnxruntime/test/util/compare_mlvalue.cc @@ -6,7 +6,7 @@ #include #include #include "core/graph/onnx_protobuf.h" - +#include "core/framework/tensorprotoutils.h" #include "Eigen/Core" #include "Eigen/src/Core/arch/CUDA/Half.h" @@ -20,35 +20,8 @@ using __half_raw = ::Eigen::half_impl::__half; } // namespace Eigen #endif -#define CASE_TYPE(X) \ - case ONNX_NAMESPACE::TensorProto_DataType_##X: \ - return ONNX_TENSOR_ELEMENT_DATA_TYPE_##X; - namespace { -ONNXTensorElementDataType CApiElementTypeFromProto(int type) { - switch (type) { - CASE_TYPE(FLOAT) - CASE_TYPE(UINT8) - CASE_TYPE(INT8) - CASE_TYPE(UINT16) - CASE_TYPE(INT16) - CASE_TYPE(INT32) - CASE_TYPE(INT64) - CASE_TYPE(STRING) - CASE_TYPE(BOOL) - CASE_TYPE(FLOAT16) - CASE_TYPE(DOUBLE) - CASE_TYPE(UINT32) - CASE_TYPE(UINT64) - CASE_TYPE(COMPLEX64) - CASE_TYPE(COMPLEX128) - CASE_TYPE(BFLOAT16) - default: - return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; - } -} - template bool IsResultCloselyMatch(const T& outvalue, const T& expected_value, const double diff, const double tol) { if (diff > tol) return false; @@ -381,7 +354,7 @@ std::pair VerifyValueInfo(const ONNX_NAMESPACE::Val info.reset(t1); } ONNXTensorElementDataType real_type = OrtGetTensorElementType(info.get()); - ONNXTensorElementDataType expected_type = CApiElementTypeFromProto(t.elem_type()); + ONNXTensorElementDataType expected_type = onnxruntime::utils::CApiElementTypeFromProtoType(t.elem_type()); if (real_type != expected_type) { std::ostringstream oss; oss << "expect " << ElementTypeToString((MLDataType)expected_type)