saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-08 17:17:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support memory mapping on Linux

Browse source
This commit is contained in:
Changming Sun 2019-03-11 13:40:25 -07:00
parent 7218be4f1f
commit 3ef273b84b
20 changed files with 365 additions and 355 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cmake/onnxruntime_unittests.cmake
View file

@ -498,10 +498,16 @@ if (onnxruntime_BUILD_SHARED_LIB)
list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_model_loading.cc)
list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_tensor_loader.cc)
endif()
set(onnxruntime_shared_lib_test_LIBS onnxruntime_mocked_allocator onnxruntime_test_utils onnxruntime_common
onnx_proto)
if(onnxruntime_USE_NSYNC)
list(APPEND onnxruntime_shared_lib_test_LIBS nsync_cpp)
endif()
AddTest(DYN
TARGET onnxruntime_shared_lib_test
SOURCES ${onnxruntime_shared_lib_test_SRC}
LIBS onnxruntime_mocked_allocator onnxruntime_test_utils onnxruntime_common onnx_proto protobuf::libprotobuf
LIBS ${onnxruntime_shared_lib_test_LIBS}
protobuf::libprotobuf
DEPENDS ${all_dependencies}
)
#demo

4
onnxruntime/core/framework/callback.h → include/onnxruntime/core/common/callback.h
View file

@ -8,8 +8,8 @@ extern "C" {
#endif
typedef struct OrtCallback {
void (ORT_API_CALL *f)(void *param) NO_EXCEPTION;
void *param;
void(ORT_API_CALL* f)(void* param) NO_EXCEPTION;
void* param;
} OrtDeleter;
#ifdef __cplusplus

2
include/onnxruntime/core/framework/data_types.h
View file

@ -179,6 +179,8 @@ class DataTypeImpl {
*/
static MLDataType TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto);
static const TensorTypeBase* TensorTypeFromONNXEnum(int type);
static const char* ToString(MLDataType type);
// Registers ONNX_NAMESPACE::DataType (internalized string) with
// MLDataType. DataType is produced by internalizing an instance of
// TypeProto contained within MLDataType

3
onnxruntime/core/framework/callback.cc
View file

@ -1,8 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/framework/callback.h"
#include "core/common/callback.h"
ORT_API(void, OrtRunCallback, _Frees_ptr_opt_ OrtCallback* f){
if(f == nullptr) return;

120
onnxruntime/core/framework/data_types.cc
View file

@ -514,6 +514,93 @@ void DataTypeImpl::RegisterDataType(MLDataType mltype) {
data_types_internal::DataTypeRegistry::instance().RegisterDataType(mltype);
}
const char* DataTypeImpl::ToString(MLDataType type) {
if (type == DataTypeImpl::GetTensorType<float>()) {
return "tensor(float)";
}
if (type == DataTypeImpl::GetTensorType<bool>()) {
return "tensor(bool)";
}
if (type == DataTypeImpl::GetTensorType<int32_t>()) {
return "tensor(int32)";
}
if (type == DataTypeImpl::GetTensorType<double>()) {
return "tensor(double)";
}
if (type == DataTypeImpl::GetTensorType<std::string>()) {
return "tensor(string)";
}
if (type == DataTypeImpl::GetTensorType<uint8_t>()) {
return "tensor(uint8)";
}
if (type == DataTypeImpl::GetTensorType<uint16_t>()) {
return "tensor(uint16)";
}
if (type == DataTypeImpl::GetTensorType<int16_t>()) {
return "tensor(int16)";
}
if (type == DataTypeImpl::GetTensorType<int64_t>()) {
return "tensor(int64)";
}
if (type == DataTypeImpl::GetTensorType<uint32_t>()) {
return "tensor(uint32)";
}
if (type == DataTypeImpl::GetTensorType<uint64_t>()) {
return "tensor(uint64)";
}
if (type == DataTypeImpl::GetTensorType<MLFloat16>()) {
return "tensor(MLFloat16)";
}
if (type == DataTypeImpl::GetTensorType<BFloat16>()) {
return "tensor(bfloat16)";
}
return "unknown";
}
const TensorTypeBase* DataTypeImpl::TensorTypeFromONNXEnum(int type) {
switch (type) {
case TensorProto_DataType_FLOAT:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<float>());
case TensorProto_DataType_BOOL:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<bool>());
case TensorProto_DataType_INT32:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<int32_t>());
case TensorProto_DataType_DOUBLE:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<double>());
case TensorProto_DataType_STRING:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<std::string>());
case TensorProto_DataType_UINT8:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<uint8_t>());
case TensorProto_DataType_UINT16:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<uint16_t>());
case TensorProto_DataType_INT8:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<int8_t>());
case TensorProto_DataType_INT16:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<int16_t>());
case TensorProto_DataType_INT64:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<int64_t>());
case TensorProto_DataType_UINT32:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<uint32_t>());
case TensorProto_DataType_UINT64:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<uint64_t>());
case TensorProto_DataType_FLOAT16:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<MLFloat16>());
case TensorProto_DataType_BFLOAT16:
return reinterpret_cast<const TensorTypeBase*>(DataTypeImpl::GetTensorType<BFloat16>());
default:
ORT_NOT_IMPLEMENTED("tensor type ", type, " is not supported");
}
}
MLDataType DataTypeImpl::TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto) {
const auto& registry = data_types_internal::DataTypeRegistry::instance();
@ -521,38 +608,7 @@ MLDataType DataTypeImpl::TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto) {
case TypeProto::ValueCase::kTensorType: {
const auto& tensor_type = proto.tensor_type();
ORT_ENFORCE(tensor_type.has_elem_type());
switch (tensor_type.elem_type()) {
case TensorProto_DataType_FLOAT:
return DataTypeImpl::GetTensorType<float>();
case TensorProto_DataType_BOOL:
return DataTypeImpl::GetTensorType<bool>();
case TensorProto_DataType_INT32:
return DataTypeImpl::GetTensorType<int32_t>();
case TensorProto_DataType_DOUBLE:
return DataTypeImpl::GetTensorType<double>();
case TensorProto_DataType_STRING:
return DataTypeImpl::GetTensorType<std::string>();
case TensorProto_DataType_UINT8:
return DataTypeImpl::GetTensorType<uint8_t>();
case TensorProto_DataType_UINT16:
return DataTypeImpl::GetTensorType<uint16_t>();
case TensorProto_DataType_INT8:
return DataTypeImpl::GetTensorType<int8_t>();
case TensorProto_DataType_INT16:
return DataTypeImpl::GetTensorType<int16_t>();
case TensorProto_DataType_INT64:
return DataTypeImpl::GetTensorType<int64_t>();
case TensorProto_DataType_UINT32:
return DataTypeImpl::GetTensorType<uint32_t>();
case TensorProto_DataType_UINT64:
return DataTypeImpl::GetTensorType<uint64_t>();
case TensorProto_DataType_FLOAT16:
return DataTypeImpl::GetTensorType<MLFloat16>();
case TensorProto_DataType_BFLOAT16:
return DataTypeImpl::GetTensorType<BFloat16>();
default:
ORT_NOT_IMPLEMENTED("tensor type ", tensor_type.elem_type(), " is not supported");
}
return TensorTypeFromONNXEnum(tensor_type.elem_type());
} break; // kTensorType
case TypeProto::ValueCase::kMapType: {
const auto& maptype = proto.map_type();

2
onnxruntime/core/framework/session_state.h
View file

@ -19,7 +19,7 @@
#include "core/framework/kernel_registry_manager.h"
#include "core/framework/mem_pattern.h"
#include "core/framework/ml_value.h"
#include "core/framework/callback.h"
#include "core/common/callback.h"
#include "core/framework/mlvalue_name_idx_map.h"
#include "core/framework/node_index_info.h"
#include "core/graph/graph_viewer.h"

38
onnxruntime/core/framework/tensor_type_and_shape.cc
View file

@ -100,42 +100,8 @@ ONNXTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType(
return type;
}
const onnxruntime::DataTypeImpl* TensorElementDataTypeToMLDataType(ONNXTensorElementDataType type) {
switch (type) {
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT:
return onnxruntime::DataTypeImpl::GetType<float>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8:
return onnxruntime::DataTypeImpl::GetType<uint8_t>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8:
return onnxruntime::DataTypeImpl::GetType<int8_t>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16:
return onnxruntime::DataTypeImpl::GetType<uint16_t>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16:
return onnxruntime::DataTypeImpl::GetType<int16_t>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32:
return onnxruntime::DataTypeImpl::GetType<int32_t>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64:
return onnxruntime::DataTypeImpl::GetType<int64_t>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING:
return onnxruntime::DataTypeImpl::GetType<std::string>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL:
return onnxruntime::DataTypeImpl::GetType<bool>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16:
return onnxruntime::DataTypeImpl::GetType<MLFloat16>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16:
return onnxruntime::DataTypeImpl::GetType<BFloat16>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE:
return onnxruntime::DataTypeImpl::GetType<double>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32:
return onnxruntime::DataTypeImpl::GetType<uint32_t>();
case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64:
return onnxruntime::DataTypeImpl::GetType<uint64_t>();
default:
return nullptr;
}
}
OrtStatus* GetTensorShapeAndType(const onnxruntime::TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out) {
OrtStatus* GetTensorShapeAndType(const onnxruntime::TensorShape* shape,
const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out) {
ONNXTensorElementDataType type = MLDataTypeToOnnxRuntimeTensorElementDataType(tensor_data_type);
if (ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED == type) {
return OrtCreateStatus(ORT_FAIL, "Not implemented");

269
onnxruntime/core/framework/tensorprotoutils.cc
View file

@ -14,7 +14,7 @@
#include "core/framework/tensor.h"
#include "core/framework/ml_value_patterns_planner.h"
#include "core/framework/allocator.h"
#include "core/framework/callback.h"
#include "core/common/callback.h"
#include "core/framework/data_types.h"
#include "core/framework/path_lib.h"
@ -23,35 +23,6 @@ using namespace ::onnxruntime::common;
namespace {
//TODO: will move OrtBuffer into env.cc and let ReadFileAsString return an OrtBuffer instead of string
//So that, we can put fclose into the destructor of OrtBuffer.
#if 0
class OrtBuffer {
public:
virtual const void* GetData() = 0;
virtual size_t GetLength() = 0;
virtual ~OrtBuffer() = default;
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OrtBuffer);
};
class OrtHeapBuffer {
public:
const void * GetData() {
}
size_t GetLength() {
}
static OrtHeapBuffer* Create(){
}
private:
void* data_;
size_t length_;
};
#endif
#ifdef __GNUC__
constexpr inline bool IsLittleEndianOrder() noexcept { return __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; }
#else
@ -314,39 +285,6 @@ std::vector<int64_t> GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::Te
return tensor_shape_vec;
}
template <typename T>
common::Status GetTensorByTypeFromTensorProto(const TensorProto& tensor_proto, const TensorShape& tensor_shape,
const OrtAllocatorInfo& alloc,
const void* raw_data, size_t raw_data_len, void* preallocated,
size_t preallocated_size, std::unique_ptr<Tensor>& out_tensor) {
int64_t tensor_size = tensor_shape.Size();
// tensor_size could be zero. see test_slice_start_out_of_bounds\test_data_set_0\output_0.pb
if (tensor_size < 0 || static_cast<uint64_t>(tensor_size) > SIZE_MAX) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid shape ", tensor_shape);
}
size_t size_to_allocate;
if (!IAllocator::CalcMemSizeForArrayWithAlignment<0>(static_cast<size_t>(tensor_size), sizeof(T),
&size_to_allocate)) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow");
}
if (preallocated && preallocated_size < size_to_allocate)
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "The buffer planner is not consistent with tensor buffer size, expected ",
size_to_allocate, ", got ", preallocated_size);
std::unique_ptr<Tensor> t;
t = std::make_unique<Tensor>(DataTypeImpl::GetType<T>(), tensor_shape, preallocated, alloc);
ORT_RETURN_IF_ERROR(
::onnxruntime::utils::UnpackTensor(tensor_proto, raw_data, raw_data_len, t->MutableData<T>(), tensor_size));
out_tensor = std::move(t);
return common::Status::OK();
}
#define CASE_PROTO(X, Y) \
case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: \
ORT_RETURN_IF_ERROR(GetTensorByTypeFromTensorProto<Y>(tensor_proto, tensor_shape, allocator, raw_data, \
raw_data_len, preallocated, preallocated_size, p_tensor)); \
break;
struct UnInitializeParam {
void* preallocated;
@ -368,7 +306,7 @@ ORT_API_STATUS(OrtInitializeBufferForTensor, _In_opt_ void* input, size_t input_
*/
ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enum ONNXTensorElementDataType type);
static void ORT_API_CALL DeleteHeapBuffer(void* param) noexcept {
static void ORT_API_CALL UnInitTensor(void* param) noexcept {
UnInitializeParam* p = reinterpret_cast<UnInitializeParam*>(param);
OrtUninitializeBuffer(p->preallocated, p->preallocated_size, p->ele_type);
delete p;
@ -399,35 +337,44 @@ ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enu
}
}
#define CASE_PROTO(X, Y) \
case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: \
ORT_RETURN_IF_ERROR( \
::onnxruntime::utils::UnpackTensor<Y>(tensor_proto, raw_data, raw_data_len, (Y*)preallocated, tensor_size)); \
break;
class AutoDelete {
public:
OrtCallback d{nullptr, nullptr};
AutoDelete() = default;
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(AutoDelete);
~AutoDelete() {
if (d.f != nullptr) {
d.f(d.param);
}
}
};
static void MoveOrtCallback(OrtCallback& from, OrtCallback& to) {
to.f = from.f;
to.param = from.param;
from.f = nullptr;
from.param = nullptr;
}
Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_proto_path,
const ONNX_NAMESPACE::TensorProto& tensor_proto, const MemBuffer& m, MLValue& value,
OrtCallback& deleter) {
const OrtAllocatorInfo& allocator = m.GetAllocInfo();
void* preallocated = m.GetBuffer();
size_t preallocated_size = m.GetLen();
ONNXTensorElementDataType ele_type = utils::GetTensorElementType(tensor_proto);
if (preallocated != nullptr && ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) {
OrtStatus* status = OrtInitializeBufferForTensor(preallocated, preallocated_size, ele_type);
if (status != nullptr) {
OrtReleaseStatus(status);
return Status(common::ONNXRUNTIME, common::FAIL, "initialize preallocated buffer failed");
}
deleter.f = DeleteHeapBuffer;
deleter.param = new UnInitializeParam{preallocated, preallocated_size, ele_type};
} else {
deleter.f = nullptr;
deleter.param = nullptr;
}
std::unique_ptr<Tensor> p_tensor;
std::string raw_data_from_file;
deleter.f = nullptr;
deleter.param = nullptr;
const void* raw_data = nullptr;
size_t raw_data_len = 0;
const DataTypeImpl* const type = DataTypeImpl::TensorTypeFromONNXEnum(tensor_proto.data_type())->GetElementType();
AutoDelete deleter_for_file_data;
void* tensor_data;
{
std::vector<int64_t> tensor_shape_vec = GetTensorShapeFromTensorProto(tensor_proto);
// Note: We permit an empty tensor_shape_vec, and treat it as a scalar (a tensor of size 1).
TensorShape tensor_shape{tensor_shape_vec};
if (tensor_proto.data_location() == TensorProto_DataLocation_EXTERNAL) {
if (ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING)
return Status(common::ONNXRUNTIME, common::FAIL, "string tensor can not have raw data");
@ -446,82 +393,120 @@ Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_proto_path,
}
// load the file
ORT_RETURN_IF_ERROR(env.ReadFileAsString(full_path.c_str(), &raw_data_from_file));
raw_data = raw_data_from_file.data();
raw_data_len = raw_data_from_file.size();
{
void* file_data;
ORT_RETURN_IF_ERROR(env.ReadFileAsString(full_path.c_str(), file_data, raw_data_len, deleter_for_file_data.d));
raw_data = file_data;
}
} else if (tensor_proto.has_raw_data()) {
if (ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING)
return Status(common::ONNXRUNTIME, common::FAIL, "string tensor can not have raw data");
raw_data = tensor_proto.raw_data().data();
raw_data_len = tensor_proto.raw_data().size();
}
switch (tensor_proto.data_type()) {
CASE_PROTO(FLOAT, float);
CASE_PROTO(DOUBLE, double);
CASE_PROTO(BOOL, bool);
CASE_PROTO(INT8, int8_t);
CASE_PROTO(INT16, int16_t);
CASE_PROTO(INT32, int32_t);
CASE_PROTO(INT64, int64_t);
CASE_PROTO(UINT8, uint8_t);
CASE_PROTO(UINT16, uint16_t);
CASE_PROTO(UINT32, uint32_t);
CASE_PROTO(UINT64, uint64_t);
CASE_PROTO(STRING, std::string);
CASE_PROTO(FLOAT16, MLFloat16);
CASE_PROTO(BFLOAT16, BFloat16);
default: {
std::ostringstream ostr;
ostr << "Initialized tensor with unexpected type: " << tensor_proto.data_type();
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str());
if (IsLittleEndianOrder() && raw_data != nullptr && deleter_for_file_data.d.f != nullptr) {
tensor_data = const_cast<void*>(raw_data);
MoveOrtCallback(deleter_for_file_data.d, deleter);
} else {
void* preallocated = m.GetBuffer();
size_t preallocated_size = m.GetLen();
int64_t tensor_size = 1;
{
for (auto i : tensor_proto.dims()) {
if (i < 0) return Status(common::ONNXRUNTIME, common::FAIL, "tensor can't contain negative dims");
tensor_size *= i;
}
}
// tensor_size could be zero. see test_slice_start_out_of_bounds\test_data_set_0\output_0.pb
if (static_cast<uint64_t>(tensor_size) > SIZE_MAX) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow");
}
size_t size_to_allocate;
if (!IAllocator::CalcMemSizeForArrayWithAlignment<0>(static_cast<size_t>(tensor_size), type->Size(),
&size_to_allocate)) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow");
}
if (preallocated && preallocated_size < size_to_allocate)
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
"The buffer planner is not consistent with tensor buffer size, expected ",
size_to_allocate, ", got ", preallocated_size);
switch (tensor_proto.data_type()) {
CASE_PROTO(FLOAT, float);
CASE_PROTO(DOUBLE, double);
CASE_PROTO(BOOL, bool);
CASE_PROTO(INT8, int8_t);
CASE_PROTO(INT16, int16_t);
CASE_PROTO(INT32, int32_t);
CASE_PROTO(INT64, int64_t);
CASE_PROTO(UINT8, uint8_t);
CASE_PROTO(UINT16, uint16_t);
CASE_PROTO(UINT32, uint32_t);
CASE_PROTO(UINT64, uint64_t);
CASE_PROTO(FLOAT16, MLFloat16);
CASE_PROTO(BFLOAT16, BFloat16);
case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_STRING:
if (preallocated != nullptr) {
OrtStatus* status = OrtInitializeBufferForTensor(preallocated, preallocated_size, ele_type);
if (status != nullptr) {
OrtReleaseStatus(status);
return Status(common::ONNXRUNTIME, common::FAIL, "initialize preallocated buffer failed");
}
deleter.f = UnInitTensor;
deleter.param = new UnInitializeParam{preallocated, preallocated_size, ele_type};
}
ORT_RETURN_IF_ERROR(::onnxruntime::utils::UnpackTensor<std::string>(tensor_proto, raw_data, raw_data_len,
(std::string*)preallocated, tensor_size));
break;
default: {
std::ostringstream ostr;
ostr << "Initialized tensor with unexpected type: " << tensor_proto.data_type();
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str());
}
}
tensor_data = preallocated;
}
}
value.Init(p_tensor.release(),
DataTypeImpl::GetType<Tensor>(),
std::vector<int64_t> tensor_shape_vec = GetTensorShapeFromTensorProto(tensor_proto);
// Note: We permit an empty tensor_shape_vec, and treat it as a scalar (a tensor of size 1).
TensorShape tensor_shape{tensor_shape_vec};
value.Init(new Tensor(type, tensor_shape, tensor_data, allocator), DataTypeImpl::GetType<Tensor>(),
DataTypeImpl::GetType<Tensor>()->GetDeleteFunc());
return Status::OK();
}
ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto& tensor_proto) {
switch (tensor_proto.data_type()) {
case TensorProto_DataType_FLOAT:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT;
case TensorProto_DataType_UINT8:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8;
case TensorProto_DataType_INT8:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8;
case TensorProto_DataType_UINT16:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16;
case TensorProto_DataType_INT16:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16;
case TensorProto_DataType_INT32:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32;
case TensorProto_DataType_INT64:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64;
case TensorProto_DataType_STRING:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING;
case TensorProto_DataType_BOOL:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL;
case TensorProto_DataType_FLOAT16:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16;
case TensorProto_DataType_DOUBLE:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE;
case TensorProto_DataType_UINT32:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32;
case TensorProto_DataType_UINT64:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64;
case TensorProto_DataType_COMPLEX64:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64;
case TensorProto_DataType_COMPLEX128:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128;
case TensorProto_DataType_BFLOAT16:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16;
#define CASE_TYPE(X) \
case ONNX_NAMESPACE::TensorProto_DataType_##X: \
return ONNX_TENSOR_ELEMENT_DATA_TYPE_##X;
ONNXTensorElementDataType CApiElementTypeFromProtoType(int type) {
switch (type) {
CASE_TYPE(FLOAT)
CASE_TYPE(UINT8)
CASE_TYPE(INT8)
CASE_TYPE(UINT16)
CASE_TYPE(INT16)
CASE_TYPE(INT32)
CASE_TYPE(INT64)
CASE_TYPE(STRING)
CASE_TYPE(BOOL)
CASE_TYPE(FLOAT16)
CASE_TYPE(DOUBLE)
CASE_TYPE(UINT32)
CASE_TYPE(UINT64)
CASE_TYPE(COMPLEX64)
CASE_TYPE(COMPLEX128)
CASE_TYPE(BFLOAT16)
default:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED;
}
}
ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto& tensor_proto) {
return CApiElementTypeFromProtoType(tensor_proto.data_type());
}
TensorProto::DataType GetTensorProtoType(const Tensor& tensor) {
auto tensor_type = tensor.DataType();
TensorProto::DataType dtype = TensorProto_DataType_UNDEFINED;

1
onnxruntime/core/framework/tensorprotoutils.h
View file

@ -37,6 +37,7 @@ common::Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_prot
// This function doesn't support string tensors
ONNX_NAMESPACE::TensorProto::DataType GetTensorProtoType(const Tensor& tensor);
ONNXTensorElementDataType CApiElementTypeFromProtoType(int type);
ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto& tensor_proto);
// How much memory it will need for putting the content of this tensor into a plain array

2
onnxruntime/core/optimizer/optimizer_execution_frame.cc
View file

@ -8,7 +8,7 @@
#include "core/framework/mldata_type_utils.h"
#include "core/framework/kernel_registry.h"
#include "core/framework/fuse_nodes_funcs.h"
#include "core/framework/callback.h"
#include "core/common/callback.h"
#include "core/optimizer/optimizer_execution_frame.h"
namespace onnxruntime {

2
onnxruntime/core/optimizer/optimizer_execution_frame.h
View file

@ -10,7 +10,7 @@
#include "core/framework/execution_frame.h"
#include "core/framework/mlvalue_name_idx_map.h"
#include "core/framework/ml_value.h"
#include "core/framework/callback.h"
#include "core/common/callback.h"
namespace onnxruntime {

14
onnxruntime/core/platform/env.h
View file

@ -24,6 +24,7 @@ limitations under the License.
#include <gsl/pointers>
#include "core/common/common.h"
#include "core/common/callback.h"
#include "core/platform/env_time.h"
#ifndef _WIN32
@ -96,11 +97,18 @@ class Env {
virtual Thread* StartThread(const ThreadOptions& thread_options, const std::string& name,
std::function<void()> fn) const = 0;
/// file_path must point to a regular file, which can't be a pipe/socket/...
#ifndef _WIN32
virtual common::Status ReadFileAsString(const char* file_path, std::string* out) const = 0;
/**
*
* \param file_path file_path must point to a regular file, which can't be a pipe/socket/...
* \param[out] p allocated buffer with the file data
* \param[out] len lenght of p
* @return
*/
virtual common::Status ReadFileAsString(const char* file_path, void*& p, size_t& len, OrtCallback& deleter) const = 0;
#else
virtual common::Status ReadFileAsString(const wchar_t* file_path, std::string* out) const = 0;
virtual common::Status ReadFileAsString(const wchar_t* file_path, void*& p, size_t& len,
OrtCallback& deleter) const = 0;
#endif
#ifdef _WIN32

103
onnxruntime/core/platform/posix/env.cc
View file

@ -17,6 +17,10 @@ limitations under the License.
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <fcntl.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <dlfcn.h>
#include <string.h>
@ -25,6 +29,7 @@ limitations under the License.
#include <assert.h>
#include "core/platform/env.h"
#include "core/common/common.h"
#include "core/common/logging/logging.h"
namespace onnxruntime {
@ -41,6 +46,26 @@ class StdThread : public Thread {
std::thread thread_;
};
static void ORT_API_CALL DeleteBuffer(void* param) noexcept { ::free(param); }
class UnmapFileParam {
public:
void* addr;
size_t len;
int fd;
};
static void ORT_API_CALL UnmapFile(void* param) noexcept {
UnmapFileParam* p = reinterpret_cast<UnmapFileParam*>(param);
int ret = munmap(p->addr, p->len);
if (ret != 0) {
int err = errno;
LOGS_DEFAULT(INFO) << "munmap failed. error code:" << err;
}
(void)close(p->fd);
delete p;
}
class PosixEnv : public Env {
public:
static PosixEnv& Instance() {
@ -55,16 +80,12 @@ class PosixEnv : public Env {
return std::thread::hardware_concurrency();
}
EnvThread* CreateThread(std::function<void()> fn) const override {
return new StdThread(fn);
}
EnvThread* CreateThread(std::function<void()> fn) const override { return new StdThread(fn); }
Task CreateTask(std::function<void()> f) const override {
return Task{std::move(f)};
}
void ExecuteTask(const Task& t) const override {
t.f();
}
void ExecuteTask(const Task& t) const override { t.f(); }
void SleepForMicroseconds(int64_t micros) const override {
while (micros > 0) {
@ -95,13 +116,36 @@ class PosixEnv : public Env {
return getpid();
}
common::Status ReadFileAsString(const char* fname, std::string* out) const override {
if (!out) {
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "ReadFileAsString: 'out' cannot be NULL");
}
static common::Status ReadBinaryFile(int fd, const char* fname, const struct stat& stbuf, void*& p, size_t& len,
OrtCallback& deleter) {
std::unique_ptr<char[]> buffer(reinterpret_cast<char*>(malloc(stbuf.st_size)));
char* wptr = reinterpret_cast<char*>(buffer.get());
auto length_remain = stbuf.st_size;
do {
size_t bytes_to_read = length_remain;
ssize_t bytes_readed = read(fd, wptr, bytes_to_read);
if (bytes_readed <= 0) {
int err = errno;
(void)close(fd);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "read file '", fname, "' fail, error code = ", err);
}
assert(static_cast<size_t>(bytes_readed) <= bytes_to_read);
wptr += bytes_readed;
length_remain -= bytes_readed;
} while (length_remain > 0);
p = buffer.release();
len = stbuf.st_size;
deleter.f = DeleteBuffer;
deleter.param = p;
return Status::OK();
}
common::Status ReadFileAsString(const char* fname, void*& p, size_t& len, OrtCallback& deleter) const override {
if (!fname) {
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "ReadFileAsString: 'fname' cannot be NULL");
}
deleter.f = nullptr;
deleter.param = nullptr;
int fd = open(fname, O_RDONLY);
if (fd < 0) {
int err = errno;
@ -113,25 +157,32 @@ class PosixEnv : public Env {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Get file '", fname, "' size fail");
}
if (stbuf.st_size == 0) {
out->clear();
p = nullptr;
len = 0;
} else {
out->resize(stbuf.st_size, '\0');
char* wptr = const_cast<char*>(out->data());
auto length_remain = stbuf.st_size;
do {
size_t bytes_to_read = length_remain;
ssize_t bytes_readed = read(fd, wptr, bytes_to_read);
if (bytes_readed <= 0) {
int err = errno;
(void)close(fd);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "read file '", fname, "' fail, error code = ", err);
if (sizeof(fname) <= 4) {
auto st = ReadBinaryFile(fd, fname, stbuf, p, len, deleter);
(void)close(fd);
if (!st.IsOK()) {
return st;
}
assert(static_cast<size_t>(bytes_readed) <= bytes_to_read);
wptr += bytes_readed;
length_remain -= bytes_readed;
} while (length_remain > 0);
(void)close(fd);
} else {
size_t flen = static_cast<size_t>(stbuf.st_size);
p = mmap(NULL, flen, PROT_READ, MAP_SHARED, fd, 0);
if (p == MAP_FAILED) {
auto st = ReadBinaryFile(fd, fname, stbuf, p, len, deleter);
(void)close(fd);
if (!st.IsOK()) {
return st;
}
} else {
len = stbuf.st_size;
deleter.f = UnmapFile;
deleter.param = new UnmapFileParam{p, flen, fd};
}
}
}
return common::Status::OK();
}

27
onnxruntime/core/platform/windows/env.cc
View file

@ -40,6 +40,7 @@ class StdThread : public Thread {
private:
std::thread thread_;
};
static void ORT_API_CALL DeleteBuffer(void* param) noexcept { ::free(param); }
class WindowsEnv : public Env {
public:
@ -93,11 +94,12 @@ class WindowsEnv : public Env {
void ExecuteTask(const Task& t) const override {
t.f();
}
common::Status ReadFileAsString(const wchar_t* fname, std::string* out) const override {
if (!fname) return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "file name is nullptr");
if (!out) {
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "'out' cannot be NULL");
common::Status ReadFileAsString(const wchar_t* fname, void*& p, size_t& len, OrtCallback& deleter) const override {
if (!fname) {
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "ReadFileAsString: 'fname' cannot be NULL");
}
deleter.f = nullptr;
deleter.param = nullptr;
HANDLE hFile = CreateFileW(fname, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (hFile == INVALID_HANDLE_VALUE) {
int err = GetLastError();
@ -111,11 +113,12 @@ class WindowsEnv : public Env {
}
// check the file file for avoiding allocating a zero length buffer
if (filesize.QuadPart == 0) { // empty file
out->clear();
p = nullptr;
len = 0;
return Status::OK();
}
out->resize(filesize.QuadPart, '\0');
char* wptr = const_cast<char*>(out->data());
std::unique_ptr<char[]> buffer(reinterpret_cast<char*>(malloc(filesize.QuadPart)));
char* wptr = reinterpret_cast<char*>(buffer.get());
auto length_remain = filesize.QuadPart;
DWORD readed = 0;
for (; length_remain > 0; wptr += readed, length_remain -= readed) {
@ -128,14 +131,20 @@ class WindowsEnv : public Env {
}
if (ReadFile(hFile, wptr, bytes_to_read, &readed, nullptr) != TRUE) {
int err = GetLastError();
out->clear();
p = nullptr;
len = 0;
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ReadFile ", ToMBString(fname), " fail, errcode =", err);
}
if (readed != bytes_to_read) {
out->clear();
p = nullptr;
len = 0;
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ReadFile ", ToMBString(fname), " fail: unexpected end");
}
}
p = buffer.release();
len = filesize.QuadPart;
deleter.f = DeleteBuffer;
deleter.param = p;
return common::Status::OK();
}

2
onnxruntime/core/providers/cpu/tensor/size.cc
View file

@ -11,8 +11,8 @@ Status Size::Compute(OpKernelContext* ctx) const {
if (input_tensor == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch");
TensorShape scalar_shape;
Tensor* p_output_tensor = ctx->Output(0, scalar_shape);
assert(p_output_tensor->Size() == sizeof(int64_t));
int64_t* p_output_scalar = p_output_tensor->template MutableData<int64_t>();
assert(p_output_tensor->Size() == sizeof(int64_t));
*p_output_scalar = input_tensor->Shape().Size();

61
onnxruntime/core/session/inference_session.cc
View file

@ -56,61 +56,6 @@
using namespace ONNX_NAMESPACE;
ONNXTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType(const onnxruntime::DataTypeImpl* cpp_type);
const onnxruntime::DataTypeImpl* TensorElementDataTypeToMLDataType(ONNXTensorElementDataType type);
namespace onnxruntime {
const char* ElementTypeToString(MLDataType type) {
if (type == DataTypeImpl::GetType<float>()) {
return "tensor(float)";
} else if (type == DataTypeImpl::GetType<bool>()) {
return "tensor(bool)";
}
else if (type == DataTypeImpl::GetType<int32_t>()) {
return "tensor(int32)";
}
else if (type == DataTypeImpl::GetType<double>()) {
return "tensor(double)";
}
else if (type == DataTypeImpl::GetType<std::string>()) {
return "tensor(string)";
}
else if (type == DataTypeImpl::GetType<uint8_t>()) {
return "tensor(uint8)";
}
else if (type == DataTypeImpl::GetType<uint16_t>()) {
return "tensor(uint16)";
}
else if (type == DataTypeImpl::GetType<int16_t>()) {
return "tensor(int16)";
}
else if (type == DataTypeImpl::GetType<int64_t>()) {
return "tensor(int64)";
}
else if (type == DataTypeImpl::GetType<uint32_t>()) {
return "tensor(uint32)";
}
else if (type == DataTypeImpl::GetType<uint64_t>()) {
return "tensor(uint64)";
}
else if (type == DataTypeImpl::GetType<MLFloat16>()) {
return "tensor(MLFloat16)";
} else if (type == DataTypeImpl::GetType<BFloat16>()) {
return "tensor(bfloat16)";
} else {
return "unknown";
}
}
} // namespace onnxruntime
ORT_API_STATUS_IMPL(OrtKernelInfoGetAttribute_float, _In_ OrtKernelInfo* info, _In_ const char* name, _Out_ float* out) {
auto status = reinterpret_cast<onnxruntime::OpKernelInfo*>(info)->GetAttr<float>(name, out);
@ -282,14 +227,16 @@ class InferenceSession::Impl {
for (size_t i = 0; i < input_count; i++) {
auto type = op->GetInputType(op, i);
schema.Input(i, "A", "Description", ElementTypeToString(TensorElementDataTypeToMLDataType(type)));
schema.Input(i, "A", "Description",
DataTypeImpl::ToString(onnxruntime::DataTypeImpl::TensorTypeFromONNXEnum(type)));
}
auto output_count = op->GetOutputTypeCount(op);
for (size_t i = 0; i < output_count; i++) {
auto type = op->GetOutputType(op, i);
schema.Output(i, "A", "Description", ElementTypeToString(TensorElementDataTypeToMLDataType(type)));
schema.Output(i, "A", "Description",
DataTypeImpl::ToString(onnxruntime::DataTypeImpl::TensorTypeFromONNXEnum(type)));
}
schema.SinceVersion(domain->op_version_start_);

5
onnxruntime/core/session/onnxruntime_c_api.cc
View file

@ -17,7 +17,7 @@
#include "core/framework/tensor.h"
#include "core/framework/ml_value.h"
#include "core/framework/environment.h"
#include "core/framework/callback.h"
#include "core/common/callback.h"
#include "core/framework/tensorprotoutils.h"
#include "core/framework/onnxruntime_typeinfo.h"
#include "core/session/inference_session.h"
@ -366,7 +366,8 @@ ORT_API_STATUS_IMPL(OrtAddCustomOpDomain, _In_ OrtSessionOptions* options, OrtCu
ORT_API_STATUS_IMPL(OrtCreateSession, _In_ OrtEnv* env, _In_ const ORTCHAR_T* model_path,
_In_ const OrtSessionOptions* options, _Out_ OrtSession** out) {
API_IMPL_BEGIN
auto sess = std::make_unique<::onnxruntime::InferenceSession>(options == nullptr ? onnxruntime::SessionOptions() : options->value, env->loggingManager);
auto sess = std::make_unique<::onnxruntime::InferenceSession>(
options == nullptr ? onnxruntime::SessionOptions() : options->value, env->loggingManager);
Status status;
if (options != nullptr) {
if (!options->custom_op_paths.empty()) {

3
onnxruntime/test/shared_lib/test_inference.cc
View file

@ -252,7 +252,8 @@ TEST_F(CApiTest, custom_op_handler) {
OrtCustomOpDomain* custom_op_domain = OrtCreateCustomOpDomain("", 5, 7);
ORT_THROW_ON_ERROR(OrtCustomOpDomain_Add(custom_op_domain, &custom_op));
TestInference<PATH_TYPE>(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, false, custom_op_domain);
TestInference<PATH_TYPE>(env, CUSTOM_OP_MODEL_URI, dims_x, values_x, expected_dims_y, expected_values_y, false, false,
custom_op_domain);
}
#ifdef ORT_RUN_EXTERNAL_ONNX_TESTS

23
onnxruntime/test/shared_lib/test_tensor_loader.cc
View file

@ -52,11 +52,14 @@ TEST_F(CApiTest, load_simple_float_tensor) {
OrtCallback* deleter;
auto st = OrtTensorProtoToOrtValue(s.data(), static_cast<int>(s.size()), nullptr, output.data(),
output.size() * sizeof(float), &value, &deleter);
// check the result
ASSERT_EQ(st, nullptr) << OrtGetErrorMessage(st);
ASSERT_EQ(output[0], 1.0f);
ASSERT_EQ(output[1], 2.2f);
ASSERT_EQ(output[2], 3.5f);
float* real_output;
st = OrtGetTensorMutableData(value, (void**)&real_output);
ASSERT_EQ(st, nullptr) << OrtGetErrorMessage(st);
// check the result
ASSERT_EQ(real_output[0], 1.0f);
ASSERT_EQ(real_output[1], 2.2f);
ASSERT_EQ(real_output[2], 3.5f);
OrtReleaseValue(value);
}
@ -104,12 +107,14 @@ static void run_external_data_test() {
}
auto st = OrtTensorProtoToOrtValue(s.data(), static_cast<int>(s.size()), cwd.empty() ? nullptr : cwd.c_str(),
output.data(), output.size() * sizeof(float), &value, &deleter);
// check the result
ASSERT_EQ(st, nullptr) << OrtGetErrorMessage(st);
ASSERT_EQ(output[0], 1.0f);
ASSERT_EQ(output[1], 2.2f);
ASSERT_EQ(output[2], 3.5f);
float* real_output;
st = OrtGetTensorMutableData(value, (void**)&real_output);
ASSERT_EQ(st, nullptr) << OrtGetErrorMessage(st);
// check the result
ASSERT_EQ(real_output[0], 1.0f);
ASSERT_EQ(real_output[1], 2.2f);
ASSERT_EQ(real_output[2], 3.5f);
OrtReleaseValue(value);
}
TEST_F(CApiTest, load_float_tensor_with_external_data) {

31
onnxruntime/test/util/compare_mlvalue.cc
View file

@ -6,7 +6,7 @@
#include <sstream>
#include <google/protobuf/text_format.h>
#include "core/graph/onnx_protobuf.h"
#include "core/framework/tensorprotoutils.h"
#include "Eigen/Core"
#include "Eigen/src/Core/arch/CUDA/Half.h"
@ -20,35 +20,8 @@ using __half_raw = ::Eigen::half_impl::__half;
} // namespace Eigen
#endif
#define CASE_TYPE(X) \
case ONNX_NAMESPACE::TensorProto_DataType_##X: \
return ONNX_TENSOR_ELEMENT_DATA_TYPE_##X;
namespace {
ONNXTensorElementDataType CApiElementTypeFromProto(int type) {
switch (type) {
CASE_TYPE(FLOAT)
CASE_TYPE(UINT8)
CASE_TYPE(INT8)
CASE_TYPE(UINT16)
CASE_TYPE(INT16)
CASE_TYPE(INT32)
CASE_TYPE(INT64)
CASE_TYPE(STRING)
CASE_TYPE(BOOL)
CASE_TYPE(FLOAT16)
CASE_TYPE(DOUBLE)
CASE_TYPE(UINT32)
CASE_TYPE(UINT64)
CASE_TYPE(COMPLEX64)
CASE_TYPE(COMPLEX128)
CASE_TYPE(BFLOAT16)
default:
return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED;
}
}
template <typename T>
bool IsResultCloselyMatch(const T& outvalue, const T& expected_value, const double diff, const double tol) {
if (diff > tol) return false;
@ -381,7 +354,7 @@ std::pair<COMPARE_RESULT, std::string> VerifyValueInfo(const ONNX_NAMESPACE::Val
info.reset(t1);
}
ONNXTensorElementDataType real_type = OrtGetTensorElementType(info.get());
ONNXTensorElementDataType expected_type = CApiElementTypeFromProto(t.elem_type());
ONNXTensorElementDataType expected_type = onnxruntime::utils::CApiElementTypeFromProtoType(t.elem_type());
if (real_type != expected_type) {
std::ostringstream oss;
oss << "expect " << ElementTypeToString((MLDataType)expected_type)