saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-24 19:43:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix perf test executable. (#1598)

Browse source 
* Mention OrtCreateSessionFromArray in C API doc

* Fix perf test executable due to removal of certain C APIs

* fix linux build

* Avoid duplication

* Fix mem leak
This commit is contained in:
Pranav Sharma 2019-08-12 09:49:29 -07:00 committed by GitHub
parent ce3c8f98dd
commit a6a4c4c079
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 386 additions and 455 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cmake/onnxruntime_unittests.cmake
View file

@ -472,6 +472,9 @@ set(onnx_test_runner_common_srcs
${onnx_test_runner_src_dir}/onnxruntime_event.h
${onnx_test_runner_src_dir}/sync_api.h
${onnx_test_runner_src_dir}/sync_api.cc
${onnx_test_runner_src_dir}/callback.h
${onnx_test_runner_src_dir}/callback.cc
${onnx_test_runner_src_dir}/mem_buffer.h
${onnx_test_runner_src_dir}/tensorprotoutils.h
${onnx_test_runner_src_dir}/tensorprotoutils.cc)
@ -566,7 +569,7 @@ onnxruntime_add_include_to_target(onnxruntime_perf_test gsl)
if (onnxruntime_BUILD_SHARED_LIB)
set(onnxruntime_perf_test_libs onnxruntime_test_utils onnx_test_runner_common onnxruntime_common
onnx_test_data_proto onnx_proto libprotobuf ${GETOPT_LIB_WIDE} onnxruntime onnxruntime_framework onnx
onnx_test_data_proto onnx_proto libprotobuf ${GETOPT_LIB_WIDE} onnxruntime
${SYS_PATH_LIB} ${CMAKE_DL_LIBS})
if(onnxruntime_USE_NSYNC)
list(APPEND onnxruntime_perf_test_libs nsync_cpp)

1
include/onnxruntime/core/session/onnxruntime_c_api.h
View file

@ -150,7 +150,6 @@ ORT_RUNTIME_CLASS(RunOptions);
ORT_RUNTIME_CLASS(TypeInfo);
ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo);
ORT_RUNTIME_CLASS(SessionOptions);
ORT_RUNTIME_CLASS(Callback);
ORT_RUNTIME_CLASS(CustomOpDomain);
ORT_RUNTIME_CLASS(Allocator);

38
onnxruntime/test/onnx/TestCase.cc
View file

@ -13,6 +13,7 @@
#include "core/platform/ort_mutex.h"
#include "core/session/onnxruntime_cxx_api.h"
#include "core/framework/path_lib.h"
#include "core/framework/allocator.h"
#include <sstream>
#include <map>
#include <regex>
@ -265,18 +266,18 @@ static void SortTensorFileNames(std::vector<std::basic_string<PATH_CHAR_TYPE>>&
}
}
OrtValue* TensorToOrtValue(const ONNX_NAMESPACE::TensorProto& t, HeapBuffer& b) {
OrtValue* TensorToOrtValue(const ONNX_NAMESPACE::TensorProto& t, onnxruntime::test::HeapBuffer& b) {
size_t len = 0;
auto status = onnxruntime::test::utils::GetSizeInBytesFromTensorProto<0>(t, &len);
auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(t, &len);
if (!status.IsOK()) {
ORT_THROW(status.ToString());
}
void* p = len == 0 ? nullptr : b.AllocMemory(len);
auto d = std::make_unique<onnxruntime::OrtCallback>();
auto temp_value = std::make_unique<OrtValue>();
Ort::Value temp_value{nullptr};
auto d = std::make_unique<onnxruntime::test::OrtCallback>();
OrtAllocatorInfo cpu_allocator_info(onnxruntime::CPU, OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeDefault);
status = onnxruntime::test::utils::TensorProtoToMLValue(Env::Default(), nullptr, t,
MemBuffer(p, len, cpu_allocator_info), *temp_value, *d);
status = onnxruntime::test::TensorProtoToMLValue(t, onnxruntime::test::MemBuffer(p, len, cpu_allocator_info),
temp_value, *d);
if (!status.IsOK()) {
ORT_THROW(status.ToString());
}
@ -287,7 +288,8 @@ OrtValue* TensorToOrtValue(const ONNX_NAMESPACE::TensorProto& t, HeapBuffer& b)
}
void LoopDataFile(int test_data_pb_fd, bool is_input, const TestModelInfo* modelinfo,
std::unordered_map<std::string, OrtValue*>& name_data_map, HeapBuffer& b, std::ostringstream& oss) {
std::unordered_map<std::string, OrtValue*>& name_data_map, onnxruntime::test::HeapBuffer& b,
std::ostringstream& oss) {
google::protobuf::io::FileInputStream f(test_data_pb_fd);
f.SetCloseOnDelete(true);
google::protobuf::io::CodedInputStream coded_input(&f);
@ -395,7 +397,8 @@ class OnnxTestCase : public ITestCase {
return std::string();
}
void ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto>& test_data_pbs, HeapBuffer& b, bool is_input,
void ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto>& test_data_pbs, onnxruntime::test::HeapBuffer& b,
bool is_input,
std::unordered_map<std::string, OrtValue*>& out);
std::once_flag model_parsed_;
@ -430,7 +433,8 @@ class OnnxTestCase : public ITestCase {
std::string GetTestCaseVersion() const override {
return model_info_->GetModelVersion();
}
void LoadTestData(size_t id, HeapBuffer& b, std::unordered_map<std::string, OrtValue*>&, bool is_input) override;
void LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::unordered_map<std::string, OrtValue*>&,
bool is_input) override;
};
ITestCase* CreateOnnxTestCase(const std::string& test_case_name, TestModelInfo* model,
@ -503,7 +507,8 @@ static void LoadTensors(const std::vector<PATH_STRING_TYPE>& pb_files,
}
}
void OnnxTestCase::LoadTestData(size_t id, HeapBuffer& b, std::unordered_map<std::string, OrtValue*>& name_data_map,
void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b,
std::unordered_map<std::string, OrtValue*>& name_data_map,
bool is_input) {
if (id >= test_data_dirs_.size()) {
ORT_THROW("index out of bound");
@ -556,7 +561,8 @@ void OnnxTestCase::LoadTestData(size_t id, HeapBuffer& b, std::unordered_map<std
ConvertTestData(test_data_pbs, b, is_input, name_data_map);
}
void OnnxTestCase::ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto>& test_data_pbs, HeapBuffer& b,
void OnnxTestCase::ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto>& test_data_pbs,
onnxruntime::test::HeapBuffer& b,
bool is_input, std::unordered_map<std::string, OrtValue*>& out) {
bool has_valid_names = true;
std::vector<std::string> var_names(test_data_pbs.size());
@ -582,16 +588,16 @@ void OnnxTestCase::ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto
const ONNX_NAMESPACE::TensorProto& input = test_data_pbs[input_index];
size_t len = 0;
auto status = onnxruntime::test::utils::GetSizeInBytesFromTensorProto<0>(input, &len);
auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(input, &len);
if (!status.IsOK()) {
ORT_THROW(status.ToString());
}
void* p = len == 0 ? nullptr : b.AllocMemory(len);
auto d = std::make_unique<onnxruntime::OrtCallback>();
Ort::Value v1{nullptr};
auto d = std::make_unique<onnxruntime::test::OrtCallback>();
OrtAllocatorInfo cpu_allocator_info(onnxruntime::CPU, OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeDefault);
auto v1 = std::make_unique<OrtValue>();
status = onnxruntime::test::utils::TensorProtoToMLValue(Env::Default(), nullptr, input,
MemBuffer(p, len, cpu_allocator_info), *v1, *d);
status = onnxruntime::test::TensorProtoToMLValue(input, onnxruntime::test::MemBuffer(p, len, cpu_allocator_info),
v1, *d);
if (!status.IsOK()) {
ORT_THROW(status.ToString());
}

2
onnxruntime/test/onnx/TestCase.h
View file

@ -19,7 +19,7 @@ class ValueInfoProto;
//One test case can contain multiple test data(input/output pairs)
class ITestCase {
public:
virtual void LoadTestData(size_t id, HeapBuffer& b, std::unordered_map<std::string, OrtValue*>& name_data_map,
virtual void LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::unordered_map<std::string, OrtValue*>& name_data_map,
bool is_input) = 0;
virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0;
virtual const std::string& GetNodeName() const = 0;

16
onnxruntime/test/onnx/callback.cc Normal file
View file

@ -0,0 +1,16 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "callback.h"
namespace onnxruntime {
namespace test {
void OrtRunCallback(OrtCallback* f) noexcept {
if (f == nullptr) return;
if (f->f != nullptr) {
f->f(f->param);
delete f;
}
}
} // namespace test
} // namespace onnxruntime

17
onnxruntime/test/onnx/callback.h Normal file
View file

@ -0,0 +1,17 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
namespace onnxruntime {
namespace test {
struct OrtCallback {
void (*f)(void* param) noexcept;
void* param;
};
/**
* f will be freed in this call
*/
void OrtRunCallback(OrtCallback* f) noexcept;
} // namespace test
} // namespace onnxruntime

12
onnxruntime/test/onnx/heap_buffer.cc
View file

@ -2,18 +2,22 @@
// Licensed under the MIT License.
#include "heap_buffer.h"
#include "core/framework/callback.h"
#include "core/session/onnxruntime_c_api.h"
#include "callback.h"
void HeapBuffer::AddDeleter(onnxruntime::OrtCallback* d) {
namespace onnxruntime {
namespace test {
void HeapBuffer::AddDeleter(OrtCallback* d) {
if (d != nullptr) deleters_.push_back(d);
}
HeapBuffer::~HeapBuffer() {
for (auto d : deleters_) {
onnxruntime::OrtRunCallback(d);
OrtRunCallback(d);
}
for (void* p : buffers_) {
free(p);
}
}
}
} // namespace test
} // namespace onnxruntime

14
onnxruntime/test/onnx/heap_buffer.h
View file

@ -4,10 +4,10 @@
#pragma once
#include <vector>
#include <memory>
namespace onnxruntime {
struct OrtCallback;
}
namespace onnxruntime {
namespace test {
struct OrtCallback;
/**
* A holder for delay freed buffers
*/
@ -23,9 +23,11 @@ class HeapBuffer {
buffers_.push_back(p);
return p;
}
void AddDeleter(onnxruntime::OrtCallback* d);
void AddDeleter(OrtCallback* d);
private:
std::vector<onnxruntime::OrtCallback*> deleters_;
std::vector<OrtCallback*> deleters_;
std::vector<void*> buffers_;
};
};
} // namespace test
} // namespace onnxruntime

27
onnxruntime/test/onnx/mem_buffer.h Normal file
View file

@ -0,0 +1,27 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/common/common.h"
namespace onnxruntime {
namespace test {
/**
* A simple POD for using with tensor deserialization
*/
class MemBuffer {
public:
MemBuffer(void* buffer, size_t len, const OrtAllocatorInfo& alloc_info)
: buffer_(buffer), len_(len), alloc_info_(alloc_info) {}
void* GetBuffer() const { return buffer_; }
size_t GetLen() const { return len_; }
const OrtAllocatorInfo& GetAllocInfo() const { return alloc_info_; }
private:
void* const buffer_;
const size_t len_;
const OrtAllocatorInfo& alloc_info_;
};
}; // namespace test
} // namespace onnxruntime

82
onnxruntime/test/onnx/runner.cc
View file

@ -27,45 +27,44 @@
using namespace onnxruntime;
using ::onnxruntime::common::Status;
// Permanently exclude following tests because ORT support only opset staring from 7,
// Permanently exclude following tests because ORT support only opset staring from 7,
// Please make no more changes to the list
const std::set<std::string> immutable_broken_tests =
{
"AvgPool1d",
"AvgPool1d_stride",
"AvgPool2d",
"AvgPool2d_stride",
"AvgPool3d",
"AvgPool3d_stride",
"AvgPool3d_stride1_pad0_gpu_input",
"BatchNorm1d_3d_input_eval",
"BatchNorm2d_eval",
"BatchNorm2d_momentum_eval",
"BatchNorm3d_eval",
"BatchNorm3d_momentum_eval",
"GLU",
"GLU_dim",
"Linear",
"PReLU_1d",
"PReLU_1d_multiparam",
"PReLU_2d",
"PReLU_2d_multiparam",
"PReLU_3d",
"PReLU_3d_multiparam",
"PoissonNLLLLoss_no_reduce",
"Softsign",
"operator_add_broadcast",
"operator_add_size1_broadcast",
"operator_add_size1_right_broadcast",
"operator_add_size1_singleton_broadcast",
"operator_addconstant",
"operator_addmm",
"operator_basic",
"operator_mm",
"operator_non_float_params",
"operator_params",
"operator_pow"
};
const std::set<std::string> immutable_broken_tests =
{
"AvgPool1d",
"AvgPool1d_stride",
"AvgPool2d",
"AvgPool2d_stride",
"AvgPool3d",
"AvgPool3d_stride",
"AvgPool3d_stride1_pad0_gpu_input",
"BatchNorm1d_3d_input_eval",
"BatchNorm2d_eval",
"BatchNorm2d_momentum_eval",
"BatchNorm3d_eval",
"BatchNorm3d_momentum_eval",
"GLU",
"GLU_dim",
"Linear",
"PReLU_1d",
"PReLU_1d_multiparam",
"PReLU_2d",
"PReLU_2d_multiparam",
"PReLU_3d",
"PReLU_3d_multiparam",
"PoissonNLLLLoss_no_reduce",
"Softsign",
"operator_add_broadcast",
"operator_add_size1_broadcast",
"operator_add_size1_right_broadcast",
"operator_add_size1_singleton_broadcast",
"operator_addconstant",
"operator_addmm",
"operator_basic",
"operator_mm",
"operator_non_float_params",
"operator_params",
"operator_pow"};
void ORT_CALLBACK RunTestCase(ORT_CALLBACK_INSTANCE pci, void* context, ORT_WORK work) {
OnnxRuntimeCloseThreadpoolWork(work);
@ -232,13 +231,13 @@ Status RunTests(TestEnv& env, int p_models, int concurrent_runs, size_t repeat_c
}
for (size_t i = 0; i != env.tests.size(); ++i) {
if (!results[i]) {
stat.AddFailedTest(std::pair<std::string,std::string>(env.tests[i]->GetTestCaseName(), env.tests[i]->GetTestCaseVersion()));
stat.AddFailedTest(std::pair<std::string, std::string>(env.tests[i]->GetTestCaseName(), env.tests[i]->GetTestCaseVersion()));
continue;
}
const TestCaseResult& r = *results[i];
for (const EXECUTE_RESULT res : r.GetExcutionResult()) {
if (res != EXECUTE_RESULT::SUCCESS && res != EXECUTE_RESULT::NOT_SUPPORT) {
stat.AddFailedTest(std::pair<std::string,std::string>(env.tests[i]->GetTestCaseName(),env.tests[i]->GetTestCaseVersion()));
stat.AddFailedTest(std::pair<std::string, std::string>(env.tests[i]->GetTestCaseName(), env.tests[i]->GetTestCaseVersion()));
}
switch (res) {
case EXECUTE_RESULT::SUCCESS:
@ -347,7 +346,7 @@ void DataRunner::RunTask(size_t task_id, ORT_CALLBACK_INSTANCE pci, bool store_r
}
EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) {
HeapBuffer holder;
onnxruntime::test::HeapBuffer holder;
std::unordered_map<std::string, OrtValue*> feeds;
c_->LoadTestData(task_id, holder, feeds, true);
@ -499,7 +498,6 @@ void SeqTestRunner::Start(ORT_CALLBACK_INSTANCE pci, size_t) {
}
void RunSingleTestCase(ITestCase* info, Ort::Env& env, const Ort::SessionOptions& sf, size_t concurrent_runs, size_t repeat_count, PThreadPool tpool, ORT_CALLBACK_INSTANCE pci, TestCaseCallBack on_finished) {
//for test in immutable list, do not even run it
if (immutable_broken_tests.find(info->GetTestCaseName()) != immutable_broken_tests.end()) {
on_finished(std::make_shared<TestCaseResult>(0, EXECUTE_RESULT::NOT_SUPPORT, info->GetNodeName()), pci);

560
onnxruntime/test/onnx/tensorprotoutils.cc
View file

@ -7,22 +7,19 @@
#include <algorithm>
#include <limits>
#include <gsl/pointers>
#include "core/common/logging/logging.h"
#include "core/graph/onnx_protobuf.h"
#include "core/framework/op_kernel.h"
#include "core/framework/tensor.h"
#include "core/framework/ort_value_pattern_planner.h"
#include "core/framework/allocator.h"
#include "core/framework/callback.h"
#include "core/framework/data_types.h"
#include "core/framework/path_lib.h"
#include "core/framework/allocator.h"
#include "core/session/onnxruntime_cxx_api.h"
#include "core/graph/onnx_protobuf.h"
#include "callback.h"
using namespace ONNX_NAMESPACE;
using namespace ::onnxruntime::common;
namespace {
struct OrtStatus {
OrtErrorCode code;
char msg[1]; // a null-terminated string
};
namespace onnxruntime {
namespace test {
#ifdef __GNUC__
constexpr inline bool IsLittleEndianOrder() noexcept { return __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; }
#else
@ -34,7 +31,38 @@ inline bool IsLittleEndianOrder() noexcept {
}
#endif
std::vector<int64_t> GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto) {
//From core common
inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept {
}
template <typename T>
inline void MakeStringInternal(std::ostringstream& ss, const T& t) noexcept {
ss << t;
}
template <typename T, typename... Args>
inline void MakeStringInternal(std::ostringstream& ss, const T& t, const Args&... args) noexcept {
::onnxruntime::MakeStringInternal(ss, t);
::onnxruntime::MakeStringInternal(ss, args...);
}
template <typename... Args>
std::string MakeString(const Args&... args) {
std::ostringstream ss;
::onnxruntime::MakeStringInternal(ss, args...);
return std::string(ss.str());
}
// Specializations for already-a-string types.
template <>
inline std::string MakeString(const std::string& str) {
return str;
}
inline std::string MakeString(const char* p_str) {
return p_str;
}
std::vector<int64_t> GetTensorShapeFromTensorProto(const onnx::TensorProto& tensor_proto) {
const auto& dims = tensor_proto.dims();
std::vector<int64_t> tensor_shape_vec(static_cast<size_t>(dims.size()));
for (int i = 0; i < dims.size(); ++i) {
@ -46,8 +74,8 @@ std::vector<int64_t> GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorP
// This function doesn't support string tensors
template <typename T>
static Status UnpackTensorWithRawData(const void* raw_data, size_t raw_data_length, size_t expected_size,
/*out*/ T* p_data) {
static void UnpackTensorWithRawData(const void* raw_data, size_t raw_data_length, size_t expected_size,
/*out*/ T* p_data) {
// allow this low level routine to be somewhat unsafe. assuming it's thoroughly tested and valid
GSL_SUPPRESS(type) // type.1 reinterpret-cast; type.4 C-style casts; type.5 'T result;' is uninitialized;
GSL_SUPPRESS(bounds .1) // pointer arithmetic
@ -55,12 +83,12 @@ static Status UnpackTensorWithRawData(const void* raw_data, size_t raw_data_leng
{
size_t expected_size_in_bytes;
if (!onnxruntime::IAllocator::CalcMemSizeForArray(expected_size, sizeof(T), &expected_size_in_bytes)) {
return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::INVALID_ARGUMENT, "size overflow");
throw Ort::Exception("size overflow", OrtErrorCode::ORT_FAIL);
}
if (raw_data_length != expected_size_in_bytes)
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"UnpackTensor: the pre-allocated size does not match the raw data size, expected ",
expected_size_in_bytes, ", got ", raw_data_length);
throw Ort::Exception(MakeString("UnpackTensor: the pre-allocated size does not match the raw data size, expected ",
expected_size_in_bytes, ", got ", raw_data_length),
OrtErrorCode::ORT_FAIL);
if (IsLittleEndianOrder()) {
memcpy(p_data, raw_data, raw_data_length);
} else {
@ -75,85 +103,81 @@ static Status UnpackTensorWithRawData(const void* raw_data, size_t raw_data_leng
p_data[i] = result;
}
}
return Status::OK();
}
}
} // namespace
namespace onnxruntime {
namespace test {
namespace utils {
// This macro doesn't work for Float16/bool/string tensors
#define DEFINE_UNPACK_TENSOR(T, Type, field_name, field_size) \
template <> \
Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, \
/*out*/ T* p_data, int64_t expected_size) { \
if (nullptr == p_data) { \
const size_t size = raw_data != nullptr ? raw_data_len : tensor.field_size(); \
if (size == 0) return Status::OK(); \
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); \
} \
if (nullptr == p_data || Type != tensor.data_type()) { \
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); \
} \
if (raw_data != nullptr) { \
return UnpackTensorWithRawData(raw_data, raw_data_len, expected_size, p_data); \
} \
if (tensor.field_size() != expected_size) \
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "corrupted protobuf data: tensor shape size(", expected_size, \
") does not match the data size(", tensor.field_size(), ") in proto"); \
auto& data = tensor.field_name(); \
for (auto data_iter = data.cbegin(); data_iter != data.cend(); ++data_iter) \
*p_data++ = *reinterpret_cast<const T*>(data_iter); \
return Status::OK(); \
#define DEFINE_UNPACK_TENSOR(T, Type, field_name, field_size) \
template <> \
void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len, \
/*out*/ T* p_data, int64_t expected_size) { \
if (nullptr == p_data) { \
const size_t size = raw_data != nullptr ? raw_data_len : tensor.field_size(); \
if (size == 0) return; \
throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); \
} \
if (nullptr == p_data || Type != tensor.data_type()) { \
throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); \
} \
if (raw_data != nullptr) { \
UnpackTensorWithRawData(raw_data, raw_data_len, expected_size, p_data); \
return; \
} \
if (tensor.field_size() != expected_size) \
throw Ort::Exception(MakeString("corrupted protobuf data: tensor shape size(", expected_size, \
") does not match the data size(", tensor.field_size(), ") in proto"), \
OrtErrorCode::ORT_FAIL); \
auto& data = tensor.field_name(); \
for (auto data_iter = data.cbegin(); data_iter != data.cend(); ++data_iter) \
*p_data++ = *reinterpret_cast<const T*>(data_iter); \
return; \
}
// TODO: complex64 complex128
DEFINE_UNPACK_TENSOR(float, ONNX_NAMESPACE::TensorProto_DataType_FLOAT, float_data, float_data_size)
DEFINE_UNPACK_TENSOR(double, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, double_data, double_data_size);
DEFINE_UNPACK_TENSOR(uint8_t, ONNX_NAMESPACE::TensorProto_DataType_UINT8, int32_data, int32_data_size)
DEFINE_UNPACK_TENSOR(int8_t, ONNX_NAMESPACE::TensorProto_DataType_INT8, int32_data, int32_data_size)
DEFINE_UNPACK_TENSOR(int16_t, ONNX_NAMESPACE::TensorProto_DataType_INT16, int32_data, int32_data_size)
DEFINE_UNPACK_TENSOR(uint16_t, ONNX_NAMESPACE::TensorProto_DataType_UINT16, int32_data, int32_data_size)
DEFINE_UNPACK_TENSOR(int32_t, ONNX_NAMESPACE::TensorProto_DataType_INT32, int32_data, int32_data_size)
DEFINE_UNPACK_TENSOR(int64_t, ONNX_NAMESPACE::TensorProto_DataType_INT64, int64_data, int64_data_size)
DEFINE_UNPACK_TENSOR(uint64_t, ONNX_NAMESPACE::TensorProto_DataType_UINT64, uint64_data, uint64_data_size)
DEFINE_UNPACK_TENSOR(uint32_t, ONNX_NAMESPACE::TensorProto_DataType_UINT32, uint64_data, uint64_data_size)
DEFINE_UNPACK_TENSOR(float, onnx::TensorProto_DataType_FLOAT, float_data, float_data_size)
DEFINE_UNPACK_TENSOR(double, onnx::TensorProto_DataType_DOUBLE, double_data, double_data_size);
DEFINE_UNPACK_TENSOR(uint8_t, onnx::TensorProto_DataType_UINT8, int32_data, int32_data_size)
DEFINE_UNPACK_TENSOR(int8_t, onnx::TensorProto_DataType_INT8, int32_data, int32_data_size)
DEFINE_UNPACK_TENSOR(int16_t, onnx::TensorProto_DataType_INT16, int32_data, int32_data_size)
DEFINE_UNPACK_TENSOR(uint16_t, onnx::TensorProto_DataType_UINT16, int32_data, int32_data_size)
DEFINE_UNPACK_TENSOR(int32_t, onnx::TensorProto_DataType_INT32, int32_data, int32_data_size)
DEFINE_UNPACK_TENSOR(int64_t, onnx::TensorProto_DataType_INT64, int64_data, int64_data_size)
DEFINE_UNPACK_TENSOR(uint64_t, onnx::TensorProto_DataType_UINT64, uint64_data, uint64_data_size)
DEFINE_UNPACK_TENSOR(uint32_t, onnx::TensorProto_DataType_UINT32, uint64_data, uint64_data_size)
// doesn't support raw data
template <>
Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* /*raw_data*/, size_t /*raw_data_len*/,
/*out*/ std::string* p_data, int64_t expected_size) {
void UnpackTensor(const onnx::TensorProto& tensor, const void* /*raw_data*/, size_t /*raw_data_len*/,
/*out*/ std::string* p_data, int64_t expected_size) {
if (nullptr == p_data) {
if (tensor.string_data_size() == 0) return Status::OK();
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT);
if (tensor.string_data_size() == 0) return;
throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT);
}
if (ONNX_NAMESPACE::TensorProto_DataType_STRING != tensor.data_type()) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT);
if (onnx::TensorProto_DataType_STRING != tensor.data_type()) {
throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT);
}
if (tensor.string_data_size() != expected_size)
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
"UnpackTensor: the pre-allocate size does not match the size in proto");
throw Ort::Exception(
"UnpackTensor: the pre-allocate size does not match the size in proto", OrtErrorCode::ORT_FAIL);
auto& string_data = tensor.string_data();
for (const auto& iter : string_data) {
*p_data++ = iter;
}
return Status::OK();
return;
}
template <>
Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len,
/*out*/ bool* p_data, int64_t expected_size) {
void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len,
/*out*/ bool* p_data, int64_t expected_size) {
if (nullptr == p_data) {
const size_t size = raw_data != nullptr ? raw_data_len : tensor.int32_data_size();
if (size == 0) return Status::OK();
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT);
if (size == 0) return;
throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT);
}
if (ONNX_NAMESPACE::TensorProto_DataType_BOOL != tensor.data_type()) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT);
if (onnx::TensorProto_DataType_BOOL != tensor.data_type()) {
throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT);
}
if (raw_data != nullptr) {
@ -161,24 +185,24 @@ Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_d
}
if (tensor.int32_data_size() != expected_size)
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
"UnpackTensor: the pre-allocate size does not match the size in proto");
throw Ort::Exception(
"UnpackTensor: the pre-allocate size does not match the size in proto", OrtErrorCode::ORT_FAIL);
for (int iter : tensor.int32_data()) {
*p_data++ = static_cast<bool>(iter);
}
return Status::OK();
return;
}
template <>
Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len,
/*out*/ MLFloat16* p_data, int64_t expected_size) {
void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len,
/*out*/ MLFloat16* p_data, int64_t expected_size) {
if (nullptr == p_data) {
const size_t size = raw_data != nullptr ? raw_data_len : tensor.int32_data_size();
if (size == 0) return Status::OK();
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT);
if (size == 0) return;
throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT);
}
if (ONNX_NAMESPACE::TensorProto_DataType_FLOAT16 != tensor.data_type()) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT);
if (onnx::TensorProto_DataType_FLOAT16 != tensor.data_type()) {
throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT);
}
if (raw_data != nullptr) {
@ -186,33 +210,34 @@ Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_d
}
if (tensor.int32_data_size() != expected_size)
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
"UnpackTensor: the pre-allocate size does not match the size in proto");
throw Ort::Exception(
"UnpackTensor: the pre-allocate size does not match the size in proto", OrtErrorCode::ORT_FAIL);
constexpr int max_value = std::numeric_limits<uint16_t>::max();
for (int i = 0; i < static_cast<int>(expected_size); i++) {
int v = tensor.int32_data()[i];
if (v < 0 || v > max_value) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "data overflow");
throw Ort::Exception(
"data overflow", OrtErrorCode::ORT_FAIL);
}
p_data[i] = MLFloat16(static_cast<uint16_t>(v));
}
return Status::OK();
return;
}
template <>
Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len,
/*out*/ BFloat16* p_data, int64_t expected_size) {
void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len,
/*out*/ BFloat16* p_data, int64_t expected_size) {
if (nullptr == p_data) {
const size_t size = raw_data != nullptr ? raw_data_len : tensor.int32_data_size();
if (size == 0)
return Status::OK();
return;
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT);
throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT);
}
if (ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16 != tensor.data_type()) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT);
if (onnx::TensorProto_DataType_BFLOAT16 != tensor.data_type()) {
throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT);
}
if (raw_data != nullptr) {
@ -220,38 +245,39 @@ Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_d
}
if (tensor.int32_data_size() != expected_size)
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
"UnpackTensor: the pre-allocate size does not match the size in proto");
throw Ort::Exception(
"UnpackTensor: the pre-allocate size does not match the size in proto", OrtErrorCode::ORT_FAIL);
constexpr int max_value = std::numeric_limits<uint16_t>::max();
for (int i = 0; i < static_cast<int>(expected_size); i++) {
int v = tensor.int32_data()[i];
if (v < 0 || v > max_value) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "data overflow");
throw Ort::Exception(
"data overflow", OrtErrorCode::ORT_FAIL);
}
p_data[i] = BFloat16(static_cast<uint16_t>(v));
}
return Status::OK();
return;
}
#define CASE_PROTO_TRACE(X, Y) \
case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: \
if (!IAllocator::CalcMemSizeForArrayWithAlignment<alignment>(size, sizeof(Y), out)) { \
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid TensorProto"); \
} \
#define CASE_PROTO_TRACE(X, Y) \
case onnx::TensorProto_DataType::TensorProto_DataType_##X: \
if (!IAllocator::CalcMemSizeForArrayWithAlignment<alignment>(size, sizeof(Y), out)) { \
throw Ort::Exception("Invalid TensorProto", OrtErrorCode::ORT_FAIL); \
} \
break;
template <size_t alignment>
common::Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out) {
Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out) {
const auto& dims = tensor_proto.dims();
size_t size = 1;
for (google::protobuf::int64 dim : dims) {
if (dim < 0 || static_cast<uint64_t>(dim) >= std::numeric_limits<size_t>::max()) {
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid TensorProto");
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid TensorProto");
}
if (!IAllocator::CalcMemSizeForArray(size, static_cast<size_t>(dim), &size)) {
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid TensorProto");
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid TensorProto");
}
}
switch (tensor_proto.data_type()) {
@ -270,49 +296,19 @@ common::Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto&
CASE_PROTO_TRACE(BFLOAT16, BFloat16);
CASE_PROTO_TRACE(STRING, std::string);
default:
return common::Status(common::ONNXRUNTIME, common::NOT_IMPLEMENTED);
return Status(common::ONNXRUNTIME, common::NOT_IMPLEMENTED);
}
return Status::OK();
}
TensorShape GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::TensorShapeProto& tensor_shape_proto) {
const auto& dims = tensor_shape_proto.dim();
std::vector<int64_t> tensor_shape_vec(static_cast<size_t>(dims.size()));
for (int i = 0; i < dims.size(); ++i) {
tensor_shape_vec[i] = dims[i].has_dim_value() ? dims[i].dim_value()
: -1; /* symbolic dimensions are represented as -1 in onnxruntime*/
}
return TensorShape(std::move(tensor_shape_vec));
}
struct UnInitializeParam {
void* preallocated;
size_t preallocated_size;
ONNXTensorElementDataType ele_type;
};
// In the future, we may make these two function as public C API
/**
* Initialize a buffer for being used with the OrtCreateTensorWithDataAsOrtValue function
*
*/
ORT_API_STATUS(OrtInitializeBufferForTensor, _In_opt_ void* input, size_t input_len,
enum ONNXTensorElementDataType type);
/**
* Uninitialize the buffer that was initialized by the OrtInitializeBufferForTensor function
*
*/
ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enum ONNXTensorElementDataType type);
static void UnInitTensor(void* param) noexcept {
UnInitializeParam* p = reinterpret_cast<UnInitializeParam*>(param);
OrtUninitializeBuffer(p->preallocated, p->preallocated_size, p->ele_type);
delete p;
}
ORT_API_STATUS_IMPL(OrtInitializeBufferForTensor, _In_opt_ void* input, size_t input_len,
enum ONNXTensorElementDataType type) {
OrtStatus* OrtInitializeBufferForTensor(void* input, size_t input_len,
ONNXTensorElementDataType type) {
try {
if (type != ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING || input == nullptr) return nullptr;
size_t tensor_size = input_len / sizeof(std::string);
@ -326,6 +322,14 @@ ORT_API_STATUS_IMPL(OrtInitializeBufferForTensor, _In_opt_ void* input, size_t i
return nullptr;
}
ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enum ONNXTensorElementDataType type);
static void UnInitTensor(void* param) noexcept {
UnInitializeParam* p = reinterpret_cast<UnInitializeParam*>(param);
OrtUninitializeBuffer(p->preallocated, p->preallocated_size, p->ele_type);
delete p;
}
ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enum ONNXTensorElementDataType type) {
if (type != ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING || input == nullptr) return;
size_t tensor_size = input_len / sizeof(std::string);
@ -336,145 +340,13 @@ ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enu
}
}
#define CASE_PROTO(X, Y) \
case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: \
ORT_RETURN_IF_ERROR( \
::onnxruntime::test::utils::UnpackTensor<Y>(tensor_proto, raw_data, raw_data_len, (Y*)preallocated, tensor_size)); \
#define CASE_PROTO(X, Y) \
case onnx::TensorProto_DataType::TensorProto_DataType_##X: \
::onnxruntime::test::UnpackTensor<Y>(tensor_proto, raw_data, raw_data_len, (Y*)preallocated, tensor_size); \
break;
class AutoDelete {
public:
OrtCallback d{nullptr, nullptr};
AutoDelete() = default;
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(AutoDelete);
~AutoDelete() {
if (d.f != nullptr) {
d.f(d.param);
}
}
};
static void MoveOrtCallback(OrtCallback& from, OrtCallback& to) {
to.f = from.f;
to.param = from.param;
from.f = nullptr;
from.param = nullptr;
}
Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_proto_path,
const ONNX_NAMESPACE::TensorProto& tensor_proto, const MemBuffer& m, OrtValue& value,
OrtCallback& deleter) {
const OrtAllocatorInfo& allocator = m.GetAllocInfo();
ONNXTensorElementDataType ele_type = onnxruntime::test::utils::GetTensorElementType(tensor_proto);
deleter.f = nullptr;
deleter.param = nullptr;
const void* raw_data = nullptr;
size_t raw_data_len = 0;
const DataTypeImpl* const type = DataTypeImpl::TensorTypeFromONNXEnum(tensor_proto.data_type())->GetElementType();
AutoDelete deleter_for_file_data;
void* tensor_data;
{
if (tensor_proto.data_location() == TensorProto_DataLocation_EXTERNAL) {
if (ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING)
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "string tensor can not have raw data");
std::unique_ptr<ExternalDataInfo> external_data_info;
ORT_RETURN_IF_ERROR(ExternalDataInfo::Create(tensor_proto.external_data(), external_data_info));
std::basic_string<ORTCHAR_T> full_path;
if (tensor_proto_path != nullptr) {
ORT_RETURN_IF_ERROR(GetDirNameFromFilePath(tensor_proto_path, full_path));
full_path = ConcatPathComponent<ORTCHAR_T>(full_path, external_data_info->GetRelPath());
} else {
full_path = external_data_info->GetRelPath();
}
raw_data_len = external_data_info->GetLength();
// load the file
{
void* file_data;
ORT_RETURN_IF_ERROR(env.ReadFileAsString(full_path.c_str(), external_data_info->GetOffset(),
file_data, raw_data_len, deleter_for_file_data.d));
raw_data = file_data;
}
} else if (tensor_proto.has_raw_data()) {
if (ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING)
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "string tensor can not have raw data");
raw_data = tensor_proto.raw_data().data();
raw_data_len = tensor_proto.raw_data().size();
}
if (IsLittleEndianOrder() && raw_data != nullptr && deleter_for_file_data.d.f != nullptr) {
tensor_data = const_cast<void*>(raw_data);
MoveOrtCallback(deleter_for_file_data.d, deleter);
} else {
void* preallocated = m.GetBuffer();
size_t preallocated_size = m.GetLen();
int64_t tensor_size = 1;
{
for (auto i : tensor_proto.dims()) {
if (i < 0) return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "tensor can't contain negative dims");
tensor_size *= i;
}
}
// tensor_size could be zero. see test_slice_start_out_of_bounds\test_data_set_0\output_0.pb
if (static_cast<uint64_t>(tensor_size) > SIZE_MAX) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow");
}
size_t size_to_allocate;
if (!IAllocator::CalcMemSizeForArrayWithAlignment<0>(static_cast<size_t>(tensor_size), type->Size(),
&size_to_allocate)) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow");
}
if (preallocated && preallocated_size < size_to_allocate)
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"The buffer planner is not consistent with tensor buffer size, expected ",
size_to_allocate, ", got ", preallocated_size);
switch (tensor_proto.data_type()) {
CASE_PROTO(FLOAT, float);
CASE_PROTO(DOUBLE, double);
CASE_PROTO(BOOL, bool);
CASE_PROTO(INT8, int8_t);
CASE_PROTO(INT16, int16_t);
CASE_PROTO(INT32, int32_t);
CASE_PROTO(INT64, int64_t);
CASE_PROTO(UINT8, uint8_t);
CASE_PROTO(UINT16, uint16_t);
CASE_PROTO(UINT32, uint32_t);
CASE_PROTO(UINT64, uint64_t);
CASE_PROTO(FLOAT16, MLFloat16);
CASE_PROTO(BFLOAT16, BFloat16);
case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_STRING:
if (preallocated != nullptr) {
OrtStatus* status = OrtInitializeBufferForTensor(preallocated, preallocated_size, ele_type);
if (status != nullptr) {
OrtReleaseStatus(status);
return Status(common::ONNXRUNTIME, common::FAIL, "initialize preallocated buffer failed");
}
deleter.f = UnInitTensor;
deleter.param = new UnInitializeParam{preallocated, preallocated_size, ele_type};
}
ORT_RETURN_IF_ERROR(::onnxruntime::test::utils::UnpackTensor<std::string>(tensor_proto, raw_data, raw_data_len,
(std::string*)preallocated, tensor_size));
break;
default: {
std::ostringstream ostr;
ostr << "Initialized tensor with unexpected type: " << tensor_proto.data_type();
return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str());
}
}
tensor_data = preallocated;
}
}
std::vector<int64_t> tensor_shape_vec = GetTensorShapeFromTensorProto(tensor_proto);
// Note: We permit an empty tensor_shape_vec, and treat it as a scalar (a tensor of size 1).
TensorShape tensor_shape{tensor_shape_vec};
value.Init(new Tensor(type, tensor_shape, tensor_data, allocator), DataTypeImpl::GetType<Tensor>(),
DataTypeImpl::GetType<Tensor>()->GetDeleteFunc());
return Status::OK();
}
#define CASE_TYPE(X) \
case ONNX_NAMESPACE::TensorProto_DataType_##X: \
#define CASE_TYPE(X) \
case onnx::TensorProto_DataType_##X: \
return ONNX_TENSOR_ELEMENT_DATA_TYPE_##X;
ONNXTensorElementDataType CApiElementTypeFromProtoType(int type) {
@ -500,72 +372,88 @@ ONNXTensorElementDataType CApiElementTypeFromProtoType(int type) {
}
}
ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto& tensor_proto) {
ONNXTensorElementDataType GetTensorElementType(const onnx::TensorProto& tensor_proto) {
return CApiElementTypeFromProtoType(tensor_proto.data_type());
}
TensorProto::DataType GetTensorProtoType(const Tensor& tensor) {
auto tensor_type = tensor.DataType();
TensorProto::DataType dtype = TensorProto_DataType_UNDEFINED;
Status TensorProtoToMLValue(const onnx::TensorProto& tensor_proto, const MemBuffer& m, Ort::Value& value,
OrtCallback& deleter) {
const OrtAllocatorInfo& allocator = m.GetAllocInfo();
ONNXTensorElementDataType ele_type = test::GetTensorElementType(tensor_proto);
const void* raw_data = nullptr;
size_t raw_data_len = 0;
void* tensor_data;
{
if (tensor_proto.data_location() == onnx::TensorProto_DataLocation::TensorProto_DataLocation_EXTERNAL) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Server doesn't support external data.");
} else if (tensor_proto.has_raw_data()) {
if (ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING)
return Status(common::ONNXRUNTIME, common::FAIL, "String tensor cannot have raw data.");
raw_data = tensor_proto.raw_data().data();
raw_data_len = tensor_proto.raw_data().size();
}
{
void* preallocated = m.GetBuffer();
size_t preallocated_size = m.GetLen();
int64_t tensor_size = 1;
{
for (auto i : tensor_proto.dims()) {
if (i < 0) return Status(common::ONNXRUNTIME, common::FAIL, "Tensor can't contain negative dims");
tensor_size *= i;
}
}
// tensor_size could be zero. see test_slice_start_out_of_bounds\test_data_set_0\output_0.pb
if (static_cast<uint64_t>(tensor_size) > SIZE_MAX) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Size overflow");
}
size_t size_to_allocate;
GetSizeInBytesFromTensorProto<0>(tensor_proto, &size_to_allocate);
if (tensor_type == DataTypeImpl::GetType<float>())
dtype = TensorProto_DataType_FLOAT;
else if (tensor_type == DataTypeImpl::GetType<double>())
dtype = TensorProto_DataType_DOUBLE;
else if (tensor_type == DataTypeImpl::GetType<int8_t>())
dtype = TensorProto_DataType_INT8;
else if (tensor_type == DataTypeImpl::GetType<int16_t>())
dtype = TensorProto_DataType_INT16;
else if (tensor_type == DataTypeImpl::GetType<int32_t>())
dtype = TensorProto_DataType_INT32;
else if (tensor_type == DataTypeImpl::GetType<int64_t>())
dtype = TensorProto_DataType_INT64;
else if (tensor_type == DataTypeImpl::GetType<uint8_t>())
dtype = TensorProto_DataType_UINT8;
else if (tensor_type == DataTypeImpl::GetType<uint16_t>())
dtype = TensorProto_DataType_UINT16;
else if (tensor_type == DataTypeImpl::GetType<uint32_t>())
dtype = TensorProto_DataType_UINT32;
else if (tensor_type == DataTypeImpl::GetType<uint64_t>())
dtype = TensorProto_DataType_UINT64;
else if (tensor_type == DataTypeImpl::GetType<bool>())
dtype = TensorProto_DataType_BOOL;
else if (tensor_type == DataTypeImpl::GetType<MLFloat16>())
dtype = TensorProto_DataType_FLOAT16;
else if (tensor_type == DataTypeImpl::GetType<BFloat16>())
dtype = TensorProto_DataType_BFLOAT16;
return dtype;
}
ONNX_NAMESPACE::TensorProto TensorToTensorProto(const Tensor& tensor, const std::string& tensor_proto_name,
const ONNX_NAMESPACE::TypeProto& tensor_proto_type) {
// Given we are using the raw_data field in the protobuf, this will work only for little-endian format.
ORT_ENFORCE(IsLittleEndianOrder());
// Set name, dimensions, type, and data of the TensorProto.
ONNX_NAMESPACE::TensorProto tensor_proto;
tensor_proto.set_name(tensor_proto_name);
for (auto& dim : tensor.Shape().GetDims()) {
tensor_proto.add_dims(dim);
if (preallocated && preallocated_size < size_to_allocate)
return Status(common::ONNXRUNTIME, common::FAIL, MakeString("The buffer planner is not consistent with tensor buffer size, expected ", size_to_allocate, ", got ", preallocated_size));
switch (tensor_proto.data_type()) {
CASE_PROTO(FLOAT, float);
CASE_PROTO(DOUBLE, double);
CASE_PROTO(BOOL, bool);
CASE_PROTO(INT8, int8_t);
CASE_PROTO(INT16, int16_t);
CASE_PROTO(INT32, int32_t);
CASE_PROTO(INT64, int64_t);
CASE_PROTO(UINT8, uint8_t);
CASE_PROTO(UINT16, uint16_t);
CASE_PROTO(UINT32, uint32_t);
CASE_PROTO(UINT64, uint64_t);
CASE_PROTO(FLOAT16, MLFloat16);
CASE_PROTO(BFLOAT16, BFloat16);
case onnx::TensorProto_DataType::TensorProto_DataType_STRING:
if (preallocated != nullptr) {
OrtStatus* status = OrtInitializeBufferForTensor(preallocated, preallocated_size, ele_type);
if (status != nullptr) {
OrtReleaseStatus(status);
return Status(common::ONNXRUNTIME, common::FAIL, "initialize preallocated buffer failed");
}
deleter.f = UnInitTensor;
deleter.param = new UnInitializeParam{preallocated, preallocated_size, ele_type};
}
::onnxruntime::test::UnpackTensor<std::string>(tensor_proto, raw_data, raw_data_len,
(std::string*)preallocated, tensor_size);
break;
default: {
std::ostringstream ostr;
ostr << "Initialized tensor with unexpected type: " << tensor_proto.data_type();
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str());
}
}
tensor_data = preallocated;
}
}
// TODO Once utils::GetTensorProtoType supports all data types, you can get the tensor proto type from the tensor,
// as follows (which will allow us to get rid of the tensor_proto_type argument).
//tensor_proto.set_data_type(utils::GetTensorProtoType(tensor));
tensor_proto.set_data_type(tensor_proto_type.tensor_type().elem_type());
tensor_proto.set_raw_data(tensor.DataRaw(), tensor.SizeInBytes());
return tensor_proto;
std::vector<int64_t> tensor_shape_vec = GetTensorShapeFromTensorProto(tensor_proto);
// Note: We permit an empty tensor_shape_vec, and treat it as a scalar (a tensor of size 1).
value = Ort::Value::CreateTensor(&allocator, tensor_data, m.GetLen(), tensor_shape_vec.data(), tensor_shape_vec.size(), (ONNXTensorElementDataType)tensor_proto.data_type());
return Status::OK();
}
template common::Status GetSizeInBytesFromTensorProto<256>(const ONNX_NAMESPACE::TensorProto& tensor_proto,
size_t* out);
template common::Status GetSizeInBytesFromTensorProto<0>(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out);
} // namespace utils
template Status GetSizeInBytesFromTensorProto<256>(const onnx::TensorProto& tensor_proto,
size_t* out);
template Status GetSizeInBytesFromTensorProto<0>(const onnx::TensorProto& tensor_proto, size_t* out);
} // namespace test
} // namespace onnxruntime
} // namespace onnxruntime

65
onnxruntime/test/onnx/tensorprotoutils.h
View file

@ -5,64 +5,35 @@
#include <vector>
#include <type_traits>
#include "core/common/common.h"
#include "core/common/status.h"
#include "core/framework/allocator.h"
#include "core/framework/ml_value.h"
#include "core/framework/mem_buffer.h"
#include "core/framework/tensor_external_data_info.h"
#include "core/session/onnxruntime_c_api.h"
#include "core/graph/onnx_protobuf.h"
#include "core/platform/env.h"
#include "core/session/onnxruntime_cxx_api.h"
#include "callback.h"
#include "mem_buffer.h"
namespace ONNX_NAMESPACE {
namespace onnx {
class TensorProto;
class TensorShapeProto;
} // namespace ONNX_NAMESPACE
}
namespace onnxruntime {
class Tensor;
namespace test {
namespace utils {
TensorShape GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::TensorShapeProto& tensor_shape_proto);
/**
* deserialize a TensorProto into a preallocated memory buffer.
* \param tensor_proto_path A local file path of where the 'input' was loaded from. Can be NULL if the tensor proto doesn't
* have any external data or it was loaded from current working dir. This path could be either a
* relative path or an absolute path.
*/
common::Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_proto_path,
const ONNX_NAMESPACE::TensorProto& input, const MemBuffer& m, OrtValue& value,
OrtCallback& deleter);
// This function doesn't support string tensors
ONNX_NAMESPACE::TensorProto::DataType GetTensorProtoType(const Tensor& tensor);
/** Creates a TensorProto from a Tensor.
@param[in] tensor the Tensor whose data and shape will be used to create the TensorProto.
@param[in] tensor_proto_name the name of the TensorProto.
@param[in] tensor_proto_type the type of the TensorProto.
@return the TensorProto.
Note: Method currently requires that data is in little-endian format.
TODO Once the GetTensorProtoType supports all data types, we can remove the tensor_proto_type parameter and
instead get the type from the tensor. */
ONNX_NAMESPACE::TensorProto TensorToTensorProto(const Tensor& tensor, const std::string& tensor_proto_name,
const ONNX_NAMESPACE::TypeProto& tensor_proto_type);
ONNXTensorElementDataType CApiElementTypeFromProtoType(int type);
ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto& tensor_proto);
// How much memory it will need for putting the content of this tensor into a plain array
// complex64/complex128 tensors are not supported.
// The output value could be zero or -1.
template <size_t alignment>
common::Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out);
common::Status GetSizeInBytesFromTensorProto(const onnx::TensorProto& tensor_proto, size_t* out);
/**
* deserialize a TensorProto into a preallocated memory buffer.
* Impl must correspond to onnxruntime/core/framework/tensorprotoutils.cc
* This implementation does not support external data so as to reduce dependency surface.
*/
common::Status TensorProtoToMLValue(const onnx::TensorProto& input, const MemBuffer& m, /* out */ Ort::Value& value,
OrtCallback& deleter);
template <typename T>
Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len,
/*out*/ T* p_data, int64_t expected_size);
void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len,
/*out*/ T* p_data, int64_t expected_size);
} // namespace utils
ONNXTensorElementDataType CApiElementTypeFromProtoType(int type);
ONNXTensorElementDataType GetTensorElementType(const onnx::TensorProto& tensor_proto);
} // namespace test
} // namespace onnxruntime
} // namespace onnxruntime

2
onnxruntime/test/perftest/performance_runner.h
View file

@ -128,7 +128,7 @@ class PerformanceRunner {
PerformanceTestConfig performance_test_config_;
TestModelInfo* test_model_info_;
std::unique_ptr<TestSession> session_;
HeapBuffer b_;
onnxruntime::test::HeapBuffer b_;
std::unique_ptr<ITestCase> test_case_;
// TODO: Convert to OrtMutex