From a6a4c4c079ef6f84d3e2b90a9d62e4c5b66a6296 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Mon, 12 Aug 2019 09:49:29 -0700 Subject: [PATCH] Fix perf test executable. (#1598) * Mention OrtCreateSessionFromArray in C API doc * Fix perf test executable due to removal of certain C APIs * fix linux build * Avoid duplication * Fix mem leak --- cmake/onnxruntime_unittests.cmake | 5 +- .../core/session/onnxruntime_c_api.h | 1 - onnxruntime/test/onnx/TestCase.cc | 38 +- onnxruntime/test/onnx/TestCase.h | 2 +- onnxruntime/test/onnx/callback.cc | 16 + onnxruntime/test/onnx/callback.h | 17 + onnxruntime/test/onnx/heap_buffer.cc | 12 +- onnxruntime/test/onnx/heap_buffer.h | 14 +- onnxruntime/test/onnx/mem_buffer.h | 27 + onnxruntime/test/onnx/runner.cc | 82 ++- onnxruntime/test/onnx/tensorprotoutils.cc | 560 +++++++----------- onnxruntime/test/onnx/tensorprotoutils.h | 65 +- .../test/perftest/performance_runner.h | 2 +- 13 files changed, 386 insertions(+), 455 deletions(-) create mode 100644 onnxruntime/test/onnx/callback.cc create mode 100644 onnxruntime/test/onnx/callback.h create mode 100644 onnxruntime/test/onnx/mem_buffer.h diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 7bc7c30578..368ee8790d 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -472,6 +472,9 @@ set(onnx_test_runner_common_srcs ${onnx_test_runner_src_dir}/onnxruntime_event.h ${onnx_test_runner_src_dir}/sync_api.h ${onnx_test_runner_src_dir}/sync_api.cc + ${onnx_test_runner_src_dir}/callback.h + ${onnx_test_runner_src_dir}/callback.cc + ${onnx_test_runner_src_dir}/mem_buffer.h ${onnx_test_runner_src_dir}/tensorprotoutils.h ${onnx_test_runner_src_dir}/tensorprotoutils.cc) @@ -566,7 +569,7 @@ onnxruntime_add_include_to_target(onnxruntime_perf_test gsl) if (onnxruntime_BUILD_SHARED_LIB) set(onnxruntime_perf_test_libs onnxruntime_test_utils onnx_test_runner_common onnxruntime_common - onnx_test_data_proto onnx_proto libprotobuf ${GETOPT_LIB_WIDE} onnxruntime onnxruntime_framework onnx + onnx_test_data_proto onnx_proto libprotobuf ${GETOPT_LIB_WIDE} onnxruntime ${SYS_PATH_LIB} ${CMAKE_DL_LIBS}) if(onnxruntime_USE_NSYNC) list(APPEND onnxruntime_perf_test_libs nsync_cpp) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index d67efd5204..fad81a5359 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -150,7 +150,6 @@ ORT_RUNTIME_CLASS(RunOptions); ORT_RUNTIME_CLASS(TypeInfo); ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo); ORT_RUNTIME_CLASS(SessionOptions); -ORT_RUNTIME_CLASS(Callback); ORT_RUNTIME_CLASS(CustomOpDomain); ORT_RUNTIME_CLASS(Allocator); diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 628ef47983..fe43b296b6 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -13,6 +13,7 @@ #include "core/platform/ort_mutex.h" #include "core/session/onnxruntime_cxx_api.h" #include "core/framework/path_lib.h" +#include "core/framework/allocator.h" #include #include #include @@ -265,18 +266,18 @@ static void SortTensorFileNames(std::vector>& } } -OrtValue* TensorToOrtValue(const ONNX_NAMESPACE::TensorProto& t, HeapBuffer& b) { +OrtValue* TensorToOrtValue(const ONNX_NAMESPACE::TensorProto& t, onnxruntime::test::HeapBuffer& b) { size_t len = 0; - auto status = onnxruntime::test::utils::GetSizeInBytesFromTensorProto<0>(t, &len); + auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(t, &len); if (!status.IsOK()) { ORT_THROW(status.ToString()); } void* p = len == 0 ? nullptr : b.AllocMemory(len); - auto d = std::make_unique(); - auto temp_value = std::make_unique(); + Ort::Value temp_value{nullptr}; + auto d = std::make_unique(); OrtAllocatorInfo cpu_allocator_info(onnxruntime::CPU, OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeDefault); - status = onnxruntime::test::utils::TensorProtoToMLValue(Env::Default(), nullptr, t, - MemBuffer(p, len, cpu_allocator_info), *temp_value, *d); + status = onnxruntime::test::TensorProtoToMLValue(t, onnxruntime::test::MemBuffer(p, len, cpu_allocator_info), + temp_value, *d); if (!status.IsOK()) { ORT_THROW(status.ToString()); } @@ -287,7 +288,8 @@ OrtValue* TensorToOrtValue(const ONNX_NAMESPACE::TensorProto& t, HeapBuffer& b) } void LoopDataFile(int test_data_pb_fd, bool is_input, const TestModelInfo* modelinfo, - std::unordered_map& name_data_map, HeapBuffer& b, std::ostringstream& oss) { + std::unordered_map& name_data_map, onnxruntime::test::HeapBuffer& b, + std::ostringstream& oss) { google::protobuf::io::FileInputStream f(test_data_pb_fd); f.SetCloseOnDelete(true); google::protobuf::io::CodedInputStream coded_input(&f); @@ -395,7 +397,8 @@ class OnnxTestCase : public ITestCase { return std::string(); } - void ConvertTestData(const std::vector& test_data_pbs, HeapBuffer& b, bool is_input, + void ConvertTestData(const std::vector& test_data_pbs, onnxruntime::test::HeapBuffer& b, + bool is_input, std::unordered_map& out); std::once_flag model_parsed_; @@ -430,7 +433,8 @@ class OnnxTestCase : public ITestCase { std::string GetTestCaseVersion() const override { return model_info_->GetModelVersion(); } - void LoadTestData(size_t id, HeapBuffer& b, std::unordered_map&, bool is_input) override; + void LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::unordered_map&, + bool is_input) override; }; ITestCase* CreateOnnxTestCase(const std::string& test_case_name, TestModelInfo* model, @@ -503,7 +507,8 @@ static void LoadTensors(const std::vector& pb_files, } } -void OnnxTestCase::LoadTestData(size_t id, HeapBuffer& b, std::unordered_map& name_data_map, +void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, + std::unordered_map& name_data_map, bool is_input) { if (id >= test_data_dirs_.size()) { ORT_THROW("index out of bound"); @@ -556,7 +561,8 @@ void OnnxTestCase::LoadTestData(size_t id, HeapBuffer& b, std::unordered_map& test_data_pbs, HeapBuffer& b, +void OnnxTestCase::ConvertTestData(const std::vector& test_data_pbs, + onnxruntime::test::HeapBuffer& b, bool is_input, std::unordered_map& out) { bool has_valid_names = true; std::vector var_names(test_data_pbs.size()); @@ -582,16 +588,16 @@ void OnnxTestCase::ConvertTestData(const std::vector(input, &len); + auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(input, &len); if (!status.IsOK()) { ORT_THROW(status.ToString()); } void* p = len == 0 ? nullptr : b.AllocMemory(len); - auto d = std::make_unique(); + Ort::Value v1{nullptr}; + auto d = std::make_unique(); OrtAllocatorInfo cpu_allocator_info(onnxruntime::CPU, OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeDefault); - auto v1 = std::make_unique(); - status = onnxruntime::test::utils::TensorProtoToMLValue(Env::Default(), nullptr, input, - MemBuffer(p, len, cpu_allocator_info), *v1, *d); + status = onnxruntime::test::TensorProtoToMLValue(input, onnxruntime::test::MemBuffer(p, len, cpu_allocator_info), + v1, *d); if (!status.IsOK()) { ORT_THROW(status.ToString()); } diff --git a/onnxruntime/test/onnx/TestCase.h b/onnxruntime/test/onnx/TestCase.h index 119f167312..66663a0450 100644 --- a/onnxruntime/test/onnx/TestCase.h +++ b/onnxruntime/test/onnx/TestCase.h @@ -19,7 +19,7 @@ class ValueInfoProto; //One test case can contain multiple test data(input/output pairs) class ITestCase { public: - virtual void LoadTestData(size_t id, HeapBuffer& b, std::unordered_map& name_data_map, + virtual void LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::unordered_map& name_data_map, bool is_input) = 0; virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0; virtual const std::string& GetNodeName() const = 0; diff --git a/onnxruntime/test/onnx/callback.cc b/onnxruntime/test/onnx/callback.cc new file mode 100644 index 0000000000..99b3b7a6bd --- /dev/null +++ b/onnxruntime/test/onnx/callback.cc @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "callback.h" + +namespace onnxruntime { +namespace test { +void OrtRunCallback(OrtCallback* f) noexcept { + if (f == nullptr) return; + if (f->f != nullptr) { + f->f(f->param); + delete f; + } +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/onnx/callback.h b/onnxruntime/test/onnx/callback.h new file mode 100644 index 0000000000..c548b57486 --- /dev/null +++ b/onnxruntime/test/onnx/callback.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +namespace onnxruntime { +namespace test { +struct OrtCallback { + void (*f)(void* param) noexcept; + void* param; +}; + +/** + * f will be freed in this call + */ +void OrtRunCallback(OrtCallback* f) noexcept; +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/onnx/heap_buffer.cc b/onnxruntime/test/onnx/heap_buffer.cc index 4d036d5033..aca75de061 100644 --- a/onnxruntime/test/onnx/heap_buffer.cc +++ b/onnxruntime/test/onnx/heap_buffer.cc @@ -2,18 +2,22 @@ // Licensed under the MIT License. #include "heap_buffer.h" -#include "core/framework/callback.h" #include "core/session/onnxruntime_c_api.h" +#include "callback.h" -void HeapBuffer::AddDeleter(onnxruntime::OrtCallback* d) { +namespace onnxruntime { +namespace test { +void HeapBuffer::AddDeleter(OrtCallback* d) { if (d != nullptr) deleters_.push_back(d); } HeapBuffer::~HeapBuffer() { for (auto d : deleters_) { - onnxruntime::OrtRunCallback(d); + OrtRunCallback(d); } for (void* p : buffers_) { free(p); } -} \ No newline at end of file +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/onnx/heap_buffer.h b/onnxruntime/test/onnx/heap_buffer.h index 3726bfe880..b4abf131b1 100644 --- a/onnxruntime/test/onnx/heap_buffer.h +++ b/onnxruntime/test/onnx/heap_buffer.h @@ -4,10 +4,10 @@ #pragma once #include #include -namespace onnxruntime { -struct OrtCallback; -} +namespace onnxruntime { +namespace test { +struct OrtCallback; /** * A holder for delay freed buffers */ @@ -23,9 +23,11 @@ class HeapBuffer { buffers_.push_back(p); return p; } - void AddDeleter(onnxruntime::OrtCallback* d); + void AddDeleter(OrtCallback* d); private: - std::vector deleters_; + std::vector deleters_; std::vector buffers_; -}; \ No newline at end of file +}; +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/onnx/mem_buffer.h b/onnxruntime/test/onnx/mem_buffer.h new file mode 100644 index 0000000000..fe2a393852 --- /dev/null +++ b/onnxruntime/test/onnx/mem_buffer.h @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/common/common.h" + +namespace onnxruntime { +namespace test { +/** + * A simple POD for using with tensor deserialization + */ +class MemBuffer { + public: + MemBuffer(void* buffer, size_t len, const OrtAllocatorInfo& alloc_info) + : buffer_(buffer), len_(len), alloc_info_(alloc_info) {} + void* GetBuffer() const { return buffer_; } + + size_t GetLen() const { return len_; } + const OrtAllocatorInfo& GetAllocInfo() const { return alloc_info_; } + + private: + void* const buffer_; + const size_t len_; + const OrtAllocatorInfo& alloc_info_; +}; +}; // namespace test +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/onnx/runner.cc b/onnxruntime/test/onnx/runner.cc index 575292fbc6..7827b4ff8a 100644 --- a/onnxruntime/test/onnx/runner.cc +++ b/onnxruntime/test/onnx/runner.cc @@ -27,45 +27,44 @@ using namespace onnxruntime; using ::onnxruntime::common::Status; -// Permanently exclude following tests because ORT support only opset staring from 7, +// Permanently exclude following tests because ORT support only opset staring from 7, // Please make no more changes to the list -const std::set immutable_broken_tests = -{ - "AvgPool1d", - "AvgPool1d_stride", - "AvgPool2d", - "AvgPool2d_stride", - "AvgPool3d", - "AvgPool3d_stride", - "AvgPool3d_stride1_pad0_gpu_input", - "BatchNorm1d_3d_input_eval", - "BatchNorm2d_eval", - "BatchNorm2d_momentum_eval", - "BatchNorm3d_eval", - "BatchNorm3d_momentum_eval", - "GLU", - "GLU_dim", - "Linear", - "PReLU_1d", - "PReLU_1d_multiparam", - "PReLU_2d", - "PReLU_2d_multiparam", - "PReLU_3d", - "PReLU_3d_multiparam", - "PoissonNLLLLoss_no_reduce", - "Softsign", - "operator_add_broadcast", - "operator_add_size1_broadcast", - "operator_add_size1_right_broadcast", - "operator_add_size1_singleton_broadcast", - "operator_addconstant", - "operator_addmm", - "operator_basic", - "operator_mm", - "operator_non_float_params", - "operator_params", - "operator_pow" -}; +const std::set immutable_broken_tests = + { + "AvgPool1d", + "AvgPool1d_stride", + "AvgPool2d", + "AvgPool2d_stride", + "AvgPool3d", + "AvgPool3d_stride", + "AvgPool3d_stride1_pad0_gpu_input", + "BatchNorm1d_3d_input_eval", + "BatchNorm2d_eval", + "BatchNorm2d_momentum_eval", + "BatchNorm3d_eval", + "BatchNorm3d_momentum_eval", + "GLU", + "GLU_dim", + "Linear", + "PReLU_1d", + "PReLU_1d_multiparam", + "PReLU_2d", + "PReLU_2d_multiparam", + "PReLU_3d", + "PReLU_3d_multiparam", + "PoissonNLLLLoss_no_reduce", + "Softsign", + "operator_add_broadcast", + "operator_add_size1_broadcast", + "operator_add_size1_right_broadcast", + "operator_add_size1_singleton_broadcast", + "operator_addconstant", + "operator_addmm", + "operator_basic", + "operator_mm", + "operator_non_float_params", + "operator_params", + "operator_pow"}; void ORT_CALLBACK RunTestCase(ORT_CALLBACK_INSTANCE pci, void* context, ORT_WORK work) { OnnxRuntimeCloseThreadpoolWork(work); @@ -232,13 +231,13 @@ Status RunTests(TestEnv& env, int p_models, int concurrent_runs, size_t repeat_c } for (size_t i = 0; i != env.tests.size(); ++i) { if (!results[i]) { - stat.AddFailedTest(std::pair(env.tests[i]->GetTestCaseName(), env.tests[i]->GetTestCaseVersion())); + stat.AddFailedTest(std::pair(env.tests[i]->GetTestCaseName(), env.tests[i]->GetTestCaseVersion())); continue; } const TestCaseResult& r = *results[i]; for (const EXECUTE_RESULT res : r.GetExcutionResult()) { if (res != EXECUTE_RESULT::SUCCESS && res != EXECUTE_RESULT::NOT_SUPPORT) { - stat.AddFailedTest(std::pair(env.tests[i]->GetTestCaseName(),env.tests[i]->GetTestCaseVersion())); + stat.AddFailedTest(std::pair(env.tests[i]->GetTestCaseName(), env.tests[i]->GetTestCaseVersion())); } switch (res) { case EXECUTE_RESULT::SUCCESS: @@ -347,7 +346,7 @@ void DataRunner::RunTask(size_t task_id, ORT_CALLBACK_INSTANCE pci, bool store_r } EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { - HeapBuffer holder; + onnxruntime::test::HeapBuffer holder; std::unordered_map feeds; c_->LoadTestData(task_id, holder, feeds, true); @@ -499,7 +498,6 @@ void SeqTestRunner::Start(ORT_CALLBACK_INSTANCE pci, size_t) { } void RunSingleTestCase(ITestCase* info, Ort::Env& env, const Ort::SessionOptions& sf, size_t concurrent_runs, size_t repeat_count, PThreadPool tpool, ORT_CALLBACK_INSTANCE pci, TestCaseCallBack on_finished) { - //for test in immutable list, do not even run it if (immutable_broken_tests.find(info->GetTestCaseName()) != immutable_broken_tests.end()) { on_finished(std::make_shared(0, EXECUTE_RESULT::NOT_SUPPORT, info->GetNodeName()), pci); diff --git a/onnxruntime/test/onnx/tensorprotoutils.cc b/onnxruntime/test/onnx/tensorprotoutils.cc index d574f3ec76..27442273f2 100644 --- a/onnxruntime/test/onnx/tensorprotoutils.cc +++ b/onnxruntime/test/onnx/tensorprotoutils.cc @@ -7,22 +7,19 @@ #include #include #include - -#include "core/common/logging/logging.h" -#include "core/graph/onnx_protobuf.h" -#include "core/framework/op_kernel.h" -#include "core/framework/tensor.h" -#include "core/framework/ort_value_pattern_planner.h" -#include "core/framework/allocator.h" -#include "core/framework/callback.h" #include "core/framework/data_types.h" -#include "core/framework/path_lib.h" +#include "core/framework/allocator.h" +#include "core/session/onnxruntime_cxx_api.h" +#include "core/graph/onnx_protobuf.h" +#include "callback.h" -using namespace ONNX_NAMESPACE; -using namespace ::onnxruntime::common; - -namespace { +struct OrtStatus { + OrtErrorCode code; + char msg[1]; // a null-terminated string +}; +namespace onnxruntime { +namespace test { #ifdef __GNUC__ constexpr inline bool IsLittleEndianOrder() noexcept { return __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__; } #else @@ -34,7 +31,38 @@ inline bool IsLittleEndianOrder() noexcept { } #endif -std::vector GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto) { +//From core common +inline void MakeStringInternal(std::ostringstream& /*ss*/) noexcept { +} + +template +inline void MakeStringInternal(std::ostringstream& ss, const T& t) noexcept { + ss << t; +} + +template +inline void MakeStringInternal(std::ostringstream& ss, const T& t, const Args&... args) noexcept { + ::onnxruntime::MakeStringInternal(ss, t); + ::onnxruntime::MakeStringInternal(ss, args...); +} + +template +std::string MakeString(const Args&... args) { + std::ostringstream ss; + ::onnxruntime::MakeStringInternal(ss, args...); + return std::string(ss.str()); +} + +// Specializations for already-a-string types. +template <> +inline std::string MakeString(const std::string& str) { + return str; +} +inline std::string MakeString(const char* p_str) { + return p_str; +} + +std::vector GetTensorShapeFromTensorProto(const onnx::TensorProto& tensor_proto) { const auto& dims = tensor_proto.dims(); std::vector tensor_shape_vec(static_cast(dims.size())); for (int i = 0; i < dims.size(); ++i) { @@ -46,8 +74,8 @@ std::vector GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorP // This function doesn't support string tensors template -static Status UnpackTensorWithRawData(const void* raw_data, size_t raw_data_length, size_t expected_size, - /*out*/ T* p_data) { +static void UnpackTensorWithRawData(const void* raw_data, size_t raw_data_length, size_t expected_size, + /*out*/ T* p_data) { // allow this low level routine to be somewhat unsafe. assuming it's thoroughly tested and valid GSL_SUPPRESS(type) // type.1 reinterpret-cast; type.4 C-style casts; type.5 'T result;' is uninitialized; GSL_SUPPRESS(bounds .1) // pointer arithmetic @@ -55,12 +83,12 @@ static Status UnpackTensorWithRawData(const void* raw_data, size_t raw_data_leng { size_t expected_size_in_bytes; if (!onnxruntime::IAllocator::CalcMemSizeForArray(expected_size, sizeof(T), &expected_size_in_bytes)) { - return Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::INVALID_ARGUMENT, "size overflow"); + throw Ort::Exception("size overflow", OrtErrorCode::ORT_FAIL); } if (raw_data_length != expected_size_in_bytes) - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "UnpackTensor: the pre-allocated size does not match the raw data size, expected ", - expected_size_in_bytes, ", got ", raw_data_length); + throw Ort::Exception(MakeString("UnpackTensor: the pre-allocated size does not match the raw data size, expected ", + expected_size_in_bytes, ", got ", raw_data_length), + OrtErrorCode::ORT_FAIL); if (IsLittleEndianOrder()) { memcpy(p_data, raw_data, raw_data_length); } else { @@ -75,85 +103,81 @@ static Status UnpackTensorWithRawData(const void* raw_data, size_t raw_data_leng p_data[i] = result; } } - return Status::OK(); } } -} // namespace - -namespace onnxruntime { -namespace test { -namespace utils { // This macro doesn't work for Float16/bool/string tensors -#define DEFINE_UNPACK_TENSOR(T, Type, field_name, field_size) \ - template <> \ - Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, \ - /*out*/ T* p_data, int64_t expected_size) { \ - if (nullptr == p_data) { \ - const size_t size = raw_data != nullptr ? raw_data_len : tensor.field_size(); \ - if (size == 0) return Status::OK(); \ - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); \ - } \ - if (nullptr == p_data || Type != tensor.data_type()) { \ - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); \ - } \ - if (raw_data != nullptr) { \ - return UnpackTensorWithRawData(raw_data, raw_data_len, expected_size, p_data); \ - } \ - if (tensor.field_size() != expected_size) \ - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "corrupted protobuf data: tensor shape size(", expected_size, \ - ") does not match the data size(", tensor.field_size(), ") in proto"); \ - auto& data = tensor.field_name(); \ - for (auto data_iter = data.cbegin(); data_iter != data.cend(); ++data_iter) \ - *p_data++ = *reinterpret_cast(data_iter); \ - return Status::OK(); \ +#define DEFINE_UNPACK_TENSOR(T, Type, field_name, field_size) \ + template <> \ + void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len, \ + /*out*/ T* p_data, int64_t expected_size) { \ + if (nullptr == p_data) { \ + const size_t size = raw_data != nullptr ? raw_data_len : tensor.field_size(); \ + if (size == 0) return; \ + throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); \ + } \ + if (nullptr == p_data || Type != tensor.data_type()) { \ + throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); \ + } \ + if (raw_data != nullptr) { \ + UnpackTensorWithRawData(raw_data, raw_data_len, expected_size, p_data); \ + return; \ + } \ + if (tensor.field_size() != expected_size) \ + throw Ort::Exception(MakeString("corrupted protobuf data: tensor shape size(", expected_size, \ + ") does not match the data size(", tensor.field_size(), ") in proto"), \ + OrtErrorCode::ORT_FAIL); \ + auto& data = tensor.field_name(); \ + for (auto data_iter = data.cbegin(); data_iter != data.cend(); ++data_iter) \ + *p_data++ = *reinterpret_cast(data_iter); \ + return; \ } // TODO: complex64 complex128 -DEFINE_UNPACK_TENSOR(float, ONNX_NAMESPACE::TensorProto_DataType_FLOAT, float_data, float_data_size) -DEFINE_UNPACK_TENSOR(double, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, double_data, double_data_size); -DEFINE_UNPACK_TENSOR(uint8_t, ONNX_NAMESPACE::TensorProto_DataType_UINT8, int32_data, int32_data_size) -DEFINE_UNPACK_TENSOR(int8_t, ONNX_NAMESPACE::TensorProto_DataType_INT8, int32_data, int32_data_size) -DEFINE_UNPACK_TENSOR(int16_t, ONNX_NAMESPACE::TensorProto_DataType_INT16, int32_data, int32_data_size) -DEFINE_UNPACK_TENSOR(uint16_t, ONNX_NAMESPACE::TensorProto_DataType_UINT16, int32_data, int32_data_size) -DEFINE_UNPACK_TENSOR(int32_t, ONNX_NAMESPACE::TensorProto_DataType_INT32, int32_data, int32_data_size) -DEFINE_UNPACK_TENSOR(int64_t, ONNX_NAMESPACE::TensorProto_DataType_INT64, int64_data, int64_data_size) -DEFINE_UNPACK_TENSOR(uint64_t, ONNX_NAMESPACE::TensorProto_DataType_UINT64, uint64_data, uint64_data_size) -DEFINE_UNPACK_TENSOR(uint32_t, ONNX_NAMESPACE::TensorProto_DataType_UINT32, uint64_data, uint64_data_size) +DEFINE_UNPACK_TENSOR(float, onnx::TensorProto_DataType_FLOAT, float_data, float_data_size) +DEFINE_UNPACK_TENSOR(double, onnx::TensorProto_DataType_DOUBLE, double_data, double_data_size); +DEFINE_UNPACK_TENSOR(uint8_t, onnx::TensorProto_DataType_UINT8, int32_data, int32_data_size) +DEFINE_UNPACK_TENSOR(int8_t, onnx::TensorProto_DataType_INT8, int32_data, int32_data_size) +DEFINE_UNPACK_TENSOR(int16_t, onnx::TensorProto_DataType_INT16, int32_data, int32_data_size) +DEFINE_UNPACK_TENSOR(uint16_t, onnx::TensorProto_DataType_UINT16, int32_data, int32_data_size) +DEFINE_UNPACK_TENSOR(int32_t, onnx::TensorProto_DataType_INT32, int32_data, int32_data_size) +DEFINE_UNPACK_TENSOR(int64_t, onnx::TensorProto_DataType_INT64, int64_data, int64_data_size) +DEFINE_UNPACK_TENSOR(uint64_t, onnx::TensorProto_DataType_UINT64, uint64_data, uint64_data_size) +DEFINE_UNPACK_TENSOR(uint32_t, onnx::TensorProto_DataType_UINT32, uint64_data, uint64_data_size) // doesn't support raw data template <> -Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* /*raw_data*/, size_t /*raw_data_len*/, - /*out*/ std::string* p_data, int64_t expected_size) { +void UnpackTensor(const onnx::TensorProto& tensor, const void* /*raw_data*/, size_t /*raw_data_len*/, + /*out*/ std::string* p_data, int64_t expected_size) { if (nullptr == p_data) { - if (tensor.string_data_size() == 0) return Status::OK(); - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + if (tensor.string_data_size() == 0) return; + throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); } - if (ONNX_NAMESPACE::TensorProto_DataType_STRING != tensor.data_type()) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + if (onnx::TensorProto_DataType_STRING != tensor.data_type()) { + throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); } if (tensor.string_data_size() != expected_size) - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, - "UnpackTensor: the pre-allocate size does not match the size in proto"); + throw Ort::Exception( + "UnpackTensor: the pre-allocate size does not match the size in proto", OrtErrorCode::ORT_FAIL); auto& string_data = tensor.string_data(); for (const auto& iter : string_data) { *p_data++ = iter; } - return Status::OK(); + return; } template <> -Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, - /*out*/ bool* p_data, int64_t expected_size) { +void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len, + /*out*/ bool* p_data, int64_t expected_size) { if (nullptr == p_data) { const size_t size = raw_data != nullptr ? raw_data_len : tensor.int32_data_size(); - if (size == 0) return Status::OK(); - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + if (size == 0) return; + throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); } - if (ONNX_NAMESPACE::TensorProto_DataType_BOOL != tensor.data_type()) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + if (onnx::TensorProto_DataType_BOOL != tensor.data_type()) { + throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); } if (raw_data != nullptr) { @@ -161,24 +185,24 @@ Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_d } if (tensor.int32_data_size() != expected_size) - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, - "UnpackTensor: the pre-allocate size does not match the size in proto"); + throw Ort::Exception( + "UnpackTensor: the pre-allocate size does not match the size in proto", OrtErrorCode::ORT_FAIL); for (int iter : tensor.int32_data()) { *p_data++ = static_cast(iter); } - return Status::OK(); + return; } template <> -Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, - /*out*/ MLFloat16* p_data, int64_t expected_size) { +void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len, + /*out*/ MLFloat16* p_data, int64_t expected_size) { if (nullptr == p_data) { const size_t size = raw_data != nullptr ? raw_data_len : tensor.int32_data_size(); - if (size == 0) return Status::OK(); - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + if (size == 0) return; + throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); } - if (ONNX_NAMESPACE::TensorProto_DataType_FLOAT16 != tensor.data_type()) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + if (onnx::TensorProto_DataType_FLOAT16 != tensor.data_type()) { + throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); } if (raw_data != nullptr) { @@ -186,33 +210,34 @@ Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_d } if (tensor.int32_data_size() != expected_size) - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, - "UnpackTensor: the pre-allocate size does not match the size in proto"); + throw Ort::Exception( + "UnpackTensor: the pre-allocate size does not match the size in proto", OrtErrorCode::ORT_FAIL); constexpr int max_value = std::numeric_limits::max(); for (int i = 0; i < static_cast(expected_size); i++) { int v = tensor.int32_data()[i]; if (v < 0 || v > max_value) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "data overflow"); + throw Ort::Exception( + "data overflow", OrtErrorCode::ORT_FAIL); } p_data[i] = MLFloat16(static_cast(v)); } - return Status::OK(); + return; } template <> -Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, - /*out*/ BFloat16* p_data, int64_t expected_size) { +void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len, + /*out*/ BFloat16* p_data, int64_t expected_size) { if (nullptr == p_data) { const size_t size = raw_data != nullptr ? raw_data_len : tensor.int32_data_size(); if (size == 0) - return Status::OK(); + return; - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); } - if (ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16 != tensor.data_type()) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT); + if (onnx::TensorProto_DataType_BFLOAT16 != tensor.data_type()) { + throw Ort::Exception("", OrtErrorCode::ORT_INVALID_ARGUMENT); } if (raw_data != nullptr) { @@ -220,38 +245,39 @@ Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_d } if (tensor.int32_data_size() != expected_size) - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, - "UnpackTensor: the pre-allocate size does not match the size in proto"); + throw Ort::Exception( + "UnpackTensor: the pre-allocate size does not match the size in proto", OrtErrorCode::ORT_FAIL); constexpr int max_value = std::numeric_limits::max(); for (int i = 0; i < static_cast(expected_size); i++) { int v = tensor.int32_data()[i]; if (v < 0 || v > max_value) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "data overflow"); + throw Ort::Exception( + "data overflow", OrtErrorCode::ORT_FAIL); } p_data[i] = BFloat16(static_cast(v)); } - return Status::OK(); + return; } -#define CASE_PROTO_TRACE(X, Y) \ - case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: \ - if (!IAllocator::CalcMemSizeForArrayWithAlignment(size, sizeof(Y), out)) { \ - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid TensorProto"); \ - } \ +#define CASE_PROTO_TRACE(X, Y) \ + case onnx::TensorProto_DataType::TensorProto_DataType_##X: \ + if (!IAllocator::CalcMemSizeForArrayWithAlignment(size, sizeof(Y), out)) { \ + throw Ort::Exception("Invalid TensorProto", OrtErrorCode::ORT_FAIL); \ + } \ break; template -common::Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out) { +Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out) { const auto& dims = tensor_proto.dims(); size_t size = 1; for (google::protobuf::int64 dim : dims) { if (dim < 0 || static_cast(dim) >= std::numeric_limits::max()) { - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid TensorProto"); + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid TensorProto"); } if (!IAllocator::CalcMemSizeForArray(size, static_cast(dim), &size)) { - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid TensorProto"); + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid TensorProto"); } } switch (tensor_proto.data_type()) { @@ -270,49 +296,19 @@ common::Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto& CASE_PROTO_TRACE(BFLOAT16, BFloat16); CASE_PROTO_TRACE(STRING, std::string); default: - return common::Status(common::ONNXRUNTIME, common::NOT_IMPLEMENTED); + return Status(common::ONNXRUNTIME, common::NOT_IMPLEMENTED); } return Status::OK(); } -TensorShape GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::TensorShapeProto& tensor_shape_proto) { - const auto& dims = tensor_shape_proto.dim(); - std::vector tensor_shape_vec(static_cast(dims.size())); - for (int i = 0; i < dims.size(); ++i) { - tensor_shape_vec[i] = dims[i].has_dim_value() ? dims[i].dim_value() - : -1; /* symbolic dimensions are represented as -1 in onnxruntime*/ - } - return TensorShape(std::move(tensor_shape_vec)); -} - struct UnInitializeParam { void* preallocated; size_t preallocated_size; ONNXTensorElementDataType ele_type; }; -// In the future, we may make these two function as public C API -/** - * Initialize a buffer for being used with the OrtCreateTensorWithDataAsOrtValue function - * - */ -ORT_API_STATUS(OrtInitializeBufferForTensor, _In_opt_ void* input, size_t input_len, - enum ONNXTensorElementDataType type); - -/** - * Uninitialize the buffer that was initialized by the OrtInitializeBufferForTensor function - * - */ -ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enum ONNXTensorElementDataType type); - -static void UnInitTensor(void* param) noexcept { - UnInitializeParam* p = reinterpret_cast(param); - OrtUninitializeBuffer(p->preallocated, p->preallocated_size, p->ele_type); - delete p; -} - -ORT_API_STATUS_IMPL(OrtInitializeBufferForTensor, _In_opt_ void* input, size_t input_len, - enum ONNXTensorElementDataType type) { +OrtStatus* OrtInitializeBufferForTensor(void* input, size_t input_len, + ONNXTensorElementDataType type) { try { if (type != ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING || input == nullptr) return nullptr; size_t tensor_size = input_len / sizeof(std::string); @@ -326,6 +322,14 @@ ORT_API_STATUS_IMPL(OrtInitializeBufferForTensor, _In_opt_ void* input, size_t i return nullptr; } +ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enum ONNXTensorElementDataType type); + +static void UnInitTensor(void* param) noexcept { + UnInitializeParam* p = reinterpret_cast(param); + OrtUninitializeBuffer(p->preallocated, p->preallocated_size, p->ele_type); + delete p; +} + ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enum ONNXTensorElementDataType type) { if (type != ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING || input == nullptr) return; size_t tensor_size = input_len / sizeof(std::string); @@ -336,145 +340,13 @@ ORT_API(void, OrtUninitializeBuffer, _In_opt_ void* input, size_t input_len, enu } } -#define CASE_PROTO(X, Y) \ - case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: \ - ORT_RETURN_IF_ERROR( \ - ::onnxruntime::test::utils::UnpackTensor(tensor_proto, raw_data, raw_data_len, (Y*)preallocated, tensor_size)); \ +#define CASE_PROTO(X, Y) \ + case onnx::TensorProto_DataType::TensorProto_DataType_##X: \ + ::onnxruntime::test::UnpackTensor(tensor_proto, raw_data, raw_data_len, (Y*)preallocated, tensor_size); \ break; -class AutoDelete { - public: - OrtCallback d{nullptr, nullptr}; - AutoDelete() = default; - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(AutoDelete); - ~AutoDelete() { - if (d.f != nullptr) { - d.f(d.param); - } - } -}; - -static void MoveOrtCallback(OrtCallback& from, OrtCallback& to) { - to.f = from.f; - to.param = from.param; - from.f = nullptr; - from.param = nullptr; -} - -Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_proto_path, - const ONNX_NAMESPACE::TensorProto& tensor_proto, const MemBuffer& m, OrtValue& value, - OrtCallback& deleter) { - const OrtAllocatorInfo& allocator = m.GetAllocInfo(); - ONNXTensorElementDataType ele_type = onnxruntime::test::utils::GetTensorElementType(tensor_proto); - deleter.f = nullptr; - deleter.param = nullptr; - const void* raw_data = nullptr; - size_t raw_data_len = 0; - const DataTypeImpl* const type = DataTypeImpl::TensorTypeFromONNXEnum(tensor_proto.data_type())->GetElementType(); - AutoDelete deleter_for_file_data; - void* tensor_data; - { - if (tensor_proto.data_location() == TensorProto_DataLocation_EXTERNAL) { - if (ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "string tensor can not have raw data"); - - std::unique_ptr external_data_info; - ORT_RETURN_IF_ERROR(ExternalDataInfo::Create(tensor_proto.external_data(), external_data_info)); - std::basic_string full_path; - if (tensor_proto_path != nullptr) { - ORT_RETURN_IF_ERROR(GetDirNameFromFilePath(tensor_proto_path, full_path)); - full_path = ConcatPathComponent(full_path, external_data_info->GetRelPath()); - } else { - full_path = external_data_info->GetRelPath(); - } - raw_data_len = external_data_info->GetLength(); - // load the file - { - void* file_data; - ORT_RETURN_IF_ERROR(env.ReadFileAsString(full_path.c_str(), external_data_info->GetOffset(), - file_data, raw_data_len, deleter_for_file_data.d)); - raw_data = file_data; - } - } else if (tensor_proto.has_raw_data()) { - if (ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "string tensor can not have raw data"); - raw_data = tensor_proto.raw_data().data(); - raw_data_len = tensor_proto.raw_data().size(); - } - if (IsLittleEndianOrder() && raw_data != nullptr && deleter_for_file_data.d.f != nullptr) { - tensor_data = const_cast(raw_data); - MoveOrtCallback(deleter_for_file_data.d, deleter); - } else { - void* preallocated = m.GetBuffer(); - size_t preallocated_size = m.GetLen(); - int64_t tensor_size = 1; - { - for (auto i : tensor_proto.dims()) { - if (i < 0) return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "tensor can't contain negative dims"); - tensor_size *= i; - } - } - // tensor_size could be zero. see test_slice_start_out_of_bounds\test_data_set_0\output_0.pb - if (static_cast(tensor_size) > SIZE_MAX) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow"); - } - size_t size_to_allocate; - if (!IAllocator::CalcMemSizeForArrayWithAlignment<0>(static_cast(tensor_size), type->Size(), - &size_to_allocate)) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "size overflow"); - } - - if (preallocated && preallocated_size < size_to_allocate) - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "The buffer planner is not consistent with tensor buffer size, expected ", - size_to_allocate, ", got ", preallocated_size); - switch (tensor_proto.data_type()) { - CASE_PROTO(FLOAT, float); - CASE_PROTO(DOUBLE, double); - CASE_PROTO(BOOL, bool); - CASE_PROTO(INT8, int8_t); - CASE_PROTO(INT16, int16_t); - CASE_PROTO(INT32, int32_t); - CASE_PROTO(INT64, int64_t); - CASE_PROTO(UINT8, uint8_t); - CASE_PROTO(UINT16, uint16_t); - CASE_PROTO(UINT32, uint32_t); - CASE_PROTO(UINT64, uint64_t); - CASE_PROTO(FLOAT16, MLFloat16); - CASE_PROTO(BFLOAT16, BFloat16); - case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_STRING: - if (preallocated != nullptr) { - OrtStatus* status = OrtInitializeBufferForTensor(preallocated, preallocated_size, ele_type); - if (status != nullptr) { - OrtReleaseStatus(status); - return Status(common::ONNXRUNTIME, common::FAIL, "initialize preallocated buffer failed"); - } - - deleter.f = UnInitTensor; - deleter.param = new UnInitializeParam{preallocated, preallocated_size, ele_type}; - } - ORT_RETURN_IF_ERROR(::onnxruntime::test::utils::UnpackTensor(tensor_proto, raw_data, raw_data_len, - (std::string*)preallocated, tensor_size)); - break; - default: { - std::ostringstream ostr; - ostr << "Initialized tensor with unexpected type: " << tensor_proto.data_type(); - return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); - } - } - tensor_data = preallocated; - } - } - std::vector tensor_shape_vec = GetTensorShapeFromTensorProto(tensor_proto); - // Note: We permit an empty tensor_shape_vec, and treat it as a scalar (a tensor of size 1). - TensorShape tensor_shape{tensor_shape_vec}; - value.Init(new Tensor(type, tensor_shape, tensor_data, allocator), DataTypeImpl::GetType(), - DataTypeImpl::GetType()->GetDeleteFunc()); - return Status::OK(); -} - -#define CASE_TYPE(X) \ - case ONNX_NAMESPACE::TensorProto_DataType_##X: \ +#define CASE_TYPE(X) \ + case onnx::TensorProto_DataType_##X: \ return ONNX_TENSOR_ELEMENT_DATA_TYPE_##X; ONNXTensorElementDataType CApiElementTypeFromProtoType(int type) { @@ -500,72 +372,88 @@ ONNXTensorElementDataType CApiElementTypeFromProtoType(int type) { } } -ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto& tensor_proto) { +ONNXTensorElementDataType GetTensorElementType(const onnx::TensorProto& tensor_proto) { return CApiElementTypeFromProtoType(tensor_proto.data_type()); } -TensorProto::DataType GetTensorProtoType(const Tensor& tensor) { - auto tensor_type = tensor.DataType(); - TensorProto::DataType dtype = TensorProto_DataType_UNDEFINED; +Status TensorProtoToMLValue(const onnx::TensorProto& tensor_proto, const MemBuffer& m, Ort::Value& value, + OrtCallback& deleter) { + const OrtAllocatorInfo& allocator = m.GetAllocInfo(); + ONNXTensorElementDataType ele_type = test::GetTensorElementType(tensor_proto); + const void* raw_data = nullptr; + size_t raw_data_len = 0; + void* tensor_data; + { + if (tensor_proto.data_location() == onnx::TensorProto_DataLocation::TensorProto_DataLocation_EXTERNAL) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Server doesn't support external data."); + } else if (tensor_proto.has_raw_data()) { + if (ele_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) + return Status(common::ONNXRUNTIME, common::FAIL, "String tensor cannot have raw data."); + raw_data = tensor_proto.raw_data().data(); + raw_data_len = tensor_proto.raw_data().size(); + } + { + void* preallocated = m.GetBuffer(); + size_t preallocated_size = m.GetLen(); + int64_t tensor_size = 1; + { + for (auto i : tensor_proto.dims()) { + if (i < 0) return Status(common::ONNXRUNTIME, common::FAIL, "Tensor can't contain negative dims"); + tensor_size *= i; + } + } + // tensor_size could be zero. see test_slice_start_out_of_bounds\test_data_set_0\output_0.pb + if (static_cast(tensor_size) > SIZE_MAX) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Size overflow"); + } + size_t size_to_allocate; + GetSizeInBytesFromTensorProto<0>(tensor_proto, &size_to_allocate); - if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_FLOAT; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_DOUBLE; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_INT8; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_INT16; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_INT32; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_INT64; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_UINT8; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_UINT16; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_UINT32; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_UINT64; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_BOOL; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_FLOAT16; - else if (tensor_type == DataTypeImpl::GetType()) - dtype = TensorProto_DataType_BFLOAT16; - - return dtype; -} - -ONNX_NAMESPACE::TensorProto TensorToTensorProto(const Tensor& tensor, const std::string& tensor_proto_name, - const ONNX_NAMESPACE::TypeProto& tensor_proto_type) { - // Given we are using the raw_data field in the protobuf, this will work only for little-endian format. - ORT_ENFORCE(IsLittleEndianOrder()); - - // Set name, dimensions, type, and data of the TensorProto. - ONNX_NAMESPACE::TensorProto tensor_proto; - - tensor_proto.set_name(tensor_proto_name); - - for (auto& dim : tensor.Shape().GetDims()) { - tensor_proto.add_dims(dim); + if (preallocated && preallocated_size < size_to_allocate) + return Status(common::ONNXRUNTIME, common::FAIL, MakeString("The buffer planner is not consistent with tensor buffer size, expected ", size_to_allocate, ", got ", preallocated_size)); + switch (tensor_proto.data_type()) { + CASE_PROTO(FLOAT, float); + CASE_PROTO(DOUBLE, double); + CASE_PROTO(BOOL, bool); + CASE_PROTO(INT8, int8_t); + CASE_PROTO(INT16, int16_t); + CASE_PROTO(INT32, int32_t); + CASE_PROTO(INT64, int64_t); + CASE_PROTO(UINT8, uint8_t); + CASE_PROTO(UINT16, uint16_t); + CASE_PROTO(UINT32, uint32_t); + CASE_PROTO(UINT64, uint64_t); + CASE_PROTO(FLOAT16, MLFloat16); + CASE_PROTO(BFLOAT16, BFloat16); + case onnx::TensorProto_DataType::TensorProto_DataType_STRING: + if (preallocated != nullptr) { + OrtStatus* status = OrtInitializeBufferForTensor(preallocated, preallocated_size, ele_type); + if (status != nullptr) { + OrtReleaseStatus(status); + return Status(common::ONNXRUNTIME, common::FAIL, "initialize preallocated buffer failed"); + } + deleter.f = UnInitTensor; + deleter.param = new UnInitializeParam{preallocated, preallocated_size, ele_type}; + } + ::onnxruntime::test::UnpackTensor(tensor_proto, raw_data, raw_data_len, + (std::string*)preallocated, tensor_size); + break; + default: { + std::ostringstream ostr; + ostr << "Initialized tensor with unexpected type: " << tensor_proto.data_type(); + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, ostr.str()); + } + } + tensor_data = preallocated; + } } - - // TODO Once utils::GetTensorProtoType supports all data types, you can get the tensor proto type from the tensor, - // as follows (which will allow us to get rid of the tensor_proto_type argument). - //tensor_proto.set_data_type(utils::GetTensorProtoType(tensor)); - - tensor_proto.set_data_type(tensor_proto_type.tensor_type().elem_type()); - - tensor_proto.set_raw_data(tensor.DataRaw(), tensor.SizeInBytes()); - - return tensor_proto; + std::vector tensor_shape_vec = GetTensorShapeFromTensorProto(tensor_proto); + // Note: We permit an empty tensor_shape_vec, and treat it as a scalar (a tensor of size 1). + value = Ort::Value::CreateTensor(&allocator, tensor_data, m.GetLen(), tensor_shape_vec.data(), tensor_shape_vec.size(), (ONNXTensorElementDataType)tensor_proto.data_type()); + return Status::OK(); } - -template common::Status GetSizeInBytesFromTensorProto<256>(const ONNX_NAMESPACE::TensorProto& tensor_proto, - size_t* out); -template common::Status GetSizeInBytesFromTensorProto<0>(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out); -} // namespace utils +template Status GetSizeInBytesFromTensorProto<256>(const onnx::TensorProto& tensor_proto, + size_t* out); +template Status GetSizeInBytesFromTensorProto<0>(const onnx::TensorProto& tensor_proto, size_t* out); } // namespace test -} // namespace onnxruntime +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/onnx/tensorprotoutils.h b/onnxruntime/test/onnx/tensorprotoutils.h index 18c171d4d3..ab3bb7dc82 100644 --- a/onnxruntime/test/onnx/tensorprotoutils.h +++ b/onnxruntime/test/onnx/tensorprotoutils.h @@ -5,64 +5,35 @@ #include #include - -#include "core/common/common.h" -#include "core/common/status.h" -#include "core/framework/allocator.h" -#include "core/framework/ml_value.h" -#include "core/framework/mem_buffer.h" -#include "core/framework/tensor_external_data_info.h" #include "core/session/onnxruntime_c_api.h" -#include "core/graph/onnx_protobuf.h" -#include "core/platform/env.h" +#include "core/session/onnxruntime_cxx_api.h" +#include "callback.h" +#include "mem_buffer.h" -namespace ONNX_NAMESPACE { +namespace onnx { class TensorProto; -class TensorShapeProto; -} // namespace ONNX_NAMESPACE +} namespace onnxruntime { -class Tensor; namespace test { -namespace utils { -TensorShape GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::TensorShapeProto& tensor_shape_proto); -/** - * deserialize a TensorProto into a preallocated memory buffer. - * \param tensor_proto_path A local file path of where the 'input' was loaded from. Can be NULL if the tensor proto doesn't - * have any external data or it was loaded from current working dir. This path could be either a - * relative path or an absolute path. - */ -common::Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_proto_path, - const ONNX_NAMESPACE::TensorProto& input, const MemBuffer& m, OrtValue& value, - OrtCallback& deleter); -// This function doesn't support string tensors -ONNX_NAMESPACE::TensorProto::DataType GetTensorProtoType(const Tensor& tensor); - -/** Creates a TensorProto from a Tensor. - @param[in] tensor the Tensor whose data and shape will be used to create the TensorProto. - @param[in] tensor_proto_name the name of the TensorProto. - @param[in] tensor_proto_type the type of the TensorProto. - @return the TensorProto. - - Note: Method currently requires that data is in little-endian format. - TODO Once the GetTensorProtoType supports all data types, we can remove the tensor_proto_type parameter and - instead get the type from the tensor. */ -ONNX_NAMESPACE::TensorProto TensorToTensorProto(const Tensor& tensor, const std::string& tensor_proto_name, - const ONNX_NAMESPACE::TypeProto& tensor_proto_type); - -ONNXTensorElementDataType CApiElementTypeFromProtoType(int type); -ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto& tensor_proto); - // How much memory it will need for putting the content of this tensor into a plain array // complex64/complex128 tensors are not supported. // The output value could be zero or -1. template -common::Status GetSizeInBytesFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto, size_t* out); +common::Status GetSizeInBytesFromTensorProto(const onnx::TensorProto& tensor_proto, size_t* out); +/** + * deserialize a TensorProto into a preallocated memory buffer. + * Impl must correspond to onnxruntime/core/framework/tensorprotoutils.cc + * This implementation does not support external data so as to reduce dependency surface. + */ +common::Status TensorProtoToMLValue(const onnx::TensorProto& input, const MemBuffer& m, /* out */ Ort::Value& value, + OrtCallback& deleter); template -Status UnpackTensor(const ONNX_NAMESPACE::TensorProto& tensor, const void* raw_data, size_t raw_data_len, - /*out*/ T* p_data, int64_t expected_size); +void UnpackTensor(const onnx::TensorProto& tensor, const void* raw_data, size_t raw_data_len, + /*out*/ T* p_data, int64_t expected_size); -} // namespace utils +ONNXTensorElementDataType CApiElementTypeFromProtoType(int type); +ONNXTensorElementDataType GetTensorElementType(const onnx::TensorProto& tensor_proto); } // namespace test -} // namespace onnxruntime +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/perftest/performance_runner.h b/onnxruntime/test/perftest/performance_runner.h index d4abaceeea..8d9cf1d808 100644 --- a/onnxruntime/test/perftest/performance_runner.h +++ b/onnxruntime/test/perftest/performance_runner.h @@ -128,7 +128,7 @@ class PerformanceRunner { PerformanceTestConfig performance_test_config_; TestModelInfo* test_model_info_; std::unique_ptr session_; - HeapBuffer b_; + onnxruntime::test::HeapBuffer b_; std::unique_ptr test_case_; // TODO: Convert to OrtMutex