From 0b4659c3fea7ef0131984bf98613e33954f9a774 Mon Sep 17 00:00:00 2001 From: Ryan Lai Date: Tue, 21 Jul 2020 17:50:15 -0700 Subject: [PATCH] Populates TensorProto with tensor from protobuf file (#4535) * Expose load tensor proto from protobuf file function * Add comment * Remove use of fstream and use parsefromzerocopystream * Close file descriptor after finish parsing it * Close input stream too * Set Close on delete only, no need to close file descriptor * Revert "Set Close on delete only, no need to close file descriptor" This reverts commit 5ba6e3c31b4695e2994db0cd71808c3514e0995a. * Revert "Close input stream too" This reverts commit 4564776733d31b9c8bc930ab57802248aaf40b2b. * Revert "Close file descriptor after finish parsing it" This reverts commit 846e550c4f80f629d0aba0869e6be127afb12025. * Revert "Remove use of fstream and use parsefromzerocopystream" This reverts commit 25a3117183986f27de5cc9ec50865a2427273503. --- winml/test/common/protobufHelpers.cpp | 13 ++++--------- winml/test/common/protobufHelpers.h | 8 +++++++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/winml/test/common/protobufHelpers.cpp b/winml/test/common/protobufHelpers.cpp index 9fb5fd19c6..3838cfa78e 100644 --- a/winml/test/common/protobufHelpers.cpp +++ b/winml/test/common/protobufHelpers.cpp @@ -11,11 +11,6 @@ #include "core/common/logging/sinks/clog_sink.h" #include "protobufHelpers.h" -#pragma warning(push) -#pragma warning(disable : 4100) -#include "onnx/onnx-ml.pb.h" -#pragma warning(pop) - #include using namespace wss; @@ -44,8 +39,8 @@ void FdClose(int fd) { } } -// Copy and pasted from LOTUS as is. temporary code to load tensors from protobufs -bool LoadTensorFromPb(onnx::TensorProto& tensor, std::wstring filePath) { +// Load Onnx TensorProto from Protobuf File +bool ProtobufHelpers::LoadOnnxTensorFromProtobufFile(onnx::TensorProto& tensor, std::wstring filePath) { // setup a string converter using convert_type = std::codecvt_utf8; std::wstring_convert converter; @@ -124,7 +119,7 @@ ITensor ProtobufHelpers::LoadTensorFromProtobufFile( bool isFp16) { // load from the file path into the onnx format onnx::TensorProto tensorProto; - if (LoadTensorFromPb(tensorProto, filePath)) { + if (LoadOnnxTensorFromProtobufFile(tensorProto, filePath)) { std::vector tensorShape = std::vector(tensorProto.dims().begin(), tensorProto.dims().end()); int64_t initialValue = 1; int64_t elementCount = std::accumulate(tensorShape.begin(), tensorShape.end(), initialValue, std::multiplies()); @@ -156,7 +151,7 @@ TensorFloat16Bit ProtobufHelpers::LoadTensorFloat16FromProtobufFile( const std::wstring& filePath) { // load from the file path into the onnx format onnx::TensorProto tensorProto; - if (LoadTensorFromPb(tensorProto, filePath)) { + if (LoadOnnxTensorFromProtobufFile(tensorProto, filePath)) { if (tensorProto.has_data_type()) { if(onnx::TensorProto::DataType::TensorProto_DataType_FLOAT16 != tensorProto.data_type()) { throw winrt::hresult_invalid_argument(L"TensorProto datatype isn't of type Float16."); diff --git a/winml/test/common/protobufHelpers.h b/winml/test/common/protobufHelpers.h index 2f51b8f547..fbc9aa6c2d 100644 --- a/winml/test/common/protobufHelpers.h +++ b/winml/test/common/protobufHelpers.h @@ -5,7 +5,10 @@ #include "std.h" #include "winrt_headers.h" - +#pragma warning(push) +#pragma warning(disable : 4100) +#include "onnx/onnx-ml.pb.h" +#pragma warning(pop) namespace ProtobufHelpers { // LoadTensorFromProtobufFile take a path to a FP32 data file and loads it into a 32bit array or @@ -18,4 +21,7 @@ namespace ProtobufHelpers winml::TensorKind kind, const std::vector& shape, uint32_t num_elements = 1); + + // Populates TensorProto with tensor from protobuf file + bool LoadOnnxTensorFromProtobufFile(onnx::TensorProto& tensor, std::wstring filePath); }