saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Move ort flatbuffers helper functions and value info r/w functions into separated lib (#5276)

Browse source 
* Move fbs include from header to cc

* add initial cmake for flatbuffers

* Move most flatbuffers util to ort_flatbuffers

* move code around

* fix

* move test/perf runner to use flatbuffer directly instead of model

* minor update

* Fix build break

* Clean up includes and foward decl

* Fix traning CI build breaks

* Addressed PR comment, replaced some include with forward decls

* Remove ORT_MUST_USE_RESULT temporarily
This commit is contained in:
Guoyu Wang 2020-09-25 05:36:29 -07:00 committed by GitHub
parent 17f1178c2e
commit 3a3f26f38e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 629 additions and 395 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmake/CMakeLists.txt
View file

@ -210,7 +210,7 @@ endif()
if(onnxruntime_MINIMAL_BUILD)
add_compile_definitions(ORT_MINIMAL_BUILD)
add_compile_definitions(ENABLE_ORT_FORMAT_LOAD)
set(onnxruntime_REDUCED_OPS_BUILD ON)
if (NOT onnxruntime_ENABLE_PYTHON)
@ -1248,7 +1248,7 @@ endif()
#names in this var must match the directory names under onnxruntime/core/providers
set(ONNXRUNTIME_PROVIDER_NAMES cpu)
foreach(target_name onnxruntime_common onnxruntime_graph onnxruntime_framework onnxruntime_util onnxruntime_providers onnxruntime_optimizer onnxruntime_session onnxruntime_mlas)
foreach(target_name onnxruntime_common onnxruntime_graph onnxruntime_framework onnxruntime_util onnxruntime_providers onnxruntime_optimizer onnxruntime_session onnxruntime_mlas onnxruntime_flatbuffers)
include(${target_name}.cmake)
if (MSVC)
target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /utf-8>" "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/utf-8>")

1
cmake/onnxruntime.cmake
View file

@ -116,6 +116,7 @@ target_link_libraries(onnxruntime PRIVATE
onnxruntime_graph
onnxruntime_common
onnxruntime_mlas
onnxruntime_flatbuffers
${onnxruntime_EXTERNAL_LIBRARIES})
if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)

19
cmake/onnxruntime_flatbuffers.cmake Normal file
View file

@ -0,0 +1,19 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
file(GLOB onnxruntime_flatbuffers_srcs CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/core/flatbuffers/*.h"
"${ONNXRUNTIME_ROOT}/core/flatbuffers/*.cc"
)
source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_flatbuffers_srcs})
add_library(onnxruntime_flatbuffers ${onnxruntime_flatbuffers_srcs})
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/flatbuffers DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core)
onnxruntime_add_include_to_target(onnxruntime_flatbuffers onnx flatbuffers)
if(onnxruntime_ENABLE_INSTRUMENT)
target_compile_definitions(onnxruntime_flatbuffers PUBLIC ONNXRUNTIME_ENABLE_INSTRUMENT)
endif()
target_include_directories(onnxruntime_flatbuffers PRIVATE ${ONNXRUNTIME_ROOT})
add_dependencies(onnxruntime_flatbuffers ${onnxruntime_EXTERNAL_DEPENDENCIES})
set_target_properties(onnxruntime_flatbuffers PROPERTIES FOLDER "ONNXRuntime")

1
cmake/onnxruntime_python.cmake
View file

@ -107,6 +107,7 @@ set(onnxruntime_pybind11_state_libs
onnxruntime_graph
onnxruntime_common
onnxruntime_mlas
onnxruntime_flatbuffers
${pybind11_lib}
)

1
cmake/onnxruntime_training.cmake
View file

@ -93,6 +93,7 @@ set(ONNXRUNTIME_LIBS
onnxruntime_graph
onnxruntime_common
onnxruntime_mlas
onnxruntime_flatbuffers
)
if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)

13
cmake/onnxruntime_unittests.cmake
View file

@ -416,6 +416,7 @@ set(ONNXRUNTIME_TEST_LIBS
onnxruntime_graph
onnxruntime_common
onnxruntime_mlas
onnxruntime_flatbuffers
)
if (onnxruntime_ENABLE_TRAINING)
@ -760,15 +761,9 @@ if (WIN32)
endif()
if (onnxruntime_BUILD_SHARED_LIB)
set(onnxruntime_perf_test_libs onnx_test_runner_common onnxruntime_test_utils onnxruntime_common re2::re2
onnx_test_data_proto onnx_proto ${PROTOBUF_LIB} ${GETOPT_LIB_WIDE} onnxruntime ${SYS_PATH_LIB}
${CMAKE_DL_LIBS})
# We want to enable onnx_test_runner/perf_test for ort minimal builds, which will need the following
# ort lib statically linked, this is a temporary solution and is tracked by,
# Product Backlog Item 895235: Re-work onnx_test_runner/perf_test for ort/onnx model
if (onnxruntime_MINIMAL_BUILD)
list(APPEND onnxruntime_perf_test_libs onnxruntime_graph onnx onnxruntime_framework onnxruntime_mlas)
endif()
set(onnxruntime_perf_test_libs onnx_test_runner_common onnxruntime_test_utils onnxruntime_common re2::re2
onnx_test_data_proto onnx_proto ${PROTOBUF_LIB} ${GETOPT_LIB_WIDE} onnxruntime onnxruntime_flatbuffers
${SYS_PATH_LIB} ${CMAKE_DL_LIBS})
if(NOT WIN32)
list(APPEND onnxruntime_perf_test_libs nsync_cpp)
endif()

15
include/onnxruntime/core/graph/graph.h
View file

@ -15,7 +15,6 @@
#include "core/common/path.h"
#include "core/common/status.h"
#include "core/common/logging/logging.h"
#include "core/flatbuffers/ort.fbs.h"
#include "core/graph/basic_types.h"
#include "core/graph/constants.h"
#include "core/graph/graph_nodes.h"
@ -24,12 +23,26 @@
#include "core/graph/function.h"
#include "gsl/gsl"
namespace flatbuffers {
class FlatBufferBuilder;
template <typename T>
struct Offset;
} // namespace flatbuffers
namespace onnxruntime {
class Graph;
struct IndexedSubGraph;
class Model;
class OpSignature;
namespace experimental {
namespace fbs {
struct Graph;
struct Node;
struct NodeEdge;
} // namespace fbs
} // namespace experimental
/**
@class Node
Class representing a node in the graph.

304
onnxruntime/core/flatbuffers/flatbuffers_utils.cc Normal file
View file

@ -0,0 +1,304 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "flatbuffers_utils.h"
#include "schema/ort.fbs.h"
#include "core/common/common.h"
#include "core/graph/constants.h"
#include "core/graph/onnx_protobuf.h"
#include "gsl/gsl"
using namespace ONNX_NAMESPACE;
using namespace ::onnxruntime::common;
using namespace ::onnxruntime::experimental;
namespace onnxruntime {
namespace experimental {
namespace utils {
#if !defined(ORT_MINIMAL_BUILD)
static Status SaveTypeInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TypeProto& type_proto,
flatbuffers::Offset<fbs::TypeInfo>& fbs_type_info);
static flatbuffers::Offset<fbs::Dimension> SaveTensorDimensionOrtFormat(
flatbuffers::FlatBufferBuilder& builder,
const TensorShapeProto_Dimension& tensor_shape_dim) {
auto denotation = builder.CreateString(tensor_shape_dim.denotation());
flatbuffers::Offset<fbs::DimensionValue> dim_val;
if (tensor_shape_dim.has_dim_param()) {
dim_val = fbs::CreateDimensionValueDirect(builder, fbs::DimensionValueType::PARAM, 0, tensor_shape_dim.dim_param().c_str());
} else if (tensor_shape_dim.has_dim_value()) {
dim_val = fbs::CreateDimensionValueDirect(builder, fbs::DimensionValueType::VALUE, tensor_shape_dim.dim_value());
} else {
dim_val = fbs::CreateDimensionValueDirect(builder);
}
return fbs::CreateDimension(builder, dim_val, denotation);
}
static Status SaveTensorShapeOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TensorShapeProto& tensor_shape_proto,
flatbuffers::Offset<fbs::Shape>& fbs_shape) {
std::vector<flatbuffers::Offset<fbs::Dimension>> dim;
dim.reserve(tensor_shape_proto.dim_size());
for (const auto& d : tensor_shape_proto.dim()) {
auto fbs_d = SaveTensorDimensionOrtFormat(builder, d);
dim.push_back(fbs_d);
}
fbs_shape = fbs::CreateShapeDirect(builder, &dim);
return Status::OK();
}
static Status SaveSequenceTypeOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TypeProto_Sequence& sequence_type_proto,
flatbuffers::Offset<fbs::SequenceType>& fbs_sequence_type) {
flatbuffers::Offset<fbs::TypeInfo> fbs_type_info;
ORT_RETURN_IF_ERROR(SaveTypeInfoOrtFormat(builder, sequence_type_proto.elem_type(), fbs_type_info));
fbs_sequence_type = fbs::CreateSequenceType(builder, fbs_type_info);
return Status::OK();
}
static Status SaveMapTypeOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TypeProto_Map& map_type_proto,
flatbuffers::Offset<fbs::MapType>& fbs_map_type) {
flatbuffers::Offset<fbs::TypeInfo> fbs_type_info;
ORT_RETURN_IF_ERROR(SaveTypeInfoOrtFormat(builder, map_type_proto.value_type(), fbs_type_info));
fbs_map_type = fbs::CreateMapType(
builder, static_cast<fbs::TensorDataType>(map_type_proto.key_type()), fbs_type_info);
return Status::OK();
}
static Status SaveTensorTypeAndShapeOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TypeProto_Tensor& tensor_type_proto,
flatbuffers::Offset<fbs::TensorTypeAndShape>& fbs_tensor_type) {
// A flatbuffers::Offset of 0 means this shape is missing (was null when serializing)
flatbuffers::Offset<fbs::Shape> shape = 0;
if (tensor_type_proto.has_shape()) {
ORT_RETURN_IF_ERROR(SaveTensorShapeOrtFormat(builder, tensor_type_proto.shape(), shape));
}
fbs_tensor_type = fbs::CreateTensorTypeAndShape(
builder, static_cast<fbs::TensorDataType>(tensor_type_proto.elem_type()), shape);
return Status::OK();
}
static Status SaveTypeInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TypeProto& type_proto,
flatbuffers::Offset<fbs::TypeInfo>& fbs_type_info) {
auto denotation = builder.CreateString(type_proto.denotation());
auto value_type = fbs::TypeInfoValue::tensor_type;
flatbuffers::Offset<void> value;
auto value_case = type_proto.value_case();
switch (value_case) {
case TypeProto::kTensorType: {
flatbuffers::Offset<fbs::TensorTypeAndShape> fbs_tensor_type;
ORT_RETURN_IF_ERROR(
SaveTensorTypeAndShapeOrtFormat(builder, type_proto.tensor_type(), fbs_tensor_type));
value = fbs_tensor_type.Union();
} break;
case TypeProto::kSequenceType: {
value_type = fbs::TypeInfoValue::sequence_type;
flatbuffers::Offset<fbs::SequenceType> fbs_sequence_type;
ORT_RETURN_IF_ERROR(
SaveSequenceTypeOrtFormat(builder, type_proto.sequence_type(), fbs_sequence_type));
value = fbs_sequence_type.Union();
} break;
case TypeProto::kMapType: {
value_type = fbs::TypeInfoValue::map_type;
flatbuffers::Offset<fbs::MapType> fbs_map_type;
ORT_RETURN_IF_ERROR(
SaveMapTypeOrtFormat(builder, type_proto.map_type(), fbs_map_type));
value = fbs_map_type.Union();
} break;
default: {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "We do not support type [", value_case, "] for now");
} break;
}
fbs::TypeInfoBuilder tb(builder);
tb.add_denotation(denotation);
tb.add_value_type(value_type);
tb.add_value(value);
fbs_type_info = tb.Finish();
return Status::OK();
}
Status SaveValueInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const ValueInfoProto& value_info_proto,
flatbuffers::Offset<fbs::ValueInfo>& fbs_value_info) {
auto name = builder.CreateSharedString(value_info_proto.name());
auto doc_string = builder.CreateString(value_info_proto.doc_string());
flatbuffers::Offset<fbs::TypeInfo> type_info = 0; // 0 indicates null
if (value_info_proto.has_type()) {
ORT_RETURN_IF_ERROR(
SaveTypeInfoOrtFormat(builder, value_info_proto.type(), type_info));
} else {
// we have a NodeArg for missing optional values (empty name, no type) so allow for that.
// everything else should have type info
if (!value_info_proto.name().empty()) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"SaveValueInfoOrtFormat: value_info_proto for ", value_info_proto.name(),
" is missing type info.");
}
}
fbs::ValueInfoBuilder vb(builder);
vb.add_name(name);
vb.add_doc_string(doc_string);
vb.add_type(type_info);
fbs_value_info = vb.Finish();
return Status::OK();
}
#endif // #if !defined(ORT_MINIMAL_BUILD)
#if defined(ENABLE_ORT_FORMAT_LOAD)
void LoadStringFromOrtFormat(std::string& dst, const flatbuffers::String* fbs_string) {
if (fbs_string)
dst = fbs_string->c_str();
}
static Status LoadTypeInfoOrtFormat(const fbs::TypeInfo& fbs_type_info,
TypeProto& type_proto);
static Status LoadTensorDimensionOrtFormat(const fbs::Dimension& fbs_dim,
TensorShapeProto_Dimension& dim) {
LoadStringFromOrtFormat(*dim.mutable_denotation(), fbs_dim.denotation());
auto fbs_dim_val = fbs_dim.value();
if (fbs_dim_val) {
auto type = fbs_dim_val->dim_type();
if (type == fbs::DimensionValueType::VALUE)
dim.set_dim_value(fbs_dim_val->dim_value());
else if (type == fbs::DimensionValueType::PARAM) {
auto fbs_dim_param = fbs_dim_val->dim_param();
ORT_RETURN_IF(nullptr == fbs_dim_param, "dim_param value with no name. Invalid ORT format model.");
dim.set_dim_param(fbs_dim_param->str());
} else {
// unknown dimension. leave dim in VALUE_NOT_SET state as this is valid
}
} else {
// tensor with unknown shape.
// e.g. output from Reshape node where shape is determined by dynamic input at runtime
}
return Status::OK();
}
static Status LoadTensorTypeAndShapeOrtFormat(const fbs::TensorTypeAndShape& fbs_tensor_type,
TypeProto_Tensor& tensor_type_proto) {
tensor_type_proto.set_elem_type(static_cast<int32_t>(fbs_tensor_type.elem_type()));
auto fbs_shape = fbs_tensor_type.shape();
if (fbs_shape) {
auto fbs_dims = fbs_shape->dim();
if (fbs_dims) {
auto dims = tensor_type_proto.mutable_shape()->mutable_dim();
dims->Reserve(fbs_dims->size());
for (const auto fbs_dim : *fbs_dims) {
ORT_RETURN_IF(nullptr == fbs_dim, "Null entry in dimensions. Invalid ORT format model.");
TensorShapeProto_Dimension dim;
ORT_RETURN_IF_ERROR(LoadTensorDimensionOrtFormat(*fbs_dim, *dims->Add()));
}
}
}
return Status::OK();
}
static Status LoadSequenceTypeOrtFormat(const fbs::SequenceType& fbs_sequence_type,
TypeProto_Sequence& sequence_type_proto) {
auto fbs_type_info = fbs_sequence_type.elem_type();
ORT_RETURN_IF(nullptr == fbs_type_info, "Null value type info in fbs::SequenceType. Invalid ORT format model.");
ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *sequence_type_proto.mutable_elem_type()));
return Status::OK();
}
static Status LoadMapTypeOrtFormat(const fbs::MapType& fbs_map_type,
TypeProto_Map& map_type_proto) {
map_type_proto.set_key_type(static_cast<int32_t>(fbs_map_type.key_type()));
auto fbs_type_info = fbs_map_type.value_type();
ORT_RETURN_IF(nullptr == fbs_type_info, "Null value type info in fbs::MapType. Invalid ORT format model.");
ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *map_type_proto.mutable_value_type()));
return Status::OK();
}
static Status LoadTypeInfoOrtFormat(const fbs::TypeInfo& fbs_type_info,
TypeProto& type_proto) {
LoadStringFromOrtFormat(*type_proto.mutable_denotation(), fbs_type_info.denotation());
auto value_type = fbs_type_info.value_type();
if (value_type == fbs::TypeInfoValue::tensor_type) {
auto fbs_tensor_type = fbs_type_info.value_as_tensor_type();
ORT_RETURN_IF(nullptr == fbs_tensor_type, "Null tensor type info. Invalid ORT format model.");
ORT_RETURN_IF_ERROR(LoadTensorTypeAndShapeOrtFormat(*fbs_tensor_type, *type_proto.mutable_tensor_type()));
} else if (value_type == fbs::TypeInfoValue::sequence_type) {
auto fbs_sequence_type = fbs_type_info.value_as_sequence_type();
ORT_RETURN_IF(nullptr == fbs_sequence_type, "Null sequence type info. Invalid ORT format model.");
ORT_RETURN_IF_ERROR(LoadSequenceTypeOrtFormat(*fbs_sequence_type, *type_proto.mutable_sequence_type()));
} else if (value_type == fbs::TypeInfoValue::map_type) {
auto fbs_map_type = fbs_type_info.value_as_map_type();
ORT_RETURN_IF(nullptr == fbs_map_type, "Null map type info. Invalid ORT format model.");
ORT_RETURN_IF_ERROR(LoadMapTypeOrtFormat(*fbs_map_type, *type_proto.mutable_map_type()));
} else {
// We do not support SparseTensor and Opaque for now
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Type:",
fbs::EnumNameTypeInfoValue(value_type), " is not supported currently");
}
return Status::OK();
}
Status LoadValueInfoOrtFormat(const fbs::ValueInfo& fbs_value_info,
ONNX_NAMESPACE::ValueInfoProto& value_info_proto) {
value_info_proto.Clear();
LoadStringFromOrtFormat(*value_info_proto.mutable_name(), fbs_value_info.name());
LoadStringFromOrtFormat(*value_info_proto.mutable_doc_string(), fbs_value_info.doc_string());
auto fbs_type_info = fbs_value_info.type();
if (fbs_type_info == nullptr) {
// there is a NodeArg with empty name for missing optional inputs that can have null type info.
// anything else should have a type
ORT_RETURN_IF(!value_info_proto.name().empty(),
"Null type info for ", value_info_proto.name(), ". Invalid ORT format model.");
} else {
ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *value_info_proto.mutable_type()));
}
return Status::OK();
}
Status LoadOpsetImportOrtFormat(const flatbuffers::Vector<flatbuffers::Offset<fbs::OperatorSetId>>* fbs_op_set_ids,
std::unordered_map<std::string, int>& domain_to_version) {
ORT_RETURN_IF(nullptr == fbs_op_set_ids, "Model must have opset imports. Invalid ORT format model.");
domain_to_version.clear();
domain_to_version.reserve(fbs_op_set_ids->size());
for (const auto* fbs_op_set_id : *fbs_op_set_ids) {
ORT_RETURN_IF(nullptr == fbs_op_set_id, "opset id is null. Invalid ORT format model.");
const auto* fbs_domain = fbs_op_set_id->domain();
ORT_RETURN_IF(nullptr == fbs_domain, "opset import domain is null. Invalid ORT format model.");
std::string domain = fbs_domain->str();
// perform same aliasing that we do when loading an ONNX format model
if (domain == kOnnxDomainAlias) {
domain_to_version[kOnnxDomain] = gsl::narrow_cast<int>(fbs_op_set_id->version());
} else {
domain_to_version[domain] = gsl::narrow_cast<int>(fbs_op_set_id->version());
}
}
return Status::OK();
}
#endif // defined(ENABLE_ORT_FORMAT_LOAD)
bool IsOrtFormatModelBytes(const void* bytes, int num_bytes) {
return num_bytes > 8 && // check buffer is large enough to contain identifier so we don't read random memory
fbs::InferenceSessionBufferHasIdentifier(bytes);
}
} // namespace utils
} // namespace experimental
} // namespace onnxruntime

70
onnxruntime/core/flatbuffers/flatbuffers_utils.h Normal file
View file

@ -0,0 +1,70 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <unordered_map>
#include <core/common/status.h>
namespace ONNX_NAMESPACE {
class ValueInfoProto;
}
namespace flatbuffers {
class FlatBufferBuilder;
template <typename T>
struct Offset;
struct String;
template <typename T>
class Vector;
} // namespace flatbuffers
namespace onnxruntime {
namespace experimental {
namespace fbs {
struct OperatorSetId;
struct ValueInfo;
} // namespace fbs
namespace utils {
// TODO, add ORT_MUST_USE_RESULT when it is moved to a different header
onnxruntime::common::Status SaveValueInfoOrtFormat(
flatbuffers::FlatBufferBuilder& builder, const ONNX_NAMESPACE::ValueInfoProto& value_info_proto,
flatbuffers::Offset<fbs::ValueInfo>& fbs_value_info);
#if defined(ENABLE_ORT_FORMAT_LOAD)
void LoadStringFromOrtFormat(std::string& dst, const flatbuffers::String* fbs_string);
onnxruntime::common::Status LoadValueInfoOrtFormat(
const fbs::ValueInfo& fbs_value_info, ONNX_NAMESPACE::ValueInfoProto& value_info_proto);
onnxruntime::common::Status LoadOpsetImportOrtFormat(
const flatbuffers::Vector<flatbuffers::Offset<fbs::OperatorSetId>>* fbs_op_set_ids,
std::unordered_map<std::string, int>& domain_to_version);
#endif
// check if filename ends in .ort
template <typename T>
bool IsOrtFormatModel(const std::basic_string<T>& filename) {
auto len = filename.size();
return len > 4 &&
filename[len - 4] == '.' &&
std::tolower(filename[len - 3]) == 'o' &&
std::tolower(filename[len - 2]) == 'r' &&
std::tolower(filename[len - 1]) == 't';
}
// check if bytes has the flatbuffer ORT identifier
bool IsOrtFormatModelBytes(const void* bytes, int num_bytes);
} // namespace utils
} // namespace experimental
} // namespace onnxruntime

0
onnxruntime/core/flatbuffers/README.md → onnxruntime/core/flatbuffers/schema/README.md
View file

2
onnxruntime/core/flatbuffers/ort.fbs → onnxruntime/core/flatbuffers/schema/ort.fbs
View file

@ -121,7 +121,7 @@ table ValueInfo {
type:TypeInfo;
}
// TODO add support of Sequence/Map/SparseTensor/Opaque
// TODO add support of SparseTensor/Opaque
union TypeInfoValue {
tensor_type:TensorTypeAndShape,
sequence_type:SequenceType,

0
onnxruntime/core/flatbuffers/ort.fbs.h → onnxruntime/core/flatbuffers/schema/ort.fbs.h
View file

1
onnxruntime/core/framework/session_state.cc
View file

@ -7,6 +7,7 @@
#include "core/common/logging/logging.h"
#include "core/common/safeint.h"
#include "core/flatbuffers/schema/ort.fbs.h"
#include "core/framework/allocator.h"
#include "core/framework/node_index_info.h"
#include "core/framework/op_kernel.h"

13
onnxruntime/core/framework/session_state.h
View file

@ -13,7 +13,6 @@
#include "core/common/common.h"
#include "core/common/logging/logging.h"
#include "core/common/profiler.h"
#include "core/flatbuffers/ort.fbs.h"
#include "core/framework/allocation_planner.h"
#include "core/framework/callback.h"
#include "core/framework/data_transfer_manager.h"
@ -33,8 +32,20 @@
#include "core/platform/path_lib.h"
#include "core/platform/threadpool.h"
namespace flatbuffers {
class FlatBufferBuilder;
template <typename T>
struct Offset;
} // namespace flatbuffers
namespace onnxruntime {
namespace experimental {
namespace fbs {
struct SessionState;
} // namespace fbs
} // namespace experimental
class ExecutionProviders;
class KernelDef;
class OpKernel;

4
onnxruntime/core/graph/graph.cc
View file

@ -15,10 +15,12 @@
#include "gsl/gsl"
#include "core/common/logging/logging.h"
#include "core/flatbuffers/schema/ort.fbs.h"
#include "core/flatbuffers/flatbuffers_utils.h"
#include "core/graph/graph_flatbuffers_utils.h"
#include "core/framework/tensor_shape.h"
#include "core/framework/tensorprotoutils.h"
#include "core/framework/utils.h"
#include "core/graph/graph_flatbuffers_utils.h"
#include "core/graph/graph_viewer.h"
#include "core/graph/indexed_sub_graph.h"
#include "core/graph/model.h"

250
onnxruntime/core/graph/graph_flatbuffers_utils.cc
View file

@ -1,8 +1,12 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "graph_flatbuffers_utils.h"
#include <core/graph/graph.h>
#include "core/flatbuffers/flatbuffers_utils.h"
#include "core/flatbuffers/schema/ort.fbs.h"
#include "core/framework/tensorprotoutils.h"
#include "graph_flatbuffers_utils.h"
#include "flatbuffers/flatbuffers.h"
using namespace ONNX_NAMESPACE;
using namespace ::onnxruntime::common;
@ -13,140 +17,6 @@ namespace experimental {
namespace utils {
#if !defined(ORT_MINIMAL_BUILD)
static Status SaveTypeInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TypeProto& type_proto,
flatbuffers::Offset<fbs::TypeInfo>& fbs_type_info) ORT_MUST_USE_RESULT;
static flatbuffers::Offset<fbs::Dimension> SaveTensorDimensionOrtFormat(
flatbuffers::FlatBufferBuilder& builder,
const TensorShapeProto_Dimension& tensor_shape_dim) {
auto denotation = builder.CreateString(tensor_shape_dim.denotation());
flatbuffers::Offset<fbs::DimensionValue> dim_val;
if (tensor_shape_dim.has_dim_param()) {
dim_val = fbs::CreateDimensionValueDirect(builder, fbs::DimensionValueType::PARAM, 0, tensor_shape_dim.dim_param().c_str());
} else if (tensor_shape_dim.has_dim_value()) {
dim_val = fbs::CreateDimensionValueDirect(builder, fbs::DimensionValueType::VALUE, tensor_shape_dim.dim_value());
} else {
dim_val = fbs::CreateDimensionValueDirect(builder);
}
return fbs::CreateDimension(builder, dim_val, denotation);
}
static Status SaveTensorShapeOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TensorShapeProto& tensor_shape_proto,
flatbuffers::Offset<fbs::Shape>& fbs_shape) {
std::vector<flatbuffers::Offset<fbs::Dimension>> dim;
dim.reserve(tensor_shape_proto.dim_size());
for (const auto& d : tensor_shape_proto.dim()) {
auto fbs_d = SaveTensorDimensionOrtFormat(builder, d);
dim.push_back(fbs_d);
}
fbs_shape = fbs::CreateShapeDirect(builder, &dim);
return Status::OK();
}
static Status SaveSequenceTypeOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TypeProto_Sequence& sequence_type_proto,
flatbuffers::Offset<fbs::SequenceType>& fbs_sequence_type) {
flatbuffers::Offset<fbs::TypeInfo> fbs_type_info;
ORT_RETURN_IF_ERROR(SaveTypeInfoOrtFormat(builder, sequence_type_proto.elem_type(), fbs_type_info));
fbs_sequence_type = fbs::CreateSequenceType(builder, fbs_type_info);
return Status::OK();
}
static Status SaveMapTypeOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TypeProto_Map& map_type_proto,
flatbuffers::Offset<fbs::MapType>& fbs_map_type) {
flatbuffers::Offset<fbs::TypeInfo> fbs_type_info;
ORT_RETURN_IF_ERROR(SaveTypeInfoOrtFormat(builder, map_type_proto.value_type(), fbs_type_info));
fbs_map_type = fbs::CreateMapType(
builder, static_cast<fbs::TensorDataType>(map_type_proto.key_type()), fbs_type_info);
return Status::OK();
}
static Status SaveTensorTypeAndShapeOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TypeProto_Tensor& tensor_type_proto,
flatbuffers::Offset<fbs::TensorTypeAndShape>& fbs_tensor_type) {
// A flatbuffers::Offset of 0 means this shape is missing (was null when serializing)
flatbuffers::Offset<fbs::Shape> shape = 0;
if (tensor_type_proto.has_shape()) {
ORT_RETURN_IF_ERROR(SaveTensorShapeOrtFormat(builder, tensor_type_proto.shape(), shape));
}
fbs_tensor_type = fbs::CreateTensorTypeAndShape(
builder, static_cast<fbs::TensorDataType>(tensor_type_proto.elem_type()), shape);
return Status::OK();
}
static Status SaveTypeInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TypeProto& type_proto,
flatbuffers::Offset<fbs::TypeInfo>& fbs_type_info) {
auto denotation = builder.CreateString(type_proto.denotation());
auto value_type = fbs::TypeInfoValue::tensor_type;
flatbuffers::Offset<void> value;
auto value_case = type_proto.value_case();
switch (value_case) {
case TypeProto::kTensorType: {
flatbuffers::Offset<fbs::TensorTypeAndShape> fbs_tensor_type;
ORT_RETURN_IF_ERROR(
SaveTensorTypeAndShapeOrtFormat(builder, type_proto.tensor_type(), fbs_tensor_type));
value = fbs_tensor_type.Union();
} break;
case TypeProto::kSequenceType: {
value_type = fbs::TypeInfoValue::sequence_type;
flatbuffers::Offset<fbs::SequenceType> fbs_sequence_type;
ORT_RETURN_IF_ERROR(
SaveSequenceTypeOrtFormat(builder, type_proto.sequence_type(), fbs_sequence_type));
value = fbs_sequence_type.Union();
} break;
case TypeProto::kMapType: {
value_type = fbs::TypeInfoValue::map_type;
flatbuffers::Offset<fbs::MapType> fbs_map_type;
ORT_RETURN_IF_ERROR(
SaveMapTypeOrtFormat(builder, type_proto.map_type(), fbs_map_type));
value = fbs_map_type.Union();
} break;
default: {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "We do not support type [", value_case, "] for now");
} break;
}
fbs::TypeInfoBuilder tb(builder);
tb.add_denotation(denotation);
tb.add_value_type(value_type);
tb.add_value(value);
fbs_type_info = tb.Finish();
return Status::OK();
}
Status SaveValueInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const ValueInfoProto& value_info_proto,
flatbuffers::Offset<fbs::ValueInfo>& fbs_value_info) {
auto name = builder.CreateSharedString(value_info_proto.name());
auto doc_string = builder.CreateString(value_info_proto.doc_string());
flatbuffers::Offset<fbs::TypeInfo> type_info = 0; // 0 indicates null
if (value_info_proto.has_type()) {
ORT_RETURN_IF_ERROR(
SaveTypeInfoOrtFormat(builder, value_info_proto.type(), type_info));
} else {
// we have a NodeArg for missing optional values (empty name, no type) so allow for that.
// everything else should have type info
if (!value_info_proto.name().empty()) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"SaveValueInfoOrtFormat: value_info_proto for ", value_info_proto.name(),
" is missing type info.");
}
}
fbs::ValueInfoBuilder vb(builder);
vb.add_name(name);
vb.add_doc_string(doc_string);
vb.add_type(type_info);
fbs_value_info = vb.Finish();
return Status::OK();
}
Status SaveInitializerOrtFormat(flatbuffers::FlatBufferBuilder& builder,
const TensorProto& initializer,
@ -264,15 +134,12 @@ Status SaveAttributeOrtFormat(flatbuffers::FlatBufferBuilder& builder,
return Status::OK();
}
#endif
#undef GET_FBS_ATTR
#undef GET_DATA_VEC
#if defined(ENABLE_ORT_FORMAT_LOAD)
#endif
static Status LoadTypeInfoOrtFormat(const fbs::TypeInfo& fbs_type_info,
TypeProto& type_proto) ORT_MUST_USE_RESULT;
#if defined(ENABLE_ORT_FORMAT_LOAD)
Status LoadInitializerOrtFormat(const fbs::Tensor& fbs_tensor,
TensorProto& initializer) {
@ -306,109 +173,6 @@ Status LoadInitializerOrtFormat(const fbs::Tensor& fbs_tensor,
return Status::OK();
}
static Status LoadTensorDimensionOrtFormat(const fbs::Dimension& fbs_dim,
TensorShapeProto_Dimension& dim) {
LoadStringFromOrtFormat(*dim.mutable_denotation(), fbs_dim.denotation());
auto fbs_dim_val = fbs_dim.value();
if (fbs_dim_val) {
auto type = fbs_dim_val->dim_type();
if (type == fbs::DimensionValueType::VALUE)
dim.set_dim_value(fbs_dim_val->dim_value());
else if (type == fbs::DimensionValueType::PARAM) {
auto fbs_dim_param = fbs_dim_val->dim_param();
ORT_RETURN_IF(nullptr == fbs_dim_param, "dim_param value with no name. Invalid ORT format model.");
dim.set_dim_param(fbs_dim_param->str());
} else {
// unknown dimension. leave dim in VALUE_NOT_SET state as this is valid
}
} else {
// tensor with unknown shape.
// e.g. output from Reshape node where shape is determined by dynamic input at runtime
}
return Status::OK();
}
static Status LoadTensorTypeAndShapeOrtFormat(const fbs::TensorTypeAndShape& fbs_tensor_type,
TypeProto_Tensor& tensor_type_proto) {
tensor_type_proto.set_elem_type(static_cast<int32_t>(fbs_tensor_type.elem_type()));
auto fbs_shape = fbs_tensor_type.shape();
if (fbs_shape) {
auto fbs_dims = fbs_shape->dim();
if (fbs_dims) {
auto dims = tensor_type_proto.mutable_shape()->mutable_dim();
dims->Reserve(fbs_dims->size());
for (const auto fbs_dim : *fbs_dims) {
ORT_RETURN_IF(nullptr == fbs_dim, "Null entry in dimensions. Invalid ORT format model.");
TensorShapeProto_Dimension dim;
ORT_RETURN_IF_ERROR(LoadTensorDimensionOrtFormat(*fbs_dim, *dims->Add()));
}
}
}
return Status::OK();
}
static Status LoadSequenceTypeOrtFormat(const fbs::SequenceType& fbs_sequence_type,
TypeProto_Sequence& sequence_type_proto) {
auto fbs_type_info = fbs_sequence_type.elem_type();
ORT_RETURN_IF(nullptr == fbs_type_info, "Null value type info in fbs::SequenceType. Invalid ORT format model.");
ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *sequence_type_proto.mutable_elem_type()));
return Status::OK();
}
static Status LoadMapTypeOrtFormat(const fbs::MapType& fbs_map_type,
TypeProto_Map& map_type_proto) {
map_type_proto.set_key_type(static_cast<int32_t>(fbs_map_type.key_type()));
auto fbs_type_info = fbs_map_type.value_type();
ORT_RETURN_IF(nullptr == fbs_type_info, "Null value type info in fbs::MapType. Invalid ORT format model.");
ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *map_type_proto.mutable_value_type()));
return Status::OK();
}
static Status LoadTypeInfoOrtFormat(const fbs::TypeInfo& fbs_type_info,
TypeProto& type_proto) {
LoadStringFromOrtFormat(*type_proto.mutable_denotation(), fbs_type_info.denotation());
auto value_type = fbs_type_info.value_type();
if (value_type == fbs::TypeInfoValue::tensor_type) {
auto fbs_tensor_type = fbs_type_info.value_as_tensor_type();
ORT_RETURN_IF(nullptr == fbs_tensor_type, "Null tensor type info. Invalid ORT format model.");
ORT_RETURN_IF_ERROR(LoadTensorTypeAndShapeOrtFormat(*fbs_tensor_type, *type_proto.mutable_tensor_type()));
} else if (value_type == fbs::TypeInfoValue::sequence_type) {
auto fbs_sequence_type = fbs_type_info.value_as_sequence_type();
ORT_RETURN_IF(nullptr == fbs_sequence_type, "Null sequence type info. Invalid ORT format model.");
ORT_RETURN_IF_ERROR(LoadSequenceTypeOrtFormat(*fbs_sequence_type, *type_proto.mutable_sequence_type()));
} else if (value_type == fbs::TypeInfoValue::map_type) {
auto fbs_map_type = fbs_type_info.value_as_map_type();
ORT_RETURN_IF(nullptr == fbs_map_type, "Null map type info. Invalid ORT format model.");
ORT_RETURN_IF_ERROR(LoadMapTypeOrtFormat(*fbs_map_type, *type_proto.mutable_map_type()));
} else {
// We do not support SparseTensor and Opaque for now
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Type:",
fbs::EnumNameTypeInfoValue(value_type), " is not supported currently");
}
return Status::OK();
}
Status LoadValueInfoOrtFormat(const fbs::ValueInfo& fbs_value_info,
ONNX_NAMESPACE::ValueInfoProto& value_info_proto) {
value_info_proto.Clear();
LoadStringFromOrtFormat(*value_info_proto.mutable_name(), fbs_value_info.name());
LoadStringFromOrtFormat(*value_info_proto.mutable_doc_string(), fbs_value_info.doc_string());
auto fbs_type_info = fbs_value_info.type();
if (fbs_type_info == nullptr) {
// there is a NodeArg with empty name for missing optional inputs that can have null type info.
// anything else should have a type
ORT_RETURN_IF(!value_info_proto.name().empty(),
"Null type info for ", value_info_proto.name(), ". Invalid ORT format model.");
} else {
ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *value_info_proto.mutable_type()));
}
return Status::OK();
}
Status LoadAttributeOrtFormat(const fbs::Attribute& fbs_attr,
ONNX_NAMESPACE::AttributeProto& attr_proto,
std::unique_ptr<onnxruntime::Graph>& sub_graph,

46
onnxruntime/core/graph/graph_flatbuffers_utils.h
View file

@ -2,19 +2,40 @@
// Licensed under the MIT License.
#pragma once
#include <core/graph/graph.h>
namespace ONNX_NAMESPACE {
class TensorProto;
class AttributeProto;
} // namespace ONNX_NAMESPACE
namespace flatbuffers {
class FlatBufferBuilder;
template <typename T>
struct Offset;
} // namespace flatbuffers
namespace onnxruntime {
class Graph;
class Node;
namespace logging {
class Logger;
}
namespace experimental {
namespace fbs {
struct Attribute;
struct Tensor;
} // namespace fbs
namespace utils {
onnxruntime::common::Status SaveValueInfoOrtFormat(
flatbuffers::FlatBufferBuilder& builder, const ONNX_NAMESPACE::ValueInfoProto& value_info_proto,
flatbuffers::Offset<fbs::ValueInfo>& fbs_value_info) ORT_MUST_USE_RESULT;
// TODO, add ORT_MUST_USE_RESULT when it is moved to a different header
onnxruntime::common::Status SaveInitializerOrtFormat(
flatbuffers::FlatBufferBuilder& builder, const ONNX_NAMESPACE::TensorProto& initializer,
flatbuffers::Offset<fbs::Tensor>& fbs_tensor) ORT_MUST_USE_RESULT;
flatbuffers::Offset<fbs::Tensor>& fbs_tensor);
// Convert a given AttributeProto into fbs::Attribute
// Note, we current do not support graphs, and sparse_tensor(s)
@ -22,19 +43,12 @@ onnxruntime::common::Status SaveInitializerOrtFormat(
// instead of the GraphProto in attr_proto
onnxruntime::common::Status SaveAttributeOrtFormat(
flatbuffers::FlatBufferBuilder& builder, const ONNX_NAMESPACE::AttributeProto& attr_proto,
flatbuffers::Offset<fbs::Attribute>& fbs_attr, const onnxruntime::Graph* graph) ORT_MUST_USE_RESULT;
flatbuffers::Offset<fbs::Attribute>& fbs_attr, const onnxruntime::Graph* graph);
#if defined(ENABLE_ORT_FORMAT_LOAD)
inline void LoadStringFromOrtFormat(std::string& dst, const flatbuffers::String* fbs_string) {
if (fbs_string)
dst = fbs_string->c_str();
}
onnxruntime::common::Status LoadInitializerOrtFormat(
const fbs::Tensor& fbs_tensor, ONNX_NAMESPACE::TensorProto& initializer) ORT_MUST_USE_RESULT;
onnxruntime::common::Status LoadValueInfoOrtFormat(
const fbs::ValueInfo& fbs_value_info, ONNX_NAMESPACE::ValueInfoProto& value_info_proto) ORT_MUST_USE_RESULT;
const fbs::Tensor& fbs_tensor, ONNX_NAMESPACE::TensorProto& initializer);
// Load a give fbs::Attribute into AttributeProto
// Note, If the attribute type is a graph, we will leave an empty graph in attr_proto,
@ -43,7 +57,7 @@ onnxruntime::common::Status LoadAttributeOrtFormat(const fbs::Attribute& fbs_att
ONNX_NAMESPACE::AttributeProto& attr_proto,
std::unique_ptr<onnxruntime::Graph>& sub_graph,
Graph& graph, Node& node,
const logging::Logger& logger) ORT_MUST_USE_RESULT;
const logging::Logger& logger);
#endif

26
onnxruntime/core/graph/model.cc
View file

@ -1,12 +1,13 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/framework/tensorprotoutils.h"
#include "core/graph/graph_flatbuffers_utils.h"
#include "core/graph/model.h"
#include "core/graph/model_load_utils.h"
#include <memory>
#include "core/common/logging/logging.h"
#include "core/flatbuffers/schema/ort.fbs.h"
#include "core/flatbuffers/flatbuffers_utils.h"
#include "core/framework/tensorprotoutils.h"
#include "core/graph/model.h"
#include "core/graph/model_load_utils.h"
#ifdef _MSC_VER
#pragma warning(push)
@ -606,22 +607,7 @@ common::Status Model::LoadFromOrtFormat(const fbs::Model& fbs_model,
#endif
std::unordered_map<std::string, int> domain_to_version;
auto fbs_op_set_ids = fbs_model.opset_import();
ORT_RETURN_IF(nullptr == fbs_op_set_ids, "Model must have opset imports. Invalid ORT format model.");
for (const auto* entry : *fbs_op_set_ids) {
const auto* fbs_domain = entry->domain();
ORT_RETURN_IF(nullptr == fbs_domain, "opset import domain is null. Invalid ORT format model.");
std::string domain = fbs_domain->str();
// perform same aliasing that we do when loading an ONNX format model
if (domain == kOnnxDomainAlias) {
domain_to_version[kOnnxDomain] = gsl::narrow_cast<int>(entry->version());
} else {
domain_to_version[domain] = gsl::narrow_cast<int>(entry->version());
}
}
ORT_RETURN_IF_ERROR(experimental::utils::LoadOpsetImportOrtFormat(fbs_model.opset_import(), domain_to_version));
auto fbs_graph = fbs_model.graph();
ORT_RETURN_IF(nullptr == fbs_graph, "Graph is null. Invalid ORT format model.");

13
onnxruntime/core/graph/model.h
View file

@ -8,13 +8,24 @@
#include <climits>
#include <string>
#include "core/common/path.h"
#include "core/flatbuffers/ort.fbs.h"
#include "core/graph/graph_viewer.h"
#include "core/session/onnxruntime_c_api.h"
#include "gsl/gsl"
namespace flatbuffers {
class FlatBufferBuilder;
template <typename T>
struct Offset;
} // namespace flatbuffers
namespace onnxruntime {
namespace experimental {
namespace fbs {
struct Model;
} // namespace fbs
} // namespace experimental
typedef std::unordered_map<std::string, std::string> ModelMetaData;
using IOnnxRuntimeOpSchemaRegistryList = std::list<std::shared_ptr<IOnnxRuntimeOpSchemaCollection>>;

10
onnxruntime/core/session/inference_session.cc
View file

@ -38,7 +38,7 @@
#include "core/platform/threadpool.h"
#include "core/providers/cpu/controlflow/utils.h"
#include "core/providers/cpu/cpu_execution_provider.h"
#include "core/flatbuffers/ort.fbs.h"
#include "core/flatbuffers/flatbuffers_utils.h"
#ifdef USE_DML // TODO: This is necessary for the workaround in TransformGraph
#include "core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h"
#endif
@ -576,7 +576,7 @@ common::Status InferenceSession::Load(const std::string& model_uri) {
bool has_explicit_type = !model_type.empty();
if ((has_explicit_type && model_type == "ORT") ||
(!has_explicit_type && inference_session_utils::IsOrtFormatModel(model_uri))) {
(!has_explicit_type && experimental::utils::IsOrtFormatModel(model_uri))) {
#if defined(ENABLE_ORT_FORMAT_LOAD)
return LoadOrtModel(model_uri);
#else
@ -603,7 +603,7 @@ common::Status InferenceSession::Load(const std::wstring& model_uri) {
bool has_explicit_type = !model_type.empty();
if ((has_explicit_type && model_type == "ORT") ||
(!has_explicit_type && inference_session_utils::IsOrtFormatModel(model_uri))) {
(!has_explicit_type && experimental::utils::IsOrtFormatModel(model_uri))) {
#if defined(ENABLE_ORT_FORMAT_LOAD)
return LoadOrtModel(model_uri);
#else
@ -631,7 +631,7 @@ common::Status InferenceSession::Load(const void* model_data, int model_data_len
if ((has_explicit_type && model_type == "ORT") ||
(!has_explicit_type &&
inference_session_utils::IsOrtFormatModelBytes(model_data, model_data_len))) {
experimental::utils::IsOrtFormatModelBytes(model_data, model_data_len))) {
#if defined(ENABLE_ORT_FORMAT_LOAD)
return LoadOrtModel(model_data, model_data_len);
#else
@ -1146,7 +1146,7 @@ common::Status InferenceSession::Initialize() {
if ((has_explicit_type && model_type == "ORT") ||
(!has_explicit_type &&
inference_session_utils::IsOrtFormatModel(session_options_.optimized_model_filepath))) {
experimental::utils::IsOrtFormatModel(session_options_.optimized_model_filepath))) {
ORT_RETURN_IF_ERROR_SESSIONID_(SaveToOrtFormat(session_options_.optimized_model_filepath));
} else {
ORT_RETURN_IF_ERROR_SESSIONID_(Model::Save(*model_, session_options_.optimized_model_filepath));

19
onnxruntime/core/session/inference_session_utils.h
View file

@ -3,7 +3,7 @@
#pragma once
#include "core/flatbuffers/ort.fbs.h"
#include "core/flatbuffers/schema/ort.fbs.h"
#if !defined(ORT_MINIMAL_BUILD)
//
@ -67,22 +67,5 @@ class JsonConfigParser {
#endif // !defined(ORT_MINIMAL_BUILD)
// check if filename ends in .ort
template <typename T>
bool IsOrtFormatModel(const std::basic_string<T>& filename) {
auto len = filename.size();
return len > 4 &&
filename[len - 4] == '.' &&
std::tolower(filename[len - 3]) == 'o' &&
std::tolower(filename[len - 2]) == 'r' &&
std::tolower(filename[len - 1]) == 't';
}
// check if bytes has the flatbuffer ORT identifier
inline bool IsOrtFormatModelBytes(const void* bytes, int num_bytes) {
return num_bytes > 8 && // check buffer is large enough to contain identifier so we don't read random memory
experimental::fbs::InferenceSessionBufferHasIdentifier(bytes);
}
} // namespace inference_session_utils
} // namespace onnxruntime

2
onnxruntime/test/framework/ort_model_only_test.cc
View file

@ -13,7 +13,7 @@
#include "test_utils.h"
#include "test/util/include/asserts.h"
#include "core/flatbuffers/ort.fbs.h"
#include "core/flatbuffers/schema/ort.fbs.h"
#include "flatbuffers/idl.h"
#include "flatbuffers/util.h"

53
onnxruntime/test/onnx/TestCase.cc
View file

@ -28,7 +28,6 @@
#include <map>
#include <regex>
using namespace onnxruntime;
using namespace onnxruntime::common;
using google::protobuf::RepeatedPtrField;
@ -41,21 +40,20 @@ namespace {
template <typename T>
inline Ort::Value CreateTensorWithDataAsOrtValue(const Ort::MemoryInfo& info,
OrtAllocator*,
const std::vector<int64_t>& dims,
OrtAllocator*,
const std::vector<int64_t>& dims,
std::vector<T>& input) {
return Ort::Value::CreateTensor<T>(static_cast<const OrtMemoryInfo*>(info), input.data(), input.size() * sizeof(T),
dims.data(), dims.size());
return Ort::Value::CreateTensor<T>(static_cast<const OrtMemoryInfo*>(info), input.data(), input.size() * sizeof(T),
dims.data(), dims.size());
}
template<>
template <>
inline Ort::Value CreateTensorWithDataAsOrtValue(const Ort::MemoryInfo&,
OrtAllocator* allocator,
const std::vector<int64_t>& dims,
std::vector<std::string>& input) {
auto tensor_value = Ort::Value::CreateTensor(allocator, dims.data(), dims.size(),
ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING);
auto tensor_value = Ort::Value::CreateTensor(allocator, dims.data(), dims.size(),
ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING);
std::vector<const char*> p_str;
for (const auto& s : input) {
@ -268,9 +266,11 @@ void LoopDataFile(int test_data_pb_fd, bool is_input, const TestModelInfo& model
std::unique_ptr<TestModelInfo> TestModelInfo::LoadOnnxModel(_In_ const PATH_CHAR_TYPE* model_url) {
return std::unique_ptr<TestModelInfo>(new OnnxModelInfo(model_url));
}
#else
#endif
#if defined(ENABLE_ORT_FORMAT_LOAD)
std::unique_ptr<TestModelInfo> TestModelInfo::LoadOrtModel(_In_ const PATH_CHAR_TYPE* model_url) {
return std::unique_ptr<TestModelInfo>(new OrtModelInfo(model_url));
return std::unique_ptr<TestModelInfo>(new OnnxModelInfo(model_url, true));
}
#endif
@ -586,14 +586,20 @@ void LoadTests(const std::vector<std::basic_string<PATH_CHAR_TYPE>>& input_paths
}
std::basic_string<PATH_CHAR_TYPE> filename_str = filename;
bool is_onnx_format = HasExtensionOf(filename_str, ORT_TSTR("onnx"));
bool is_ort_format = HasExtensionOf(filename_str, ORT_TSTR("ort"));
bool is_valid_model = false;
#if !defined(ORT_MINIMAL_BUILD)
if (!HasExtensionOf(filename_str, ORT_TSTR("onnx")))
return true;
#else
if( !HasExtensionOf(filename_str, ORT_TSTR("ort")) )
return true;
is_valid_model = is_onnx_format;
#endif
#if defined(ENABLE_ORT_FORMAT_LOAD)
is_valid_model = is_valid_model || is_ort_format;
#endif
if (!is_valid_model)
return true;
std::basic_string<PATH_CHAR_TYPE> test_case_name = my_dir_name;
if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) test_case_name = test_case_name.substr(5);
@ -606,11 +612,22 @@ void LoadTests(const std::vector<std::basic_string<PATH_CHAR_TYPE>>& input_paths
std::basic_string<PATH_CHAR_TYPE> p = ConcatPathComponent<PATH_CHAR_TYPE>(node_data_root_path, filename_str);
std::unique_ptr<TestModelInfo> model_info;
if (is_onnx_format) {
#if !defined(ORT_MINIMAL_BUILD)
model_info = TestModelInfo::LoadOnnxModel(p.c_str());
model_info = TestModelInfo::LoadOnnxModel(p.c_str());
#else
model_info = TestModelInfo::LoadOrtModel(p.c_str());
ORT_THROW("onnx model is not supported in this build");
#endif
} else if (is_ort_format) {
#if defined(ENABLE_ORT_FORMAT_LOAD)
model_info = TestModelInfo::LoadOrtModel(p.c_str());
#else
ORT_THROW("ort model is not supported in this build");
#endif
} else {
ORT_NOT_IMPLEMENTED(ToMBString(filename_str), " is not supported");
}
std::unique_ptr<ITestCase> l = CreateOnnxTestCase(ToMBString(test_case_name), std::move(model_info),
default_per_sample_tolerance,

5
onnxruntime/test/onnx/TestCase.h
View file

@ -69,9 +69,12 @@ class TestModelInfo {
#if !defined(ORT_MINIMAL_BUILD)
static std::unique_ptr<TestModelInfo> LoadOnnxModel(_In_ const PATH_CHAR_TYPE* model_url);
#else
#endif
#if defined(ENABLE_ORT_FORMAT_LOAD)
static std::unique_ptr<TestModelInfo> LoadOrtModel(_In_ const PATH_CHAR_TYPE* model_url);
#endif
static const std::string unknown_version;
};

109
onnxruntime/test/onnx/onnx_model_info.cc
View file

@ -6,17 +6,36 @@
#include "re2/re2.h"
#include "pb_helper.h"
#if defined(ORT_MINIMAL_BUILD)
#if defined(ENABLE_ORT_FORMAT_LOAD)
#include <fstream>
#include "core/graph/model.h"
#include "core/common/logging/sinks/clog_sink.h"
#include "core/flatbuffers/schema/ort.fbs.h"
#include "core/flatbuffers/flatbuffers_utils.h"
using namespace onnxruntime::experimental;
using namespace onnxruntime::logging;
#endif
using namespace onnxruntime;
OnnxModelInfo::OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url, bool is_ort_model)
: model_url_(model_url) {
if (is_ort_model) {
#if defined(ENABLE_ORT_FORMAT_LOAD)
InitOrtModelInfo(model_url);
#else
ORT_THROW("ort model is not supported in this build");
#endif
} else {
#if !defined(ORT_MINIMAL_BUILD)
InitOnnxModelInfo(model_url);
#else
ORT_THROW("onnx model is not supported in this build");
#endif
}
}
#if !defined(ORT_MINIMAL_BUILD)
static constexpr int protobuf_block_size_in_bytes = 4 * 1024 * 1024;
template <typename T>
static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField<T>& input_value_info,
@ -26,9 +45,7 @@ static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField<
}
}
OnnxModelInfo::OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url)
: BaseModelInfo(model_url) {
// parse model
void OnnxModelInfo::InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { // parse model
int model_fd;
auto st = Env::Default().FileOpenRd(model_url, model_fd);
if (!st.IsOK()) {
@ -81,10 +98,11 @@ OnnxModelInfo::OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url)
RepeatedPtrFieldToVector(graph.output(), output_value_info_);
}
#else
#endif // #if !defined(ORT_MINIMAL_BUILD)
OrtModelInfo::OrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url)
: BaseModelInfo(model_url) {
#if defined(ENABLE_ORT_FORMAT_LOAD)
void OnnxModelInfo::InitOrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url) {
std::vector<uint8_t> bytes;
size_t num_bytes = 0;
const auto model_location = ToWideString(model_url);
@ -93,6 +111,9 @@ OrtModelInfo::OrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url)
std::ifstream bytes_stream(model_location, std::ifstream::in | std::ifstream::binary);
bytes_stream.read(reinterpret_cast<char*>(bytes.data()), num_bytes);
// TODO use ort format version here?
onnx_commit_tag_ = TestModelInfo::unknown_version;
// TODO, verify it is a valid ort format
// TODO, version matches the ORT version
const auto* fbs_session = fbs::GetInferenceSession(bytes.data());
@ -103,34 +124,56 @@ OrtModelInfo::OrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url)
if (nullptr == fbs_model)
ORT_THROW("Missing Model. Invalid ORT format model.");
std::unique_ptr<Model> model;
// create scoped manager so sink gets destroyed once done
{
LoggingManager manager{std::unique_ptr<ISink>{new CLogSink{}}, Severity::kVERBOSE, false,
LoggingManager::InstanceType::Temporal};
const auto* fbs_graph = fbs_model->graph();
if (nullptr == fbs_graph)
ORT_THROW("Missing Graph. Invalid ORT format model.");
auto logger = manager.CreateLogger("CreateOrtModelInfo");
ORT_THROW_IF_ERROR(Model::LoadFromOrtFormat(*fbs_model, *logger, model));
}
// TODO use ort format version here?
onnx_commit_tag_ = TestModelInfo::unknown_version;
Graph& graph = model->MainGraph();
for (const auto entry : graph.DomainToVersionMap()) {
std::unordered_map<std::string, int> _opset_import;
ORT_THROW_IF_ERROR(experimental::utils::LoadOpsetImportOrtFormat(fbs_model->opset_import(), _opset_import));
for (const auto& entry : _opset_import)
domain_to_version_[entry.first] = entry.second;
// Load all node args from fbs_graph
std::unordered_map<std::string, ONNX_NAMESPACE::ValueInfoProto> _node_args;
auto fbs_node_args = fbs_graph->node_args();
if (fbs_node_args) {
_node_args.reserve(fbs_node_args->size());
for (const auto* fbs_value_info : *fbs_node_args) {
if (nullptr == fbs_value_info)
ORT_THROW("NodeArg is missing. Invalid ORT format model.");
ONNX_NAMESPACE::ValueInfoProto node_arg_info;
ORT_THROW_IF_ERROR(experimental::utils::LoadValueInfoOrtFormat(*fbs_value_info, node_arg_info));
// NodeArg ctor is private, cannot use make_unique
_node_args[fbs_value_info->name()->str()] = std::move(node_arg_info);
}
}
if (graph.NumberOfNodes() == 1) {
node_name_ = graph.Nodes().cbegin()->OpType();
}
if (fbs_graph->nodes() && fbs_graph->nodes()->size() == 1) {
const auto* node = fbs_graph->nodes()->Get(0);
if (!node)
ORT_THROW("Missing Node. Invalid ORT format model.");
for (const auto* node_arg : graph.GetInputs()) {
input_value_info_.push_back(node_arg->ToProto());
}
if (!node->op_type())
ORT_THROW("Missing op_type. Invalid ORT format model.");
for (const auto* node_arg : graph.GetOutputs()) {
output_value_info_.push_back(node_arg->ToProto());
node_name_ = node->op_type()->str();
}
// Load input and output node args
auto add_node_args = [&](const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>* fbs_node_args,
std::vector<ONNX_NAMESPACE::ValueInfoProto> node_args) -> Status {
if (fbs_node_args != nullptr) {
node_args.reserve(fbs_node_args->size());
for (const auto* fbs_node_arg_name : *fbs_node_args) {
if (!fbs_node_arg_name)
ORT_THROW("NodeArg Name is missing. Invalid ORT format model.");
node_args.push_back(_node_args.at(fbs_node_arg_name->str()));
}
}
return Status::OK();
};
ORT_THROW_IF_ERROR(add_node_args(fbs_graph->inputs(), input_value_info_));
ORT_THROW_IF_ERROR(add_node_args(fbs_graph->outputs(), output_value_info_));
}
#endif //#if !defined(ORT_MINIMAL_BUILD)
#endif //#if defined(ENABLE_ORT_FORMAT_LOAD)

28
onnxruntime/test/onnx/onnx_model_info.h
View file

@ -5,10 +5,8 @@
#include "core/graph/onnx_protobuf.h"
#include "TestCase.h"
// This is a temporary solution to enable onnx_test_runner for ort minimal build and ort file format
// It is tracked by Product Backlog Item 895235: Re-work onnx_test_runner/perf_test for ort/onnx model
class BaseModelInfo : public TestModelInfo {
protected:
class OnnxModelInfo : public TestModelInfo {
private:
std::string node_name_;
std::string onnx_commit_tag_;
std::vector<ONNX_NAMESPACE::ValueInfoProto> input_value_info_;
@ -16,8 +14,16 @@ class BaseModelInfo : public TestModelInfo {
std::unordered_map<std::string, int64_t> domain_to_version_;
const std::basic_string<PATH_CHAR_TYPE> model_url_;
#if !defined(ORT_MINIMAL_BUILD)
void InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url);
#endif
#if defined(ENABLE_ORT_FORMAT_LOAD)
void InitOrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url);
#endif
public:
BaseModelInfo(_In_ const PATH_CHAR_TYPE* model_url) : model_url_(model_url) {}
OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url, bool is_ort_model = false);
bool HasDomain(const std::string& name) const {
return domain_to_version_.find(name) != domain_to_version_.end();
}
@ -39,15 +45,3 @@ class BaseModelInfo : public TestModelInfo {
const std::string& GetInputName(size_t i) const override { return input_value_info_[i].name(); }
const std::string& GetOutputName(size_t i) const override { return output_value_info_[i].name(); }
};
#if !defined(ORT_MINIMAL_BUILD)
class OnnxModelInfo : public BaseModelInfo {
public:
OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url);
};
#else
class OrtModelInfo : public BaseModelInfo {
public:
OrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url);
};
#endif

4
onnxruntime/test/perftest/performance_runner.cc
View file

@ -237,7 +237,9 @@ static std::unique_ptr<TestModelInfo> CreateModelInfo(const PerformanceTestConfi
if (HasExtensionOf(file_path, ORT_TSTR("onnx"))) {
return TestModelInfo::LoadOnnxModel(performance_test_config_.model_info.model_file_path.c_str());
}
#else
#endif
#if defined(ENABLE_ORT_FORMAT_LOAD)
if (HasExtensionOf(file_path, ORT_TSTR("ort"))) {
return TestModelInfo::LoadOrtModel(performance_test_config_.model_info.model_file_path.c_str());
}

11
orttraining/orttraining/core/graph/optimizer_graph_builder.cc
View file

@ -256,7 +256,7 @@ Status OptimizerGraphBuilder::AddGradientNorm(
"Unsupport gradient type: it has to be either float, MLFloat16 or BFloat16.");
}
for (const auto argdef : grad_argdefs) {
for (const auto& argdef : grad_argdefs) {
ONNX_NAMESPACE::TensorProto_DataType elem_type =
static_cast<ONNX_NAMESPACE::TensorProto_DataType>(argdef.type_proto->tensor_type().elem_type());
if (elem_type != grad_type) {
@ -324,8 +324,7 @@ OptimizerGraphBuilder::OptimizerGraphBuilder(
std::transform(
weight_names_.begin(), weight_names_.end(), std::back_inserter(gradient_names_),
[&weight_names_to_opt_configs](const std::string& weight_name) {
return GradientBuilderBase::GradientName(weight_names_to_opt_configs.at(weight_name).mixed_precision_weight_arg != nullptr ?
weight_names_to_opt_configs.at(weight_name).mixed_precision_weight_arg->Name() : weight_name);
return GradientBuilderBase::GradientName(weight_names_to_opt_configs.at(weight_name).mixed_precision_weight_arg != nullptr ? weight_names_to_opt_configs.at(weight_name).mixed_precision_weight_arg->Name() : weight_name);
});
// add optimizer configurations
@ -412,9 +411,9 @@ Status OptimizerGraphBuilder::BuildInternal(
ORT_RETURN_IF_ERROR(AddGradientNorm(
nodearg_name_generator, gradient_argdefs, graph_defs, global_grad_norm_argdef));
optimizer_graph_outputs[OptimizerOutputKey::GlobalGradientNorm] = global_grad_norm_argdef.name;
ORT_RETURN_IF_ERROR(AddFiniteGradientCheck(
nodearg_name_generator, {global_grad_norm_argdef}, graph_defs, global_grad_norm_finite_argdef));
optimizer_graph_outputs[OptimizerOutputKey::GradientAllIsFinite] = global_grad_norm_finite_argdef.name;
ORT_RETURN_IF_ERROR(AddFiniteGradientCheck(
nodearg_name_generator, {global_grad_norm_argdef}, graph_defs, global_grad_norm_finite_argdef));
optimizer_graph_outputs[OptimizerOutputKey::GradientAllIsFinite] = global_grad_norm_finite_argdef.name;
}
}