diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index a26cb89536..ccf63e6ae5 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -210,7 +210,7 @@ endif() if(onnxruntime_MINIMAL_BUILD) add_compile_definitions(ORT_MINIMAL_BUILD) add_compile_definitions(ENABLE_ORT_FORMAT_LOAD) - + set(onnxruntime_REDUCED_OPS_BUILD ON) if (NOT onnxruntime_ENABLE_PYTHON) @@ -1248,7 +1248,7 @@ endif() #names in this var must match the directory names under onnxruntime/core/providers set(ONNXRUNTIME_PROVIDER_NAMES cpu) -foreach(target_name onnxruntime_common onnxruntime_graph onnxruntime_framework onnxruntime_util onnxruntime_providers onnxruntime_optimizer onnxruntime_session onnxruntime_mlas) +foreach(target_name onnxruntime_common onnxruntime_graph onnxruntime_framework onnxruntime_util onnxruntime_providers onnxruntime_optimizer onnxruntime_session onnxruntime_mlas onnxruntime_flatbuffers) include(${target_name}.cmake) if (MSVC) target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 5123671744..759a381741 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -116,6 +116,7 @@ target_link_libraries(onnxruntime PRIVATE onnxruntime_graph onnxruntime_common onnxruntime_mlas + onnxruntime_flatbuffers ${onnxruntime_EXTERNAL_LIBRARIES}) if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) diff --git a/cmake/onnxruntime_flatbuffers.cmake b/cmake/onnxruntime_flatbuffers.cmake new file mode 100644 index 0000000000..8d0ab23734 --- /dev/null +++ b/cmake/onnxruntime_flatbuffers.cmake @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +file(GLOB onnxruntime_flatbuffers_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/core/flatbuffers/*.h" + "${ONNXRUNTIME_ROOT}/core/flatbuffers/*.cc" + ) + +source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_flatbuffers_srcs}) + +add_library(onnxruntime_flatbuffers ${onnxruntime_flatbuffers_srcs}) +install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/flatbuffers DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core) +onnxruntime_add_include_to_target(onnxruntime_flatbuffers onnx flatbuffers) +if(onnxruntime_ENABLE_INSTRUMENT) + target_compile_definitions(onnxruntime_flatbuffers PUBLIC ONNXRUNTIME_ENABLE_INSTRUMENT) +endif() +target_include_directories(onnxruntime_flatbuffers PRIVATE ${ONNXRUNTIME_ROOT}) +add_dependencies(onnxruntime_flatbuffers ${onnxruntime_EXTERNAL_DEPENDENCIES}) +set_target_properties(onnxruntime_flatbuffers PROPERTIES FOLDER "ONNXRuntime") diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 76f6273e98..ce1213a1f8 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -107,6 +107,7 @@ set(onnxruntime_pybind11_state_libs onnxruntime_graph onnxruntime_common onnxruntime_mlas + onnxruntime_flatbuffers ${pybind11_lib} ) diff --git a/cmake/onnxruntime_training.cmake b/cmake/onnxruntime_training.cmake index 042a57c1ee..6d3ebd17cb 100644 --- a/cmake/onnxruntime_training.cmake +++ b/cmake/onnxruntime_training.cmake @@ -93,6 +93,7 @@ set(ONNXRUNTIME_LIBS onnxruntime_graph onnxruntime_common onnxruntime_mlas + onnxruntime_flatbuffers ) if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index e2a87e443b..db67b2f927 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -416,6 +416,7 @@ set(ONNXRUNTIME_TEST_LIBS onnxruntime_graph onnxruntime_common onnxruntime_mlas + onnxruntime_flatbuffers ) if (onnxruntime_ENABLE_TRAINING) @@ -760,15 +761,9 @@ if (WIN32) endif() if (onnxruntime_BUILD_SHARED_LIB) - set(onnxruntime_perf_test_libs onnx_test_runner_common onnxruntime_test_utils onnxruntime_common re2::re2 - onnx_test_data_proto onnx_proto ${PROTOBUF_LIB} ${GETOPT_LIB_WIDE} onnxruntime ${SYS_PATH_LIB} - ${CMAKE_DL_LIBS}) - # We want to enable onnx_test_runner/perf_test for ort minimal builds, which will need the following - # ort lib statically linked, this is a temporary solution and is tracked by, - # Product Backlog Item 895235: Re-work onnx_test_runner/perf_test for ort/onnx model - if (onnxruntime_MINIMAL_BUILD) - list(APPEND onnxruntime_perf_test_libs onnxruntime_graph onnx onnxruntime_framework onnxruntime_mlas) - endif() + set(onnxruntime_perf_test_libs onnx_test_runner_common onnxruntime_test_utils onnxruntime_common re2::re2 + onnx_test_data_proto onnx_proto ${PROTOBUF_LIB} ${GETOPT_LIB_WIDE} onnxruntime onnxruntime_flatbuffers + ${SYS_PATH_LIB} ${CMAKE_DL_LIBS}) if(NOT WIN32) list(APPEND onnxruntime_perf_test_libs nsync_cpp) endif() diff --git a/include/onnxruntime/core/graph/graph.h b/include/onnxruntime/core/graph/graph.h index f71c2ace69..6eb39db064 100644 --- a/include/onnxruntime/core/graph/graph.h +++ b/include/onnxruntime/core/graph/graph.h @@ -15,7 +15,6 @@ #include "core/common/path.h" #include "core/common/status.h" #include "core/common/logging/logging.h" -#include "core/flatbuffers/ort.fbs.h" #include "core/graph/basic_types.h" #include "core/graph/constants.h" #include "core/graph/graph_nodes.h" @@ -24,12 +23,26 @@ #include "core/graph/function.h" #include "gsl/gsl" +namespace flatbuffers { +class FlatBufferBuilder; +template +struct Offset; +} // namespace flatbuffers + namespace onnxruntime { class Graph; struct IndexedSubGraph; class Model; class OpSignature; +namespace experimental { +namespace fbs { +struct Graph; +struct Node; +struct NodeEdge; +} // namespace fbs +} // namespace experimental + /** @class Node Class representing a node in the graph. diff --git a/onnxruntime/core/flatbuffers/flatbuffers_utils.cc b/onnxruntime/core/flatbuffers/flatbuffers_utils.cc new file mode 100644 index 0000000000..faf5a5d8cf --- /dev/null +++ b/onnxruntime/core/flatbuffers/flatbuffers_utils.cc @@ -0,0 +1,304 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "flatbuffers_utils.h" +#include "schema/ort.fbs.h" + +#include "core/common/common.h" +#include "core/graph/constants.h" +#include "core/graph/onnx_protobuf.h" +#include "gsl/gsl" + +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::common; +using namespace ::onnxruntime::experimental; + +namespace onnxruntime { +namespace experimental { +namespace utils { + +#if !defined(ORT_MINIMAL_BUILD) +static Status SaveTypeInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder, + const TypeProto& type_proto, + flatbuffers::Offset& fbs_type_info); + +static flatbuffers::Offset SaveTensorDimensionOrtFormat( + flatbuffers::FlatBufferBuilder& builder, + const TensorShapeProto_Dimension& tensor_shape_dim) { + auto denotation = builder.CreateString(tensor_shape_dim.denotation()); + flatbuffers::Offset dim_val; + if (tensor_shape_dim.has_dim_param()) { + dim_val = fbs::CreateDimensionValueDirect(builder, fbs::DimensionValueType::PARAM, 0, tensor_shape_dim.dim_param().c_str()); + } else if (tensor_shape_dim.has_dim_value()) { + dim_val = fbs::CreateDimensionValueDirect(builder, fbs::DimensionValueType::VALUE, tensor_shape_dim.dim_value()); + } else { + dim_val = fbs::CreateDimensionValueDirect(builder); + } + + return fbs::CreateDimension(builder, dim_val, denotation); +} + +static Status SaveTensorShapeOrtFormat(flatbuffers::FlatBufferBuilder& builder, + const TensorShapeProto& tensor_shape_proto, + flatbuffers::Offset& fbs_shape) { + std::vector> dim; + dim.reserve(tensor_shape_proto.dim_size()); + for (const auto& d : tensor_shape_proto.dim()) { + auto fbs_d = SaveTensorDimensionOrtFormat(builder, d); + dim.push_back(fbs_d); + } + fbs_shape = fbs::CreateShapeDirect(builder, &dim); + return Status::OK(); +} + +static Status SaveSequenceTypeOrtFormat(flatbuffers::FlatBufferBuilder& builder, + const TypeProto_Sequence& sequence_type_proto, + flatbuffers::Offset& fbs_sequence_type) { + flatbuffers::Offset fbs_type_info; + ORT_RETURN_IF_ERROR(SaveTypeInfoOrtFormat(builder, sequence_type_proto.elem_type(), fbs_type_info)); + fbs_sequence_type = fbs::CreateSequenceType(builder, fbs_type_info); + return Status::OK(); +} + +static Status SaveMapTypeOrtFormat(flatbuffers::FlatBufferBuilder& builder, + const TypeProto_Map& map_type_proto, + flatbuffers::Offset& fbs_map_type) { + flatbuffers::Offset fbs_type_info; + ORT_RETURN_IF_ERROR(SaveTypeInfoOrtFormat(builder, map_type_proto.value_type(), fbs_type_info)); + fbs_map_type = fbs::CreateMapType( + builder, static_cast(map_type_proto.key_type()), fbs_type_info); + return Status::OK(); +} + +static Status SaveTensorTypeAndShapeOrtFormat(flatbuffers::FlatBufferBuilder& builder, + const TypeProto_Tensor& tensor_type_proto, + flatbuffers::Offset& fbs_tensor_type) { + // A flatbuffers::Offset of 0 means this shape is missing (was null when serializing) + flatbuffers::Offset shape = 0; + if (tensor_type_proto.has_shape()) { + ORT_RETURN_IF_ERROR(SaveTensorShapeOrtFormat(builder, tensor_type_proto.shape(), shape)); + } + + fbs_tensor_type = fbs::CreateTensorTypeAndShape( + builder, static_cast(tensor_type_proto.elem_type()), shape); + + return Status::OK(); +} + +static Status SaveTypeInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder, + const TypeProto& type_proto, + flatbuffers::Offset& fbs_type_info) { + auto denotation = builder.CreateString(type_proto.denotation()); + auto value_type = fbs::TypeInfoValue::tensor_type; + flatbuffers::Offset value; + auto value_case = type_proto.value_case(); + switch (value_case) { + case TypeProto::kTensorType: { + flatbuffers::Offset fbs_tensor_type; + ORT_RETURN_IF_ERROR( + SaveTensorTypeAndShapeOrtFormat(builder, type_proto.tensor_type(), fbs_tensor_type)); + value = fbs_tensor_type.Union(); + } break; + case TypeProto::kSequenceType: { + value_type = fbs::TypeInfoValue::sequence_type; + flatbuffers::Offset fbs_sequence_type; + ORT_RETURN_IF_ERROR( + SaveSequenceTypeOrtFormat(builder, type_proto.sequence_type(), fbs_sequence_type)); + value = fbs_sequence_type.Union(); + } break; + case TypeProto::kMapType: { + value_type = fbs::TypeInfoValue::map_type; + flatbuffers::Offset fbs_map_type; + ORT_RETURN_IF_ERROR( + SaveMapTypeOrtFormat(builder, type_proto.map_type(), fbs_map_type)); + value = fbs_map_type.Union(); + } break; + default: { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "We do not support type [", value_case, "] for now"); + } break; + } + + fbs::TypeInfoBuilder tb(builder); + tb.add_denotation(denotation); + tb.add_value_type(value_type); + tb.add_value(value); + fbs_type_info = tb.Finish(); + return Status::OK(); +} + +Status SaveValueInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder, + const ValueInfoProto& value_info_proto, + flatbuffers::Offset& fbs_value_info) { + auto name = builder.CreateSharedString(value_info_proto.name()); + auto doc_string = builder.CreateString(value_info_proto.doc_string()); + flatbuffers::Offset type_info = 0; // 0 indicates null + if (value_info_proto.has_type()) { + ORT_RETURN_IF_ERROR( + SaveTypeInfoOrtFormat(builder, value_info_proto.type(), type_info)); + } else { + // we have a NodeArg for missing optional values (empty name, no type) so allow for that. + // everything else should have type info + if (!value_info_proto.name().empty()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "SaveValueInfoOrtFormat: value_info_proto for ", value_info_proto.name(), + " is missing type info."); + } + } + + fbs::ValueInfoBuilder vb(builder); + vb.add_name(name); + vb.add_doc_string(doc_string); + vb.add_type(type_info); + fbs_value_info = vb.Finish(); + return Status::OK(); +} + +#endif // #if !defined(ORT_MINIMAL_BUILD) + +#if defined(ENABLE_ORT_FORMAT_LOAD) + +void LoadStringFromOrtFormat(std::string& dst, const flatbuffers::String* fbs_string) { + if (fbs_string) + dst = fbs_string->c_str(); +} + +static Status LoadTypeInfoOrtFormat(const fbs::TypeInfo& fbs_type_info, + TypeProto& type_proto); + +static Status LoadTensorDimensionOrtFormat(const fbs::Dimension& fbs_dim, + TensorShapeProto_Dimension& dim) { + LoadStringFromOrtFormat(*dim.mutable_denotation(), fbs_dim.denotation()); + auto fbs_dim_val = fbs_dim.value(); + if (fbs_dim_val) { + auto type = fbs_dim_val->dim_type(); + if (type == fbs::DimensionValueType::VALUE) + dim.set_dim_value(fbs_dim_val->dim_value()); + else if (type == fbs::DimensionValueType::PARAM) { + auto fbs_dim_param = fbs_dim_val->dim_param(); + ORT_RETURN_IF(nullptr == fbs_dim_param, "dim_param value with no name. Invalid ORT format model."); + dim.set_dim_param(fbs_dim_param->str()); + } else { + // unknown dimension. leave dim in VALUE_NOT_SET state as this is valid + } + } else { + // tensor with unknown shape. + // e.g. output from Reshape node where shape is determined by dynamic input at runtime + } + return Status::OK(); +} + +static Status LoadTensorTypeAndShapeOrtFormat(const fbs::TensorTypeAndShape& fbs_tensor_type, + TypeProto_Tensor& tensor_type_proto) { + tensor_type_proto.set_elem_type(static_cast(fbs_tensor_type.elem_type())); + auto fbs_shape = fbs_tensor_type.shape(); + if (fbs_shape) { + auto fbs_dims = fbs_shape->dim(); + if (fbs_dims) { + auto dims = tensor_type_proto.mutable_shape()->mutable_dim(); + dims->Reserve(fbs_dims->size()); + for (const auto fbs_dim : *fbs_dims) { + ORT_RETURN_IF(nullptr == fbs_dim, "Null entry in dimensions. Invalid ORT format model."); + TensorShapeProto_Dimension dim; + ORT_RETURN_IF_ERROR(LoadTensorDimensionOrtFormat(*fbs_dim, *dims->Add())); + } + } + } + return Status::OK(); +} + +static Status LoadSequenceTypeOrtFormat(const fbs::SequenceType& fbs_sequence_type, + TypeProto_Sequence& sequence_type_proto) { + auto fbs_type_info = fbs_sequence_type.elem_type(); + ORT_RETURN_IF(nullptr == fbs_type_info, "Null value type info in fbs::SequenceType. Invalid ORT format model."); + ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *sequence_type_proto.mutable_elem_type())); + return Status::OK(); +} + +static Status LoadMapTypeOrtFormat(const fbs::MapType& fbs_map_type, + TypeProto_Map& map_type_proto) { + map_type_proto.set_key_type(static_cast(fbs_map_type.key_type())); + auto fbs_type_info = fbs_map_type.value_type(); + ORT_RETURN_IF(nullptr == fbs_type_info, "Null value type info in fbs::MapType. Invalid ORT format model."); + ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *map_type_proto.mutable_value_type())); + return Status::OK(); +} + +static Status LoadTypeInfoOrtFormat(const fbs::TypeInfo& fbs_type_info, + TypeProto& type_proto) { + LoadStringFromOrtFormat(*type_proto.mutable_denotation(), fbs_type_info.denotation()); + auto value_type = fbs_type_info.value_type(); + if (value_type == fbs::TypeInfoValue::tensor_type) { + auto fbs_tensor_type = fbs_type_info.value_as_tensor_type(); + ORT_RETURN_IF(nullptr == fbs_tensor_type, "Null tensor type info. Invalid ORT format model."); + ORT_RETURN_IF_ERROR(LoadTensorTypeAndShapeOrtFormat(*fbs_tensor_type, *type_proto.mutable_tensor_type())); + } else if (value_type == fbs::TypeInfoValue::sequence_type) { + auto fbs_sequence_type = fbs_type_info.value_as_sequence_type(); + ORT_RETURN_IF(nullptr == fbs_sequence_type, "Null sequence type info. Invalid ORT format model."); + ORT_RETURN_IF_ERROR(LoadSequenceTypeOrtFormat(*fbs_sequence_type, *type_proto.mutable_sequence_type())); + } else if (value_type == fbs::TypeInfoValue::map_type) { + auto fbs_map_type = fbs_type_info.value_as_map_type(); + ORT_RETURN_IF(nullptr == fbs_map_type, "Null map type info. Invalid ORT format model."); + ORT_RETURN_IF_ERROR(LoadMapTypeOrtFormat(*fbs_map_type, *type_proto.mutable_map_type())); + } else { + // We do not support SparseTensor and Opaque for now + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Type:", + fbs::EnumNameTypeInfoValue(value_type), " is not supported currently"); + } + + return Status::OK(); +} + +Status LoadValueInfoOrtFormat(const fbs::ValueInfo& fbs_value_info, + ONNX_NAMESPACE::ValueInfoProto& value_info_proto) { + value_info_proto.Clear(); + + LoadStringFromOrtFormat(*value_info_proto.mutable_name(), fbs_value_info.name()); + LoadStringFromOrtFormat(*value_info_proto.mutable_doc_string(), fbs_value_info.doc_string()); + + auto fbs_type_info = fbs_value_info.type(); + if (fbs_type_info == nullptr) { + // there is a NodeArg with empty name for missing optional inputs that can have null type info. + // anything else should have a type + ORT_RETURN_IF(!value_info_proto.name().empty(), + "Null type info for ", value_info_proto.name(), ". Invalid ORT format model."); + } else { + ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *value_info_proto.mutable_type())); + } + + return Status::OK(); +} + +Status LoadOpsetImportOrtFormat(const flatbuffers::Vector>* fbs_op_set_ids, + std::unordered_map& domain_to_version) { + ORT_RETURN_IF(nullptr == fbs_op_set_ids, "Model must have opset imports. Invalid ORT format model."); + + domain_to_version.clear(); + domain_to_version.reserve(fbs_op_set_ids->size()); + for (const auto* fbs_op_set_id : *fbs_op_set_ids) { + ORT_RETURN_IF(nullptr == fbs_op_set_id, "opset id is null. Invalid ORT format model."); + + const auto* fbs_domain = fbs_op_set_id->domain(); + ORT_RETURN_IF(nullptr == fbs_domain, "opset import domain is null. Invalid ORT format model."); + + std::string domain = fbs_domain->str(); + + // perform same aliasing that we do when loading an ONNX format model + if (domain == kOnnxDomainAlias) { + domain_to_version[kOnnxDomain] = gsl::narrow_cast(fbs_op_set_id->version()); + } else { + domain_to_version[domain] = gsl::narrow_cast(fbs_op_set_id->version()); + } + } + return Status::OK(); +} + +#endif // defined(ENABLE_ORT_FORMAT_LOAD) + +bool IsOrtFormatModelBytes(const void* bytes, int num_bytes) { + return num_bytes > 8 && // check buffer is large enough to contain identifier so we don't read random memory + fbs::InferenceSessionBufferHasIdentifier(bytes); +} + +} // namespace utils +} // namespace experimental +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/flatbuffers/flatbuffers_utils.h b/onnxruntime/core/flatbuffers/flatbuffers_utils.h new file mode 100644 index 0000000000..f72a919fd3 --- /dev/null +++ b/onnxruntime/core/flatbuffers/flatbuffers_utils.h @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +namespace ONNX_NAMESPACE { +class ValueInfoProto; +} + +namespace flatbuffers { +class FlatBufferBuilder; + +template +struct Offset; + +struct String; + +template +class Vector; +} // namespace flatbuffers + +namespace onnxruntime { + +namespace experimental { + +namespace fbs { +struct OperatorSetId; +struct ValueInfo; +} // namespace fbs + +namespace utils { + +// TODO, add ORT_MUST_USE_RESULT when it is moved to a different header +onnxruntime::common::Status SaveValueInfoOrtFormat( + flatbuffers::FlatBufferBuilder& builder, const ONNX_NAMESPACE::ValueInfoProto& value_info_proto, + flatbuffers::Offset& fbs_value_info); + +#if defined(ENABLE_ORT_FORMAT_LOAD) + +void LoadStringFromOrtFormat(std::string& dst, const flatbuffers::String* fbs_string); + +onnxruntime::common::Status LoadValueInfoOrtFormat( + const fbs::ValueInfo& fbs_value_info, ONNX_NAMESPACE::ValueInfoProto& value_info_proto); + +onnxruntime::common::Status LoadOpsetImportOrtFormat( + const flatbuffers::Vector>* fbs_op_set_ids, + std::unordered_map& domain_to_version); + +#endif + +// check if filename ends in .ort +template +bool IsOrtFormatModel(const std::basic_string& filename) { + auto len = filename.size(); + return len > 4 && + filename[len - 4] == '.' && + std::tolower(filename[len - 3]) == 'o' && + std::tolower(filename[len - 2]) == 'r' && + std::tolower(filename[len - 1]) == 't'; +} + +// check if bytes has the flatbuffer ORT identifier +bool IsOrtFormatModelBytes(const void* bytes, int num_bytes); + +} // namespace utils +} // namespace experimental +} // namespace onnxruntime diff --git a/onnxruntime/core/flatbuffers/README.md b/onnxruntime/core/flatbuffers/schema/README.md similarity index 100% rename from onnxruntime/core/flatbuffers/README.md rename to onnxruntime/core/flatbuffers/schema/README.md diff --git a/onnxruntime/core/flatbuffers/ort.fbs b/onnxruntime/core/flatbuffers/schema/ort.fbs similarity index 98% rename from onnxruntime/core/flatbuffers/ort.fbs rename to onnxruntime/core/flatbuffers/schema/ort.fbs index 163a0da6df..6e7a41acfe 100644 --- a/onnxruntime/core/flatbuffers/ort.fbs +++ b/onnxruntime/core/flatbuffers/schema/ort.fbs @@ -121,7 +121,7 @@ table ValueInfo { type:TypeInfo; } -// TODO add support of Sequence/Map/SparseTensor/Opaque +// TODO add support of SparseTensor/Opaque union TypeInfoValue { tensor_type:TensorTypeAndShape, sequence_type:SequenceType, diff --git a/onnxruntime/core/flatbuffers/ort.fbs.h b/onnxruntime/core/flatbuffers/schema/ort.fbs.h similarity index 100% rename from onnxruntime/core/flatbuffers/ort.fbs.h rename to onnxruntime/core/flatbuffers/schema/ort.fbs.h diff --git a/onnxruntime/core/framework/session_state.cc b/onnxruntime/core/framework/session_state.cc index edb54925c8..2069987faa 100644 --- a/onnxruntime/core/framework/session_state.cc +++ b/onnxruntime/core/framework/session_state.cc @@ -7,6 +7,7 @@ #include "core/common/logging/logging.h" #include "core/common/safeint.h" +#include "core/flatbuffers/schema/ort.fbs.h" #include "core/framework/allocator.h" #include "core/framework/node_index_info.h" #include "core/framework/op_kernel.h" diff --git a/onnxruntime/core/framework/session_state.h b/onnxruntime/core/framework/session_state.h index f954959329..e7c90e9746 100644 --- a/onnxruntime/core/framework/session_state.h +++ b/onnxruntime/core/framework/session_state.h @@ -13,7 +13,6 @@ #include "core/common/common.h" #include "core/common/logging/logging.h" #include "core/common/profiler.h" -#include "core/flatbuffers/ort.fbs.h" #include "core/framework/allocation_planner.h" #include "core/framework/callback.h" #include "core/framework/data_transfer_manager.h" @@ -33,8 +32,20 @@ #include "core/platform/path_lib.h" #include "core/platform/threadpool.h" +namespace flatbuffers { +class FlatBufferBuilder; +template +struct Offset; +} // namespace flatbuffers + namespace onnxruntime { +namespace experimental { +namespace fbs { +struct SessionState; +} // namespace fbs +} // namespace experimental + class ExecutionProviders; class KernelDef; class OpKernel; diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 65031dfffa..a1c7341113 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -15,10 +15,12 @@ #include "gsl/gsl" #include "core/common/logging/logging.h" +#include "core/flatbuffers/schema/ort.fbs.h" +#include "core/flatbuffers/flatbuffers_utils.h" +#include "core/graph/graph_flatbuffers_utils.h" #include "core/framework/tensor_shape.h" #include "core/framework/tensorprotoutils.h" #include "core/framework/utils.h" -#include "core/graph/graph_flatbuffers_utils.h" #include "core/graph/graph_viewer.h" #include "core/graph/indexed_sub_graph.h" #include "core/graph/model.h" diff --git a/onnxruntime/core/graph/graph_flatbuffers_utils.cc b/onnxruntime/core/graph/graph_flatbuffers_utils.cc index 871432b8c4..bdac2bdce8 100644 --- a/onnxruntime/core/graph/graph_flatbuffers_utils.cc +++ b/onnxruntime/core/graph/graph_flatbuffers_utils.cc @@ -1,8 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "graph_flatbuffers_utils.h" +#include +#include "core/flatbuffers/flatbuffers_utils.h" +#include "core/flatbuffers/schema/ort.fbs.h" #include "core/framework/tensorprotoutils.h" +#include "graph_flatbuffers_utils.h" +#include "flatbuffers/flatbuffers.h" using namespace ONNX_NAMESPACE; using namespace ::onnxruntime::common; @@ -13,140 +17,6 @@ namespace experimental { namespace utils { #if !defined(ORT_MINIMAL_BUILD) -static Status SaveTypeInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder, - const TypeProto& type_proto, - flatbuffers::Offset& fbs_type_info) ORT_MUST_USE_RESULT; - -static flatbuffers::Offset SaveTensorDimensionOrtFormat( - flatbuffers::FlatBufferBuilder& builder, - const TensorShapeProto_Dimension& tensor_shape_dim) { - auto denotation = builder.CreateString(tensor_shape_dim.denotation()); - flatbuffers::Offset dim_val; - if (tensor_shape_dim.has_dim_param()) { - dim_val = fbs::CreateDimensionValueDirect(builder, fbs::DimensionValueType::PARAM, 0, tensor_shape_dim.dim_param().c_str()); - } else if (tensor_shape_dim.has_dim_value()) { - dim_val = fbs::CreateDimensionValueDirect(builder, fbs::DimensionValueType::VALUE, tensor_shape_dim.dim_value()); - } else { - dim_val = fbs::CreateDimensionValueDirect(builder); - } - - return fbs::CreateDimension(builder, dim_val, denotation); -} - -static Status SaveTensorShapeOrtFormat(flatbuffers::FlatBufferBuilder& builder, - const TensorShapeProto& tensor_shape_proto, - flatbuffers::Offset& fbs_shape) { - std::vector> dim; - dim.reserve(tensor_shape_proto.dim_size()); - for (const auto& d : tensor_shape_proto.dim()) { - auto fbs_d = SaveTensorDimensionOrtFormat(builder, d); - dim.push_back(fbs_d); - } - fbs_shape = fbs::CreateShapeDirect(builder, &dim); - return Status::OK(); -} - -static Status SaveSequenceTypeOrtFormat(flatbuffers::FlatBufferBuilder& builder, - const TypeProto_Sequence& sequence_type_proto, - flatbuffers::Offset& fbs_sequence_type) { - flatbuffers::Offset fbs_type_info; - ORT_RETURN_IF_ERROR(SaveTypeInfoOrtFormat(builder, sequence_type_proto.elem_type(), fbs_type_info)); - fbs_sequence_type = fbs::CreateSequenceType(builder, fbs_type_info); - return Status::OK(); -} - -static Status SaveMapTypeOrtFormat(flatbuffers::FlatBufferBuilder& builder, - const TypeProto_Map& map_type_proto, - flatbuffers::Offset& fbs_map_type) { - flatbuffers::Offset fbs_type_info; - ORT_RETURN_IF_ERROR(SaveTypeInfoOrtFormat(builder, map_type_proto.value_type(), fbs_type_info)); - fbs_map_type = fbs::CreateMapType( - builder, static_cast(map_type_proto.key_type()), fbs_type_info); - return Status::OK(); -} - -static Status SaveTensorTypeAndShapeOrtFormat(flatbuffers::FlatBufferBuilder& builder, - const TypeProto_Tensor& tensor_type_proto, - flatbuffers::Offset& fbs_tensor_type) { - // A flatbuffers::Offset of 0 means this shape is missing (was null when serializing) - flatbuffers::Offset shape = 0; - if (tensor_type_proto.has_shape()) { - ORT_RETURN_IF_ERROR(SaveTensorShapeOrtFormat(builder, tensor_type_proto.shape(), shape)); - } - - fbs_tensor_type = fbs::CreateTensorTypeAndShape( - builder, static_cast(tensor_type_proto.elem_type()), shape); - - return Status::OK(); -} - -static Status SaveTypeInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder, - const TypeProto& type_proto, - flatbuffers::Offset& fbs_type_info) { - auto denotation = builder.CreateString(type_proto.denotation()); - auto value_type = fbs::TypeInfoValue::tensor_type; - flatbuffers::Offset value; - auto value_case = type_proto.value_case(); - switch (value_case) { - case TypeProto::kTensorType: { - flatbuffers::Offset fbs_tensor_type; - ORT_RETURN_IF_ERROR( - SaveTensorTypeAndShapeOrtFormat(builder, type_proto.tensor_type(), fbs_tensor_type)); - value = fbs_tensor_type.Union(); - } break; - case TypeProto::kSequenceType: { - value_type = fbs::TypeInfoValue::sequence_type; - flatbuffers::Offset fbs_sequence_type; - ORT_RETURN_IF_ERROR( - SaveSequenceTypeOrtFormat(builder, type_proto.sequence_type(), fbs_sequence_type)); - value = fbs_sequence_type.Union(); - } break; - case TypeProto::kMapType: { - value_type = fbs::TypeInfoValue::map_type; - flatbuffers::Offset fbs_map_type; - ORT_RETURN_IF_ERROR( - SaveMapTypeOrtFormat(builder, type_proto.map_type(), fbs_map_type)); - value = fbs_map_type.Union(); - } break; - default: { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "We do not support type [", value_case, "] for now"); - } break; - } - - fbs::TypeInfoBuilder tb(builder); - tb.add_denotation(denotation); - tb.add_value_type(value_type); - tb.add_value(value); - fbs_type_info = tb.Finish(); - return Status::OK(); -} - -Status SaveValueInfoOrtFormat(flatbuffers::FlatBufferBuilder& builder, - const ValueInfoProto& value_info_proto, - flatbuffers::Offset& fbs_value_info) { - auto name = builder.CreateSharedString(value_info_proto.name()); - auto doc_string = builder.CreateString(value_info_proto.doc_string()); - flatbuffers::Offset type_info = 0; // 0 indicates null - if (value_info_proto.has_type()) { - ORT_RETURN_IF_ERROR( - SaveTypeInfoOrtFormat(builder, value_info_proto.type(), type_info)); - } else { - // we have a NodeArg for missing optional values (empty name, no type) so allow for that. - // everything else should have type info - if (!value_info_proto.name().empty()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "SaveValueInfoOrtFormat: value_info_proto for ", value_info_proto.name(), - " is missing type info."); - } - } - - fbs::ValueInfoBuilder vb(builder); - vb.add_name(name); - vb.add_doc_string(doc_string); - vb.add_type(type_info); - fbs_value_info = vb.Finish(); - return Status::OK(); -} Status SaveInitializerOrtFormat(flatbuffers::FlatBufferBuilder& builder, const TensorProto& initializer, @@ -264,15 +134,12 @@ Status SaveAttributeOrtFormat(flatbuffers::FlatBufferBuilder& builder, return Status::OK(); } -#endif - #undef GET_FBS_ATTR #undef GET_DATA_VEC -#if defined(ENABLE_ORT_FORMAT_LOAD) +#endif -static Status LoadTypeInfoOrtFormat(const fbs::TypeInfo& fbs_type_info, - TypeProto& type_proto) ORT_MUST_USE_RESULT; +#if defined(ENABLE_ORT_FORMAT_LOAD) Status LoadInitializerOrtFormat(const fbs::Tensor& fbs_tensor, TensorProto& initializer) { @@ -306,109 +173,6 @@ Status LoadInitializerOrtFormat(const fbs::Tensor& fbs_tensor, return Status::OK(); } -static Status LoadTensorDimensionOrtFormat(const fbs::Dimension& fbs_dim, - TensorShapeProto_Dimension& dim) { - LoadStringFromOrtFormat(*dim.mutable_denotation(), fbs_dim.denotation()); - auto fbs_dim_val = fbs_dim.value(); - if (fbs_dim_val) { - auto type = fbs_dim_val->dim_type(); - if (type == fbs::DimensionValueType::VALUE) - dim.set_dim_value(fbs_dim_val->dim_value()); - else if (type == fbs::DimensionValueType::PARAM) { - auto fbs_dim_param = fbs_dim_val->dim_param(); - ORT_RETURN_IF(nullptr == fbs_dim_param, "dim_param value with no name. Invalid ORT format model."); - dim.set_dim_param(fbs_dim_param->str()); - } else { - // unknown dimension. leave dim in VALUE_NOT_SET state as this is valid - } - } else { - // tensor with unknown shape. - // e.g. output from Reshape node where shape is determined by dynamic input at runtime - } - return Status::OK(); -} - -static Status LoadTensorTypeAndShapeOrtFormat(const fbs::TensorTypeAndShape& fbs_tensor_type, - TypeProto_Tensor& tensor_type_proto) { - tensor_type_proto.set_elem_type(static_cast(fbs_tensor_type.elem_type())); - auto fbs_shape = fbs_tensor_type.shape(); - if (fbs_shape) { - auto fbs_dims = fbs_shape->dim(); - if (fbs_dims) { - auto dims = tensor_type_proto.mutable_shape()->mutable_dim(); - dims->Reserve(fbs_dims->size()); - for (const auto fbs_dim : *fbs_dims) { - ORT_RETURN_IF(nullptr == fbs_dim, "Null entry in dimensions. Invalid ORT format model."); - TensorShapeProto_Dimension dim; - ORT_RETURN_IF_ERROR(LoadTensorDimensionOrtFormat(*fbs_dim, *dims->Add())); - } - } - } - return Status::OK(); -} - -static Status LoadSequenceTypeOrtFormat(const fbs::SequenceType& fbs_sequence_type, - TypeProto_Sequence& sequence_type_proto) { - auto fbs_type_info = fbs_sequence_type.elem_type(); - ORT_RETURN_IF(nullptr == fbs_type_info, "Null value type info in fbs::SequenceType. Invalid ORT format model."); - ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *sequence_type_proto.mutable_elem_type())); - return Status::OK(); -} - -static Status LoadMapTypeOrtFormat(const fbs::MapType& fbs_map_type, - TypeProto_Map& map_type_proto) { - map_type_proto.set_key_type(static_cast(fbs_map_type.key_type())); - auto fbs_type_info = fbs_map_type.value_type(); - ORT_RETURN_IF(nullptr == fbs_type_info, "Null value type info in fbs::MapType. Invalid ORT format model."); - ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *map_type_proto.mutable_value_type())); - return Status::OK(); -} - -static Status LoadTypeInfoOrtFormat(const fbs::TypeInfo& fbs_type_info, - TypeProto& type_proto) { - LoadStringFromOrtFormat(*type_proto.mutable_denotation(), fbs_type_info.denotation()); - auto value_type = fbs_type_info.value_type(); - if (value_type == fbs::TypeInfoValue::tensor_type) { - auto fbs_tensor_type = fbs_type_info.value_as_tensor_type(); - ORT_RETURN_IF(nullptr == fbs_tensor_type, "Null tensor type info. Invalid ORT format model."); - ORT_RETURN_IF_ERROR(LoadTensorTypeAndShapeOrtFormat(*fbs_tensor_type, *type_proto.mutable_tensor_type())); - } else if (value_type == fbs::TypeInfoValue::sequence_type) { - auto fbs_sequence_type = fbs_type_info.value_as_sequence_type(); - ORT_RETURN_IF(nullptr == fbs_sequence_type, "Null sequence type info. Invalid ORT format model."); - ORT_RETURN_IF_ERROR(LoadSequenceTypeOrtFormat(*fbs_sequence_type, *type_proto.mutable_sequence_type())); - } else if (value_type == fbs::TypeInfoValue::map_type) { - auto fbs_map_type = fbs_type_info.value_as_map_type(); - ORT_RETURN_IF(nullptr == fbs_map_type, "Null map type info. Invalid ORT format model."); - ORT_RETURN_IF_ERROR(LoadMapTypeOrtFormat(*fbs_map_type, *type_proto.mutable_map_type())); - } else { - // We do not support SparseTensor and Opaque for now - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Type:", - fbs::EnumNameTypeInfoValue(value_type), " is not supported currently"); - } - - return Status::OK(); -} - -Status LoadValueInfoOrtFormat(const fbs::ValueInfo& fbs_value_info, - ONNX_NAMESPACE::ValueInfoProto& value_info_proto) { - value_info_proto.Clear(); - - LoadStringFromOrtFormat(*value_info_proto.mutable_name(), fbs_value_info.name()); - LoadStringFromOrtFormat(*value_info_proto.mutable_doc_string(), fbs_value_info.doc_string()); - - auto fbs_type_info = fbs_value_info.type(); - if (fbs_type_info == nullptr) { - // there is a NodeArg with empty name for missing optional inputs that can have null type info. - // anything else should have a type - ORT_RETURN_IF(!value_info_proto.name().empty(), - "Null type info for ", value_info_proto.name(), ". Invalid ORT format model."); - } else { - ORT_RETURN_IF_ERROR(LoadTypeInfoOrtFormat(*fbs_type_info, *value_info_proto.mutable_type())); - } - - return Status::OK(); -} - Status LoadAttributeOrtFormat(const fbs::Attribute& fbs_attr, ONNX_NAMESPACE::AttributeProto& attr_proto, std::unique_ptr& sub_graph, diff --git a/onnxruntime/core/graph/graph_flatbuffers_utils.h b/onnxruntime/core/graph/graph_flatbuffers_utils.h index c938020969..11d81bc134 100644 --- a/onnxruntime/core/graph/graph_flatbuffers_utils.h +++ b/onnxruntime/core/graph/graph_flatbuffers_utils.h @@ -2,19 +2,40 @@ // Licensed under the MIT License. #pragma once -#include + +namespace ONNX_NAMESPACE { +class TensorProto; +class AttributeProto; +} // namespace ONNX_NAMESPACE + +namespace flatbuffers { +class FlatBufferBuilder; +template +struct Offset; +} // namespace flatbuffers namespace onnxruntime { + +class Graph; +class Node; + +namespace logging { +class Logger; +} + namespace experimental { + +namespace fbs { +struct Attribute; +struct Tensor; +} // namespace fbs + namespace utils { -onnxruntime::common::Status SaveValueInfoOrtFormat( - flatbuffers::FlatBufferBuilder& builder, const ONNX_NAMESPACE::ValueInfoProto& value_info_proto, - flatbuffers::Offset& fbs_value_info) ORT_MUST_USE_RESULT; - +// TODO, add ORT_MUST_USE_RESULT when it is moved to a different header onnxruntime::common::Status SaveInitializerOrtFormat( flatbuffers::FlatBufferBuilder& builder, const ONNX_NAMESPACE::TensorProto& initializer, - flatbuffers::Offset& fbs_tensor) ORT_MUST_USE_RESULT; + flatbuffers::Offset& fbs_tensor); // Convert a given AttributeProto into fbs::Attribute // Note, we current do not support graphs, and sparse_tensor(s) @@ -22,19 +43,12 @@ onnxruntime::common::Status SaveInitializerOrtFormat( // instead of the GraphProto in attr_proto onnxruntime::common::Status SaveAttributeOrtFormat( flatbuffers::FlatBufferBuilder& builder, const ONNX_NAMESPACE::AttributeProto& attr_proto, - flatbuffers::Offset& fbs_attr, const onnxruntime::Graph* graph) ORT_MUST_USE_RESULT; + flatbuffers::Offset& fbs_attr, const onnxruntime::Graph* graph); #if defined(ENABLE_ORT_FORMAT_LOAD) -inline void LoadStringFromOrtFormat(std::string& dst, const flatbuffers::String* fbs_string) { - if (fbs_string) - dst = fbs_string->c_str(); -} onnxruntime::common::Status LoadInitializerOrtFormat( - const fbs::Tensor& fbs_tensor, ONNX_NAMESPACE::TensorProto& initializer) ORT_MUST_USE_RESULT; - -onnxruntime::common::Status LoadValueInfoOrtFormat( - const fbs::ValueInfo& fbs_value_info, ONNX_NAMESPACE::ValueInfoProto& value_info_proto) ORT_MUST_USE_RESULT; + const fbs::Tensor& fbs_tensor, ONNX_NAMESPACE::TensorProto& initializer); // Load a give fbs::Attribute into AttributeProto // Note, If the attribute type is a graph, we will leave an empty graph in attr_proto, @@ -43,7 +57,7 @@ onnxruntime::common::Status LoadAttributeOrtFormat(const fbs::Attribute& fbs_att ONNX_NAMESPACE::AttributeProto& attr_proto, std::unique_ptr& sub_graph, Graph& graph, Node& node, - const logging::Logger& logger) ORT_MUST_USE_RESULT; + const logging::Logger& logger); #endif diff --git a/onnxruntime/core/graph/model.cc b/onnxruntime/core/graph/model.cc index 1fea9ba4e8..896e9f371d 100644 --- a/onnxruntime/core/graph/model.cc +++ b/onnxruntime/core/graph/model.cc @@ -1,12 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/framework/tensorprotoutils.h" -#include "core/graph/graph_flatbuffers_utils.h" -#include "core/graph/model.h" -#include "core/graph/model_load_utils.h" #include #include "core/common/logging/logging.h" +#include "core/flatbuffers/schema/ort.fbs.h" +#include "core/flatbuffers/flatbuffers_utils.h" +#include "core/framework/tensorprotoutils.h" +#include "core/graph/model.h" +#include "core/graph/model_load_utils.h" #ifdef _MSC_VER #pragma warning(push) @@ -606,22 +607,7 @@ common::Status Model::LoadFromOrtFormat(const fbs::Model& fbs_model, #endif std::unordered_map domain_to_version; - auto fbs_op_set_ids = fbs_model.opset_import(); - ORT_RETURN_IF(nullptr == fbs_op_set_ids, "Model must have opset imports. Invalid ORT format model."); - - for (const auto* entry : *fbs_op_set_ids) { - const auto* fbs_domain = entry->domain(); - ORT_RETURN_IF(nullptr == fbs_domain, "opset import domain is null. Invalid ORT format model."); - - std::string domain = fbs_domain->str(); - - // perform same aliasing that we do when loading an ONNX format model - if (domain == kOnnxDomainAlias) { - domain_to_version[kOnnxDomain] = gsl::narrow_cast(entry->version()); - } else { - domain_to_version[domain] = gsl::narrow_cast(entry->version()); - } - } + ORT_RETURN_IF_ERROR(experimental::utils::LoadOpsetImportOrtFormat(fbs_model.opset_import(), domain_to_version)); auto fbs_graph = fbs_model.graph(); ORT_RETURN_IF(nullptr == fbs_graph, "Graph is null. Invalid ORT format model."); diff --git a/onnxruntime/core/graph/model.h b/onnxruntime/core/graph/model.h index 9f236e8dce..53d42475c1 100644 --- a/onnxruntime/core/graph/model.h +++ b/onnxruntime/core/graph/model.h @@ -8,13 +8,24 @@ #include #include #include "core/common/path.h" -#include "core/flatbuffers/ort.fbs.h" #include "core/graph/graph_viewer.h" #include "core/session/onnxruntime_c_api.h" #include "gsl/gsl" +namespace flatbuffers { +class FlatBufferBuilder; +template +struct Offset; +} // namespace flatbuffers + namespace onnxruntime { +namespace experimental { +namespace fbs { +struct Model; +} // namespace fbs +} // namespace experimental + typedef std::unordered_map ModelMetaData; using IOnnxRuntimeOpSchemaRegistryList = std::list>; diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 52720e0bbf..d61396694e 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -38,7 +38,7 @@ #include "core/platform/threadpool.h" #include "core/providers/cpu/controlflow/utils.h" #include "core/providers/cpu/cpu_execution_provider.h" -#include "core/flatbuffers/ort.fbs.h" +#include "core/flatbuffers/flatbuffers_utils.h" #ifdef USE_DML // TODO: This is necessary for the workaround in TransformGraph #include "core/providers/dml/DmlExecutionProvider/src/GraphTransformer.h" #endif @@ -576,7 +576,7 @@ common::Status InferenceSession::Load(const std::string& model_uri) { bool has_explicit_type = !model_type.empty(); if ((has_explicit_type && model_type == "ORT") || - (!has_explicit_type && inference_session_utils::IsOrtFormatModel(model_uri))) { + (!has_explicit_type && experimental::utils::IsOrtFormatModel(model_uri))) { #if defined(ENABLE_ORT_FORMAT_LOAD) return LoadOrtModel(model_uri); #else @@ -603,7 +603,7 @@ common::Status InferenceSession::Load(const std::wstring& model_uri) { bool has_explicit_type = !model_type.empty(); if ((has_explicit_type && model_type == "ORT") || - (!has_explicit_type && inference_session_utils::IsOrtFormatModel(model_uri))) { + (!has_explicit_type && experimental::utils::IsOrtFormatModel(model_uri))) { #if defined(ENABLE_ORT_FORMAT_LOAD) return LoadOrtModel(model_uri); #else @@ -631,7 +631,7 @@ common::Status InferenceSession::Load(const void* model_data, int model_data_len if ((has_explicit_type && model_type == "ORT") || (!has_explicit_type && - inference_session_utils::IsOrtFormatModelBytes(model_data, model_data_len))) { + experimental::utils::IsOrtFormatModelBytes(model_data, model_data_len))) { #if defined(ENABLE_ORT_FORMAT_LOAD) return LoadOrtModel(model_data, model_data_len); #else @@ -1146,7 +1146,7 @@ common::Status InferenceSession::Initialize() { if ((has_explicit_type && model_type == "ORT") || (!has_explicit_type && - inference_session_utils::IsOrtFormatModel(session_options_.optimized_model_filepath))) { + experimental::utils::IsOrtFormatModel(session_options_.optimized_model_filepath))) { ORT_RETURN_IF_ERROR_SESSIONID_(SaveToOrtFormat(session_options_.optimized_model_filepath)); } else { ORT_RETURN_IF_ERROR_SESSIONID_(Model::Save(*model_, session_options_.optimized_model_filepath)); diff --git a/onnxruntime/core/session/inference_session_utils.h b/onnxruntime/core/session/inference_session_utils.h index 6d7a1d7da7..5899afac8d 100644 --- a/onnxruntime/core/session/inference_session_utils.h +++ b/onnxruntime/core/session/inference_session_utils.h @@ -3,7 +3,7 @@ #pragma once -#include "core/flatbuffers/ort.fbs.h" +#include "core/flatbuffers/schema/ort.fbs.h" #if !defined(ORT_MINIMAL_BUILD) // @@ -67,22 +67,5 @@ class JsonConfigParser { #endif // !defined(ORT_MINIMAL_BUILD) -// check if filename ends in .ort -template -bool IsOrtFormatModel(const std::basic_string& filename) { - auto len = filename.size(); - return len > 4 && - filename[len - 4] == '.' && - std::tolower(filename[len - 3]) == 'o' && - std::tolower(filename[len - 2]) == 'r' && - std::tolower(filename[len - 1]) == 't'; -} - -// check if bytes has the flatbuffer ORT identifier -inline bool IsOrtFormatModelBytes(const void* bytes, int num_bytes) { - return num_bytes > 8 && // check buffer is large enough to contain identifier so we don't read random memory - experimental::fbs::InferenceSessionBufferHasIdentifier(bytes); -} - } // namespace inference_session_utils } // namespace onnxruntime diff --git a/onnxruntime/test/framework/ort_model_only_test.cc b/onnxruntime/test/framework/ort_model_only_test.cc index c7000cd046..6c9240103c 100644 --- a/onnxruntime/test/framework/ort_model_only_test.cc +++ b/onnxruntime/test/framework/ort_model_only_test.cc @@ -13,7 +13,7 @@ #include "test_utils.h" #include "test/util/include/asserts.h" -#include "core/flatbuffers/ort.fbs.h" +#include "core/flatbuffers/schema/ort.fbs.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 68cc572903..bb26a99363 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -28,7 +28,6 @@ #include #include - using namespace onnxruntime; using namespace onnxruntime::common; using google::protobuf::RepeatedPtrField; @@ -41,21 +40,20 @@ namespace { template inline Ort::Value CreateTensorWithDataAsOrtValue(const Ort::MemoryInfo& info, - OrtAllocator*, - const std::vector& dims, + OrtAllocator*, + const std::vector& dims, std::vector& input) { - return Ort::Value::CreateTensor(static_cast(info), input.data(), input.size() * sizeof(T), - dims.data(), dims.size()); + return Ort::Value::CreateTensor(static_cast(info), input.data(), input.size() * sizeof(T), + dims.data(), dims.size()); } -template<> +template <> inline Ort::Value CreateTensorWithDataAsOrtValue(const Ort::MemoryInfo&, OrtAllocator* allocator, const std::vector& dims, std::vector& input) { - - auto tensor_value = Ort::Value::CreateTensor(allocator, dims.data(), dims.size(), - ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING); + auto tensor_value = Ort::Value::CreateTensor(allocator, dims.data(), dims.size(), + ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING); std::vector p_str; for (const auto& s : input) { @@ -268,9 +266,11 @@ void LoopDataFile(int test_data_pb_fd, bool is_input, const TestModelInfo& model std::unique_ptr TestModelInfo::LoadOnnxModel(_In_ const PATH_CHAR_TYPE* model_url) { return std::unique_ptr(new OnnxModelInfo(model_url)); } -#else +#endif + +#if defined(ENABLE_ORT_FORMAT_LOAD) std::unique_ptr TestModelInfo::LoadOrtModel(_In_ const PATH_CHAR_TYPE* model_url) { - return std::unique_ptr(new OrtModelInfo(model_url)); + return std::unique_ptr(new OnnxModelInfo(model_url, true)); } #endif @@ -586,14 +586,20 @@ void LoadTests(const std::vector>& input_paths } std::basic_string filename_str = filename; + bool is_onnx_format = HasExtensionOf(filename_str, ORT_TSTR("onnx")); + bool is_ort_format = HasExtensionOf(filename_str, ORT_TSTR("ort")); + bool is_valid_model = false; + #if !defined(ORT_MINIMAL_BUILD) - if (!HasExtensionOf(filename_str, ORT_TSTR("onnx"))) - return true; -#else - if( !HasExtensionOf(filename_str, ORT_TSTR("ort")) ) - return true; + is_valid_model = is_onnx_format; #endif +#if defined(ENABLE_ORT_FORMAT_LOAD) + is_valid_model = is_valid_model || is_ort_format; +#endif + if (!is_valid_model) + return true; + std::basic_string test_case_name = my_dir_name; if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) test_case_name = test_case_name.substr(5); @@ -606,11 +612,22 @@ void LoadTests(const std::vector>& input_paths std::basic_string p = ConcatPathComponent(node_data_root_path, filename_str); std::unique_ptr model_info; + + if (is_onnx_format) { #if !defined(ORT_MINIMAL_BUILD) - model_info = TestModelInfo::LoadOnnxModel(p.c_str()); + model_info = TestModelInfo::LoadOnnxModel(p.c_str()); #else - model_info = TestModelInfo::LoadOrtModel(p.c_str()); + ORT_THROW("onnx model is not supported in this build"); #endif + } else if (is_ort_format) { +#if defined(ENABLE_ORT_FORMAT_LOAD) + model_info = TestModelInfo::LoadOrtModel(p.c_str()); +#else + ORT_THROW("ort model is not supported in this build"); +#endif + } else { + ORT_NOT_IMPLEMENTED(ToMBString(filename_str), " is not supported"); + } std::unique_ptr l = CreateOnnxTestCase(ToMBString(test_case_name), std::move(model_info), default_per_sample_tolerance, diff --git a/onnxruntime/test/onnx/TestCase.h b/onnxruntime/test/onnx/TestCase.h index 0432c5ab3b..3cfc979438 100644 --- a/onnxruntime/test/onnx/TestCase.h +++ b/onnxruntime/test/onnx/TestCase.h @@ -69,9 +69,12 @@ class TestModelInfo { #if !defined(ORT_MINIMAL_BUILD) static std::unique_ptr LoadOnnxModel(_In_ const PATH_CHAR_TYPE* model_url); -#else +#endif + +#if defined(ENABLE_ORT_FORMAT_LOAD) static std::unique_ptr LoadOrtModel(_In_ const PATH_CHAR_TYPE* model_url); #endif + static const std::string unknown_version; }; diff --git a/onnxruntime/test/onnx/onnx_model_info.cc b/onnxruntime/test/onnx/onnx_model_info.cc index d97995f614..600978e0ca 100644 --- a/onnxruntime/test/onnx/onnx_model_info.cc +++ b/onnxruntime/test/onnx/onnx_model_info.cc @@ -6,17 +6,36 @@ #include "re2/re2.h" #include "pb_helper.h" -#if defined(ORT_MINIMAL_BUILD) +#if defined(ENABLE_ORT_FORMAT_LOAD) + #include -#include "core/graph/model.h" -#include "core/common/logging/sinks/clog_sink.h" +#include "core/flatbuffers/schema/ort.fbs.h" +#include "core/flatbuffers/flatbuffers_utils.h" using namespace onnxruntime::experimental; -using namespace onnxruntime::logging; + #endif using namespace onnxruntime; +OnnxModelInfo::OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url, bool is_ort_model) + : model_url_(model_url) { + if (is_ort_model) { +#if defined(ENABLE_ORT_FORMAT_LOAD) + InitOrtModelInfo(model_url); +#else + ORT_THROW("ort model is not supported in this build"); +#endif + } else { #if !defined(ORT_MINIMAL_BUILD) + InitOnnxModelInfo(model_url); +#else + ORT_THROW("onnx model is not supported in this build"); +#endif + } +} + +#if !defined(ORT_MINIMAL_BUILD) + static constexpr int protobuf_block_size_in_bytes = 4 * 1024 * 1024; template static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField& input_value_info, @@ -26,9 +45,7 @@ static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField< } } -OnnxModelInfo::OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) - : BaseModelInfo(model_url) { - // parse model +void OnnxModelInfo::InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { // parse model int model_fd; auto st = Env::Default().FileOpenRd(model_url, model_fd); if (!st.IsOK()) { @@ -81,10 +98,11 @@ OnnxModelInfo::OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) RepeatedPtrFieldToVector(graph.output(), output_value_info_); } -#else +#endif // #if !defined(ORT_MINIMAL_BUILD) -OrtModelInfo::OrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url) - : BaseModelInfo(model_url) { +#if defined(ENABLE_ORT_FORMAT_LOAD) + +void OnnxModelInfo::InitOrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { std::vector bytes; size_t num_bytes = 0; const auto model_location = ToWideString(model_url); @@ -93,6 +111,9 @@ OrtModelInfo::OrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url) std::ifstream bytes_stream(model_location, std::ifstream::in | std::ifstream::binary); bytes_stream.read(reinterpret_cast(bytes.data()), num_bytes); + // TODO use ort format version here? + onnx_commit_tag_ = TestModelInfo::unknown_version; + // TODO, verify it is a valid ort format // TODO, version matches the ORT version const auto* fbs_session = fbs::GetInferenceSession(bytes.data()); @@ -103,34 +124,56 @@ OrtModelInfo::OrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url) if (nullptr == fbs_model) ORT_THROW("Missing Model. Invalid ORT format model."); - std::unique_ptr model; - // create scoped manager so sink gets destroyed once done - { - LoggingManager manager{std::unique_ptr{new CLogSink{}}, Severity::kVERBOSE, false, - LoggingManager::InstanceType::Temporal}; + const auto* fbs_graph = fbs_model->graph(); + if (nullptr == fbs_graph) + ORT_THROW("Missing Graph. Invalid ORT format model."); - auto logger = manager.CreateLogger("CreateOrtModelInfo"); - ORT_THROW_IF_ERROR(Model::LoadFromOrtFormat(*fbs_model, *logger, model)); - } - - // TODO use ort format version here? - onnx_commit_tag_ = TestModelInfo::unknown_version; - - Graph& graph = model->MainGraph(); - for (const auto entry : graph.DomainToVersionMap()) { + std::unordered_map _opset_import; + ORT_THROW_IF_ERROR(experimental::utils::LoadOpsetImportOrtFormat(fbs_model->opset_import(), _opset_import)); + for (const auto& entry : _opset_import) domain_to_version_[entry.first] = entry.second; + + // Load all node args from fbs_graph + std::unordered_map _node_args; + auto fbs_node_args = fbs_graph->node_args(); + if (fbs_node_args) { + _node_args.reserve(fbs_node_args->size()); + for (const auto* fbs_value_info : *fbs_node_args) { + if (nullptr == fbs_value_info) + ORT_THROW("NodeArg is missing. Invalid ORT format model."); + ONNX_NAMESPACE::ValueInfoProto node_arg_info; + ORT_THROW_IF_ERROR(experimental::utils::LoadValueInfoOrtFormat(*fbs_value_info, node_arg_info)); + // NodeArg ctor is private, cannot use make_unique + _node_args[fbs_value_info->name()->str()] = std::move(node_arg_info); + } } - if (graph.NumberOfNodes() == 1) { - node_name_ = graph.Nodes().cbegin()->OpType(); - } + if (fbs_graph->nodes() && fbs_graph->nodes()->size() == 1) { + const auto* node = fbs_graph->nodes()->Get(0); + if (!node) + ORT_THROW("Missing Node. Invalid ORT format model."); - for (const auto* node_arg : graph.GetInputs()) { - input_value_info_.push_back(node_arg->ToProto()); - } + if (!node->op_type()) + ORT_THROW("Missing op_type. Invalid ORT format model."); - for (const auto* node_arg : graph.GetOutputs()) { - output_value_info_.push_back(node_arg->ToProto()); + node_name_ = node->op_type()->str(); } + // Load input and output node args + auto add_node_args = [&](const flatbuffers::Vector>* fbs_node_args, + std::vector node_args) -> Status { + if (fbs_node_args != nullptr) { + node_args.reserve(fbs_node_args->size()); + for (const auto* fbs_node_arg_name : *fbs_node_args) { + if (!fbs_node_arg_name) + ORT_THROW("NodeArg Name is missing. Invalid ORT format model."); + node_args.push_back(_node_args.at(fbs_node_arg_name->str())); + } + } + return Status::OK(); + }; + + ORT_THROW_IF_ERROR(add_node_args(fbs_graph->inputs(), input_value_info_)); + ORT_THROW_IF_ERROR(add_node_args(fbs_graph->outputs(), output_value_info_)); } -#endif //#if !defined(ORT_MINIMAL_BUILD) + +#endif //#if defined(ENABLE_ORT_FORMAT_LOAD) diff --git a/onnxruntime/test/onnx/onnx_model_info.h b/onnxruntime/test/onnx/onnx_model_info.h index 3d9ba1ca2f..a8a048a7e7 100644 --- a/onnxruntime/test/onnx/onnx_model_info.h +++ b/onnxruntime/test/onnx/onnx_model_info.h @@ -5,10 +5,8 @@ #include "core/graph/onnx_protobuf.h" #include "TestCase.h" -// This is a temporary solution to enable onnx_test_runner for ort minimal build and ort file format -// It is tracked by Product Backlog Item 895235: Re-work onnx_test_runner/perf_test for ort/onnx model -class BaseModelInfo : public TestModelInfo { - protected: +class OnnxModelInfo : public TestModelInfo { + private: std::string node_name_; std::string onnx_commit_tag_; std::vector input_value_info_; @@ -16,8 +14,16 @@ class BaseModelInfo : public TestModelInfo { std::unordered_map domain_to_version_; const std::basic_string model_url_; +#if !defined(ORT_MINIMAL_BUILD) + void InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url); +#endif + +#if defined(ENABLE_ORT_FORMAT_LOAD) + void InitOrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url); +#endif + public: - BaseModelInfo(_In_ const PATH_CHAR_TYPE* model_url) : model_url_(model_url) {} + OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url, bool is_ort_model = false); bool HasDomain(const std::string& name) const { return domain_to_version_.find(name) != domain_to_version_.end(); } @@ -39,15 +45,3 @@ class BaseModelInfo : public TestModelInfo { const std::string& GetInputName(size_t i) const override { return input_value_info_[i].name(); } const std::string& GetOutputName(size_t i) const override { return output_value_info_[i].name(); } }; - -#if !defined(ORT_MINIMAL_BUILD) -class OnnxModelInfo : public BaseModelInfo { - public: - OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url); -}; -#else -class OrtModelInfo : public BaseModelInfo { - public: - OrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url); -}; -#endif diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index c25fd2aa77..d94bf40929 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -237,7 +237,9 @@ static std::unique_ptr CreateModelInfo(const PerformanceTestConfi if (HasExtensionOf(file_path, ORT_TSTR("onnx"))) { return TestModelInfo::LoadOnnxModel(performance_test_config_.model_info.model_file_path.c_str()); } -#else +#endif + +#if defined(ENABLE_ORT_FORMAT_LOAD) if (HasExtensionOf(file_path, ORT_TSTR("ort"))) { return TestModelInfo::LoadOrtModel(performance_test_config_.model_info.model_file_path.c_str()); } diff --git a/orttraining/orttraining/core/graph/optimizer_graph_builder.cc b/orttraining/orttraining/core/graph/optimizer_graph_builder.cc index 32d6564e0c..0e8f4e7219 100644 --- a/orttraining/orttraining/core/graph/optimizer_graph_builder.cc +++ b/orttraining/orttraining/core/graph/optimizer_graph_builder.cc @@ -256,7 +256,7 @@ Status OptimizerGraphBuilder::AddGradientNorm( "Unsupport gradient type: it has to be either float, MLFloat16 or BFloat16."); } - for (const auto argdef : grad_argdefs) { + for (const auto& argdef : grad_argdefs) { ONNX_NAMESPACE::TensorProto_DataType elem_type = static_cast(argdef.type_proto->tensor_type().elem_type()); if (elem_type != grad_type) { @@ -324,8 +324,7 @@ OptimizerGraphBuilder::OptimizerGraphBuilder( std::transform( weight_names_.begin(), weight_names_.end(), std::back_inserter(gradient_names_), [&weight_names_to_opt_configs](const std::string& weight_name) { - return GradientBuilderBase::GradientName(weight_names_to_opt_configs.at(weight_name).mixed_precision_weight_arg != nullptr ? - weight_names_to_opt_configs.at(weight_name).mixed_precision_weight_arg->Name() : weight_name); + return GradientBuilderBase::GradientName(weight_names_to_opt_configs.at(weight_name).mixed_precision_weight_arg != nullptr ? weight_names_to_opt_configs.at(weight_name).mixed_precision_weight_arg->Name() : weight_name); }); // add optimizer configurations @@ -412,9 +411,9 @@ Status OptimizerGraphBuilder::BuildInternal( ORT_RETURN_IF_ERROR(AddGradientNorm( nodearg_name_generator, gradient_argdefs, graph_defs, global_grad_norm_argdef)); optimizer_graph_outputs[OptimizerOutputKey::GlobalGradientNorm] = global_grad_norm_argdef.name; - ORT_RETURN_IF_ERROR(AddFiniteGradientCheck( - nodearg_name_generator, {global_grad_norm_argdef}, graph_defs, global_grad_norm_finite_argdef)); - optimizer_graph_outputs[OptimizerOutputKey::GradientAllIsFinite] = global_grad_norm_finite_argdef.name; + ORT_RETURN_IF_ERROR(AddFiniteGradientCheck( + nodearg_name_generator, {global_grad_norm_argdef}, graph_defs, global_grad_norm_finite_argdef)); + optimizer_graph_outputs[OptimizerOutputKey::GradientAllIsFinite] = global_grad_norm_finite_argdef.name; } }