From d4d52056be03fc4aa63039ca305d4b3cef24d581 Mon Sep 17 00:00:00 2001 From: gwang-msft <62914304+gwang-msft@users.noreply.github.com> Date: Thu, 20 Aug 2020 18:45:43 -0700 Subject: [PATCH] Flatbuffers schema for serialization of the onnxruntime::model/graph (#4870) * add flatbuffers submodule * test version of flat buffer schema * test version of flat buffer schema * minor updates * add serialization of the value info, group defs in different namespace * update comments * update cgmanifest.json * update namespace, changed typeinfovalue to use union, added root_type and file_identifier * add new container type, add max_node_index to graph * add serializing session state * addressed review comments * minor updates Co-authored-by: gwang0000 <62914304+gwang0000@users.noreply.github.com> --- .gitmodules | 3 + cgmanifests/submodules/cgmanifest.json | 16 +- flatbuffers | 1 + onnxruntime/core/flatbuffers/ort.fbs | 205 +++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 3 deletions(-) create mode 160000 flatbuffers create mode 100644 onnxruntime/core/flatbuffers/ort.fbs diff --git a/.gitmodules b/.gitmodules index 986e2538a0..de7959418f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -62,3 +62,6 @@ [submodule "cmake/external/onnx-tensorrt"] path = cmake/external/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git +[submodule "flatbuffers"] + path = flatbuffers + url = https://github.com/google/flatbuffers.git diff --git a/cgmanifests/submodules/cgmanifest.json b/cgmanifests/submodules/cgmanifest.json index 018b1837fb..de7d36acc8 100644 --- a/cgmanifests/submodules/cgmanifest.json +++ b/cgmanifests/submodules/cgmanifest.json @@ -335,7 +335,7 @@ "component": { "type": "git", "git": { - "commitHash": "0c070abb0c40fec649f81a73a75b0098662ec486", + "commitHash": "a82c6a7010e2e332d8f74ad5b0c726fd47c85376", "repositoryUrl": "https://github.com/onnx/onnx" }, "comments": "git submodule at cmake/external/onnx" @@ -375,8 +375,8 @@ "component": { "type": "git", "git": { - "commitHash": "77a8d522e582efaac3166e912e96215acbf3f129", - "repositoryUrl": "https://github.com/stevenlix/onnx-tensorrt.git" + "commitHash": "088554a5fbee9ba183c05c09c1abe986034e9208", + "repositoryUrl": "https://github.com/onnx/onnx-tensorrt.git" }, "comments": "git submodule at cmake/external/onnx-tensorrt" } @@ -531,6 +531,16 @@ "comments": "git submodule at cmake/external/wil" } }, + { + "component": { + "type": "git", + "git": { + "commitHash": "f3003e08d02ca4810924a53ca463a8b3150926d8", + "repositoryUrl": "https://github.com/google/flatbuffers.git" + }, + "comments": "git submodule at flatbuffers" + } + }, { "component": { "type": "git", diff --git a/flatbuffers b/flatbuffers new file mode 160000 index 0000000000..f3003e08d0 --- /dev/null +++ b/flatbuffers @@ -0,0 +1 @@ +Subproject commit f3003e08d02ca4810924a53ca463a8b3150926d8 diff --git a/onnxruntime/core/flatbuffers/ort.fbs b/onnxruntime/core/flatbuffers/ort.fbs new file mode 100644 index 0000000000..c016ccfede --- /dev/null +++ b/onnxruntime/core/flatbuffers/ort.fbs @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace onnxruntime.experimental.fbs.Attribute; + +enum Type : int { + UNDEFINED = 0, + FLOAT = 1, + INT = 2, + STRING = 3, + TENSOR = 4, + GRAPH = 5, + FLOATS = 6, + INTS = 7, + STRINGS = 8, + TENSORS = 9, + GRAPHS = 10, + SPARSE_TENSOR = 11, + SPARSE_TENSORS = 12, +} + +namespace onnxruntime.experimental.fbs.Tensor.Shape; + +table Info { +dim:[Dimension]; +} + +table Dimension { + value:DimensionValue; + denotation:string; +} + +table DimensionValue { + dim_value:long; + dim_param:string; +} + +namespace onnxruntime.experimental.fbs.Tensor; + +enum DataType : int { + UNDEFINED = 0, + FLOAT = 1, + UINT8 = 2, + INT8 = 3, + UINT16 = 4, + INT16 = 5, + INT32 = 6, + INT64 = 7, + STRING = 8, + BOOL = 9, + FLOAT16 = 10, + DOUBLE = 11, + UINT32 = 12, + UINT64 = 13, + COMPLEX64 = 14, + COMPLEX128 = 15, + BFLOAT16 = 16, +} + +table TypeInfo{ + elem_type:DataType; + shape:Shape.Info; +} + +namespace onnxruntime.experimental.fbs.Graph; + +enum NodeType : int { + Primitive = 0, + Fused = 1, +} + +struct EdgeEnd { + node_index:uint; + src_arg_index:int; + dst_arg_index:int; +} + +table Node { + name:string; + doc_string:string; + domain:string; + since_version:int; + + index:uint; + op_type:string; + type:NodeType; + execution_provider_type:string; + + inputs:[string]; + outputs:[string]; + attributes:[onnxruntime.experimental.fbs.Attribute]; + + input_arg_counts:[int]; + implicit_inputs:[string]; +} + +namespace onnxruntime.experimental.fbs; + +table OperatorSetId { + domain:string; + version:long; +} + +table ValueInfo { + name:string; + doc_string:string; + type:TypeInfo; +} + +// TODO add support of Sequence/Map/SparseTensor/Opaque +union TypeInfoValue { + tensor_type:Tensor.TypeInfo, +} + +table TypeInfo { + denotation:string; + value:TypeInfoValue; +} + +// For simplicity, we will have only two data fields +// - string_data for string +// - raw_data for all other types +table Tensor { + name:string; + doc_string:string; + + dims:[long]; + data_type:Tensor.DataType; + + raw_data:[ubyte]; + + // string_data is least used, leave it at the end + string_data:[string]; +} + +table Attribute{ + name:string; + doc_string:string; + + type:Attribute.Type; + + f:float; + i:long; + s:[ubyte]; + t:Tensor; + g:Graph; + + floats:[float]; + ints:[long]; + strings:[string]; + tensors:[Tensor]; + graphs:[Graph]; +} + +table Graph{ + initializers:[Tensor]; + + node_args:[ValueInfo]; + nodes:[Graph.Node]; + max_node_index:uint; + + input_edges:[Graph.EdgeEnd]; + output_edges:[Graph.EdgeEnd]; + + inputs:[string]; + outputs:[string]; + + outer_scope_node_args:[string]; +} + +table Model { + ir_version:long; + opset_import:OperatorSetId; + producer_name:string; + producer_version:string; + domain:string; + model_version:long; + doc_string:string; + + graph:Graph; +} + +table KernelCreateInfos { + node_indices:[uint]; + kernel_def_hashes:[ulong]; +} + +table SubGraphSessionState { + graph_id:string (key); + session_state:SessionState; +} + +table SessionState { + kernels:KernelCreateInfos; + sub_graph_session_states:[SubGraphSessionState]; +} + +table InferenceSession { + ort_version:string; + model:Model; + session_state:SessionState; +} + +root_type InferenceSession; +file_identifier "ORTM";