Flatbuffers schema for serialization of the onnxruntime::model/graph (#4870)

* add flatbuffers submodule

* test version of flat buffer schema

* test version of flat buffer schema

* minor updates

* add serialization of the value info, group defs in different namespace

* update comments

* update cgmanifest.json

* update namespace, changed typeinfovalue to use union, added root_type and file_identifier

* add new container type, add max_node_index to graph

* add serializing session state

* addressed review comments

* minor updates

Co-authored-by: gwang0000 <62914304+gwang0000@users.noreply.github.com>

.gitmodules

@@ -62,3 +62,6 @@
 [submodule "cmake/external/onnx-tensorrt"]
 	path = cmake/external/onnx-tensorrt
 	url = https://github.com/onnx/onnx-tensorrt.git
+[submodule "flatbuffers"]
+	path = flatbuffers
+	url = https://github.com/google/flatbuffers.git

cgmanifests/submodules/cgmanifest.json

@@ -335,7 +335,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "0c070abb0c40fec649f81a73a75b0098662ec486",
+          "commitHash": "a82c6a7010e2e332d8f74ad5b0c726fd47c85376",
           "repositoryUrl": "https://github.com/onnx/onnx"
         },
         "comments": "git submodule at cmake/external/onnx"
@@ -375,8 +375,8 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "77a8d522e582efaac3166e912e96215acbf3f129",
-          "repositoryUrl": "https://github.com/stevenlix/onnx-tensorrt.git"
+          "commitHash": "088554a5fbee9ba183c05c09c1abe986034e9208",
+          "repositoryUrl": "https://github.com/onnx/onnx-tensorrt.git"
         },
         "comments": "git submodule at cmake/external/onnx-tensorrt"
       }
@@ -531,6 +531,16 @@
         "comments": "git submodule at cmake/external/wil"
       }
     },
+    {
+      "component": {
+        "type": "git",
+        "git": {
+          "commitHash": "f3003e08d02ca4810924a53ca463a8b3150926d8",
+          "repositoryUrl": "https://github.com/google/flatbuffers.git"
+        },
+        "comments": "git submodule at flatbuffers"
+      }
+    },
     {
       "component": {
         "type": "git",

flatbuffers

@@ -0,0 +1 @@
+Subproject commit f3003e08d02ca4810924a53ca463a8b3150926d8

onnxruntime/core/flatbuffers/ort.fbs

@@ -0,0 +1,205 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+namespace onnxruntime.experimental.fbs.Attribute;
+
+enum Type : int {
+  UNDEFINED = 0,
+  FLOAT = 1,
+  INT = 2,
+  STRING = 3,
+  TENSOR = 4,
+  GRAPH = 5,
+  FLOATS = 6,
+  INTS = 7,
+  STRINGS = 8,
+  TENSORS = 9,
+  GRAPHS = 10,
+  SPARSE_TENSOR = 11,
+  SPARSE_TENSORS = 12,
+}
+
+namespace onnxruntime.experimental.fbs.Tensor.Shape;
+
+table Info {
+dim:[Dimension];
+}
+
+table Dimension {
+  value:DimensionValue;
+  denotation:string;
+}
+
+table DimensionValue {
+  dim_value:long;
+  dim_param:string;
+}
+
+namespace onnxruntime.experimental.fbs.Tensor;
+
+enum DataType : int {
+  UNDEFINED = 0,
+  FLOAT = 1,
+  UINT8 = 2,
+  INT8 = 3,
+  UINT16 = 4,
+  INT16 = 5,
+  INT32 = 6,
+  INT64 = 7,
+  STRING = 8,
+  BOOL = 9,
+  FLOAT16 = 10,
+  DOUBLE = 11,
+  UINT32 = 12,
+  UINT64 = 13,
+  COMPLEX64 = 14,
+  COMPLEX128 = 15,
+  BFLOAT16 = 16,
+}
+
+table TypeInfo{
+  elem_type:DataType;
+  shape:Shape.Info;
+}
+
+namespace onnxruntime.experimental.fbs.Graph;
+
+enum NodeType : int {
+  Primitive = 0,
+  Fused = 1,
+}
+
+struct EdgeEnd {
+  node_index:uint;
+  src_arg_index:int;
+  dst_arg_index:int;
+}
+
+table Node {
+  name:string;
+  doc_string:string;
+  domain:string;
+  since_version:int;
+
+  index:uint;
+  op_type:string;
+  type:NodeType;
+  execution_provider_type:string;
+  
+  inputs:[string];
+  outputs:[string];
+  attributes:[onnxruntime.experimental.fbs.Attribute];
+
+  input_arg_counts:[int];
+  implicit_inputs:[string];
+}
+
+namespace onnxruntime.experimental.fbs;
+
+table OperatorSetId {
+  domain:string;
+  version:long;
+}
+
+table ValueInfo {
+  name:string;
+  doc_string:string;
+  type:TypeInfo;
+}
+
+// TODO add support of Sequence/Map/SparseTensor/Opaque
+union TypeInfoValue {
+  tensor_type:Tensor.TypeInfo,
+}
+
+table TypeInfo {
+  denotation:string;
+  value:TypeInfoValue;
+}
+
+// For simplicity, we will have only two data fields
+// - string_data for string
+// - raw_data for all other types
+table Tensor {
+  name:string;
+  doc_string:string;
+  
+  dims:[long];
+  data_type:Tensor.DataType;
+
+  raw_data:[ubyte];
+
+  // string_data is least used, leave it at the end
+  string_data:[string];
+}
+
+table Attribute{
+  name:string;
+  doc_string:string;
+  
+  type:Attribute.Type;
+  
+  f:float;
+  i:long;
+  s:[ubyte];
+  t:Tensor;
+  g:Graph;
+  
+  floats:[float];
+  ints:[long];
+  strings:[string];
+  tensors:[Tensor];
+  graphs:[Graph];
+}
+
+table Graph{
+  initializers:[Tensor];
+
+  node_args:[ValueInfo];
+  nodes:[Graph.Node];
+  max_node_index:uint;
+
+  input_edges:[Graph.EdgeEnd];
+  output_edges:[Graph.EdgeEnd];
+
+  inputs:[string];
+  outputs:[string];
+
+  outer_scope_node_args:[string];
+}
+
+table Model {
+  ir_version:long;
+  opset_import:OperatorSetId;
+  producer_name:string;
+  producer_version:string;
+  domain:string;
+  model_version:long;
+  doc_string:string;
+
+  graph:Graph;
+}
+
+table KernelCreateInfos {
+  node_indices:[uint];
+  kernel_def_hashes:[ulong];
+}
+
+table SubGraphSessionState {
+  graph_id:string (key);
+  session_state:SessionState;
+}
+
+table SessionState {
+  kernels:KernelCreateInfos;
+  sub_graph_session_states:[SubGraphSessionState];
+}
+
+table InferenceSession {
+  ort_version:string;
+  model:Model;
+  session_state:SessionState;
+}
+
+root_type InferenceSession;
+file_identifier "ORTM";