From 2c46fff69aa3748095ec9654e7a75d701136422e Mon Sep 17 00:00:00 2001 From: shahasad <43590019+shahasad@users.noreply.github.com> Date: Wed, 1 May 2019 14:58:21 -0700 Subject: [PATCH] Enable gen-doc on windows CI (#716) * add --gen_doc to ci_build * make gen-doc conditional to build/test step * some fix in the git diff check * some more trick on doc diff * updated for input/output * updated the contrib operator doc * fix on missing input output descriptions * fixed the problem of missing doc string, due to protobuf optimization * fix * revert last change * moved gen_doc.py to /tools/python * fixed typo --- docs/ContribOperators.md | 717 ++++-------------- tools/ci_build/build.py | 18 +- .../azure-pipelines/win-ci-pipeline.yml | 4 +- .../python/tools => tools/python}/gen_doc.py | 45 +- 4 files changed, 183 insertions(+), 601 deletions(-) rename {onnxruntime/python/tools => tools/python}/gen_doc.py (94%) diff --git a/docs/ContribOperators.md b/docs/ContribOperators.md index 4f6aac66e5..9f33e67cf5 100644 --- a/docs/ContribOperators.md +++ b/docs/ContribOperators.md @@ -5,20 +5,12 @@ * com.microsoft * com.microsoft.AttnLSTM - * com.microsoft.ConvInteger - * com.microsoft.DequantizeLinear * com.microsoft.ExpandDims * com.microsoft.FusedConv * com.microsoft.FusedGemm * com.microsoft.GatherND - * com.microsoft.MatMulInteger * com.microsoft.MaxpoolWithMask * com.microsoft.MurmurHash3 - * com.microsoft.NonMaxSuppression - * com.microsoft.QLinearConv - * com.microsoft.QLinearMatMul - * com.microsoft.QuantizeLinear - * com.microsoft.ROIAlign * com.microsoft.Range * com.microsoft.ReduceSumInteger * com.microsoft.SampleOp @@ -184,158 +176,54 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs (3 - 14)
-
:
-
-
:
-
-
:
-
-
(optional) :
-
-
(optional) :
-
-
(optional) :
-
-
(optional) :
-
-
(optional) :
-
-
(optional) :
-
-
(optional) :
-
-
(optional) :
-
-
(optional) :
-
-
(optional) :
-
-
(optional) :
-
+
X : T
+
The input sequences packed (and potentially padded) into one 3-D tensor with the shape of `[seq_length, batch_size, input_size]`
+
W : T
+
The weight tensor for the gates. Concatenation of `W[iofc]` and `WB[iofc]` (if bidirectional) along dimension 0. The tensor has shape `[num_directions, 4*hidden_size, input_size]`.
+
R : T
+
The recurrence weight tensor. Concatenation of `R[iofc]` and `RB[iofc]` (if bidirectional) along dimension 0. This tensor has shape `[num_directions, 4*hidden_size, hidden_size]`.
+
B (optional) : T
+
The bias tensor for input gate. Concatenation of `[Wb[iofc], Rb[iofc]]`, and `[WBb[iofc], RBb[iofc]]` (if bidirectional) along dimension 0. This tensor has shape `[num_directions, 8*hidden_size]`. Optional: If not specified - assumed to be 0.
+
sequence_lens (optional) : T1
+
Optional tensor specifying lengths of the sequences in a batch. If not specified - assumed all sequences in the batch to have length `seq_length`. It has shape `[batch_size]`
+
initial_h (optional) : T
+
Optional initial value of the hidden. If not specified - assumed to be 0. It has shape `[num_directions, batch_size, hidden_size]`.
+
initial_c (optional) : T
+
Optional initial value of the cell. If not specified - assumed to be 0. It has shape `[num_directions, batch_size, hidden_size]`.
+
P (optional) : T
+
The weight tensor for peepholes. Concatenation of `P[iof]` and `PB[iof]` (if bidirectional) along dimension 0. It has shape `[num_directions, 3*hidde_size]`. Optional: If not specified - assumed to be 0.
+
QW (optional) : T
+
The weight tensor of the query layer in the attention mechanism. Should be of shape `[num_directions, am_query_depth(hidden_size of lstm), am_attn_size]`
+
MW (optional) : T
+
The weight tensor of the memory layer in the attention mechanism. Should be of shape `[num_directions, memory_depth, am_attn_size]`
+
V (optional) : T
+
The attention_v tensor in the attention mechanism. Should be of shape `[num_directions, am_attn_size]`
+
M (optional) : T
+
The sequence of the memory (input) for attention mechanism. Should be of `[batch_size, max_memory_step, memory_depth]`
+
memory_seq_lens (optional) : T1
+
The sequence length of the input memory for the attention mechanism. Should be of `[batch_size]`
+
AW (optional) : T
+
The weights of attention layer in the attention wrapper. If exists, should be of shape `[num_directions, memory_depth+hidden_size, aw_attn_size]. Please note that attention mechanism context depth is also memory_depth in the attention mechanism.`
#### Outputs (0 - 3)
-
(optional) :
-
-
(optional) :
-
-
(optional) :
-
+
Y (optional) : T
+
A tensor that concats all the intermediate output values of the hidden. It has shape `[seq_length, num_directions, batch_size, hidden_size]`
+
Y_h (optional) : T
+
The last output value of the hidden. It has shape `[num_directions, batch_size, hidden_size]`.
+
Y_c (optional) : T
+
The last output value of the cell. It has shape `[num_directions, batch_size, hidden_size]`.
#### Type Constraints
-
:
-
-
:
-
-
- - -### **com.microsoft.ConvInteger** - - The integer convolution operator consumes an input tensor, a filter, and a padding value, - and computes the output. The production MUST never overflow. The accumulation may overflow - if and only if in 32 bits. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft' operator set. - -#### Attributes - -
-
auto_pad : string
-
auto_pad must be either NOTSET, SAME_UPPER, SAME_LOWER or VALID. Where default value is NOTSET, which means explicit padding is used. SAME_UPPER or SAME_LOWER mean pad the input so that the output size match the input.In case of odd number add the extra padding at the end for SAME_UPPER and at the beginning for SAME_LOWER. VALID mean no padding.
-
dilations : list of ints
-
dilation value along each axis of the filter. If not present, the dilation defaults to 1 along each axis.
-
group : int
-
number of groups input channels and output channels are divided into. default is 1.
-
kernel_shape : list of ints
-
The shape of the convolution kernel. If not present, should be inferred from input 'w'.
-
pads : list of ints
-
Padding for the beginning and ending along each axis, it can take any value greater than or equal to 0.The value represent the number of pixels added to the beginning and end part of the corresponding axis.`pads` format should be as follow [x1_begin, x2_begin...x1_end, x2_end,...], where xi_begin the number ofpixels added at the beginning of axis `i` and xi_end, the number of pixels added at the end of axis `i`.This attribute cannot be used simultaneously with auto_pad attribute. If not present, the padding defaultsto 0 along start and end of each axis.
-
strides : list of ints
-
Stride along each axis. If not present, the stride defaults to 1 along each axis.
-
- -#### Inputs (2 - 4) - -
-
:
-
-
:
-
-
(optional) :
-
-
(optional) :
-
-
- -#### Outputs - -
-
:
-
-
- -#### Type Constraints - -
-
:
-
-
:
-
-
:
-
-
- - -### **com.microsoft.DequantizeLinear** - - The linear de-quantization operator. It consumes a quantized data, a scale, a zero point and computes the full precision data. - The dequantization formula is y = (x - x_zero_point) * x_scale. - Scale and zero point must have same shape. They must be either scalar (per tensor) or 1-D tensor (per 'axis'). - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft' operator set. - -#### Attributes - -
-
axis : int
-
the axis along which same quantization parameters are applied. It's optional. If it's not specified, it means per-tensor quantization and input 'x_scale' and 'x_zero_point' must be scalars. If it's specified, it means per 'axis' quantization and input 'x_scale' and 'x_zero_point' must be 1-D tensors.
-
- -#### Inputs - -
-
:
-
-
:
-
-
:
-
-
- -#### Outputs - -
-
:
-
-
- -#### Type Constraints - -
-
:
-
-
:
-
+
T : tensor(float), tensor(double)
+
Constrain input and output types to float tensors.
+
T1 : tensor(int32)
+
Constrain seq_lens to integral tensors.
@@ -350,24 +238,24 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs
-
:
-
-
:
-
+
X : T
+
input
+
axis : tensor(int32)
+
Specified axis to insert a dimension
#### Outputs
-
:
-
+
Y : T
+
output
#### Type Constraints
-
:
-
+
T : tensor(uint8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(int8), tensor(int16), tensor(int32), tensor(int64), tensor(float16), tensor(float), tensor(double), tensor(string), tensor(bool), tensor(complex64), tensor(complex128)
+
Constrain to any tensor type. If the dtype attribute is not provided this must be a valid output type.
@@ -404,26 +292,26 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs (2 - 3)
-
:
+
X : T
-
:
+
W : T
-
(optional) :
+
B (optional) : T
#### Outputs
-
:
+
Y : T
#### Type Constraints
-
:
-
+
T : tensor(float16), tensor(float), tensor(double)
+
Constrain input and output types to float tensors
@@ -456,26 +344,26 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs
-
:
-
-
:
-
-
:
-
+
A : T
+
Input tensor A. The shape of A should be (M, K) if transA is 0, or (K, M) if transA is non-zero.
+
B : T
+
Input tensor B. The shape of B should be (K, N) if transB is 0, or (N, K) if transB is non-zero.
+
C : T
+
Input tensor C. The shape of C should be unidirectional broadcastable to (M, N).
#### Outputs
-
:
-
+
Y : T
+
Output tensor of shape (M, N).
#### Type Constraints
-
:
-
+
T : tensor(float16), tensor(float), tensor(double), tensor(uint32), tensor(uint64), tensor(int32), tensor(int64)
+
Constrain input and output types to float/int tensors.
@@ -507,67 +395,26 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs
-
:
-
-
:
-
+
data : T
+
Tensor of rank r >= 1.
+
indices : Tind
+
Tensor of rank q >= 1.
#### Outputs
-
:
-
+
output : T
+
Tensor of rank q-1+r-indices[-1].
#### Type Constraints
-
:
-
-
:
-
-
- - -### **com.microsoft.MatMulInteger** - - Matrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html. - The production MUST never overflow. The accumulation may overflow if and only if in 32 bits. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft' operator set. - -#### Inputs (2 - 4) - -
-
:
-
-
:
-
-
(optional) :
-
-
(optional) :
-
-
- -#### Outputs - -
-
:
-
-
- -#### Type Constraints - -
-
:
-
-
:
-
-
:
-
+
T : tensor(uint8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(int8), tensor(int16), tensor(int32), tensor(int64), tensor(float16), tensor(float), tensor(double), tensor(string), tensor(bool), tensor(complex64), tensor(complex128)
+
Constrain input and output types to any tensor type.
+
Tind : tensor(int32), tensor(int64)
+
Constrain indice type to int32 or int64
@@ -597,24 +444,24 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs
-
:
-
-
:
+
X : T
+
M : tensor(int32)
+
mask
#### Outputs
-
:
+
Y : T
#### Type Constraints
-
:
-
+
T : tensor(float)
+
Constrain input0 and output types to float tensors
@@ -638,313 +485,24 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs
-
:
-
+
X : T1
+
An input tensor to hash.
#### Outputs
-
:
-
+
Y : T2
+
32-bit hash value.
#### Type Constraints
-
:
-
-
:
-
-
- - -### **com.microsoft.NonMaxSuppression** - - Pruning away boxes that have high intersection-over-union (IOU) overlap with previously selected boxes. - Bounding boxes with score less than score_threshold are removed. Bounding boxes are supplied as [y1, x1, y2, x2], - where (y1, x1) and (y2, x2) are the coordinates of any diagonal pair of box corners and the coordinates can be provided - as normalized (i.e., lying in the interval [0, 1]) or absolute. - Note that this algorithm is agnostic to where the origin is in the coordinate system and more generally is invariant to - orthogonal transformations and translations of the coordinate system; - thus translating or reflections of the coordinate system result in the same boxes being selected by the algorithm. - The output of this operation is a set of integers indexing into the input collection of bounding boxes representing the selected boxes. - The bounding box coordinates corresponding to the selected indices can then be obtained using the gather operation. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft' operator set. - -#### Attributes - -
-
iou_threshold : float
-
Float representing the threshold for deciding whether boxes overlap too much with respect to IOU. Value range [0, 1]. The default is 0.0
-
max_output_size : int (required)
-
Integer representing the maximum number of boxes to be selected by non max suppression.
-
pad_to_max_output_size : int
-
Optional. 1(true) - the output selected_indices is padded to be of length max_output_size. Defaults to 0(false).
-
score_threshold : float (required)
-
Float tensor representing the threshold for deciding when to remove boxes based on score.
-
- -#### Inputs - -
-
:
-
-
:
-
-
- -#### Outputs (1 - 2) - -
-
:
-
-
(optional) :
-
-
- -#### Type Constraints - -
-
:
-
-
:
-
-
- - -### **com.microsoft.QLinearConv** - - The convolution operator consumes a quantized input tensor, its scale and zero point, - a quantized filter, its scale and zero point, and output's scale and zero point, - and computes the quantized output. Each scale and zero point pair must have same shape. - It means they must be either scalars (per tensor) or 1-D tensors (per channel). - The production MUST never overflow. The accumulation may overflow in 32 bits - if the input is 8 bits or in 64 bits if the input is 16 bits. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft' operator set. - -#### Attributes - -
-
auto_pad : string
-
auto_pad must be either NOTSET, SAME_UPPER, SAME_LOWER or VALID. Where default value is NOTSET, which means explicit padding is used. SAME_UPPER or SAME_LOWER mean pad the input so that the output size match the input.In case of odd number add the extra padding at the end for SAME_UPPER and at the beginning for SAME_LOWER. VALID mean no padding.
-
dilations : list of ints
-
dilation value along each axis of the filter. If not present, the dilation defaults to 1 along each axis.
-
group : int
-
number of groups input channels and output channels are divided into. default is 1.
-
kernel_shape : list of ints
-
The shape of the convolution kernel. If not present, should be inferred from input 'w'.
-
pads : list of ints
-
Padding for the beginning and ending along each axis, it can take any value greater than or equal to 0.The value represent the number of pixels added to the beginning and end part of the corresponding axis.`pads` format should be as follow [x1_begin, x2_begin...x1_end, x2_end,...], where xi_begin the number ofpixels added at the beginning of axis `i` and xi_end, the number of pixels added at the end of axis `i`.This attribute cannot be used simultaneously with auto_pad attribute. If not present, the padding defaultsto 0 along start and end of each axis.
-
strides : list of ints
-
Stride along each axis. If not present, the stride defaults to 1 along each axis.
-
- -#### Inputs (8 - 9) - -
-
:
-
-
:
-
-
:
-
-
:
-
-
:
-
-
:
-
-
:
-
-
:
-
-
(optional) :
-
-
- -#### Outputs - -
-
:
-
-
- -#### Type Constraints - -
-
:
-
-
:
-
-
:
-
-
:
-
-
- - -### **com.microsoft.QLinearMatMul** - - Matrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html. - It consumes two quantized input tensors, their scales and zero points, and output's scale and zero point, and computes - the quantized output. The quantization formula is x_quantized = (x_fp32 / x_scale) + x_zero_point. For (x_fp32 / x_scale), - it computes the nearest integer value to arg (in floating-point format), rounding halfway cases away from zero. - Scale and zero point must have same shape. They must be either scalar (per tensor) or 1-D tensor (per row for a and per column for b). - If scale and zero point are 1D tensor, the number of elements of scale and zero point tensor of input 'a' and output 'y' - should be equal to the number of rows of input 'a', and the number of elements of scale and zero point tensor of input 'b' - should be equal to the number of columns of input 'b'. The production MUST never overflow. The accumulation may overflow in 32 bits - if the input is 8 bits or in 64 bits if the input is 16 bits. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft' operator set. - -#### Inputs - -
-
:
-
-
:
-
-
:
-
-
:
-
-
:
-
-
:
-
-
:
-
-
:
-
-
- -#### Outputs - -
-
:
-
-
- -#### Type Constraints - -
-
:
-
-
:
-
-
:
-
-
- - -### **com.microsoft.QuantizeLinear** - - The linear quantization operator. It consumes a full precision data, a scale, a zero point and computes the quantized data. - The quantization formula is y = (x / y_scale) + y_zero_point. For (x / y_scale), it computes the nearest integer value to arg (in floating-point format), - rounding halfway cases away from zero. Scale and zero point must have same shape. They must be either scalar (per tensor) or 1-D tensor (per 'axis'). - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft' operator set. - -#### Attributes - -
-
axis : int
-
The axis along which same quantization parameters are applied. It's optional. If it's not specified, it means per-tensor quantization and input 'x_scale' and 'x_zero_point' must be scalars. If it's specified, it means per 'axis' quantization and input 'x_scale' and 'x_zero_point' must be 1-D tensors.
-
- -#### Inputs - -
-
:
-
-
:
-
-
:
-
-
- -#### Outputs - -
-
:
-
-
- -#### Type Constraints - -
-
:
-
-
:
-
-
- - -### **com.microsoft.ROIAlign** - - Region of Interest (RoI) align operation described in the - [Mask R-CNN paper](https://arxiv.org/abs/1703.06870). - RoIAlign consumes an input tensor X and region of interests (rois) - to apply pooling across each RoI; it produces a 4-D tensor of shape - (num_rois, C, pooled_h, pooled_w). - - RoIAlign is proposed to avoid the misalignment by removing - quantizations while converting from original image into feature - map and from feature map into RoI feature; in each ROI bin, - the value of the sampled locations are computed directly - through bilinear interpolation. - -#### Version - -This version of the operator has been available since version 1 of the 'com.microsoft' operator set. - -#### Attributes - -
-
mode : string
-
The pooling method. Two modes are supported: 'avg' and 'max'. Default is 'avg'.
-
pooled_h : int
-
default 1; Pooled output Y's height.
-
pooled_w : int
-
default 1; Pooled output Y's width.
-
sampling_ratio : int
-
Number of sampling points in the interpolation grid used to compute the output value of each pooled output bin. If > 0, then exactly sampling_ratio x sampling_ratio grid points are used. If == 0, then an adaptive number of grid points are used (computed as ceil(roi_width / pooled_w), and likewise for height). Default is 0.
-
spatial_scale : float
-
Multiplicative spatial scale factor to translate ROI coordinates from their input spatial scale to the scale used when pooling, i.e., spatial scale of the input feature map X relative to the input image. E.g.; default is 1.0f.
-
- -#### Inputs - -
-
:
-
-
:
-
-
- -#### Outputs - -
-
:
-
-
- -#### Type Constraints - -
-
:
-
+
T1 : tensor(uint32), tensor(int32), tensor(string)
+
Constrain input type to unsigned or signed 32-bit integer tensor, or string tensor. It should be utf-8 encoded if using unicode.
+
T2 : tensor(uint32), tensor(int32)
+
Constrain output type to unsigned and signed 32-bit integer tensor.
@@ -960,26 +518,26 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs (2 - 3)
-
:
-
-
:
-
-
(optional) :
-
+
start : T
+
Tensor(scalar, or dims=[1]). First entry in the range.
+
limit : T
+
Tensor(scalar, or dims=[1]). Upper limit of sequence, exclusive.
+
delta (optional) : T
+
Tensor(scalar, or dims=[1]). Number that increments start. Defaults to 1.
#### Outputs
-
:
-
+
Y : T
+
1-D Tensor of the range.
#### Type Constraints
-
:
-
+
T : tensor(float), tensor(double), tensor(int16), tensor(int32), tensor(int64)
+
Constrain input and output types.
@@ -1006,24 +564,24 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs
-
:
-
+
data : T1
+
An input tensor.
#### Outputs
-
:
-
+
reduced : T2
+
Reduced output tensor.
#### Type Constraints
-
:
-
-
:
-
+
T1 : tensor(int8), tensor(uint8)
+
Constrain input type to 8-bit integer tensor.
+
T2 : tensor(int32), tensor(uint32)
+
Constrain output data type to 32-bit integer tensor.T2 must be tensor(uint32) when T1 is tensor(uint8),or must be tensor(int32) when T1 is tensor(int8).
@@ -1038,22 +596,22 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs
-
:
-
+
X : T
+
input
#### Outputs
-
:
-
+
Y : T
+
output
#### Type Constraints
-
:
-
+
T : tensor(uint32), tensor(uint64), tensor(int32), tensor(int64), tensor(float16), tensor(float), tensor(double)
+
Constrain to any tensor type. If the dtype attribute is not provided this must be a valid output type.
@@ -1063,7 +621,8 @@ This version of the operator has been available since version 1 of the 'com.micr If the maximum number of tokens found per input string is D, the output shape would be [N, C, D] when input shape is [N, C]. Similarly, if input shape is [C] then the output should be [C, D]. Tokenizer has two different operation modes. The first mode is selected when "tokenexp" is not set and "separators" is set. If "tokenexp" is set and "separators" is not set, - the second mode will be used. The first mode breaks each input string into tokens by removing separators. + the second mode will be used. The first mode breaks each input string into tokens by matching and removing separators. + "separators" is a list of strings which are regular expressions. "tokenexp" is a single regular expression. Let's assume "separators" is [" "] and consider an example. If input is @@ -1078,6 +637,9 @@ This version of the operator has been available since version 1 of the 'com.micr whose shape is [2, 5] because you can find at most 5 tokens per input string. Note that the input at most can have two axes, so 3-D and higher dimension are not supported. + If "separators" contains a single empty string, the Tokenizer will enter into character tokenezation mode. This means all strings + will be broken part into individual characters. + For each input string, the second mode searches matches of "tokenexp" and each match will be a token in Y. The matching of "tokenexp" is conducted greedily (i.e., a match should be as long as possible). This operator searches for the first match starting from the beginning of the considered string, @@ -1089,6 +651,13 @@ This version of the operator has been available since version 1 of the 'com.micr If input is ["Hello", "World"], then the corresponding output would be [0x02, "Hello", "World", 0x03]. This implies that if mark is true, [C]/[N, C] - input's output shape becomes [C, D+2]/[N, C, D+2]. + + If tokenizer removes the entire content of [C]-input, it will produce [[]]. + I.e. the output shape should be [C][0] or [N][C][0] if input shape was [N][C]. + + If the tokenizer receives empty input of [0] then the output is [0] if empty input + of [N, 0] then [N, 0]. + #### Version @@ -1104,7 +673,7 @@ This version of the operator has been available since version 1 of the 'com.micr
pad_value : string (required)
The string used to pad output tensors when the tokens extracted doesn't match the maximum number of tokens found. If start/end markers are needed, padding will appear outside the markers.
separators : list of strings
-
an optional list of strings (type: AttributeProto::STRINGS), each single string in this attribute is a separator. Two consecutive segments in X connected by a separator would be divided into two tokens. For example, if the input is "Hello World!" and this attribute contains only one space character, the corresponding output would be ["Hello", "World!"]. To achieve character-level tokenization, one should set the separators to [""], which contains only one empty string. If 'separators' is a L-element array, there will be L rounds of tokenization using one stop word. More specifically, in the first round, the first element in 'separators' is used to tokenize each string in the input. Then, the second element in 'separators' will be used to tokenize the resulted strings produced at the first round.
+
an optional list of strings attribute that contains a list of separators - regular expressions to match separators Two consecutive segments in X connected by a separator would be divided into two tokens. For example, if the input is "Hello World!" and this attribute contains only one space character, the corresponding output would be ["Hello", "World!"]. To achieve character-level tokenization, one should set the 'separators' to [""], which contains an empty string.
tokenexp : string
An optional string. Token's regular expression in basic POSIX format (http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03). If set, tokenizer may produce tokens matching the specified pattern. Note that one and only of 'tokenexp' and 'separators' should be set.
@@ -1112,22 +681,22 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs
-
:
-
+
X : T
+
Strings to tokenize
#### Outputs
-
:
-
+
Y : T
+
Tokenized strings
#### Type Constraints
-
:
-
+
T : tensor(string)
+
Input/Output is a string tensor
@@ -1153,30 +722,30 @@ This version of the operator has been available since version 1 of the 'com.micr #### Inputs
-
:
-
-
:
-
-
:
-
-
:
-
+
Sequence : T
+
Specify batchs of sequence words to embedding
+
W : T1
+
Specify weights of conv
+
B : T1
+
Specify bias of conv
+
C : T1
+
Specify embedding vector of char
#### Outputs
-
:
-
+
Y : T1
+
output
#### Type Constraints
-
:
-
-
:
-
+
T : tensor(int32)
+
Constrain to tensor(int32).
+
T1 : tensor(float)
+
Constrain to tensor(float).
diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index e293e26802..930167a07a 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -331,7 +331,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_BUILD_SERVER=" + ("ON" if args.build_server else "OFF"), "-Donnxruntime_BUILD_x86=" + ("ON" if args.x86 else "OFF"), # nGraph and TensorRT providers currently only supports full_protobuf option. - "-Donnxruntime_USE_FULL_PROTOBUF=" + ("ON" if args.use_full_protobuf or args.use_ngraph or args.use_tensorrt or args.build_server else "OFF"), + "-Donnxruntime_USE_FULL_PROTOBUF=" + ("ON" if args.use_full_protobuf or args.use_ngraph or args.use_tensorrt or args.build_server or args.gen_doc else "OFF"), "-Donnxruntime_DISABLE_CONTRIB_OPS=" + ("ON" if args.disable_contrib_ops else "OFF"), "-Donnxruntime_MSVC_STATIC_RUNTIME=" + ("ON" if args.enable_msvc_static_runtime else "OFF"), ] @@ -642,7 +642,7 @@ def generate_documentation(source_dir, build_dir, configs): operator_doc_path = os.path.join(source_dir, 'docs', 'ContribOperators.md') for config in configs: #copy the gen_doc.py - shutil.copy(os.path.join(source_dir,'onnxruntime','python','tools','gen_doc.py'), + shutil.copy(os.path.join(source_dir,'tools','python','gen_doc.py'), os.path.join(build_dir,config, config)) run_subprocess([ sys.executable, @@ -650,10 +650,16 @@ def generate_documentation(source_dir, build_dir, configs): '--output_path', operator_doc_path ], cwd = os.path.join(build_dir,config, config)) - - docdiff = run_subprocess(['git', 'diff', operator_doc_path], capture=True).stdout + docdiff = '' + try: + docdiff = subprocess.check_output(['git', 'diff', operator_doc_path]) + except subprocess.CalledProcessError: + print('git diff returned non-zero error code') + + if len(docdiff) > 0: - raise BuildError("The updated operator document file "+operator_doc_path+" must be checked in") + raise BuildError("The updated operator document file "+operator_doc_path+" must be checked in.\n diff:\n"+docdiff) + def main(): args = parse_arguments() @@ -792,7 +798,7 @@ def main(): nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1') build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_ngraph, args.use_tensorrt, nightly_build) - if args.gen_doc: + if args.gen_doc and (args.build or args.test): generate_documentation(source_dir, build_dir, configs) log.info("Build complete") diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index b63664792a..b796042d13 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -13,7 +13,7 @@ jobs: displayName: 'Download test data and generate cmake config' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --update' workingDirectory: "$(Build.BinariesDirectory)" - task: VSBuild@1 @@ -30,7 +30,7 @@ jobs: displayName: 'Test Debug' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --test' workingFolder: '$(Build.BinariesDirectory)' - task: VSBuild@1 displayName: 'Build C# Debug' diff --git a/onnxruntime/python/tools/gen_doc.py b/tools/python/gen_doc.py similarity index 94% rename from onnxruntime/python/tools/gen_doc.py rename to tools/python/gen_doc.py index ff84b69a6a..28580ed9fc 100644 --- a/onnxruntime/python/tools/gen_doc.py +++ b/tools/python/gen_doc.py @@ -114,10 +114,11 @@ def display_schema(schema, versions): # type: (OpSchema, Sequence[OpSchema]) -> s = '' # doc - if schema.doc: + schemadoc = schema.doc + if schemadoc: s += '\n' s += '\n'.join(' ' + line - for line in schema.doc.lstrip().splitlines()) + for line in schemadoc.lstrip().splitlines()) s += '\n' # since version @@ -138,10 +139,11 @@ def display_schema(schema, versions): # type: (OpSchema, Sequence[OpSchema]) -> return s # attributes - if schema.attributes: + attribs = schema.attributes + if attribs: s += '\n#### Attributes\n\n' s += '
\n' - for _, attr in sorted(schema.attributes.items()): + for _, attr in sorted(attribs.items()): # option holds either required or default value opt = '' if attr.required: @@ -175,20 +177,23 @@ def display_schema(schema, versions): # type: (OpSchema, Sequence[OpSchema]) -> s += ' ({} - {})'.format(display_number(schema.min_input), display_number(schema.max_input)) s += '\n\n' - if schema.inputs: + + inputs = schema.inputs + if inputs: s += '
\n' - for input in schema.inputs: + for inp in inputs: option_str = "" - if OpSchema.FormalParameterOption.Optional == input.option: + if OpSchema.FormalParameterOption.Optional == inp.option: option_str = " (optional)" - elif OpSchema.FormalParameterOption.Variadic == input.option: - if input.isHomogeneous: + elif OpSchema.FormalParameterOption.Variadic == inp.option: + if inp.isHomogeneous: option_str = " (variadic)" else: option_str = " (variadic, heterogeneous)" - s += '
{}{} : {}
\n'.format(input.name, option_str, input.typeStr) - s += '
{}
\n'.format(input.description) - s += '
\n' + s += '
{}{} : {}
\n'.format(inp.name, option_str, inp.typeStr) + s += '
{}
\n'.format(inp.description) + + s += '
\n' # outputs s += '\n#### Outputs' @@ -196,10 +201,10 @@ def display_schema(schema, versions): # type: (OpSchema, Sequence[OpSchema]) -> s += ' ({} - {})'.format(display_number(schema.min_output), display_number(schema.max_output)) s += '\n\n' - - if schema.outputs: + outputs = schema.outputs + if outputs: s += '
\n' - for output in schema.outputs: + for output in outputs: option_str = "" if OpSchema.FormalParameterOption.Optional == output.option: option_str = " (optional)" @@ -210,14 +215,16 @@ def display_schema(schema, versions): # type: (OpSchema, Sequence[OpSchema]) -> option_str = " (variadic, heterogeneous)" s += '
{}{} : {}
\n'.format(output.name, option_str, output.typeStr) s += '
{}
\n'.format(output.description) - s += '
\n' + + s += '\n' # type constraints s += '\n#### Type Constraints' s += '\n\n' - if schema.type_constraints: + typecons = schema.type_constraints + if typecons: s += '
\n' - for type_constraint in schema.type_constraints: + for type_constraint in typecons: allowedTypes = type_constraint.allowed_type_strs allowedTypeStr = '' if (len(allowedTypes) > 0): @@ -295,7 +302,7 @@ def support_level_str(level): # type: (OpSchema.SupportType) -> Text def main(args): # type: (Type[Args]) -> None - + with io.open(args.output, 'w', newline='', encoding="utf-8") as fout: fout.write('## Contrib Operator Schemas\n') fout.write(