diff --git a/docs/ContribOperators.md b/docs/ContribOperators.md index f16aa6b079..494b6cdb66 100644 --- a/docs/ContribOperators.md +++ b/docs/ContribOperators.md @@ -1,10 +1,13 @@ ## Contrib Operator Schemas *This file is automatically generated from the - [def files](/onnxruntime/core/graph/contrib_ops/contrib_defs.cc) via [this script](/tools/python/gen_doc.py). + [def files](/onnxruntime/core/graph/contrib_ops/contrib_defs.cc) via [this script](/tools/python/gen_contrib_doc.py). Do not modify directly and instead edit operator definitions.* * com.microsoft + * com.microsoft.Attention * com.microsoft.AttnLSTM + * com.microsoft.BiasGelu + * com.microsoft.BiasSoftmax * com.microsoft.CDist * com.microsoft.ComplexMul * com.microsoft.ComplexMulConj @@ -12,38 +15,40 @@ * com.microsoft.CropAndResize * com.microsoft.DequantizeLinear * com.microsoft.DynamicQuantizeMatMul + * com.microsoft.EmbedLayerNormalization * com.microsoft.ExpandDims + * com.microsoft.FastGelu * com.microsoft.FusedConv * com.microsoft.FusedGemm + * com.microsoft.FusedMatMul * com.microsoft.GatherND + * com.microsoft.Gelu + * com.microsoft.Inverse * com.microsoft.Irfft * com.microsoft.MatMulInteger16 + * com.microsoft.MatMulIntegerToFloat * com.microsoft.MaxpoolWithMask * com.microsoft.MulInteger * com.microsoft.MurmurHash3 * com.microsoft.Pad + * com.microsoft.QAttention * com.microsoft.QLinearAdd * com.microsoft.QLinearAveragePool * com.microsoft.QLinearLeakyRelu * com.microsoft.QLinearMul * com.microsoft.QLinearReduceMean + * com.microsoft.QLinearSigmoid * com.microsoft.QuantizeLinear * com.microsoft.Range * com.microsoft.ReduceSumInteger * com.microsoft.Rfft * com.microsoft.SampleOp + * com.microsoft.SkipLayerNormalization * com.microsoft.Tokenizer + * com.microsoft.TransposeMatMul + * com.microsoft.Trilu * com.microsoft.Unique * com.microsoft.WordConvEmbedding - * experimental com.microsoft.Attention - * experimental com.microsoft.BiasGelu - * experimental com.microsoft.EmbedLayerNormalization - * experimental com.microsoft.FastGelu - * experimental com.microsoft.Gelu - * experimental com.microsoft.Inverse - * experimental com.microsoft.QAttention - * experimental com.microsoft.SkipLayerNormalization - * experimental com.microsoft.TransposeMatMul * com.microsoft.nchwc * com.microsoft.nchwc.AveragePool * com.microsoft.nchwc.Conv @@ -55,6 +60,63 @@ * com.microsoft.nchwc.Upsample ## com.microsoft +### **com.microsoft.Attention** + + Multi-Head Self Attention that can be either unidirectional (like GPT-2) or bidirectional (like BERT). + The mask_index input is optional. Besides raw attention mask with shape (batch_size, past_sequence_length + sequence_length), + we also support other two formats: When input has right-side padding, mask_index is one dimension with shape (batch_size), + where value of each element is the end position, or valid length of actual sequence excluding padding. When input has + left-side padding, mask_index has shape (2 * batch_size), where the values are the exclusive end positions followed by + the inclusive start positions. When unidirectional is 1, and each token only attend to previous tokens. For GPT-2, both past + and present state are optional. Present state could appear in output even when past state is not in input. + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Attributes + +
+
num_heads : int (required)
+
Number of attention heads
+
unidirectional : int
+
Whether every token can only attend to previous tokens. Default value is 0.
+
+ +#### Inputs (3 - 5) + +
+
input : T
+
3D input tensor with shape (batch_size, sequence_length, hidden_size), hidden_size = num_heads * head_size
+
weight : T
+
2D input tensor with shape (hidden_size, 3 * hidden_size)
+
bias : T
+
1D input tensor with shape (3 * hidden_size)
+
mask_index (optional) : M
+
Attention mask with shape (batch_size, past_sequence_length + sequence_length), or index with shape (batch_size) or (2 * batch_size).
+
past (optional) : T
+
past state for key and value with shape (2, batch_size, num_heads, past_sequence_length, head_size).
+
+ +#### Outputs (1 - 2) + +
+
output : T
+
3D output tensor with shape (batch_size, append_length, hidden_size)
+
present (optional) : T
+
present state for key and value with shape (2, batch_size, num_heads, past_sequence_length + sequence_length, head_size)
+
+ +#### Type Constraints + +
+
T : tensor(float), tensor(float16)
+
Constrain input and output types to float tensors.
+
M : tensor(int32)
+
Constrain mask index to integer types
+
+ + ### **com.microsoft.AttnLSTM** Computes an one-layer RNN where its RNN Cell is an AttentionWrapper wrapped a LSTM Cell. The RNN layer @@ -264,6 +326,80 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.BiasGelu** + + Bias Gelu. + It's an extension of Gelu. It takes the sum of input A and bias input B as the input of Gelu activation. + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Inputs + +
+
A : T
+
The normal input data.
+
B : T
+
The bias input data that is a 1D tensor.
+
+ +#### Outputs + +
+
C : T
+
The output.
+
+ +#### Type Constraints + +
+
T : tensor(float16), tensor(float), tensor(double)
+
Constrain input and output types to float tensors.
+
+ + +### **com.microsoft.BiasSoftmax** + + Y = softmax(scores + bias)) with simple broadcast on bias. Intended to specialize softmax(scores + additive_mask) commonly found in transformer models. + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Attributes + +
+
broadcast_axis : int
+
broadcast bias across input for dimensions broadcast_axis to softmax_axis-1
+
softmax_axis : int
+
apply softmax to elements for dimensions softmax_axis or higher
+
+ +#### Inputs + +
+
data : T
+
The input data as Tensor.
+
bias : T
+
The bias (or mask) as Tensor.
+
+ +#### Outputs + +
+
output : T
+
The output.
+
+ +#### Type Constraints + +
+
T : tensor(float16), tensor(float), tensor(double)
+
Constrain input and output types to float tensors.
+
+ + ### **com.microsoft.CDist** #### Version @@ -515,7 +651,7 @@ This version of the operator has been available since version 1 of the 'com.micr This version of the operator has been available since version 1 of the 'com.microsoft' operator set. -#### Inputs (3 - 4) +#### Inputs (3 - 5)
A : T1
@@ -526,6 +662,8 @@ This version of the operator has been available since version 1 of the 'com.micr
Scale of quantized input 'B'. It could be a scalar or a 1-D tensor, which means a per-tensor or per-column quantization. If it's a 1-D tensor, its number of elements should be equal to the number of columns of input 'B'.
b_zero_point (optional) : T2
Zero point tensor for input 'B'. It's optional and default value is 0. It could be a scalar or a 1-D tensor, which means a per-tensor or per-column quantization. If it's a 1-D tensor, its number of elements should be equal to the number of columns of input 'B'.
+
bias (optional) : T1
+
1D input tensor, whose dimension is same as B's last dimension
#### Outputs @@ -545,6 +683,65 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.EmbedLayerNormalization** + + EmbedLayerNormalization is the fusion of embedding layer in BERT model, with optional mask processing. + The embedding layer takes input_ids (word IDs) and segment_ids (sentence IDs) to look up word_embedding, position_embedding, + and segment_emedding; the embeddings are added then applied layer normalization using gamma and beta tensors. + The last input mask is optional. If mask is provided, mask index (that is position of first 0 in mask, or number of words) + will be calculated. + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Attributes + +
+
epsilon : float
+
The epsilon value to use to avoid division by zero.
+
+ +#### Inputs (7 - 8) + +
+
input_ids : T1
+
2D words IDs with shape (batch_size, sequence_length)
+
segment_ids (optional) : T1
+
2D segment IDs with shape (batch_size, sequence_length)
+
word_embedding : T
+
2D with shape (,hidden_size)
+
position_embedding : T
+
2D with shape (, hidden_size)
+
segment_embedding (optional) : T
+
2D with shape (, hidden_size)
+
gamma : T
+
1D gamma tensor for layer normalization with shape (hidden_size)
+
beta : T
+
1D beta tensor for layer normalization with shape (hidden_size)
+
mask (optional) : T1
+
2D attention mask with shape (batch_size, sequence_length)
+
+ +#### Outputs + +
+
output : T
+
3D output tensor with shape (batch_size, sequence_length, hidden_size)
+
mask_index : T1
+
1D mask_index tensor with shape (batch_size)
+
+ +#### Type Constraints + +
+
T1 : tensor(int32)
+
Constrain input and output integer tensors types
+
T : tensor(float), tensor(float16)
+
Constrain input and output float tensors types.
+
+ + ### **com.microsoft.ExpandDims** ExpandDims echo operator. @@ -577,6 +774,38 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.FastGelu** + + GELU (Gaussian Error Linear Unit) approximation: Y=0.5*X*(1+tanh(0.797885*X+0.035677*X*X*X)) with an optional input of bias that will be added to X before GELU. + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Inputs (1 - 2) + +
+
X : T
+
input tensor
+
bias (optional) : T
+
bias tensor
+
+ +#### Outputs + +
+
Y : T
+
output tensor
+
+ +#### Type Constraints + +
+
T : tensor(float), tensor(float16), tensor(bfloat16)
+
Constrain input and output types to float or half tensors.
+
+ + ### **com.microsoft.FusedConv** The fused convolution operator schema is the same as Conv besides it includes an attribute @@ -689,6 +918,49 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.FusedMatMul** + + Matrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Attributes + +
+
alpha : float
+
Scalar multiplier for the product of the input tensors.
+
transA : int
+
Whether A should be transposed on the last two dimensions before doing multiplication
+
transB : int
+
Whether B should be transposed on the last two dimensions before doing multiplication
+
+ +#### Inputs + +
+
A : T
+
N-dimensional matrix A
+
B : T
+
N-dimensional matrix B
+
+ +#### Outputs + +
+
Y : T
+
Matrix multiply results
+
+ +#### Type Constraints + +
+
T : tensor(float16), tensor(float), tensor(double), tensor(bfloat16)
+
Constrain input and output types to float tensors.
+
+ + ### **com.microsoft.GatherND** Given `data` tensor of rank r >= 1, and `indices` tensor of rank q >= 1, gather @@ -740,6 +1012,68 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.Gelu** + + Gaussian Error Linear Unit. + A high-performing neural network activation function.The GELU nonlinearity is + the expected transformation of a stochastic regularizer which randomly applies + the identity or zero map to a neuron's input. The GELU nonlinearity weights + inputs by their magnitude, rather than gates inputs by their sign as in ReLUs. + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Inputs + +
+
X : T
+
The input data as Tensor.
+
+ +#### Outputs + +
+
Y : T
+
The output.
+
+ +#### Type Constraints + +
+
T : tensor(float16), tensor(float), tensor(double), tensor(bfloat16)
+
Constrain input and output types to float tensors.
+
+ + +### **com.microsoft.Inverse** + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Inputs + +
+
X : T
+
Input tensor. Every matrix in the batch must be invertible.
+
+ +#### Outputs + +
+
Y : T
+
Output tensor of the same type and shape as the input tensor.
+
+ +#### Type Constraints + +
+
T : tensor(float16), tensor(float), tensor(double)
+
Constrain input and output types to float tensors.
+
+ + ### **com.microsoft.Irfft** #### Version @@ -816,6 +1150,50 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.MatMulIntegerToFloat** + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Inputs (4 - 7) + +
+
A : T1
+
N-dimensional matrix A
+
B : T2
+
N-dimensional matrix B
+
a_scale : T3
+
Scale of quantized input 'A'. It could be a scalar or a 1-D tensor, which means a per-tensor or per-column quantization. If it's a 1-D tensor, its number of elements should be equal to the number of columns of input 'A'.
+
b_scale : T3
+
Scale of quantized input 'B'. It could be a scalar or a 1-D tensor, which means a per-tensor or per-column quantization. If it's a 1-D tensor, its number of elements should be equal to the number of columns of input 'B'.
+
a_zero_point (optional) : T1
+
Zero point tensor for input 'A'. It's optional and default value is 0. It could be a scalar or a 1-D tensor, which means a per-tensor or per-column quantization. If it's a 1-D tensor, its number of elements should be equal to the number of columns of input 'A'.
+
b_zero_point (optional) : T2
+
Zero point tensor for input 'B'. It's optional and default value is 0. It could be a scalar or a 1-D tensor, which means a per-tensor or per-column quantization. If it's a 1-D tensor, its number of elements should be equal to the number of columns of input 'B'.
+
bias (optional) : T3
+
1D input tensor, whose dimension is same as B's last dimension
+
+ +#### Outputs + +
+
Y : T3
+
Matrix multiply results from A * B
+
+ +#### Type Constraints + +
+
T1 : tensor(int8), tensor(uint8)
+
Constrain input A data type to 8-bit integer tensor.
+
T2 : tensor(int8), tensor(uint8)
+
Constrain input B data type to 8-bit integer tensor.
+
T3 : tensor(float)
+
Constrain input a_scale, b_scale and output Y data type as float tensor.
+
+ + ### **com.microsoft.MaxpoolWithMask** For internal use. @@ -1006,6 +1384,69 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.QAttention** + + Quantization of Multi-Head Self Attention. + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Attributes + +
+
num_heads : int (required)
+
Number of attention heads
+
unidirectional : int
+
Whether every token can only attend to previous tokens. Default value is 0.
+
+ +#### Inputs (5 - 9) + +
+
input : T1
+
3D input tensor with shape (batch_size, sequence_length, hidden_size), hidden_size = num_heads * head_size
+
weight : T2
+
2D input tensor with shape (hidden_size, 3 * hidden_size)
+
bias : T3
+
1D input tensor with shape (3 * hidden_size)
+
input_scale : T3
+
scale of quantized input tensor. It's a scalar, which means a per-tensor/layer quantization.
+
weight_scale : T3
+
scale of weight scale. It's a scalar, which means a per-tensor/layer quantization.
+
mask_index (optional) : T4
+
Attention mask index with shape (batch_size)
+
input_zero_point (optional) : T1
+
zero point of quantized input tensor.It's a scalar, which means a per-tensor/layer quantization.
+
weight_zero_point (optional) : T2
+
zero point of quantized weight tensor. It's a scalar, which means a per-tensor/layer quantization.
+
past (optional) : T3
+
past state for key and value with shape (2, batch_size, num_heads, past_sequence_length, head_size).
+
+ +#### Outputs (1 - 2) + +
+
output : T3
+
3D output tensor with shape (batch_size, sequence_length, hidden_size)
+
present (optional) : T3
+
present state for key and value with shape (2, batch_size, num_heads, past_sequence_length + sequence_length, head_size)
+
+ +#### Type Constraints + +
+
T1 : tensor(int8), tensor(uint8)
+
Constrain input and output types to int8 tensors.
+
T2 : tensor(int8), tensor(uint8)
+
Constrain input and output types to int8 tensors.
+
T3 : tensor(float), tensor(float16)
+
Constrain input and output types to float tensors.
+
T4 : tensor(int32)
+
Constrain mask index to integer types
+
+ + ### **com.microsoft.QLinearAdd** Performs element-wise binary addition on 8 bit data types (with Numpy-style broadcasting support). @@ -1288,6 +1729,46 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.QLinearSigmoid** + + QLinearSigmoid takes quantized input data (Tensor), and quantize parameter for output, and produces one output data + (Tensor) where the function `f(x) = quantize(Sigmoid(dequantize(x)))`, is applied to the data tensor elementwise. + Wwhere the function `Sigmoid(x) = 1 / (1 + exp(-x))` + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Inputs (4 - 5) + +
+
X : T
+
Input tensor
+
X_scale : tensor(float)
+
Input X's scale. It's a scalar, which means a per-tensor/layer quantization.
+
X_zero_point (optional) : T
+
Input X's zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.
+
Y_scale : tensor(float)
+
Output Y's scale. It's a scalar, which means a per-tensor/layer quantization.
+
Y_zero_point (optional) : T
+
Output Y's zero point. Default value is 0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization.
+
+ +#### Outputs + +
+
Y : T
+
Output tensor
+
+ +#### Type Constraints + +
+
T : tensor(uint8), tensor(int8)
+
Constrain input and output types to 8 bit tensors.
+
+ + ### **com.microsoft.QuantizeLinear** The linear quantization operator. It consumes a full precision data, a scale, a zero point to compute the low precision / quantized tensor. @@ -1482,6 +1963,57 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.SkipLayerNormalization** + + Skip and Layer Normalization Fusion + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Attributes + +
+
epsilon : float
+
The epsilon value to use to avoid division by zero.
+
+ +#### Inputs (4 - 5) + +
+
input : T
+
3D input tensor with shape (batch_size, sequence_length, hidden_size)
+
skip : T
+
3D skip tensor with shape (batch_size, sequence_length, hidden_size)
+
gamma : T
+
1D input tensor with shape (hidden_size)
+
beta : T
+
1D skip tensor with shape (hidden_size
+
bias (optional) : T
+
1D bias tensor with shape (hidden_size
+
+ +#### Outputs (1 - 3) + +
+
output : T
+
3D output tensor with shape (batch_size, sequence_length, hidden_size)
+
mean (optional) : U
+
Saved mean used during training to speed up gradient computation
+
inv_std_var (optional) : U
+
Saved inverse standard variance used during training to speed up gradient computation.
+
+ +#### Type Constraints + +
+
T : tensor(float), tensor(float16)
+
Constrain input and output types to float or half tensors.
+
U : tensor(float)
+
Constrain mean and inv_std_var to float tensors.
+
+ + ### **com.microsoft.Tokenizer** Tokenizer divides each string in X into a vector of strings along the last axis. Allowed input shapes are [C] and [N, C]. @@ -1531,7 +2063,7 @@ This version of the operator has been available since version 1 of the 'com.micr
separators : list of strings
an optional list of strings attribute that contains a list of separators - regular expressions to match separators Two consecutive segments in X connected by a separator would be divided into two tokens. For example, if the input is "Hello World!" and this attribute contains only one space character, the corresponding output would be ["Hello", "World!"]. To achieve character-level tokenization, one should set the 'separators' to [""], which contains an empty string.
tokenexp : string
-
An optional string. Token's regular expression in basic POSIX format (http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03). If set, tokenizer may produce tokens matching the specified pattern. Note that one and only of 'tokenexp' and 'separators' should be set.
+
An optional string. Token's regular expression in basic POSIX format (pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03). If set, tokenizer may produce tokens matching the specified pattern. Note that one and only of 'tokenexp' and 'separators' should be set.
#### Inputs @@ -1556,6 +2088,67 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.TransposeMatMul** (deprecated) + + Deprecated. Going forward FusedMatMul should be used. This OP will be supported for backward compatibility. + Matrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html + +#### Version + +This version of the operator has been deprecated since version 2 of the 'com.microsoft' operator set. + +Other versions of this operator: com.microsoft.TransposeMatMul-1 + + +### **com.microsoft.Trilu** + + Returns the upper or lower triangular part of a 2-D matrix, or batches of 2-D matrices. If the attribute "upper" is set to true, + the upper triangular matrix is retained. Lower triangular matrix is retained otherwise. Default value for upper is true. + Trilu takes one input tensor of shape [*, N, M], where * is zero or more batch dimensions. The upper triangular part consists + of the elements on and above the given diagonal (k). The lower triangular part consists of elements on and below the diagonal. + All other elements in the matrix are set to zero. + If k = 0, the triangular part on and above/below the main diagonal is retained. + If upper is set to true, a positive k retains the upper triangular matrix excluding k diagonals above + the main diagonal. A negative k value includes as many diagonals below the main diagonal. + If upper is set to false, a positive k retains the lower triangular matrix including k diagonals above + the main diagonal. A negative k value excludes as many diagonals below the main diagonal. + + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Attributes + +
+
upper : int
+
Boolean. Indicates whether upper or lower part of matrix is retained. Default is true.
+
+ +#### Inputs (1 - 2) + +
+
X : T
+
Input tensor of rank 2 or higher.
+
k (optional) : tensor(int64)
+
A 0-D tensor containing a single value corresponding to the number diagonals above or the main diagonal to exclude or include.Default value is 0 if it's not specified.
+
+ +#### Outputs + +
+
Y : T
+
Output tensor of the same type and shape as the input tensor.
+
+ +#### Type Constraints + +
+
T : tensor(float16), tensor(float), tensor(double), tensor(bfloat16), tensor(uint8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(int8), tensor(int16), tensor(int32), tensor(int64), tensor(bool)
+
Constrain input and output types to all numeric tensors and bool tensors.
+
+ + ### **com.microsoft.Unique** Finds all the unique values (deduped list) present in the given input tensor. @@ -1651,395 +2244,6 @@ This version of the operator has been available since version 1 of the 'com.micr -### experimental **com.microsoft.Attention** - - Multi-Head Self Attention that can be either unidirectional (like GPT-2) or bidirectional (like BERT). - The mask_index input is optional. Besides raw attention mask with shape (batch_size, past_sequence_length + sequence_length), - we also support other two formats: When input has right-side padding, mask_index is one dimension with shape (batch_size), - where value of each element is the end position, or valid length of actual sequence excluding padding. When input has - left-side padding, mask_index has shape (2 * batch_size), where the values are the exclusive end positions followed by - the inclusive start positions. When unidirectional is 1, and each token only attend to previous tokens. For GPT-2, both past - and present state are optional. Present state could appear in output even when past state is not in input. - -#### Version - -No versioning maintained for experimental ops. -#### Attributes - -
-
num_heads : int (required)
-
Number of attention heads
-
unidirectional : int
-
Whether every token can only attend to previous tokens. Default value is 0.
-
- -#### Inputs (3 - 5) - -
-
input : T
-
3D input tensor with shape (batch_size, sequence_length, hidden_size), hidden_size = num_heads * head_size
-
weight : T
-
2D input tensor with shape (hidden_size, 3 * hidden_size)
-
bias : T
-
1D input tensor with shape (3 * hidden_size)
-
mask_index (optional) : M
-
Attention mask with shape (batch_size, past_sequence_length + sequence_length), or index with shape (batch_size) or (2 * batch_size).
-
past (optional) : T
-
past state for key and value with shape (2, batch_size, num_heads, past_sequence_length, head_size).
-
- -#### Outputs (1 - 2) - -
-
output : T
-
3D output tensor with shape (batch_size, append_length, hidden_size)
-
present (optional) : T
-
present state for key and value with shape (2, batch_size, num_heads, past_sequence_length + sequence_length, head_size)
-
- -#### Type Constraints - -
-
T : tensor(float), tensor(float16)
-
Constrain input and output types to float tensors.
-
M : tensor(int32)
-
Constrain mask index to integer types
-
- - -### experimental **com.microsoft.BiasGelu** - - Bias Gelu. - It's an extension of Gelu. It takes the sum of input A and bias input B as the input of Gelu activation. - -#### Version - -No versioning maintained for experimental ops. -#### Inputs - -
-
A : T
-
The normal input data.
-
B : T
-
The bias input data that is a 1D tensor.
-
- -#### Outputs - -
-
C : T
-
The output.
-
- -#### Type Constraints - -
-
T : tensor(float16), tensor(float), tensor(double)
-
Constrain input and output types to float tensors.
-
- - -### experimental **com.microsoft.EmbedLayerNormalization** - - EmbedLayerNormalization is the fusion of embedding layer in BERT model, with optional mask processing. - The embedding layer takes input_ids (word IDs) and segment_ids (sentence IDs) to look up word_embedding, position_embedding, - and segment_emedding; the embeddings are added then applied layer normalization using gamma and beta tensors. - The last input mask is optional. If mask is provided, mask index (that is position of first 0 in mask, or number of words) - will be calculated. - -#### Version - -No versioning maintained for experimental ops. -#### Attributes - -
-
epsilon : float
-
The epsilon value to use to avoid division by zero.
-
- -#### Inputs (7 - 8) - -
-
input_ids : T1
-
2D words IDs with shape (batch_size, sequence_length)
-
segment_ids : T1
-
2D segment IDs with shape (batch_size, sequence_length)
-
word_embedding : T
-
2D with shape (,hidden_size)
-
position_embedding : T
-
2D with shape (, hidden_size)
-
segment_embedding : T
-
2D with shape (, hidden_size)
-
gamma : T
-
1D gamma tensor for layer normalization with shape (hidden_size)
-
beta : T
-
1D beta tensor for layer normalization with shape (hidden_size)
-
mask (optional) : T1
-
2D attention mask with shape (batch_size, sequence_length)
-
- -#### Outputs - -
-
output : T
-
3D output tensor with shape (batch_size, sequence_length, hidden_size)
-
mask_index : T1
-
1D mask_index tensor with shape (batch_size)
-
- -#### Type Constraints - -
-
T1 : tensor(int32)
-
Constrain input and output integer tensors types
-
T : tensor(float), tensor(float16)
-
Constrain input and output float tensors types.
-
- - -### experimental **com.microsoft.FastGelu** - - GELU (Gaussian Error Linear Unit) approximation: Y=0.5*X*(1+tanh(0.797885*X+0.035677*X*X*X)) with an optional input of bias that will be added to X before GELU. - -#### Version - -No versioning maintained for experimental ops. -#### Inputs (1 - 2) - -
-
X : T
-
input tensor
-
bias (optional) : T
-
bias tensor
-
- -#### Outputs - -
-
Y : T
-
output tensor
-
- -#### Type Constraints - -
-
T : tensor(float), tensor(float16)
-
Constrain input and output types to float or half tensors.
-
- - -### experimental **com.microsoft.Gelu** - - Gaussian Error Linear Unit. - A high-performing neural network activation function.The GELU nonlinearity is - the expected transformation of a stochastic regularizer which randomly applies - the identity or zero map to a neuron's input. The GELU nonlinearity weights - inputs by their magnitude, rather than gates inputs by their sign as in ReLUs. - -#### Version - -No versioning maintained for experimental ops. -#### Inputs - -
-
X : T
-
The input data as Tensor.
-
- -#### Outputs - -
-
Y : T
-
The output.
-
- -#### Type Constraints - -
-
T : tensor(float16), tensor(float), tensor(double)
-
Constrain input and output types to float tensors.
-
- - -### experimental **com.microsoft.Inverse** - -#### Version - -No versioning maintained for experimental ops. -#### Inputs - -
-
X : T
-
Input tensor. Every matrix in the batch must be invertible.
-
- -#### Outputs - -
-
Y : T
-
Output tensor of the same type and shape as the input tensor.
-
- -#### Type Constraints - -
-
T : tensor(float16), tensor(float), tensor(double)
-
Constrain input and output types to float tensors.
-
- - -### experimental **com.microsoft.QAttention** - - Quantization of Multi-Head Self Attention. - -#### Version - -No versioning maintained for experimental ops. -#### Attributes - -
-
num_heads : int (required)
-
Number of attention heads
-
unidirectional : int
-
Whether every token can only attend to previous tokens. Default value is 0.
-
- -#### Inputs (5 - 9) - -
-
input : T1
-
3D input tensor with shape (batch_size, sequence_length, hidden_size), hidden_size = num_heads * head_size
-
weight : T2
-
2D input tensor with shape (hidden_size, 3 * hidden_size)
-
bias : T3
-
1D input tensor with shape (3 * hidden_size)
-
input_scale : T3
-
scale of quantized input tensor. It's a scalar, which means a per-tensor/layer quantization.
-
weight_scale : T3
-
scale of weight scale. It's a scalar, which means a per-tensor/layer quantization.
-
mask_index (optional) : T4
-
Attention mask index with shape (batch_size)
-
input_zero_point (optional) : T1
-
zero point of quantized input tensor.It's a scalar, which means a per-tensor/layer quantization.
-
weight_zero_point (optional) : T2
-
zero point of quantized weight tensor. It's a scalar, which means a per-tensor/layer quantization.
-
past (optional) : T3
-
past state for key and value with shape (2, batch_size, num_heads, past_sequence_length, head_size).
-
- -#### Outputs (1 - 2) - -
-
output : T3
-
3D output tensor with shape (batch_size, sequence_length, hidden_size)
-
present (optional) : T3
-
present state for key and value with shape (2, batch_size, num_heads, past_sequence_length + sequence_length, head_size)
-
- -#### Type Constraints - -
-
T1 : tensor(int8), tensor(uint8)
-
Constrain input and output types to int8 tensors.
-
T2 : tensor(int8), tensor(uint8)
-
Constrain input and output types to int8 tensors.
-
T3 : tensor(float), tensor(float16)
-
Constrain input and output types to float tensors.
-
T4 : tensor(int32)
-
Constrain mask index to integer types
-
- - -### experimental **com.microsoft.SkipLayerNormalization** - - Skip and Layer Normalization Fusion - -#### Version - -No versioning maintained for experimental ops. -#### Attributes - -
-
epsilon : float
-
The epsilon value to use to avoid division by zero.
-
- -#### Inputs (4 - 5) - -
-
input : T
-
3D input tensor with shape (batch_size, sequence_length, hidden_size)
-
skip : T
-
3D skip tensor with shape (batch_size, sequence_length, hidden_size)
-
gamma : T
-
1D input tensor with shape (hidden_size)
-
beta : T
-
1D skip tensor with shape (hidden_size
-
bias (optional) : T
-
1D bias tensor with shape (hidden_size
-
- -#### Outputs (1 - 3) - -
-
output : T
-
3D output tensor with shape (batch_size, sequence_length, hidden_size)
-
mean (optional) : U
-
Saved mean used during training to speed up gradient computation
-
inv_std_var (optional) : U
-
Saved inverse standard variance used during training to speed up gradient computation.
-
- -#### Type Constraints - -
-
T : tensor(float), tensor(float16)
-
Constrain input and output types to float or half tensors.
-
U : tensor(float)
-
Constrain mean and inv_std_var to float tensors.
-
- - -### experimental **com.microsoft.TransposeMatMul** - - Matrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html - -#### Version - -No versioning maintained for experimental ops. -#### Attributes - -
-
transA : int
-
Whether A should be transposed on the last two dimensions before doing multiplication
-
transB : int
-
Whether B should be transposed on the last two dimensions before doing multiplication
-
- -#### Inputs - -
-
A : T
-
N-dimensional matrix A
-
B : T
-
N-dimensional matrix B
-
- -#### Outputs - -
-
Y : T
-
Matrix multiply results
-
- -#### Type Constraints - -
-
T : tensor(float16), tensor(float), tensor(double)
-
Constrain input and output types to float tensors.
-
- - ## com.microsoft.nchwc ### **com.microsoft.nchwc.AveragePool** diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index 8cf0320d4c..5010a057dd 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -412,7 +412,6 @@ and present state are optional. Present state could appear in output even when p ONNX_CONTRIB_OPERATOR_SCHEMA(Attention) .SetDomain(kMSDomain) .SinceVersion(1) - .SetSupportLevel(OpSchema::SupportType::EXPERIMENTAL) .SetDoc(Attention_ver1_doc) .Attr("num_heads", "Number of attention heads", AttributeProto::INT) .Attr("unidirectional", @@ -470,7 +469,6 @@ and present state are optional. Present state could appear in output even when p ONNX_CONTRIB_OPERATOR_SCHEMA(QAttention) .SetDomain(kMSDomain) .SinceVersion(1) - .SetSupportLevel(OpSchema::SupportType::EXPERIMENTAL) .SetDoc("Quantization of Multi-Head Self Attention.") .Attr("num_heads", "Number of attention heads", AttributeProto::INT) .Attr("unidirectional", @@ -566,7 +564,6 @@ will be calculated.)DOC"; ONNX_CONTRIB_OPERATOR_SCHEMA(EmbedLayerNormalization) .SetDomain(kMSDomain) .SinceVersion(1) - .SetSupportLevel(OpSchema::SupportType::EXPERIMENTAL) .SetDoc(EmbedLayerNormalization_ver1_doc) .Attr("epsilon", "The epsilon value to use to avoid division by zero.", AttributeProto::FLOAT, kDefaultEmbedLayerNormEpsilon) .Input(0, "input_ids", "2D words IDs with shape (batch_size, sequence_length)", "T1") @@ -628,7 +625,6 @@ GELU (Gaussian Error Linear Unit) approximation: Y=0.5*X*(1+tanh(0.797885*X+0.03 ONNX_CONTRIB_OPERATOR_SCHEMA(FastGelu) .SetDomain(kMSDomain) .SinceVersion(1) - .SetSupportLevel(OpSchema::SupportType::EXPERIMENTAL) .SetDoc(FastGelu_ver1_doc) .Input(0, "X", "input tensor", "T") .Input(1, "bias", "bias tensor", "T", OpSchema::Optional) @@ -639,7 +635,6 @@ GELU (Gaussian Error Linear Unit) approximation: Y=0.5*X*(1+tanh(0.797885*X+0.03 ONNX_CONTRIB_OPERATOR_SCHEMA(SkipLayerNormalization) .SetDomain(kMSDomain) .SinceVersion(1) - .SetSupportLevel(OpSchema::SupportType::EXPERIMENTAL) .SetDoc("Skip and Layer Normalization Fusion") .Attr("epsilon", "The epsilon value to use to avoid division by zero.", AttributeProto::FLOAT, kDefaultSkipLayerNormEpsilon) .Input(0, "input", "3D input tensor with shape (batch_size, sequence_length, hidden_size)", "T") @@ -1922,6 +1917,37 @@ Matrix product that behaves like numpy.matmul: https://docs.scipy.org/doc/numpy- FusedMatMulShapeInference(ctx); }); + ONNX_CONTRIB_OPERATOR_SCHEMA(TransposeMatMul) + .SetDomain(kMSDomain) + .SinceVersion(2) + .Deprecate() + .Input(0, "A", "N-dimensional matrix A", "T") + .Input(1, "B", "N-dimensional matrix B", "T") + .Attr( + "alpha", + "Scalar multiplier for the product of the input tensors.", + AttributeProto::FLOAT, + 1.0f) + .Attr( + "transA", + "Whether A should be transposed on the last two dimensions before doing multiplication", + AttributeProto::INT, + static_cast(0)) + .Attr( + "transB", + "Whether B should be transposed on the last two dimensions before doing multiplication", + AttributeProto::INT, + static_cast(0)) + .Output(0, "Y", "Matrix multiply results", "T") + .TypeConstraint( + "T", + {"tensor(float16)", "tensor(float)", "tensor(double)", "tensor(bfloat16)"}, + "Constrain input and output types to float tensors.") + .SetDoc(TransposeMatMul_doc) + .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + FusedMatMulShapeInference(ctx); + }); + ONNX_CONTRIB_OPERATOR_SCHEMA(FusedMatMul) .SetDomain(kMSDomain) .SinceVersion(1) @@ -2843,7 +2869,6 @@ inputs by their magnitude, rather than gates inputs by their sign as in ReLUs.)D ONNX_CONTRIB_OPERATOR_SCHEMA(Gelu) .SetDomain(kMSDomain) .SinceVersion(1) - .SetSupportLevel(OpSchema::SupportType::EXPERIMENTAL) .SetDoc(Gelu_ver1_doc) .Input(0, "X", "The input data as Tensor.", "T") .Output(0, "Y", "The output.", "T") @@ -2859,7 +2884,6 @@ It's an extension of Gelu. It takes the sum of input A and bias input B as the i ONNX_CONTRIB_OPERATOR_SCHEMA(BiasGelu) .SetDomain(kMSDomain) .SinceVersion(1) - .SetSupportLevel(OpSchema::SupportType::EXPERIMENTAL) .SetDoc(BiasGelu_ver1_doc) .Input(0, "A", "The normal input data.", "T") .Input(1, "B", "The bias input data that is a 1D tensor.", "T") @@ -2885,7 +2909,6 @@ It's an extension of Gelu. It takes the sum of input A and bias input B as the i ONNX_CONTRIB_OPERATOR_SCHEMA(Inverse) .SetDomain(kMSDomain) .SinceVersion(1) - .SetSupportLevel(OpSchema::SupportType::EXPERIMENTAL) .Input(0, "X", "Input tensor. Every matrix in the batch must be invertible.", "T") .Output(0, "Y", "Output tensor of the same type and shape as the input tensor.", "T") .TypeConstraint( @@ -2943,7 +2966,6 @@ It's an extension of Gelu. It takes the sum of input A and bias input B as the i .SetDomain(kMSDomain) .SinceVersion(1) .SetDoc(Trilu_ver1_doc) - .SetSupportLevel(OpSchema::SupportType::EXPERIMENTAL) .Attr("upper", "Boolean. Indicates whether upper or lower part of matrix is retained. Default is true.", AttributeProto::INT, @@ -2999,7 +3021,6 @@ It's an extension of Gelu. It takes the sum of input A and bias input B as the i ONNX_CONTRIB_OPERATOR_SCHEMA(BiasSoftmax) .SetDomain(kMSDomain) .SinceVersion(1) - .SetSupportLevel(OpSchema::SupportType::EXPERIMENTAL) .SetDoc( "Y = softmax(scores + bias)) with simple broadcast on bias. " "Intended to specialize softmax(scores + additive_mask) commonly found in transformer models.") diff --git a/onnxruntime/core/session/environment.cc b/onnxruntime/core/session/environment.cc index d42785e314..4e183f4157 100644 --- a/onnxruntime/core/session/environment.cc +++ b/onnxruntime/core/session/environment.cc @@ -92,7 +92,7 @@ Status Environment::Initialize(std::unique_ptr logging_ #if !defined(ORT_MINIMAL_BUILD) // Register Microsoft domain with min/max op_set version as 1/1. std::call_once(schemaRegistrationOnceFlag, []() { - ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSDomain, 1, 1); + ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSDomain, 1, 2); ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSNchwcDomain, 1, 1); ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSFeaturizersDomain, 1, 1); #ifdef USE_DML diff --git a/onnxruntime/test/contrib_ops/fused_matmul_op_test.cc b/onnxruntime/test/contrib_ops/fused_matmul_op_test.cc index 1a4dcc755b..c41c6cbc1b 100644 --- a/onnxruntime/test/contrib_ops/fused_matmul_op_test.cc +++ b/onnxruntime/test/contrib_ops/fused_matmul_op_test.cc @@ -218,6 +218,24 @@ TEST(TransposeMatMulOpTest, FloatTypeScale) { RunFusedMatMulTest("TransposeMatMul", 1, true, true, 4.0f, true); } +TEST(TransposeMatMulOpTest, DeprecatedSchema) { + OpTester test("TransposeMatMul", 2, onnxruntime::kMSDomain); + + std::vector input0_dims{3, 4}; + std::vector input0_vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + + std::vector input1_dims{4, 3}; + std::vector input1_vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + + test.AddInput("A", input0_dims, input0_vals); + test.AddInput("B", input1_dims, input1_vals); + test.AddOutput("Y", {3, 3}, {42, 48, 54, 114, 136, 158, 186, 224, 262}); + + // TransposeMatMul is deprecated from version 2 onwards of ms domain. + test.Run(OpTester::ExpectResult::kExpectFailure); +} + + } // namespace transpose_matmul } // namespace test } // namespace onnxruntime diff --git a/tools/python/gen_contrib_doc.py b/tools/python/gen_contrib_doc.py index a76309d67f..f4e3f0cfea 100644 --- a/tools/python/gen_contrib_doc.py +++ b/tools/python/gen_contrib_doc.py @@ -50,6 +50,10 @@ def format_name_with_domain(domain, schema_name): # type: (Text, Text) -> Text return schema_name +def format_name_with_version(schema_name, version): # type: (Text, Text) -> Text + return '{}-{}'.format(schema_name, version) + + def display_attr_type(v): # type: (OpSchema.AttrType) -> Text assert isinstance(v, OpSchema.AttrType) s = Text(v) @@ -132,8 +136,9 @@ def display_schema(schema, versions): # type: (OpSchema, Sequence[OpSchema]) -> if len(versions) > 1: # TODO: link to the Changelog.md s += '\nOther versions of this operator: {}\n'.format( - ', '.join(display_version_link(format_name_with_domain(v.domain, v.name), - v.since_version) for v in versions[:-1])) + ', '.join(format_name_with_version( + format_name_with_domain(v.domain, v.name), v.since_version) + for v in versions[:-1])) # If this schema is deprecated, don't display any of the following sections if schema.deprecated: @@ -310,7 +315,7 @@ def main(args): # type: (Type[Args]) -> None fout.write( "*This file is automatically generated from the\n" " [def files](/onnxruntime/core/graph/contrib_ops/contrib_defs.cc) via " - "[this script](/tools/python/gen_doc.py).\n" + "[this script](/tools/python/gen_contrib_doc.py).\n" " Do not modify directly and instead edit operator definitions.*\n") # domain -> support level -> name -> [schema]