diff --git a/docs/ContribOperators.md b/docs/ContribOperators.md index a6f3f845d6..22607e543a 100644 --- a/docs/ContribOperators.md +++ b/docs/ContribOperators.md @@ -371,7 +371,7 @@ This version of the operator has been available since version 1 of the 'com.micr
The id of the padding token
-#### Inputs (6 - 10) +#### Inputs (5 - 9)
input_ids : I
@@ -384,8 +384,6 @@ This version of the operator has been available since version 1 of the 'com.micr
Number of beams for beam search. 1 means no beam search. Shape is (1)
num_return_sequences : I
The number of returned sequences in the batch. Shape is (1)
-
temperature : T
-
The value used to module the next token probabilities. Accepts value > 0.0. Shape is (1)
length_penalty (optional) : T
Exponential penalty to the length. Default value 1.0 means no penalty.Value > 1.0 encourages longer sequences, while values < 1.0 produces shorter sequences.Shape is (1,)
repetition_penalty (optional) : T
diff --git a/docs/OperatorKernels.md b/docs/OperatorKernels.md index b64fd3eef5..d4a9483d77 100644 --- a/docs/OperatorKernels.md +++ b/docs/OperatorKernels.md @@ -389,7 +389,7 @@ Do not modify directly.* |**Operator Domain:** *com.microsoft*|||| |Attention|*in* input:**T**
*in* weight:**T**
*in* bias:**T**
*in* mask_index:**M**
*in* past:**T**
*in* extra_add:**T**
*out* output:**T**
*out* present:**T**|1+|**T** = tensor(float)| |AttnLSTM|*in* X:**T**
*in* W:**T**
*in* R:**T**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*in* initial_c:**T**
*in* P:**T**
*in* QW:**T**
*in* MW:**T**
*in* V:**T**
*in* M:**T**
*in* memory_seq_lens:**T1**
*in* AW:**T**
*out* Y:**T**
*out* Y_h:**T**
*out* Y_c:**T**|1+|**T** = tensor(double), tensor(float)
**T1** = tensor(int32)| -|BeamSearch|*in* input_ids:**I**
*in* max_length:**I**
*in* min_length:**I**
*in* num_beams:**I**
*in* num_return_sequences:**I**
*in* temperature:**T**
*in* length_penalty:**T**
*in* repetition_penalty:**T**
*in* vocab_mask:**M**
*in* prefix_vocab_mask:**M**
*out* sequences:**I**
*out* sequences_scores:**T**
*out* scores:**T**|1+|**T** = tensor(float)| +|BeamSearch|*in* input_ids:**I**
*in* max_length:**I**
*in* min_length:**I**
*in* num_beams:**I**
*in* num_return_sequences:**I**
*in* length_penalty:**T**
*in* repetition_penalty:**T**
*in* vocab_mask:**M**
*in* prefix_vocab_mask:**M**
*out* sequences:**I**
*out* sequences_scores:**T**
*out* scores:**T**|1+|**T** = tensor(float)| |BiasGelu|*in* A:**T**
*in* B:**T**
*out* C:**T**|1+|**T** = tensor(float)| |BifurcationDetector|*in* src_tokens:**T**
*in* cur_tokens:**T**
*in* prev_suffix_match_idx:**T**
*in* pred_tokens:**T**
*out* tokens:**T**
*out* suffix_match_idx:**T**|1+|**T** = tensor(int64)| |CDist|*in* A:**T**
*in* B:**T**
*out* C:**T**|1+|**T** = tensor(double), tensor(float)| @@ -747,7 +747,7 @@ Do not modify directly.* | | |**Operator Domain:** *com.microsoft*|||| |Attention|*in* input:**T**
*in* weight:**T**
*in* bias:**T**
*in* mask_index:**M**
*in* past:**T**
*in* extra_add:**T**
*out* output:**T**
*out* present:**T**|1+|**T** = tensor(float), tensor(float16)| -|BeamSearch|*in* input_ids:**I**
*in* max_length:**I**
*in* min_length:**I**
*in* num_beams:**I**
*in* num_return_sequences:**I**
*in* temperature:**T**
*in* length_penalty:**T**
*in* repetition_penalty:**T**
*in* vocab_mask:**M**
*in* prefix_vocab_mask:**M**
*out* sequences:**I**
*out* sequences_scores:**T**
*out* scores:**T**|1+|**T** = tensor(float), tensor(float16)| +|BeamSearch|*in* input_ids:**I**
*in* max_length:**I**
*in* min_length:**I**
*in* num_beams:**I**
*in* num_return_sequences:**I**
*in* length_penalty:**T**
*in* repetition_penalty:**T**
*in* vocab_mask:**M**
*in* prefix_vocab_mask:**M**
*out* sequences:**I**
*out* sequences_scores:**T**
*out* scores:**T**|1+|**T** = tensor(float), tensor(float16)| |BiasDropout|*in* data:**T**
*in* bias:**T**
*in* residual:**T**
*in* ratio:**T1**
*in* training_mode:**T2**
*out* output:**T**
*out* mask:**T2**|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**T1** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)
**T2** = tensor(bool)| |BiasGelu|*in* A:**T**
*in* B:**T**
*out* C:**T**|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)| |BiasSoftmax|*in* data:**T**
*in* bias:**T**
*out* output:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| diff --git a/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_base.h b/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_base.h index 7c6e4739d9..9885b4ae54 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_base.h +++ b/onnxruntime/contrib_ops/cpu/transformers/beam_search_impl_base.h @@ -220,7 +220,7 @@ Status BeamSearchBase::CheckInputs(const OpKernelContextInternal& context) { "Input 'input_ids' is expected to have 2 dimensions, got ", dims.size()); } - const Tensor* vocab_mask = context.Input(8); + const Tensor* vocab_mask = context.Input(7); if (vocab_mask != nullptr) { // vocab_mask is optional const auto& vocab_mask_dims = vocab_mask->Shape().GetDims(); if (vocab_mask_dims.size() != 1) { @@ -238,9 +238,8 @@ Status BeamSearchBase::CheckInputs(const OpKernelContextInternal& context) { parameters_->vocab_mask = vocab_mask->DataAsSpan(); } - const Tensor* prefix_vocab_mask = context.Input(9); - if (prefix_vocab_mask != nullptr) { - // prefix_vocab_mask is optional + const Tensor* prefix_vocab_mask = context.Input(8); + if (prefix_vocab_mask != nullptr) { // prefix_vocab_mask is optional const auto& vocab_mask_dims = prefix_vocab_mask->Shape().GetDims(); if (vocab_mask_dims.size() != 2) { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, diff --git a/onnxruntime/contrib_ops/cpu/transformers/beam_search_parameters.cc b/onnxruntime/contrib_ops/cpu/transformers/beam_search_parameters.cc index d5375e6be5..0bbb406146 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/beam_search_parameters.cc +++ b/onnxruntime/contrib_ops/cpu/transformers/beam_search_parameters.cc @@ -59,14 +59,10 @@ void BeamSearchParameters::ParseFromInputs(OpKernelContext* context) { ORT_ENFORCE(num_beams >= num_return_sequences, "num_return_sequences (", num_return_sequences, ") shall be be no more than num_beams (", num_beams, ")"); - auto* temperature_tensor = context->Input(5); - temperature = temperature_tensor ? static_cast(*temperature_tensor->Data()) : 1; - ORT_ENFORCE(temperature > 0.0f, "temperature shall be greater than 0, got ", temperature); - - auto* length_penalty_tensor = context->Input(6); + auto* length_penalty_tensor = context->Input(5); length_penalty = length_penalty_tensor ? static_cast(*length_penalty_tensor->Data()) : 1; - auto* repetition_penalty_tensor = context->Input(7); + auto* repetition_penalty_tensor = context->Input(6); repetition_penalty = repetition_penalty_tensor ? static_cast(*repetition_penalty_tensor->Data()) : 1.0f; ORT_ENFORCE(repetition_penalty > 0.0f, "repetition_penalty shall be greater than 0, got ", repetition_penalty); } diff --git a/onnxruntime/contrib_ops/cpu/transformers/beam_search_shared.h b/onnxruntime/contrib_ops/cpu/transformers/beam_search_shared.h index 2643ca3c1b..2179c7eba2 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/beam_search_shared.h +++ b/onnxruntime/contrib_ops/cpu/transformers/beam_search_shared.h @@ -91,7 +91,6 @@ struct IBeamSearchParameters { int max_length; int num_beams; int num_return_sequences; - float temperature; float length_penalty; float repetition_penalty; int batch_size; // deduce from first dimension of input_ids diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index 06fc4f2273..469e0b7d5a 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -968,15 +968,14 @@ ONNX_MS_OPERATOR_SET_SCHEMA(BeamSearch, 1, .Input(2, "min_length", "The minimum length below which the score of eos_token_id is set to -Inf. Shape is (1)", "I", OpSchema::Optional) .Input(3, "num_beams", "Number of beams for beam search. 1 means no beam search. Shape is (1)", "I") .Input(4, "num_return_sequences", "The number of returned sequences in the batch. Shape is (1)", "I") - .Input(5, "temperature", "The value used to module the next token probabilities. Accepts value > 0.0. Shape is (1)", "T") - .Input(6, "length_penalty", + .Input(5, "length_penalty", "Exponential penalty to the length. Default value 1.0 means no penalty." "Value > 1.0 encourages longer sequences, while values < 1.0 produces shorter sequences." "Shape is (1,)", "T", OpSchema::Optional) - .Input(7, "repetition_penalty", "The parameter for repetition penalty. Default value 1.0 means no penalty. Accepts value > 0.0. Shape is (1)", "T", OpSchema::Optional) - .Input(8, "vocab_mask", "Mask of vocabulary. Words that masked with 0 are not allowed to be generated, and 1 is allowed. Shape is (vacab_size)", "M", OpSchema::Optional) - .Input(9, "prefix_vocab_mask", "Mask of vocabulary for first step. Words that masked with 0 are not allowed to be generated, and 1 is allowed. Shape is (batch_size, vocab_size)", "M", OpSchema::Optional) + .Input(6, "repetition_penalty", "The parameter for repetition penalty. Default value 1.0 means no penalty. Accepts value > 0.0. Shape is (1)", "T", OpSchema::Optional) + .Input(7, "vocab_mask", "Mask of vocabulary. Words that masked with 0 are not allowed to be generated, and 1 is allowed. Shape is (vacab_size)", "M", OpSchema::Optional) + .Input(8, "prefix_vocab_mask", "Mask of vocabulary for first step. Words that masked with 0 are not allowed to be generated, and 1 is allowed. Shape is (batch_size, vocab_size)", "M", OpSchema::Optional) .Output(0, "sequences", "Word IDs of generated sequences. Shape is (batch_size, num_return_sequences, max_sequence_length)", "I") .Output(1, "sequences_scores", "Final beam score of the generated sequences. Shape is (batch_size, num_return_sequences)", "T", OpSchema::Optional) .Output(2, "scores", diff --git a/onnxruntime/python/tools/transformers/convert_beam_search.py b/onnxruntime/python/tools/transformers/convert_beam_search.py index 70283d2852..1f37f9ef71 100644 --- a/onnxruntime/python/tools/transformers/convert_beam_search.py +++ b/onnxruntime/python/tools/transformers/convert_beam_search.py @@ -199,14 +199,6 @@ def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace: help="Number of return sequence <= num_beams", ) - beam_search_group.add_argument( - "--temperature", - type=float, - required=False, - default=1, - help="Softmax temperature for output logits.", - ) - beam_search_group.add_argument( "--length_penalty", type=float, @@ -305,6 +297,7 @@ def t5_to_onnx(args: argparse.Namespace): disable_auto_mixed_precision=False, use_int32_inputs=True, ) + args.encoder_decoder_init_onnx = paths[0] args.decoder_onnx = paths[1] @@ -610,7 +603,6 @@ def convert_model(args: argparse.Namespace): "min_length", "num_beams", "num_return_sequences", - "temperature", "length_penalty", "repetition_penalty", "vocab_mask", @@ -668,7 +660,6 @@ def convert_model(args: argparse.Namespace): min_length = onnx.helper.make_tensor_value_info("min_length", TensorProto.INT32, [1]) num_beams = onnx.helper.make_tensor_value_info("num_beams", TensorProto.INT32, [1]) num_return_sequences = onnx.helper.make_tensor_value_info("num_return_sequences", TensorProto.INT32, [1]) - temperature = onnx.helper.make_tensor_value_info("temperature", TensorProto.FLOAT, [1]) length_penalty = onnx.helper.make_tensor_value_info("length_penalty", TensorProto.FLOAT, [1]) repetition_penalty = onnx.helper.make_tensor_value_info("repetition_penalty", TensorProto.FLOAT, [1]) vocab_mask = onnx.helper.make_tensor_value_info("vocab_mask", TensorProto.INT32, [vocab_size]) @@ -679,7 +670,6 @@ def convert_model(args: argparse.Namespace): min_length, num_beams, num_return_sequences, - temperature, length_penalty, repetition_penalty, vocab_mask, @@ -790,7 +780,6 @@ def test_torch_performance( eos_token_id=eos_token_id, pad_token_id=pad_token_id, num_return_sequences=args.num_return_sequences, - temperature=args.temperature, length_penalty=args.length_penalty, repetition_penalty=args.repetition_penalty, bad_words_ids=bad_words_ids, @@ -817,11 +806,6 @@ def test_gpt_model(args: argparse.Namespace, use_vocab_mask: bool = False, sente """ assert args.model_type == "gpt2" - if args.temperature != 1.0: - # TODO(tianleiwu): implement temperature in BeamSearch operator. - print("Skipping parity test as temperature is not implemented in BeamSearch operator") - return None - if args.prefix_vocab_mask: print("Skipping parity test as prefix vocab mask is not implemented by Hugging Face") return None @@ -877,7 +861,6 @@ def test_gpt_model(args: argparse.Namespace, use_vocab_mask: bool = False, sente eos_token_id=eos_token_id, pad_token_id=pad_token_id, num_return_sequences=args.num_return_sequences, - temperature=args.temperature, length_penalty=args.length_penalty, repetition_penalty=args.repetition_penalty, bad_words_ids=bad_words_ids, @@ -912,7 +895,6 @@ def test_gpt_model(args: argparse.Namespace, use_vocab_mask: bool = False, sente "min_length": np.array([args.min_length], dtype=np.int32), "num_beams": np.array([args.num_beams], dtype=np.int32), "num_return_sequences": np.array([args.num_return_sequences], dtype=np.int32), - "temperature": np.array([args.temperature], dtype=np.float32), "length_penalty": np.array([args.length_penalty], dtype=np.float32), "repetition_penalty": np.array([args.repetition_penalty], dtype=np.float32), "vocab_mask": vocab_mask, @@ -1003,11 +985,6 @@ def test_t5_model(args: argparse.Namespace, use_vocab_mask: bool = False, senten """ assert args.model_type == "t5" - if args.temperature != 1.0: - # TODO(tianleiwu): implement temperature in BeamSearch operator. - print("Skipping parity test as temperature is not implemented in BeamSearch operator") - return None - if args.prefix_vocab_mask: print("Skipping parity test as prefix vocab mask is not implemented by Hugging Face") return None @@ -1063,7 +1040,6 @@ def test_t5_model(args: argparse.Namespace, use_vocab_mask: bool = False, senten eos_token_id=eos_token_id, pad_token_id=pad_token_id, num_return_sequences=args.num_return_sequences, - temperature=args.temperature, length_penalty=args.length_penalty, repetition_penalty=args.repetition_penalty, bad_words_ids=bad_words_ids, @@ -1099,7 +1075,6 @@ def test_t5_model(args: argparse.Namespace, use_vocab_mask: bool = False, senten "min_length": np.array([args.min_length], dtype=np.int32), "num_beams": np.array([args.num_beams], dtype=np.int32), "num_return_sequences": np.array([args.num_return_sequences], dtype=np.int32), - "temperature": np.array([args.temperature], dtype=np.float32), "length_penalty": np.array([args.length_penalty], dtype=np.float32), "repetition_penalty": np.array([args.repetition_penalty], dtype=np.float32), "vocab_mask": vocab_mask, diff --git a/onnxruntime/test/python/transformers/test_beam_search.py b/onnxruntime/test/python/transformers/test_beam_search.py index 3aa3f8bdbf..efb7a19739 100644 --- a/onnxruntime/test/python/transformers/test_beam_search.py +++ b/onnxruntime/test/python/transformers/test_beam_search.py @@ -82,10 +82,6 @@ class TestBeamSearchGpt(unittest.TestCase): def test_early_stopping(self): self.run_beam_search("--early_stopping") - # @pytest.mark.slow - # def test_temperature(self): - # self.run_beam_search("--temperature 0.5") - @pytest.mark.slow def test_length_penalty(self): for length_penalty in [0.5, 2.0]: @@ -180,10 +176,6 @@ class TestBeamSearchT5(unittest.TestCase): def test_early_stopping(self): self.run_beam_search("--early_stopping") - # @pytest.mark.slow - # def test_temperature(self): - # self.run_beam_search("--temperature 0.5") - @pytest.mark.slow def test_length_penalty(self): for length_penalty in [0.5, 2.0]: