mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Remove temperature input from BeamSearch operator (#11896)
* remove temperature input * update index of remaining inputs
This commit is contained in:
Tianlei Wu
GitHub
parent
eb41bfb7b5
commit
6ee2c1b5fc
8 changed files with 13 additions and 55 deletions
|
|
@ -371,7 +371,7 @@ This version of the operator has been available since version 1 of the 'com.micr
|
|||
<dd>The id of the padding token</dd>
|
||||
</dl>
|
||||
|
||||
#### Inputs (6 - 10)
|
||||
#### Inputs (5 - 9)
|
||||
|
||||
<dl>
|
||||
<dt><tt>input_ids</tt> : I</dt>
|
||||
|
|
@ -384,8 +384,6 @@ This version of the operator has been available since version 1 of the 'com.micr
|
|||
<dd>Number of beams for beam search. 1 means no beam search. Shape is (1)</dd>
|
||||
<dt><tt>num_return_sequences</tt> : I</dt>
|
||||
<dd>The number of returned sequences in the batch. Shape is (1)</dd>
|
||||
<dt><tt>temperature</tt> : T</dt>
|
||||
<dd>The value used to module the next token probabilities. Accepts value > 0.0. Shape is (1)</dd>
|
||||
<dt><tt>length_penalty</tt> (optional) : T</dt>
|
||||
<dd>Exponential penalty to the length. Default value 1.0 means no penalty.Value > 1.0 encourages longer sequences, while values < 1.0 produces shorter sequences.Shape is (1,)</dd>
|
||||
<dt><tt>repetition_penalty</tt> (optional) : T</dt>
|
||||
|
|
|
|||
|
|
@ -389,7 +389,7 @@ Do not modify directly.*
|
|||
|**Operator Domain:** *com.microsoft*||||
|
||||
|Attention|*in* input:**T**<br> *in* weight:**T**<br> *in* bias:**T**<br> *in* mask_index:**M**<br> *in* past:**T**<br> *in* extra_add:**T**<br> *out* output:**T**<br> *out* present:**T**|1+|**T** = tensor(float)|
|
||||
|AttnLSTM|*in* X:**T**<br> *in* W:**T**<br> *in* R:**T**<br> *in* B:**T**<br> *in* sequence_lens:**T1**<br> *in* initial_h:**T**<br> *in* initial_c:**T**<br> *in* P:**T**<br> *in* QW:**T**<br> *in* MW:**T**<br> *in* V:**T**<br> *in* M:**T**<br> *in* memory_seq_lens:**T1**<br> *in* AW:**T**<br> *out* Y:**T**<br> *out* Y_h:**T**<br> *out* Y_c:**T**|1+|**T** = tensor(double), tensor(float)<br/> **T1** = tensor(int32)|
|
||||
|BeamSearch|*in* input_ids:**I**<br> *in* max_length:**I**<br> *in* min_length:**I**<br> *in* num_beams:**I**<br> *in* num_return_sequences:**I**<br> *in* temperature:**T**<br> *in* length_penalty:**T**<br> *in* repetition_penalty:**T**<br> *in* vocab_mask:**M**<br> *in* prefix_vocab_mask:**M**<br> *out* sequences:**I**<br> *out* sequences_scores:**T**<br> *out* scores:**T**|1+|**T** = tensor(float)|
|
||||
|BeamSearch|*in* input_ids:**I**<br> *in* max_length:**I**<br> *in* min_length:**I**<br> *in* num_beams:**I**<br> *in* num_return_sequences:**I**<br> *in* length_penalty:**T**<br> *in* repetition_penalty:**T**<br> *in* vocab_mask:**M**<br> *in* prefix_vocab_mask:**M**<br> *out* sequences:**I**<br> *out* sequences_scores:**T**<br> *out* scores:**T**|1+|**T** = tensor(float)|
|
||||
|BiasGelu|*in* A:**T**<br> *in* B:**T**<br> *out* C:**T**|1+|**T** = tensor(float)|
|
||||
|BifurcationDetector|*in* src_tokens:**T**<br> *in* cur_tokens:**T**<br> *in* prev_suffix_match_idx:**T**<br> *in* pred_tokens:**T**<br> *out* tokens:**T**<br> *out* suffix_match_idx:**T**|1+|**T** = tensor(int64)|
|
||||
|CDist|*in* A:**T**<br> *in* B:**T**<br> *out* C:**T**|1+|**T** = tensor(double), tensor(float)|
|
||||
|
|
@ -747,7 +747,7 @@ Do not modify directly.*
|
|||
| |
|
||||
|**Operator Domain:** *com.microsoft*||||
|
||||
|Attention|*in* input:**T**<br> *in* weight:**T**<br> *in* bias:**T**<br> *in* mask_index:**M**<br> *in* past:**T**<br> *in* extra_add:**T**<br> *out* output:**T**<br> *out* present:**T**|1+|**T** = tensor(float), tensor(float16)|
|
||||
|BeamSearch|*in* input_ids:**I**<br> *in* max_length:**I**<br> *in* min_length:**I**<br> *in* num_beams:**I**<br> *in* num_return_sequences:**I**<br> *in* temperature:**T**<br> *in* length_penalty:**T**<br> *in* repetition_penalty:**T**<br> *in* vocab_mask:**M**<br> *in* prefix_vocab_mask:**M**<br> *out* sequences:**I**<br> *out* sequences_scores:**T**<br> *out* scores:**T**|1+|**T** = tensor(float), tensor(float16)|
|
||||
|BeamSearch|*in* input_ids:**I**<br> *in* max_length:**I**<br> *in* min_length:**I**<br> *in* num_beams:**I**<br> *in* num_return_sequences:**I**<br> *in* length_penalty:**T**<br> *in* repetition_penalty:**T**<br> *in* vocab_mask:**M**<br> *in* prefix_vocab_mask:**M**<br> *out* sequences:**I**<br> *out* sequences_scores:**T**<br> *out* scores:**T**|1+|**T** = tensor(float), tensor(float16)|
|
||||
|BiasDropout|*in* data:**T**<br> *in* bias:**T**<br> *in* residual:**T**<br> *in* ratio:**T1**<br> *in* training_mode:**T2**<br> *out* output:**T**<br> *out* mask:**T2**|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)<br/> **T1** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)<br/> **T2** = tensor(bool)|
|
||||
|BiasGelu|*in* A:**T**<br> *in* B:**T**<br> *out* C:**T**|1+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)|
|
||||
|BiasSoftmax|*in* data:**T**<br> *in* bias:**T**<br> *out* output:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)|
|
||||
|
|
|
|||
|
|
@ -220,7 +220,7 @@ Status BeamSearchBase<T>::CheckInputs(const OpKernelContextInternal& context) {
|
|||
"Input 'input_ids' is expected to have 2 dimensions, got ", dims.size());
|
||||
}
|
||||
|
||||
const Tensor* vocab_mask = context.Input<Tensor>(8);
|
||||
const Tensor* vocab_mask = context.Input<Tensor>(7);
|
||||
if (vocab_mask != nullptr) { // vocab_mask is optional
|
||||
const auto& vocab_mask_dims = vocab_mask->Shape().GetDims();
|
||||
if (vocab_mask_dims.size() != 1) {
|
||||
|
|
@ -238,9 +238,8 @@ Status BeamSearchBase<T>::CheckInputs(const OpKernelContextInternal& context) {
|
|||
parameters_->vocab_mask = vocab_mask->DataAsSpan<int32_t>();
|
||||
}
|
||||
|
||||
const Tensor* prefix_vocab_mask = context.Input<Tensor>(9);
|
||||
if (prefix_vocab_mask != nullptr) {
|
||||
// prefix_vocab_mask is optional
|
||||
const Tensor* prefix_vocab_mask = context.Input<Tensor>(8);
|
||||
if (prefix_vocab_mask != nullptr) { // prefix_vocab_mask is optional
|
||||
const auto& vocab_mask_dims = prefix_vocab_mask->Shape().GetDims();
|
||||
if (vocab_mask_dims.size() != 2) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
|
||||
|
|
|
|||
|
|
@ -59,14 +59,10 @@ void BeamSearchParameters::ParseFromInputs(OpKernelContext* context) {
|
|||
ORT_ENFORCE(num_beams >= num_return_sequences,
|
||||
"num_return_sequences (", num_return_sequences, ") shall be be no more than num_beams (", num_beams, ")");
|
||||
|
||||
auto* temperature_tensor = context->Input<Tensor>(5);
|
||||
temperature = temperature_tensor ? static_cast<float>(*temperature_tensor->Data<float>()) : 1;
|
||||
ORT_ENFORCE(temperature > 0.0f, "temperature shall be greater than 0, got ", temperature);
|
||||
|
||||
auto* length_penalty_tensor = context->Input<Tensor>(6);
|
||||
auto* length_penalty_tensor = context->Input<Tensor>(5);
|
||||
length_penalty = length_penalty_tensor ? static_cast<float>(*length_penalty_tensor->Data<float>()) : 1;
|
||||
|
||||
auto* repetition_penalty_tensor = context->Input<Tensor>(7);
|
||||
auto* repetition_penalty_tensor = context->Input<Tensor>(6);
|
||||
repetition_penalty = repetition_penalty_tensor ? static_cast<float>(*repetition_penalty_tensor->Data<float>()) : 1.0f;
|
||||
ORT_ENFORCE(repetition_penalty > 0.0f, "repetition_penalty shall be greater than 0, got ", repetition_penalty);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,7 +91,6 @@ struct IBeamSearchParameters {
|
|||
int max_length;
|
||||
int num_beams;
|
||||
int num_return_sequences;
|
||||
float temperature;
|
||||
float length_penalty;
|
||||
float repetition_penalty;
|
||||
int batch_size; // deduce from first dimension of input_ids
|
||||
|
|
|
|||
|
|
@ -968,15 +968,14 @@ ONNX_MS_OPERATOR_SET_SCHEMA(BeamSearch, 1,
|
|||
.Input(2, "min_length", "The minimum length below which the score of eos_token_id is set to -Inf. Shape is (1)", "I", OpSchema::Optional)
|
||||
.Input(3, "num_beams", "Number of beams for beam search. 1 means no beam search. Shape is (1)", "I")
|
||||
.Input(4, "num_return_sequences", "The number of returned sequences in the batch. Shape is (1)", "I")
|
||||
.Input(5, "temperature", "The value used to module the next token probabilities. Accepts value > 0.0. Shape is (1)", "T")
|
||||
.Input(6, "length_penalty",
|
||||
.Input(5, "length_penalty",
|
||||
"Exponential penalty to the length. Default value 1.0 means no penalty."
|
||||
"Value > 1.0 encourages longer sequences, while values < 1.0 produces shorter sequences."
|
||||
"Shape is (1,)",
|
||||
"T", OpSchema::Optional)
|
||||
.Input(7, "repetition_penalty", "The parameter for repetition penalty. Default value 1.0 means no penalty. Accepts value > 0.0. Shape is (1)", "T", OpSchema::Optional)
|
||||
.Input(8, "vocab_mask", "Mask of vocabulary. Words that masked with 0 are not allowed to be generated, and 1 is allowed. Shape is (vacab_size)", "M", OpSchema::Optional)
|
||||
.Input(9, "prefix_vocab_mask", "Mask of vocabulary for first step. Words that masked with 0 are not allowed to be generated, and 1 is allowed. Shape is (batch_size, vocab_size)", "M", OpSchema::Optional)
|
||||
.Input(6, "repetition_penalty", "The parameter for repetition penalty. Default value 1.0 means no penalty. Accepts value > 0.0. Shape is (1)", "T", OpSchema::Optional)
|
||||
.Input(7, "vocab_mask", "Mask of vocabulary. Words that masked with 0 are not allowed to be generated, and 1 is allowed. Shape is (vacab_size)", "M", OpSchema::Optional)
|
||||
.Input(8, "prefix_vocab_mask", "Mask of vocabulary for first step. Words that masked with 0 are not allowed to be generated, and 1 is allowed. Shape is (batch_size, vocab_size)", "M", OpSchema::Optional)
|
||||
.Output(0, "sequences", "Word IDs of generated sequences. Shape is (batch_size, num_return_sequences, max_sequence_length)", "I")
|
||||
.Output(1, "sequences_scores", "Final beam score of the generated sequences. Shape is (batch_size, num_return_sequences)", "T", OpSchema::Optional)
|
||||
.Output(2, "scores",
|
||||
|
|
|
|||
|
|
@ -199,14 +199,6 @@ def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace:
|
|||
help="Number of return sequence <= num_beams",
|
||||
)
|
||||
|
||||
beam_search_group.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
required=False,
|
||||
default=1,
|
||||
help="Softmax temperature for output logits.",
|
||||
)
|
||||
|
||||
beam_search_group.add_argument(
|
||||
"--length_penalty",
|
||||
type=float,
|
||||
|
|
@ -305,6 +297,7 @@ def t5_to_onnx(args: argparse.Namespace):
|
|||
disable_auto_mixed_precision=False,
|
||||
use_int32_inputs=True,
|
||||
)
|
||||
|
||||
args.encoder_decoder_init_onnx = paths[0]
|
||||
args.decoder_onnx = paths[1]
|
||||
|
||||
|
|
@ -610,7 +603,6 @@ def convert_model(args: argparse.Namespace):
|
|||
"min_length",
|
||||
"num_beams",
|
||||
"num_return_sequences",
|
||||
"temperature",
|
||||
"length_penalty",
|
||||
"repetition_penalty",
|
||||
"vocab_mask",
|
||||
|
|
@ -668,7 +660,6 @@ def convert_model(args: argparse.Namespace):
|
|||
min_length = onnx.helper.make_tensor_value_info("min_length", TensorProto.INT32, [1])
|
||||
num_beams = onnx.helper.make_tensor_value_info("num_beams", TensorProto.INT32, [1])
|
||||
num_return_sequences = onnx.helper.make_tensor_value_info("num_return_sequences", TensorProto.INT32, [1])
|
||||
temperature = onnx.helper.make_tensor_value_info("temperature", TensorProto.FLOAT, [1])
|
||||
length_penalty = onnx.helper.make_tensor_value_info("length_penalty", TensorProto.FLOAT, [1])
|
||||
repetition_penalty = onnx.helper.make_tensor_value_info("repetition_penalty", TensorProto.FLOAT, [1])
|
||||
vocab_mask = onnx.helper.make_tensor_value_info("vocab_mask", TensorProto.INT32, [vocab_size])
|
||||
|
|
@ -679,7 +670,6 @@ def convert_model(args: argparse.Namespace):
|
|||
min_length,
|
||||
num_beams,
|
||||
num_return_sequences,
|
||||
temperature,
|
||||
length_penalty,
|
||||
repetition_penalty,
|
||||
vocab_mask,
|
||||
|
|
@ -790,7 +780,6 @@ def test_torch_performance(
|
|||
eos_token_id=eos_token_id,
|
||||
pad_token_id=pad_token_id,
|
||||
num_return_sequences=args.num_return_sequences,
|
||||
temperature=args.temperature,
|
||||
length_penalty=args.length_penalty,
|
||||
repetition_penalty=args.repetition_penalty,
|
||||
bad_words_ids=bad_words_ids,
|
||||
|
|
@ -817,11 +806,6 @@ def test_gpt_model(args: argparse.Namespace, use_vocab_mask: bool = False, sente
|
|||
"""
|
||||
assert args.model_type == "gpt2"
|
||||
|
||||
if args.temperature != 1.0:
|
||||
# TODO(tianleiwu): implement temperature in BeamSearch operator.
|
||||
print("Skipping parity test as temperature is not implemented in BeamSearch operator")
|
||||
return None
|
||||
|
||||
if args.prefix_vocab_mask:
|
||||
print("Skipping parity test as prefix vocab mask is not implemented by Hugging Face")
|
||||
return None
|
||||
|
|
@ -877,7 +861,6 @@ def test_gpt_model(args: argparse.Namespace, use_vocab_mask: bool = False, sente
|
|||
eos_token_id=eos_token_id,
|
||||
pad_token_id=pad_token_id,
|
||||
num_return_sequences=args.num_return_sequences,
|
||||
temperature=args.temperature,
|
||||
length_penalty=args.length_penalty,
|
||||
repetition_penalty=args.repetition_penalty,
|
||||
bad_words_ids=bad_words_ids,
|
||||
|
|
@ -912,7 +895,6 @@ def test_gpt_model(args: argparse.Namespace, use_vocab_mask: bool = False, sente
|
|||
"min_length": np.array([args.min_length], dtype=np.int32),
|
||||
"num_beams": np.array([args.num_beams], dtype=np.int32),
|
||||
"num_return_sequences": np.array([args.num_return_sequences], dtype=np.int32),
|
||||
"temperature": np.array([args.temperature], dtype=np.float32),
|
||||
"length_penalty": np.array([args.length_penalty], dtype=np.float32),
|
||||
"repetition_penalty": np.array([args.repetition_penalty], dtype=np.float32),
|
||||
"vocab_mask": vocab_mask,
|
||||
|
|
@ -1003,11 +985,6 @@ def test_t5_model(args: argparse.Namespace, use_vocab_mask: bool = False, senten
|
|||
"""
|
||||
assert args.model_type == "t5"
|
||||
|
||||
if args.temperature != 1.0:
|
||||
# TODO(tianleiwu): implement temperature in BeamSearch operator.
|
||||
print("Skipping parity test as temperature is not implemented in BeamSearch operator")
|
||||
return None
|
||||
|
||||
if args.prefix_vocab_mask:
|
||||
print("Skipping parity test as prefix vocab mask is not implemented by Hugging Face")
|
||||
return None
|
||||
|
|
@ -1063,7 +1040,6 @@ def test_t5_model(args: argparse.Namespace, use_vocab_mask: bool = False, senten
|
|||
eos_token_id=eos_token_id,
|
||||
pad_token_id=pad_token_id,
|
||||
num_return_sequences=args.num_return_sequences,
|
||||
temperature=args.temperature,
|
||||
length_penalty=args.length_penalty,
|
||||
repetition_penalty=args.repetition_penalty,
|
||||
bad_words_ids=bad_words_ids,
|
||||
|
|
@ -1099,7 +1075,6 @@ def test_t5_model(args: argparse.Namespace, use_vocab_mask: bool = False, senten
|
|||
"min_length": np.array([args.min_length], dtype=np.int32),
|
||||
"num_beams": np.array([args.num_beams], dtype=np.int32),
|
||||
"num_return_sequences": np.array([args.num_return_sequences], dtype=np.int32),
|
||||
"temperature": np.array([args.temperature], dtype=np.float32),
|
||||
"length_penalty": np.array([args.length_penalty], dtype=np.float32),
|
||||
"repetition_penalty": np.array([args.repetition_penalty], dtype=np.float32),
|
||||
"vocab_mask": vocab_mask,
|
||||
|
|
|
|||
|
|
@ -82,10 +82,6 @@ class TestBeamSearchGpt(unittest.TestCase):
|
|||
def test_early_stopping(self):
|
||||
self.run_beam_search("--early_stopping")
|
||||
|
||||
# @pytest.mark.slow
|
||||
# def test_temperature(self):
|
||||
# self.run_beam_search("--temperature 0.5")
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_length_penalty(self):
|
||||
for length_penalty in [0.5, 2.0]:
|
||||
|
|
@ -180,10 +176,6 @@ class TestBeamSearchT5(unittest.TestCase):
|
|||
def test_early_stopping(self):
|
||||
self.run_beam_search("--early_stopping")
|
||||
|
||||
# @pytest.mark.slow
|
||||
# def test_temperature(self):
|
||||
# self.run_beam_search("--temperature 0.5")
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_length_penalty(self):
|
||||
for length_penalty in [0.5, 2.0]:
|
||||
|
|
|
|||
Loading…
Reference in a new issue