saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

activate past_present_share_buffer for sampling node (#14166)

Browse source
This commit is contained in:
Zhang Lei 2023-01-07 19:36:39 -08:00 committed by GitHub
parent be879c11ee
commit 74fe45bf09
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
onnxruntime/python/tools/transformers/convert_generation.py
View file

@ -245,7 +245,7 @@ def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace:
"--past_present_share_buffer",
required=False,
action="store_true",
help="Use shared buffer for past and present, currently work for gpt2 greedy search.",
help="Use shared buffer for past and present, currently work for gpt2 greedy/sampling search.",
)
model_group.set_defaults(past_present_share_buffer=False)
@ -1365,7 +1365,7 @@ def convert_generation_model(args: argparse.Namespace, generation_type: Generati
is_beamsearch: bool = generation_type == GenerationType.BEAMSEARCH
is_greedysearch: bool = generation_type == GenerationType.GREEDYSEARCH
is_sampling: bool = generation_type == GenerationType.SAMPLING
past_present_share_buffer: bool = args.past_present_share_buffer and is_greedysearch
past_present_share_buffer: bool = args.past_present_share_buffer and (is_greedysearch or is_sampling)
logger.info(f"**** past_present_share_buffer={past_present_share_buffer}, is_greedysearch={is_greedysearch}")