From 07e9a4c16404dac4a9bd57f671af764294a35a82 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Sat, 16 May 2020 20:17:40 -0700 Subject: [PATCH] Update benchmark to reflect those used in our latest results (#3967) Update optimizer for GPT2 models exported from PyTorch 1.5. Update benchmark to use GPT2 models without Past State inputs/outputs Update bert_perf_test to allow setting omp_num_threads etc to test only one setting --- .../tools/transformers/Gpt2OnnxModel.py | 64 +++-- .../python/tools/transformers/benchmark.py | 117 ++++---- .../tools/transformers/bert_perf_test.py | 257 ++++++++++++------ 3 files changed, 271 insertions(+), 167 deletions(-) diff --git a/onnxruntime/python/tools/transformers/Gpt2OnnxModel.py b/onnxruntime/python/tools/transformers/Gpt2OnnxModel.py index ede7dd62de..e6922bce3b 100644 --- a/onnxruntime/python/tools/transformers/Gpt2OnnxModel.py +++ b/onnxruntime/python/tools/transformers/Gpt2OnnxModel.py @@ -15,7 +15,6 @@ logger = logging.getLogger(__name__) class Gpt2OnnxModel(BertOnnxModel): - def __init(self, model, num_heads, hidden_size): super().__init__(model, num_heads, hidden_size) @@ -28,7 +27,6 @@ class Gpt2OnnxModel(BertOnnxModel): input_name_to_nodes = self.input_name_to_nodes() output_name_to_node = self.output_name_to_node() - nodes_to_remove = [] attention_count = 0 for normalize_node in self.get_nodes_by_op_type("LayerNormalization"): @@ -65,10 +63,39 @@ class Gpt2OnnxModel(BertOnnxModel): continue qk_nodes = self.match_parent_path(matmul_qkv, ['Softmax', 'Sub', 'Mul', 'Div', 'MatMul'], [0, 0, 0, 0, 0]) - if qk_nodes is None: - logger.debug("fuse_attention: failed to match qk path") - continue - (softmax_qk, sub_qk, mul_qk, div_qk, matmul_qk) = qk_nodes + if qk_nodes is not None: + (softmax_qk, sub_qk, mul_qk, div_qk, matmul_qk) = qk_nodes + mask_nodes = self.match_parent_path( + sub_qk, + ['Mul', 'Sub', 'Slice', 'Slice', 'Unsqueeze', 'Sub', 'Squeeze', 'Slice', 'Shape', 'Div'], + [1, 0, 1, 0, 1, 0, 0, 0, 0, 0]) # yapf: disable + if mask_nodes is None: + logger.debug("fuse_attention: failed to match mask path") + continue + div_mask = mask_nodes[-1] + + if div_qk != div_mask: + logger.debug("fuse_attention: skip since div_qk != div_mask") + continue + else: + # New pattern for gpt2 from PyTorch 1.5.0 and Transformers 2.9.0. + qk_nodes = self.match_parent_path(matmul_qkv, ['Softmax', 'Where', 'Div', 'MatMul'], [0, 0, 1, 0]) + if qk_nodes is None: + logger.debug("fuse_attention: failed to match qk path") + continue + (softmax_qk, where_qk, div_qk, matmul_qk) = qk_nodes + mask_nodes = self.match_parent_path( + where_qk, + ['Cast', 'Slice', 'Slice', 'Unsqueeze', 'Sub', 'Squeeze', 'Slice', 'Shape', 'Div'], + [ 0, 0, 0, 1, 0, 0, 0, 0, 0]) # yapf: disable + if mask_nodes is None: + logger.debug("fuse_attention: failed to match mask path") + continue + div_mask = mask_nodes[-1] + + if div_qk != div_mask: + logger.debug("fuse_attention: skip since div_qk != div_mask") + continue q_nodes = self.match_parent_path(matmul_qk, ['Transpose', 'Reshape', 'Split'], [0, 0, 0]) if q_nodes is None: @@ -88,31 +115,12 @@ class Gpt2OnnxModel(BertOnnxModel): logger.debug("fuse_attention: skip since split_v != split_k") continue - mask_nodes = self.match_parent_path( - sub_qk, - ['Mul', 'Sub', 'Slice', 'Slice', 'Unsqueeze', 'Sub', 'Squeeze', 'Slice', 'Shape', 'Div'], - [1, 0, 1, 0, 1, 0, 0, 0, 0, 0]) # yapf: disable - if mask_nodes is None: - logger.debug("fuse_attention: failed to match mask path") - continue - (mul_mask, sub_mask, slice_mask, slice_mask_0, unsqueeze_mask, sub_mask, squeeze_mask, slice_mask_1, - shape_mask, div_mask) = mask_nodes - - if div_qk != div_mask: - logger.debug("fuse_attention: skip since div_qk != div_mask") - continue - self.create_attention_node(gemm, gemm_qkv, layernorm_before_attention.output[0], reshape_qkv.output[0], attention_count == 0) - nodes_to_remove.extend([reshape_qkv, transpose_qkv, matmul_qkv]) - nodes_to_remove.extend(qk_nodes) - nodes_to_remove.extend(q_nodes) - nodes_to_remove.extend(k_nodes) - nodes_to_remove.extend(v_nodes) - nodes_to_remove.extend(mask_nodes) + # we rely on prune_graph() to clean old subgraph nodes: + # qk_nodes + q_nodes + k_nodes + v_nodes + mask_nodes + [reshape_qkv, transpose_qkv, matmul_qkv] attention_count += 1 - self.remove_nodes(nodes_to_remove) self.prune_graph() logger.info(f"Fused Attention count:{attention_count}") @@ -187,4 +195,4 @@ class Gpt2OnnxModel(BertOnnxModel): reshape_count += 2 self.prune_graph() - logger.info(f"Remove Reshape count:{reshape_count}") + logger.info(f"postprocess: remove Reshape count:{reshape_count}") diff --git a/onnxruntime/python/tools/transformers/benchmark.py b/onnxruntime/python/tools/transformers/benchmark.py index 8b8d1bef93..46ecb5af35 100644 --- a/onnxruntime/python/tools/transformers/benchmark.py +++ b/onnxruntime/python/tools/transformers/benchmark.py @@ -13,11 +13,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -""" Benchmarking the inference of pretrained transformer models +""" Benchmarking the inference of pretrained transformer models. PyTorch/TorchScript benchmark is based on https://github.com/huggingface/transformers/blob/master/examples/benchmarks.py. + One difference is that random input_ids is generated in this benchmark. - This will automatically export a pretrained model to ONNX, and do optimization (optional) + For onnxruntime, this script will convert a pretrained model to ONNX, and optimize it when -o parameter is used. Example commands: Export all models to ONNX, optimize and validate them: @@ -48,24 +48,25 @@ from packaging import version logger = logging.getLogger('') -DEFAULT_MODELS = ["bert-base-cased", "distilbert-base-uncased", "roberta-base", "gpt2"] - # List of pretrained models: https://huggingface.co/transformers/pretrained_models.html -# Pretrained model name to a tuple of input names, opset_version and optimization model type +# Pretrained model name to a tuple of input names, opset_version, use_external_data_format and optimization model type MODELS = { - "bert-base-cased": (["input_ids", "attention_mask", "token_type_ids"], 11, "bert"), - "distilbert-base-uncased": (["input_ids", "attention_mask"], 11, "bert"), - "roberta-base": (["input_ids", "attention_mask"], 11, "bert"), + "bert-base-cased": (["input_ids", "attention_mask", "token_type_ids"], 11, False, "bert"), + "distilbert-base-uncased": (["input_ids", "attention_mask"], 11, False, "bert"), + "roberta-base": (["input_ids", "attention_mask"], 11, False, "bert"), - # No past state inputs for GPT models. You might set enable_cache=False in modeling_gpt2.py (need installing transformers from source) to disable past state output. - "gpt2": (["input_ids"], 11, "gpt2"), - "distilgpt2": (["input_ids"], 11, "gpt2"), - "openai-gpt": (["input_ids"], 11, "gpt2"), + # No past state inputs for GPT models. + "gpt2": (["input_ids"], 11, False, "gpt2"), # no past state inputs & outputs + "distilgpt2": (["input_ids"], 11, False, "gpt2"), # no past state inputs & outputs + "openai-gpt": (["input_ids"], 11, False, "gpt2"), # no past state inputs # Models uses Einsum, which need opset version 12 and PyTorch 1.5.0 or above. # Currently OnnxRuntime lacks cuda op for Einsum. GPU inference will be very slow. - "albert-base-v2": (["input_ids"], 12, "bert"), - "xlnet-base-cased": (["input_ids"], 12, "bert"), + "albert-base-v2": (["input_ids"], 12, False, "bert"), + "xlnet-base-cased": (["input_ids"], 12, False, "bert"), + + # This model is very large. Need use_external_data_format=True to export it. + "xlm-mlm-en-2048": (["input_ids"], 11, True, "bert"), } cpu_count = psutil.cpu_count(logical=True) @@ -73,11 +74,23 @@ cpu_count = psutil.cpu_count(logical=True) if "OMP_NUM_THREADS" not in os.environ: os.environ["OMP_NUM_THREADS"] = str(cpu_count) -from transformers import (AutoConfig, AutoTokenizer, is_torch_available) +import torch +from transformers import (AutoConfig, AutoTokenizer, AutoModel, GPT2Model) -if is_torch_available(): - import torch - from transformers import AutoModel + +# use_cache is True by default in GPT2Model. Here we wrap a class to disable past state output. +class GPT2ModelNoPastState(GPT2Model): + def __init__(self, config): + super().__init__(config) + + def forward(self, input_ids): + return super().forward(input_ids, use_cache=False) + + +def load_pretrained_model(model_name, config, cache_dir): + if model_name in ["gpt2", "distilgpt2"]: + return GPT2ModelNoPastState.from_pretrained(model_name, config=config, cache_dir=cache_dir) + return AutoModel.from_pretrained(model_name, config=config, cache_dir=cache_dir) def create_onnxruntime_session(onnx_model_path, use_gpu): @@ -176,16 +189,22 @@ def validate_onnx_model(onnx_model_filename, example_inputs, example_outputs_fla optimize_model_statistics = {} -def optimize_onnx_model(onnx_model_filename, model_type, num_attention_heads, hidden_size, fp16): +def optimize_onnx_model(onnx_model_filename, model_type, num_attention_heads, hidden_size, fp16, overwrite): optimized_model_filename = onnx_model_filename.replace(".onnx", "_fp16.onnx" if fp16 else "_fp32.onnx") - if not os.path.exists(optimized_model_filename): + if overwrite or not os.path.exists(optimized_model_filename): from optimizer import optimize_model + from BertOnnxModel import BertOptimizationOptions + optimization_options = BertOptimizationOptions(model_type) + if fp16: + optimization_options.enable_gelu_approximation = True + # Use onnxruntime to optimize model, which will be saved to *_ort_cpu.onnx opt_model = optimize_model(onnx_model_filename, model_type, num_heads=num_attention_heads, hidden_size=hidden_size, opt_level=99, + optimization_options=optimization_options, only_onnxruntime=True) optimize_model_statistics[onnx_model_filename] = opt_model.get_fused_operator_statistics() @@ -194,7 +213,7 @@ def optimize_onnx_model(onnx_model_filename, model_type, num_attention_heads, hi model_type, num_heads=num_attention_heads, hidden_size=hidden_size, - opt_level=0) + opt_level=99) optimize_model_statistics[optimized_model_filename] = opt_model.get_fused_operator_statistics() if fp16: @@ -205,9 +224,9 @@ def optimize_onnx_model(onnx_model_filename, model_type, num_attention_heads, hi return optimized_model_filename -def export_onnx_model(model_name, cache_dir, input_names, fp16, optimize_onnx, validate_onnx): +def export_onnx_model(model_name, cache_dir, input_names, fp16, optimize_onnx, validate_onnx, overwrite): config = AutoConfig.from_pretrained(model_name, cache_dir=cache_dir) - model = AutoModel.from_pretrained(model_name, config=config, cache_dir=cache_dir) + model = load_pretrained_model(model_name, config=config, cache_dir=cache_dir) model.cpu() tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir=cache_dir) @@ -223,7 +242,7 @@ def export_onnx_model(model_name, cache_dir, input_names, fp16, optimize_onnx, v example_outputs_flatten = update_flatten_list(example_outputs_flatten, []) onnx_model_filename = "{}_{}.onnx".format(model_name, str(len(input_names))) - if not os.path.exists(onnx_model_filename): + if overwrite or not os.path.exists(onnx_model_filename): logger.info("Exporting ONNX model to {}".format(onnx_model_filename)) dynamic_axes, output_names = build_dynamic_axes(example_inputs, example_outputs_flatten) @@ -236,7 +255,8 @@ def export_onnx_model(model_name, cache_dir, input_names, fp16, optimize_onnx, v example_outputs=example_outputs, dynamic_axes=dynamic_axes, do_constant_folding=True, - opset_version=MODELS[model_name][1]) + opset_version=MODELS[model_name][1], + use_external_data_format=MODELS[model_name][2]) else: logger.info(f"Skip export since model existed: {onnx_model_filename}") @@ -245,9 +265,9 @@ def export_onnx_model(model_name, cache_dir, input_names, fp16, optimize_onnx, v is_valid_onnx_model = validate_onnx_model(onnx_model_filename, example_inputs, example_outputs_flatten) if optimize_onnx or fp16: - model_type = MODELS[model_name][2] + model_type = MODELS[model_name][3] onnx_model_filename = optimize_onnx_model(onnx_model_filename, model_type, config.num_attention_heads, - config.hidden_size, fp16) + config.hidden_size, fp16, overwrite) if validate_onnx: is_valid_onnx_model = validate_onnx_model(onnx_model_filename, example_inputs, example_outputs_flatten) @@ -272,7 +292,7 @@ def get_latency_result(runtimes, batch_size): def run_onnxruntime(use_gpu, model_names, fp16, batch_sizes, sequence_lengths, repeat_times, input_counts, - optimize_onnx, validate_onnx, cache_dir, verbose): + optimize_onnx, validate_onnx, cache_dir, verbose, overwrite): import onnxruntime results = [] @@ -295,7 +315,7 @@ def run_onnxruntime(use_gpu, model_names, fp16, batch_sizes, sequence_lengths, r with torch.no_grad(): onnx_model_file, is_valid_onnx_model, vocab_size, max_sequence_length = export_onnx_model( - model_name, cache_dir, input_names, fp16, optimize_onnx, validate_onnx) + model_name, cache_dir, input_names, fp16, optimize_onnx, validate_onnx, overwrite) if not is_valid_onnx_model: continue @@ -347,19 +367,21 @@ def run_pytorch(use_gpu, model_names, fp16, batch_sizes, sequence_lengths, repea for model_name in model_names: config = AutoConfig.from_pretrained(model_name, torchscript=torchscript, cache_dir=cache_dir) - model = AutoModel.from_pretrained(model_name, config=config, cache_dir=cache_dir) + model = load_pretrained_model(model_name, config=config, cache_dir=cache_dir) tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir=cache_dir) max_input_size = tokenizer.max_model_input_sizes[model_name] logger.debug(f"Model {model}") logger.debug(f"Number of parameters {model.num_parameters()}") + if fp16: + model.half() + + device = torch.device("cuda:0" if use_gpu else "cpu") + model.to(device) + for batch_size in batch_sizes: if batch_size <= 0: continue - if fp16: - model.half() - device = torch.device("cuda:0" if use_gpu else "cpu") - model.to(device) for sequence_length in sequence_lengths: if max_input_size is not None and sequence_length > max_input_size: @@ -372,13 +394,8 @@ def run_pytorch(use_gpu, model_names, fp16, batch_sizes, sequence_lengths, repea dtype=torch.long, device=device) try: - if torchscript: - logger.debug("Tracing model with input shape {}".format(input_ids.shape)) - inference = torch.jit.trace(model, input_ids) - inference(input_ids) - else: - inference = model - inference(input_ids) + inference = torch.jit.trace(model, input_ids) if torchscript else model + inference(input_ids) runtimes = timeit.repeat(lambda: inference(input_ids), repeat=repeat_times, number=1) @@ -452,11 +469,15 @@ def output_summary(results, csv_filename, args): def output_fusion_statistics(optimize_model_statistics, csv_filename): + from transformers import __version__ as transformers_version with open(csv_filename, mode="a", newline='') as csv_file: - column_names = ["model_filename"] + list(next(iter(optimize_model_statistics.values())).keys()) + column_names = ["model_filename", "transformers", "torch"] + list( + next(iter(optimize_model_statistics.values())).keys()) csv_writer = csv.DictWriter(csv_file, fieldnames=column_names) csv_writer.writeheader() for key in optimize_model_statistics.keys(): + optimize_model_statistics[key]["transformers"] = transformers_version + optimize_model_statistics[key]["torch"] = torch.__version__ optimize_model_statistics[key]["model_filename"] = key csv_writer.writerow(optimize_model_statistics[key]) logger.info(f"Fusion statistics is saved to csv file: {csv_filename}") @@ -496,6 +517,8 @@ def parse_arguments(): parser.add_argument("--verbose", required=False, action="store_true", help="Print more information") + parser.add_argument("--overwrite", required=False, action="store_true", help="Overwrite existing models") + parser.add_argument("-o", "--optimize_onnx", required=False, @@ -543,7 +566,7 @@ def setup_logger(verbose): coloredlogs.install(level='DEBUG', fmt='[%(filename)s:%(lineno)s - %(funcName)20s()] %(message)s') else: coloredlogs.install(fmt='%(message)s') - logging.getLogger("transformers").setLevel(logging.ERROR) + logging.getLogger("transformers").setLevel(logging.WARNING) def main(): @@ -569,10 +592,6 @@ def main(): results = [] if enable_torch or enable_torchscript: - if not is_torch_available(): - logger.error("Trying to run a PyTorch benchmark but PyTorch was not found in the environment.") - return - if args.input_counts != [1]: logger.warning("--input_counts is not implemented for torch or torchscript engine.") @@ -588,7 +607,7 @@ def main(): try: results += run_onnxruntime(args.use_gpu, args.models, args.fp16, args.batch_sizes, args.sequence_lengths, args.test_times, args.input_counts, args.optimize_onnx, args.validate_onnx, - args.cache_dir, args.verbose) + args.cache_dir, args.verbose, args.overwrite) except: logger.error(f"Exception", exc_info=True) diff --git a/onnxruntime/python/tools/transformers/bert_perf_test.py b/onnxruntime/python/tools/transformers/bert_perf_test.py index 62de94635d..353ffb97d5 100644 --- a/onnxruntime/python/tools/transformers/bert_perf_test.py +++ b/onnxruntime/python/tools/transformers/bert_perf_test.py @@ -26,6 +26,62 @@ from datetime import datetime import multiprocessing from bert_test_data import get_bert_inputs, generate_test_data +from dataclasses import dataclass + + +@dataclass +class TestSetting: + batch_size: int + sequence_length: int + test_cases: int + test_times: int + contiguous: bool + use_gpu: bool + warmup: bool + omp_num_threads: int + omp_wait_policy: str + intra_op_num_threads: int + seed: int + verbose: bool + contiguous: bool + inclusive: bool + extra_latency: float = 0 + + def get_setting(self) -> str: + return f"batch_size={self.batch_size},sequence_length={self.sequence_length},test_cases={self.test_cases},test_times={self.test_times},contiguous={self.contiguous},use_gpu={self.use_gpu},warmup={self.warmup}" + + def check(self, intra_op_threads, omp_threads, omp_policy) -> bool: + if intra_op_threads is None: + if self.intra_op_num_threads > 0: + return False + else: + assert intra_op_threads > 0 + if not (self.intra_op_num_threads is None or self.intra_op_num_threads == intra_op_threads): + return False + + if omp_threads is None: + if self.omp_num_threads > 0: + return False + else: + assert omp_threads > 0 + if not (self.omp_num_threads is None or self.omp_num_threads == omp_threads): + return False + + if self.omp_wait_policy is not None: + if omp_policy != self.omp_wait_policy: + return False + + return True + + +@dataclass +class ModelSetting: + model_path: str + input_ids: str + segment_ids: str + input_mask: str + opt_level: int + def create_session(model_path, use_gpu, intra_op_num_threads, graph_optimization_level=None): # Import onnxruntime shall be after OpenMP environment variable setting. @@ -43,7 +99,7 @@ def create_session(model_path, use_gpu, intra_op_num_threads, graph_optimization session = onnxruntime.InferenceSession(model_path) else: execution_providers = ['CPUExecutionProvider' - ] if not use_gpu else ['CUDAExecutionProvider', 'CPUExecutionProvider'] + ] if not use_gpu else ['CUDAExecutionProvider', 'CPUExecutionProvider'] sess_options = onnxruntime.SessionOptions() sess_options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL @@ -113,7 +169,7 @@ def to_string(model_path, session, test_setting): 'GraphOptimizationLevel.ORT_', '') option += ",OMP_NUM_THREADS={}".format(os.environ["OMP_NUM_THREADS"] if "OMP_NUM_THREADS" in os.environ else "") option += ",OMP_WAIT_POLICY={}".format(os.environ["OMP_WAIT_POLICY"] if "OMP_WAIT_POLICY" in os.environ else "") - option += ",{}".format(test_setting) + option += ",{}".format(test_setting.get_setting()) return option @@ -128,23 +184,20 @@ def setup_openmp_environ(omp_num_threads, omp_wait_policy): if "OMP_WAIT_POLICY" in os.environ: del os.environ["OMP_WAIT_POLICY"] else: - assert omp_wait_policy in ["ACTIVE", "PASSIVE"] + assert omp_wait_policy in ["ACTIVE", "PASSIVE"], f"{omp_wait_policy} is not a valid policy" os.environ["OMP_WAIT_POLICY"] = omp_wait_policy -def run_one_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, - contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, opt_level, - extra_latency): +def run_one_test(model_setting, test_setting, perf_results, all_inputs, intra_op_num_threads, omp_num_threads, + omp_wait_policy): # Environment variable shall be set before import onnxruntime. setup_openmp_environ(omp_num_threads, omp_wait_policy) - test_setting = "batch_size={},sequence_length={},test_cases={},test_times={},contiguous={},use_gpu={},warmup={}".format( - batch_size, sequence_length, test_cases, test_times, contiguous, use_gpu, not no_warmup) - - session = create_session(model_path, use_gpu, intra_op_num_threads, opt_level) + session = create_session(model_setting.model_path, test_setting.use_gpu, intra_op_num_threads, + model_setting.opt_level) output_names = [output.name for output in session.get_outputs()] - key = to_string(model_path, session, test_setting) + key = to_string(model_setting.model_path, session, test_setting) if key in perf_results: print("skip duplicated test:", key) return @@ -152,12 +205,12 @@ def run_one_test(perf_results, model_path, all_inputs, batch_size, sequence_leng print("Running test:", key) all_latency_list = [] - for i in range(test_times): - results, latency_list = onnxruntime_inference(session, all_inputs, output_names, not no_warmup) + for i in range(test_setting.test_times): + results, latency_list = onnxruntime_inference(session, all_inputs, output_names, test_setting.warmup) all_latency_list.extend(latency_list) # latency in miliseconds - latency_ms = np.array(all_latency_list) * 1000 + extra_latency + latency_ms = np.array(all_latency_list) * 1000 + test_setting.extra_latency average_latency = statistics.mean(latency_ms) latency_50 = np.percentile(latency_ms, 50) @@ -165,7 +218,7 @@ def run_one_test(perf_results, model_path, all_inputs, batch_size, sequence_leng latency_90 = np.percentile(latency_ms, 90) latency_95 = np.percentile(latency_ms, 95) latency_99 = np.percentile(latency_ms, 99) - throughput = batch_size * (1000.0 / average_latency) + throughput = test_setting.batch_size * (1000.0 / average_latency) perf_results[key] = (average_latency, latency_50, latency_75, latency_90, latency_95, latency_99, throughput) @@ -173,48 +226,67 @@ def run_one_test(perf_results, model_path, all_inputs, batch_size, sequence_leng format(throughput, '.2f'))) -def launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, - contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, opt_level, - extra_latency): +def launch_test(model_setting, test_setting, perf_results, all_inputs, intra_op_num_threads, omp_num_threads, + omp_wait_policy): + if not test_setting.check(intra_op_num_threads, omp_num_threads, omp_wait_policy): + return + process = multiprocessing.Process(target=run_one_test, - args=(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, - test_cases, test_times, contiguous, intra_op_num_threads, omp_num_threads, - omp_wait_policy, no_warmup, opt_level, extra_latency)) + args=(model_setting, test_setting, perf_results, all_inputs, intra_op_num_threads, + omp_num_threads, omp_wait_policy)) process.start() process.join() -def run_perf_tests(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, - all_inputs, test_all, no_warmup, opt_level, extra_latency): +def run_perf_tests(model_setting, test_setting, perf_results, test_all, all_inputs): cpu_count = psutil.cpu_count(logical=False) logical_cores = psutil.cpu_count(logical=True) - # Test a setting without any setting as baseline 1. - launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, - contiguous, None, None, None, no_warmup, opt_level, extra_latency) + candidate_threads = list(set([1, logical_cores, cpu_count])) - if not use_gpu: + if (test_setting.intra_op_num_threads is not None) or (test_setting.omp_num_threads is not None): + + if test_setting.intra_op_num_threads is not None: + intra_op_threads = [test_setting.intra_op_num_threads] + else: + intra_op_threads = [None] + candidate_threads + + if test_setting.omp_num_threads is not None: + omp_threads = [test_setting.omp_num_threads] + else: + omp_threads = [None] + candidate_threads + + if test_setting.omp_wait_policy is not None: + omp_policies = [test_setting.omp_wait_policy] + else: + omp_policies = [None, 'PASSIVE', 'ACTIVE'] + + for it in intra_op_threads: + for ot in omp_threads: + for op in omp_policies: + launch_test(model_setting, test_setting, perf_results, all_inputs, it, ot, op) + return + + # Test a setting without any setting as baseline 1. + launch_test(model_setting, test_setting, perf_results, all_inputs, None, None, None) + + if not test_setting.use_gpu: # For CPU: intra_op_num_threads = 1, omp_num_threads=None, omp_wait_policy=None # Another setting without environment variable as baseline 2. - launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, - contiguous, 1, None, None, no_warmup, opt_level, extra_latency) + launch_test(model_setting, test_setting, perf_results, all_inputs, 1, None, None) else: # For GPU, we test two more settings by default: # (1) intra_op_num_threads = 1, omp_num_threads=cpu_count, omp_wait_policy=PASSIVE # (2) intra_op_num_threads = logical_cores, omp_num_threads=1, omp_wait_policy=ACTIVE - launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, - contiguous, 1, cpu_count, 'PASSIVE', no_warmup, opt_level, extra_latency) + launch_test(model_setting, test_setting, perf_results, all_inputs, 1, cpu_count, 'PASSIVE') - launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, - contiguous, logical_cores, 1, 'ACTIVE', no_warmup, opt_level, extra_latency) + launch_test(model_setting, test_setting, perf_results, all_inputs, logical_cores, 1, 'ACTIVE') # GPU latency is not sensitive to these settings. No need to test many combinations. # Skip remaining settings for GPU without --all flag. - if use_gpu and not test_all: + if test_setting.use_gpu and not test_all: return - candidates = list(set([1, logical_cores, cpu_count])) - for intra_op_num_threads in candidates: for omp_num_threads in candidates: # skip settings that are very slow @@ -234,66 +306,33 @@ def run_perf_tests(perf_results, model_path, batch_size, sequence_length, use_gp continue for omp_wait_policy in ['ACTIVE', 'PASSIVE']: - launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, - test_times, contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, - opt_level, extra_latency) + launch_test(model_setting, test_setting, perf_results, all_inputs, intra_op_num_threads, + omp_num_threads, omp_wait_policy) -def run_performance(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, - verbose, inclusive, test_all, no_warmup, opt_level, input_ids_name, segment_ids_name, - input_mask_name): - - input_ids, segment_ids, input_mask = get_bert_inputs(model_path, input_ids_name, segment_ids_name, input_mask_name) +def run_performance(model_setting, test_setting, perf_results, test_all): + input_ids, segment_ids, input_mask = get_bert_inputs(model_setting.model_path, model_setting.input_ids, + model_setting.segment_ids, model_setting.input_mask) # Do not generate random mask for performance test. - print(f"Generating {test_cases} samples for batch_size={batch_size} sequence_length={sequence_length}") - all_inputs = generate_test_data(batch_size, - sequence_length, - test_cases, - seed, - verbose, + print( + f"Generating {test_setting.test_cases} samples for batch_size={test_setting.batch_size} sequence_length={test_setting.sequence_length}" + ) + all_inputs = generate_test_data(test_setting.batch_size, + test_setting.sequence_length, + test_setting.test_cases, + test_setting.seed, + test_setting.verbose, input_ids, segment_ids, input_mask, random_mask_length=False) + if test_setting.contiguous: + all_inputs, contiguous_latency = get_contiguous_inputs(all_inputs) + print("Extra latency for converting inputs to contiguous: {} ms".format(format(contiguous_latency, '.2f'))) + test_setting.extra_latency = contiguous_latency if test_setting.inclusive else 0 - contiguous = False - run_perf_tests(perf_results, - model_path, - batch_size, - sequence_length, - use_gpu, - test_cases, - test_times, - contiguous, - all_inputs, - test_all, - no_warmup, - opt_level, - extra_latency=0) - - # only test contiguous array when the --all flag is set. - if not test_all: - return - - # Convert inputs to contiguous array, which could improve inference performance - all_inputs, contiguous_latency = get_contiguous_inputs(all_inputs) - print("Extra latency for converting inputs to contiguous: {} ms".format(format(contiguous_latency, '.2f'))) - - contiguous = True - run_perf_tests(perf_results, - model_path, - batch_size, - sequence_length, - use_gpu, - test_cases, - test_times, - contiguous, - all_inputs, - test_all, - no_warmup, - opt_level, - extra_latency=contiguous_latency if inclusive else 0) + run_perf_tests(model_setting, test_setting, perf_results, test_all, all_inputs) def parse_arguments(): @@ -345,6 +384,28 @@ def parse_arguments(): parser.add_argument('--all', required=False, action='store_true', help="test all candidate settings") parser.set_defaults(all=False) + parser.add_argument('--omp_num_threads', + required=False, + type=int, + default=None, + help=">0, set OMP_NUM_THREADS value. 0, do not set") + + parser.add_argument('--intra_op_num_threads', + required=False, + type=int, + default=None, + help=">=0, set intra_op_num_threads") + + parser.add_argument('--omp_wait_policy', + required=False, + type=str, + default=None, + choices=['ACTIVE', 'PASSIVE'], + help="OMP_WAIT_POLICY") + + parser.add_argument('--contiguous', required=False, type=bool, default=False, help="contiguous input") + parser.set_defaults(contiguous=False) + parser.add_argument('--no_warmup', required=False, action='store_true', help="do not use one sample for warm-up.") parser.set_defaults(no_warmup=False) @@ -369,10 +430,26 @@ def main(): if not min(batch_size_set) >= 1 and max(batch_size_set) <= 128: raise Exception("batch_size not in range [1, 128]") + model_setting = ModelSetting(args.model, args.input_ids, args.segment_ids, args.input_mask, args.opt_level) + for batch_size in batch_size_set: - run_performance(perf_results, args.model, batch_size, args.sequence_length, args.use_gpu, args.samples, - args.test_times, args.seed, args.verbose, args.inclusive, args.all, args.no_warmup, - args.opt_level, args.input_ids, args.segment_ids, args.input_mask) + test_setting = TestSetting( + batch_size, + args.sequence_length, + args.samples, + args.test_times, + None, #contiguous + args.use_gpu, + not args.no_warmup, + args.omp_num_threads, + args.omp_wait_policy, + args.intra_op_num_threads, + args.seed, + args.verbose, + args.contiguous, + args.inclusive) + print("test setting", test_setting) + run_performance(model_setting, test_setting, perf_results, args.all) # Sort the results so that the first one has smallest latency. sorted_results = sorted(perf_results.items(), reverse=False, key=lambda x: x[1])