From d19955fd89a18dbe82bed4396c7f2923995095fc Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Tue, 6 Sep 2022 16:15:16 -0700 Subject: [PATCH] fix transformers script issues (#12802) Fix a few obvious issues: (1) bert_perf_test.py create session without provider in line 65. (2) compare_bert_results.py miss a parameter in create_session in line 37 (3) onnx_exporter.py returns value mismatch in lines 667, 690. (4) remove some imports not used in the scripts. (5) fusion_utils need not print "Removed 0 cast nodes" or "Removed 0 Identity nodes"... (6) update requirements for numpy version since gpt2 parity tool use equal_nan in numpy v1.19+ --- .../tools/transformers/bert_perf_test.py | 81 +++++++++---------- .../transformers/compare_bert_results.py | 14 +--- .../python/tools/transformers/float16.py | 4 + .../python/tools/transformers/fusion_utils.py | 8 +- .../tools/transformers/onnx_exporter.py | 19 ++--- .../tools/transformers/requirements.txt | 6 +- 6 files changed, 61 insertions(+), 71 deletions(-) diff --git a/onnxruntime/python/tools/transformers/bert_perf_test.py b/onnxruntime/python/tools/transformers/bert_perf_test.py index 0d5e18e8fc..7162c7621b 100644 --- a/onnxruntime/python/tools/transformers/bert_perf_test.py +++ b/onnxruntime/python/tools/transformers/bert_perf_test.py @@ -18,7 +18,6 @@ import multiprocessing import os import random import statistics -import sys import timeit from dataclasses import dataclass from datetime import datetime @@ -61,53 +60,49 @@ def create_session(model_path, use_gpu, provider, intra_op_num_threads, graph_op "Warning: Please install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu performance." ) - if intra_op_num_threads is None and graph_optimization_level is None: - session = onnxruntime.InferenceSession(model_path) + if use_gpu: + if provider == "dml": + execution_providers = ["DmlExecutionProvider", "CPUExecutionProvider"] + elif provider == "rocm": + execution_providers = ["ROCMExecutionProvider", "CPUExecutionProvider"] + elif provider == "migraphx": + execution_providers = [ + "MIGraphXExecutionProvider", + "ROCMExecutionProvider", + "CPUExecutionProvider", + ] + elif provider == "cuda": + execution_providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] + elif provider == "tensorrt": + execution_providers = [ + "TensorrtExecutionProvider", + "CUDAExecutionProvider", + "CPUExecutionProvider", + ] + else: + execution_providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] else: - if use_gpu: - if provider == "dml": - execution_providers = ["DmlExecutionProvider", "CPUExecutionProvider"] - elif provider == "rocm": - execution_providers = ["ROCMExecutionProvider", "CPUExecutionProvider"] - elif provider == "migraphx": - execution_providers = [ - "MIGraphXExecutionProvider", - "ROCMExecutionProvider", - "CPUExecutionProvider", - ] - elif provider == "cuda": - execution_providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] - elif provider == "tensorrt": - execution_providers = [ - "TensorrtExecutionProvider", - "CUDAExecutionProvider", - "CPUExecutionProvider", - ] - else: - execution_providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] - else: - execution_providers = ["CPUExecutionProvider"] + execution_providers = ["CPUExecutionProvider"] - sess_options = onnxruntime.SessionOptions() - sess_options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL + sess_options = onnxruntime.SessionOptions() - if graph_optimization_level is None: - sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL - elif graph_optimization_level == 0: - sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL - elif graph_optimization_level == 1: - sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC - elif graph_optimization_level == 2: - sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_EXTENDED - elif graph_optimization_level == 99: - sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL - else: - sess_options.graph_optimization_level = graph_optimization_level + if graph_optimization_level is None: + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + elif graph_optimization_level == 0: + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL + elif graph_optimization_level == 1: + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC + elif graph_optimization_level == 2: + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_EXTENDED + elif graph_optimization_level == 99: + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + else: + sess_options.graph_optimization_level = graph_optimization_level - if intra_op_num_threads is not None: - sess_options.intra_op_num_threads = intra_op_num_threads + if intra_op_num_threads is not None: + sess_options.intra_op_num_threads = intra_op_num_threads - session = onnxruntime.InferenceSession(model_path, sess_options, providers=execution_providers) + session = onnxruntime.InferenceSession(model_path, sess_options, providers=execution_providers) if use_gpu: if provider == "dml": diff --git a/onnxruntime/python/tools/transformers/compare_bert_results.py b/onnxruntime/python/tools/transformers/compare_bert_results.py index 337b96b89d..07f5dba880 100644 --- a/onnxruntime/python/tools/transformers/compare_bert_results.py +++ b/onnxruntime/python/tools/transformers/compare_bert_results.py @@ -6,23 +6,13 @@ # It is a tool to compare the inference results of the original model and optimized model. import argparse -import csv -import os -import random import statistics -import sys -import timeit -from datetime import datetime from pathlib import Path import numpy as np -import onnx -import onnx.utils import psutil from bert_perf_test import create_session, onnxruntime_inference from bert_test_data import generate_test_data, get_bert_inputs, output_test_data -from onnx import ModelProto, TensorProto, numpy_helper -from onnx_model import OnnxModel def run_model(model_path, all_inputs, use_gpu, disable_optimization): @@ -34,7 +24,9 @@ def run_model(model_path, all_inputs, use_gpu, disable_optimization): intra_op_num_threads = psutil.cpu_count(logical=False) - session = create_session(model_path, use_gpu, intra_op_num_threads, graph_optimization_level) + session = create_session( + model_path, use_gpu, "cuda" if use_gpu else "cpu", intra_op_num_threads, graph_optimization_level + ) output_names = [output.name for output in session.get_outputs()] results, latency_list = onnxruntime_inference(session, all_inputs, output_names) diff --git a/onnxruntime/python/tools/transformers/float16.py b/onnxruntime/python/tools/transformers/float16.py index bff689bb33..437e72fce0 100644 --- a/onnxruntime/python/tools/transformers/float16.py +++ b/onnxruntime/python/tools/transformers/float16.py @@ -380,11 +380,15 @@ def float_to_float16_max_diff(tensor, min_positive_val=5.96e-08, max_finite_val= if tensor.data_type != onnx_proto.TensorProto.FLOAT: raise ValueError("Expected tensor data type is float.") + float32_data = None if tensor.float_data: float32_data = np.array(tensor.float_data) if tensor.raw_data: float32_data = np.frombuffer(tensor.raw_data, dtype="float32") + if float32_data is None: + raise RuntimeError("external data not loaded!") + float16_data = convert_np_to_float16(float32_data, min_positive_val, max_finite_val) return np.amax(np.abs(float32_data - np.float32(float16_data))) diff --git a/onnxruntime/python/tools/transformers/fusion_utils.py b/onnxruntime/python/tools/transformers/fusion_utils.py index 59d67cda56..9f2fd05172 100644 --- a/onnxruntime/python/tools/transformers/fusion_utils.py +++ b/onnxruntime/python/tools/transformers/fusion_utils.py @@ -153,8 +153,9 @@ class FusionUtils: self.model.replace_input_of_all_nodes(node.output[0], node.input[0]) nodes_to_remove.append(node) - self.model.remove_nodes(nodes_to_remove) - logger.info(f"Removed {len(nodes_to_remove)} Identity nodes") + if nodes_to_remove: + self.model.remove_nodes(nodes_to_remove) + logger.info(f"Removed {len(nodes_to_remove)} Identity nodes") def remove_useless_cast_nodes(self): """Remove cast nodes that are not needed: input and output has same data type.""" @@ -182,7 +183,8 @@ class FusionUtils: else: self.model.replace_input_of_all_nodes(node.output[0], node.input[0]) self.model.remove_node(node) - logger.info(f"Removed {len(nodes_to_remove)} Cast nodes with output type same as input") + + logger.info(f"Removed {len(nodes_to_remove)} Cast nodes with output type same as input") def remove_useless_reshape_nodes(self): """Remove reshape node that is not needed based on symbolic shape inference: input and output has same shape""" diff --git a/onnxruntime/python/tools/transformers/onnx_exporter.py b/onnxruntime/python/tools/transformers/onnx_exporter.py index 50673716a7..c4dda99496 100644 --- a/onnxruntime/python/tools/transformers/onnx_exporter.py +++ b/onnxruntime/python/tools/transformers/onnx_exporter.py @@ -25,7 +25,7 @@ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" logger = logging.getLogger(__name__) -# Walkaround by replacing torch.triu using self-defined op +# Workaround by replacing torch.triu using self-defined op # Since torch.triu cannot be exported to ONNX. See https://github.com/pytorch/pytorch/issues/32968 torch_func = {"triu": torch.triu} @@ -202,7 +202,7 @@ def optimize_onnx_model_by_ort(onnx_model_path, ort_model_path, use_gpu, overwri from optimizer import get_fusion_statistics, optimize_by_onnxruntime # Use onnxruntime to optimize model, which will be saved to *_ort.onnx - opt_model = optimize_by_onnxruntime( + _ = optimize_by_onnxruntime( onnx_model_path, use_gpu=use_gpu, optimized_model_path=ort_model_path, @@ -214,7 +214,6 @@ def optimize_onnx_model_by_ort(onnx_model_path, ort_model_path, use_gpu, overwri def optimize_onnx_model( - model_name, onnx_model_path, optimized_model_path, model_type, @@ -234,7 +233,7 @@ def optimize_onnx_model( from fusion_options import FusionOptions from optimizer import optimize_model - if optimization_options == None: + if optimization_options is None: optimization_options = FusionOptions(model_type) optimization_options.use_raw_attention_mask(use_raw_attention_mask) if Precision.FLOAT16 == precision: @@ -327,8 +326,8 @@ def load_tf_model(model_name, model_class, cache_dir, config_modifier): config_modifier.modify(config) # Loading tf model from transformers limits the cpu affinity to {0} when KMP_AFFINITY is set # Restore the affinity after model loading for expected ORT performance - affi_helper = AffinitySetting() - affi_helper.get_affinity() + affinity_setting = AffinitySetting() + affinity_setting.get_affinity() model = load_pretrained_model( model_name, config=config, @@ -336,7 +335,7 @@ def load_tf_model(model_name, model_class, cache_dir, config_modifier): custom_model_class=model_class, is_tf_model=True, ) - affi_helper.set_affinity() + affinity_setting.set_affinity() return config, model @@ -399,7 +398,6 @@ def validate_and_optimize_onnx( use_external_data_format, ) optimize_onnx_model( - model_name, onnx_model_path, optimized_model_path, model_type, @@ -664,7 +662,7 @@ def export_onnx_model_from_tf( logger.info(f"Skip export since model existed: {onnx_model_path}") model_type = model_type + "_tf" - (opt_onnx_model_file, onnx_model_file, is_valid_onnx_model, vocab_size,) = validate_and_optimize_onnx( + optimized_onnx_path, is_valid_onnx_model, vocab_size = validate_and_optimize_onnx( model_name, use_external_data_format, model_type, @@ -686,8 +684,7 @@ def export_onnx_model_from_tf( ) return ( - opt_onnx_model_file, - onnx_model_file, + optimized_onnx_path, is_valid_onnx_model, vocab_size, max_input_size, diff --git a/onnxruntime/python/tools/transformers/requirements.txt b/onnxruntime/python/tools/transformers/requirements.txt index a299aa2374..7c43e373ec 100644 --- a/onnxruntime/python/tools/transformers/requirements.txt +++ b/onnxruntime/python/tools/transformers/requirements.txt @@ -1,13 +1,13 @@ onnx >= 1.8 -numpy +numpy >= 1.19.0 coloredlogs psutil py-cpuinfo py3nvml packaging -transformers >= 4.0 +transformers >= 4.18.0 scipy sentencepiece # please follow https://pytorch.org/ to install PyTorch for your OS -torch >= 1.8 \ No newline at end of file +torch >= 1.8