fix transformers script issues (#12802)

Fix a few obvious issues:
(1) bert_perf_test.py create session without provider in line 65.
(2) compare_bert_results.py miss a parameter in create_session in line 37
(3) onnx_exporter.py returns value mismatch in lines 667, 690.
(4) remove some imports not used in the scripts.
(5) fusion_utils need not print "Removed 0 cast nodes" or "Removed 0 Identity nodes"...
(6) update requirements for numpy version since gpt2 parity tool use equal_nan in numpy v1.19+

onnxruntime/python/tools/transformers/bert_perf_test.py

@@ -18,7 +18,6 @@ import multiprocessing
 import os
 import random
 import statistics
-import sys
 import timeit
 from dataclasses import dataclass
 from datetime import datetime
@@ -61,53 +60,49 @@ def create_session(model_path, use_gpu, provider, intra_op_num_threads, graph_op
             "Warning: Please install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu performance."
         )
 
-    if intra_op_num_threads is None and graph_optimization_level is None:
-        session = onnxruntime.InferenceSession(model_path)
+    if use_gpu:
+        if provider == "dml":
+            execution_providers = ["DmlExecutionProvider", "CPUExecutionProvider"]
+        elif provider == "rocm":
+            execution_providers = ["ROCMExecutionProvider", "CPUExecutionProvider"]
+        elif provider == "migraphx":
+            execution_providers = [
+                "MIGraphXExecutionProvider",
+                "ROCMExecutionProvider",
+                "CPUExecutionProvider",
+            ]
+        elif provider == "cuda":
+            execution_providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+        elif provider == "tensorrt":
+            execution_providers = [
+                "TensorrtExecutionProvider",
+                "CUDAExecutionProvider",
+                "CPUExecutionProvider",
+            ]
+        else:
+            execution_providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
     else:
-        if use_gpu:
-            if provider == "dml":
-                execution_providers = ["DmlExecutionProvider", "CPUExecutionProvider"]
-            elif provider == "rocm":
-                execution_providers = ["ROCMExecutionProvider", "CPUExecutionProvider"]
-            elif provider == "migraphx":
-                execution_providers = [
-                    "MIGraphXExecutionProvider",
-                    "ROCMExecutionProvider",
-                    "CPUExecutionProvider",
-                ]
-            elif provider == "cuda":
-                execution_providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
-            elif provider == "tensorrt":
-                execution_providers = [
-                    "TensorrtExecutionProvider",
-                    "CUDAExecutionProvider",
-                    "CPUExecutionProvider",
-                ]
-            else:
-                execution_providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
-        else:
-            execution_providers = ["CPUExecutionProvider"]
+        execution_providers = ["CPUExecutionProvider"]
 
-        sess_options = onnxruntime.SessionOptions()
-        sess_options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
+    sess_options = onnxruntime.SessionOptions()
 
-        if graph_optimization_level is None:
-            sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
-        elif graph_optimization_level == 0:
-            sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL
-        elif graph_optimization_level == 1:
-            sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
-        elif graph_optimization_level == 2:
-            sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
-        elif graph_optimization_level == 99:
-            sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
-        else:
-            sess_options.graph_optimization_level = graph_optimization_level
+    if graph_optimization_level is None:
+        sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
+    elif graph_optimization_level == 0:
+        sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL
+    elif graph_optimization_level == 1:
+        sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
+    elif graph_optimization_level == 2:
+        sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
+    elif graph_optimization_level == 99:
+        sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
+    else:
+        sess_options.graph_optimization_level = graph_optimization_level
 
-        if intra_op_num_threads is not None:
-            sess_options.intra_op_num_threads = intra_op_num_threads
+    if intra_op_num_threads is not None:
+        sess_options.intra_op_num_threads = intra_op_num_threads
 
-        session = onnxruntime.InferenceSession(model_path, sess_options, providers=execution_providers)
+    session = onnxruntime.InferenceSession(model_path, sess_options, providers=execution_providers)
 
     if use_gpu:
         if provider == "dml":

onnxruntime/python/tools/transformers/compare_bert_results.py

@@ -6,23 +6,13 @@
 # It is a tool to compare the inference results of the original model and optimized model.
 
 import argparse
-import csv
-import os
-import random
 import statistics
-import sys
-import timeit
-from datetime import datetime
 from pathlib import Path
 
 import numpy as np
-import onnx
-import onnx.utils
 import psutil
 from bert_perf_test import create_session, onnxruntime_inference
 from bert_test_data import generate_test_data, get_bert_inputs, output_test_data
-from onnx import ModelProto, TensorProto, numpy_helper
-from onnx_model import OnnxModel
 
 
 def run_model(model_path, all_inputs, use_gpu, disable_optimization):
@@ -34,7 +24,9 @@ def run_model(model_path, all_inputs, use_gpu, disable_optimization):
 
     intra_op_num_threads = psutil.cpu_count(logical=False)
 
-    session = create_session(model_path, use_gpu, intra_op_num_threads, graph_optimization_level)
+    session = create_session(
+        model_path, use_gpu, "cuda" if use_gpu else "cpu", intra_op_num_threads, graph_optimization_level
+    )
 
     output_names = [output.name for output in session.get_outputs()]
     results, latency_list = onnxruntime_inference(session, all_inputs, output_names)

onnxruntime/python/tools/transformers/float16.py

@@ -380,11 +380,15 @@ def float_to_float16_max_diff(tensor, min_positive_val=5.96e-08, max_finite_val=
     if tensor.data_type != onnx_proto.TensorProto.FLOAT:
         raise ValueError("Expected tensor data type is float.")
 
+    float32_data = None
     if tensor.float_data:
         float32_data = np.array(tensor.float_data)
 
     if tensor.raw_data:
         float32_data = np.frombuffer(tensor.raw_data, dtype="float32")
 
+    if float32_data is None:
+        raise RuntimeError("external data not loaded!")
+
     float16_data = convert_np_to_float16(float32_data, min_positive_val, max_finite_val)
     return np.amax(np.abs(float32_data - np.float32(float16_data)))

onnxruntime/python/tools/transformers/fusion_utils.py

@@ -153,8 +153,9 @@ class FusionUtils:
                     self.model.replace_input_of_all_nodes(node.output[0], node.input[0])
                     nodes_to_remove.append(node)
 
-        self.model.remove_nodes(nodes_to_remove)
-        logger.info(f"Removed {len(nodes_to_remove)} Identity nodes")
+        if nodes_to_remove:
+            self.model.remove_nodes(nodes_to_remove)
+            logger.info(f"Removed {len(nodes_to_remove)} Identity nodes")
 
     def remove_useless_cast_nodes(self):
         """Remove cast nodes that are not needed: input and output has same data type."""
@@ -182,7 +183,8 @@ class FusionUtils:
                 else:
                     self.model.replace_input_of_all_nodes(node.output[0], node.input[0])
                 self.model.remove_node(node)
-        logger.info(f"Removed {len(nodes_to_remove)} Cast nodes with output type same as input")
+
+            logger.info(f"Removed {len(nodes_to_remove)} Cast nodes with output type same as input")
 
     def remove_useless_reshape_nodes(self):
         """Remove reshape node that is not needed based on symbolic shape inference: input and output has same shape"""

onnxruntime/python/tools/transformers/onnx_exporter.py

@@ -25,7 +25,7 @@ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
 
 logger = logging.getLogger(__name__)
 
-# Walkaround by replacing torch.triu using self-defined op
+# Workaround by replacing torch.triu using self-defined op
 # Since torch.triu cannot be exported to ONNX. See https://github.com/pytorch/pytorch/issues/32968
 torch_func = {"triu": torch.triu}
 
@@ -202,7 +202,7 @@ def optimize_onnx_model_by_ort(onnx_model_path, ort_model_path, use_gpu, overwri
         from optimizer import get_fusion_statistics, optimize_by_onnxruntime
 
         # Use onnxruntime to optimize model, which will be saved to *_ort.onnx
-        opt_model = optimize_by_onnxruntime(
+        _ = optimize_by_onnxruntime(
             onnx_model_path,
             use_gpu=use_gpu,
             optimized_model_path=ort_model_path,
@@ -214,7 +214,6 @@ def optimize_onnx_model_by_ort(onnx_model_path, ort_model_path, use_gpu, overwri
 
 
 def optimize_onnx_model(
-    model_name,
     onnx_model_path,
     optimized_model_path,
     model_type,
@@ -234,7 +233,7 @@ def optimize_onnx_model(
         from fusion_options import FusionOptions
         from optimizer import optimize_model
 
-        if optimization_options == None:
+        if optimization_options is None:
             optimization_options = FusionOptions(model_type)
         optimization_options.use_raw_attention_mask(use_raw_attention_mask)
         if Precision.FLOAT16 == precision:
@@ -327,8 +326,8 @@ def load_tf_model(model_name, model_class, cache_dir, config_modifier):
     config_modifier.modify(config)
     # Loading tf model from transformers limits the cpu affinity to {0} when KMP_AFFINITY is set
     # Restore the affinity after model loading for expected ORT performance
-    affi_helper = AffinitySetting()
-    affi_helper.get_affinity()
+    affinity_setting = AffinitySetting()
+    affinity_setting.get_affinity()
     model = load_pretrained_model(
         model_name,
         config=config,
@@ -336,7 +335,7 @@ def load_tf_model(model_name, model_class, cache_dir, config_modifier):
         custom_model_class=model_class,
         is_tf_model=True,
     )
-    affi_helper.set_affinity()
+    affinity_setting.set_affinity()
 
     return config, model
 
@@ -399,7 +398,6 @@ def validate_and_optimize_onnx(
             use_external_data_format,
         )
         optimize_onnx_model(
-            model_name,
             onnx_model_path,
             optimized_model_path,
             model_type,
@@ -664,7 +662,7 @@ def export_onnx_model_from_tf(
         logger.info(f"Skip export since model existed: {onnx_model_path}")
 
     model_type = model_type + "_tf"
-    (opt_onnx_model_file, onnx_model_file, is_valid_onnx_model, vocab_size,) = validate_and_optimize_onnx(
+    optimized_onnx_path, is_valid_onnx_model, vocab_size = validate_and_optimize_onnx(
         model_name,
         use_external_data_format,
         model_type,
@@ -686,8 +684,7 @@ def export_onnx_model_from_tf(
     )
 
     return (
-        opt_onnx_model_file,
-        onnx_model_file,
+        optimized_onnx_path,
         is_valid_onnx_model,
         vocab_size,
         max_input_size,

onnxruntime/python/tools/transformers/requirements.txt

@@ -1,13 +1,13 @@
 onnx >= 1.8
-numpy
+numpy >= 1.19.0
 coloredlogs
 psutil
 py-cpuinfo
 py3nvml
 packaging
-transformers >= 4.0
+transformers >= 4.18.0
 scipy
 sentencepiece
 
 # please follow https://pytorch.org/ to install PyTorch for your OS
-torch >= 1.8
+torch >= 1.8