A list of changes in transformers tool (#6224)

* longformer fp16 e2e

* add fp16/fp32 parity check helper file

* excludes nodes with subgraph in profiling

* use onnxconverter_common to do fp32->fp16

* add version check for onnxconverter_common

* remove helper file

* add pkg installation on notebooks and script

onnxruntime/python/tools/transformers/dev_benchmark.cmd

@@ -71,6 +71,7 @@ if %run_install% == true (
     )
   )
 
+  pip install --upgrade onnxconverter_common
   pip install --upgrade onnxruntime-tools
   pip install --upgrade git+https://github.com/huggingface/transformers
 )
@@ -145,7 +146,7 @@ if %run_torch% == true (
   >>benchmark.log echo python %optimizer_script% -e torch -m %1 %benchmark_options% %2 %3 %4
   if %run_tests%==true python %optimizer_script% -e torch -m %1 %benchmark_options% %2 %3 %4
 )
-  
+
 if %run_torchscript% == true (
   >>benchmark.log echo python %optimizer_script% -e torchscript -m %1 %benchmark_options% %2 %3 %4
   if %run_tests%==true python %optimizer_script% -e torchscript -m %1 %benchmark_options% %2 %3 %4

onnxruntime/python/tools/transformers/longformer/benchmark_longformer.py

@@ -197,9 +197,9 @@ def test_all(args):
 
             if "onnxruntime" in args.engines:
                 session = benchmark_helper.create_onnxruntime_session(onnx_model_path,
-                                                                    use_gpu=True,
-                                                                    enable_all_optimization=True,
-                                                                    num_threads=num_threads)
+                                                                      use_gpu=True,
+                                                                      enable_all_optimization=True,
+                                                                      num_threads=num_threads)
                 results += test_onnxruntime(device, model, model_name, session, args.batch_sizes, args.sequence_lengths, args.global_lengths,
                                             args.test_times, num_threads, optimized, precision)
     return results

onnxruntime/python/tools/transformers/longformer/convert_longformer_to_onnx.py

@@ -1,5 +1,5 @@
 # Before running this script, please run "python setup.py install" to build the longformer_attention.cpp
-# under a python environment with PyTorch installed. Then you can update the path of longformer_attention.cpython-*.so 
+# under a python environment with PyTorch installed. Then you can update the path of longformer_attention.cpython-*.so
 # and run this script in same environment.
 # Conversion tested in Ubuntu 18.04 in WSL (Windows Subsystem for Linux), python 3.6, onnxruntime 1.5.2, PyTorch 1.6.0+cpu, transformers 3.0.2
 # GPU is not needed for this script. You can run it in CPU.
@@ -25,7 +25,7 @@ torch.ops.load_library(r'build/lib.linux-x86_64-3.6/longformer_attention.cpython
 
 # mapping from model name to pretrained model name
 MODELS = {
-    "longformer-base-4096": "allenai/longformer-base-4096", 
+    "longformer-base-4096": "allenai/longformer-base-4096",
     "longformer-random-tiny": "patrickvonplaten/longformer-random-tiny"  # A tiny model for debugging
 }
 
@@ -42,7 +42,7 @@ def parse_arguments():
                         choices=list(MODELS.keys()),
                         help="Pre-trained models in the list: " + ", ".join(MODELS.keys()))
 
-    # Sequence length shall choose properly. 
+    # Sequence length shall choose properly.
     # If multiple of windows size is used, there is no padding in ONNX model so you will need padding by yourself before running onnx model.
     parser.add_argument("-s",
                         "--sequence_length",
@@ -199,7 +199,7 @@ if args.precision != 'fp32' or args.optimize_onnx:
     print(f"optimized fp32 model saved to {optimized_model_path}")
 
     if args.precision == 'fp16':
-        optimizer.convert_model_float32_to_float16(cast_input_output=False)
+        optimizer.convert_model_float32_to_float16(cast_input_output=True)
         optimized_model_path = model_name + "_fp16.onnx"
         optimizer.save_model_to_file(optimized_model_path)
         print(f"optimized fp16 model saved to {optimized_model_path}")

onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb

@@ -103,7 +103,7 @@
     "\n",
     "# Install other packages used in this notebook.\n",
     "!{sys.executable} -m pip install transformers==3.0.2\n",
-    "!{sys.executable} -m pip install onnx psutil pytz pandas py-cpuinfo py3nvml netron"
+    "!{sys.executable} -m pip install onnx onnxconverter_common psutil pytz pandas py-cpuinfo py3nvml netron"
    ]
   },
   {

onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb

@@ -111,6 +111,7 @@
     "else:\n",
     "    !{sys.executable} -m pip install --upgrade torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html\n",
     "!{sys.executable} -m pip install --upgrade onnxruntime==1.4.0\n",
+    "!{sys.executable} -m pip install --upgrade onnxconverter_common\n",
     "!{sys.executable} -m pip install --upgrade onnxruntime-tools\n",
     "\n",
     "# Install other packages used in this notebook.\n",

onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb

@@ -59,6 +59,7 @@
     "!{sys.executable} -m pip uninstall --quiet --yes onnxruntime-gpu\n",
     "!{sys.executable} -m pip install --quiet onnxruntime-gpu\n",
     "!{sys.executable} -m pip install --quiet --upgrade transformers\n",
+    "!{sys.executable} -m pip install --quiet --upgrade onnxconverter_common\n",
     "!{sys.executable} -m pip install --quiet --upgrade onnxruntime-tools\n",
     "!{sys.executable} -m pip install --quiet wget netron pandas"
    ]

onnxruntime/python/tools/transformers/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb

@@ -59,6 +59,7 @@
     "!{sys.executable} -m pip install --quiet --upgrade onnxruntime-tools==1.4.0\n",
     "!{sys.executable} -m pip install --quiet --upgrade keras2onnx==1.7.0\n",
     "!{sys.executable} -m pip install --quiet transformers==3.0.2\n",
+    "!{sys.executable} -m pip install --quiet onnxconverter_common\n",
     "!{sys.executable} -m pip install --quiet wget pandas"
    ]
   },

onnxruntime/python/tools/transformers/onnx_model.py

@@ -407,6 +407,12 @@ class OnnxModel:
     def convert_model_float32_to_float16(self, cast_input_output=True):
         """ Convert a graph to FLOAT16
         """
+        from packaging.version import Version
+        import onnxconverter_common.float16 as oc
+        if Version(oc.__version__) > Version("1.7.0"):
+            self.model = oc.convert_float_to_float16(self.model, keep_io_types = cast_input_output)
+            return
+
         graph = self.model.graph
         initializers = graph.initializer
 

onnxruntime/python/tools/transformers/profiler.py

@@ -10,6 +10,7 @@ Example of profiling of longformer model:
     python profiler.py --model longformer-base-4096_fp32.onnx --batch_size 1 --sequence_length 4096 --global_length 8 --samples 1000 --thread_num 8 --dummy_inputs longformer --use_gpu
 """
 
+NODES_TYPE_CONTAINING_SUBGRAPH = ['Scan', 'Loop', 'If']
 
 def parse_arguments(argv=None):
     parser = argparse.ArgumentParser()
@@ -156,6 +157,10 @@ def parse_profile_results(sess_time, kernel_time_only=False, threshold=0):
             elif kernel_time_only:
                 continue
 
+            op_name = item["args"]["op_name"]
+            if op_name in NODES_TYPE_CONTAINING_SUBGRAPH:
+                continue
+
             if item["name"] in node_time:
                 node_time[item["name"]] += item["dur"]
             else:
@@ -188,6 +193,10 @@ def group_profile_results(sess_time, kernel_time_only=False, threshold=0):
                 continue
 
             op_name = item["args"]["op_name"]
+
+            if op_name in NODES_TYPE_CONTAINING_SUBGRAPH:
+                continue
+
             if op_name in op_time:
                 op_time[op_name] += item["dur"]
                 op_records[op_name] += 1

onnxruntime/python/tools/transformers/run_benchmark.sh

@@ -89,6 +89,7 @@ if [ "$run_install" = true ] ; then
   else
     pip install onnxruntime-gpu
   fi
+  pip install --upgrade onnxconverter_common
   pip install --upgrade onnxruntime-tools
   pip install --upgrade transformers
 fi