diff --git a/docs/How_To_Update_ONNX_Dev_Notes.md b/docs/How_To_Update_ONNX_Dev_Notes.md index 3c13e50862..be1b9c4cfe 100644 --- a/docs/How_To_Update_ONNX_Dev_Notes.md +++ b/docs/How_To_Update_ONNX_Dev_Notes.md @@ -28,7 +28,7 @@ This file should be generated. See [cgmanifests/README](/cgmanifests/README.md) 1. If you are updating ONNX from a released tag to a new commit, please ask Changming (@snnn) to deploy the new test data along with other test models to our CI build machines. This is to ensure that our tests cover every ONNX opset. -1. Send you PR, and **manually** queue a build for every packaging pipeline for your branch. +1. Send your PR, and **manually** queue a build for every packaging pipeline for your branch. 1. If there is a build failure in stage "Check out of dated documents" in WebAssembly CI pipeline, update ONNX Runtime Web WebGL operator support document: @@ -42,3 +42,7 @@ This file should be generated. See [cgmanifests/README](/cgmanifests/README.md) - [csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs](/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs) - [onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc](/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc) - [onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc](/onnxruntime/test/testdata/onnx_backend_test_series_overrides.jsonc) + +1. If an operator has changed we may need to update optimizers involving that operator. +- Run [find_optimizer_opset_version_updates_required.py](/tools/python/find_optimizer_opset_version_updates_required.py), compare with the output from the current main branch, and check for any new warnings. +- If there are new warnings contact the optimizer owner (which can usually be determined by looking at who edited the file most recently) or failing that ask the 'ONNX Runtime Shared Core' mailing list. diff --git a/tools/python/FindOptimizerOpsetVersionUpdatesRequired.py b/tools/python/find_optimizer_opset_version_updates_required.py similarity index 73% rename from tools/python/FindOptimizerOpsetVersionUpdatesRequired.py rename to tools/python/find_optimizer_opset_version_updates_required.py index d0c940ae99..412b25df90 100644 --- a/tools/python/FindOptimizerOpsetVersionUpdatesRequired.py +++ b/tools/python/find_optimizer_opset_version_updates_required.py @@ -7,6 +7,7 @@ import glob import logging import os import re +import typing logging.basicConfig(format="[%(levelname)s] - %(message)s", level=logging.DEBUG) log = logging.getLogger() @@ -29,14 +30,17 @@ def parse_args(): return args -def get_call_args_from_file(filename, function_or_declaration): - """Search a file for all function calls or declarations that match the provided name. - Currently requires both the opening '(' and closing ')' to be on the same line.""" +def get_call_args_from_file(filename: str, function_or_declaration: str) -> typing.List[str]: + """ + Search a file for all function calls or declarations that match the provided name. + Requires both the opening '(' and closing ')' to be on the same line. + Handles multiple calls being on the same line. + """ results = [] with open(filename) as f: line_num = 0 - for line in f.readlines(): + for line in f: for match in re.finditer(function_or_declaration, line): # check we have both the opening and closing brackets for the function call/declaration. # if we do we have all the arguments @@ -59,18 +63,51 @@ def get_call_args_from_file(filename, function_or_declaration): return results +def get_multiline_call_args_from_file(filename: str, function_or_declaration: str) -> typing.List[str]: + """ + Search a file for all function calls or declarations that match the provided name. + Allows the opening '(' and closing ')' to be split across multiple lines. + Supports a single call per line. + """ + + results = [] + with open(filename) as f: + function_and_args = None + + for line in f: + if not function_and_args: + # look for new match + start = line.find(function_or_declaration) + if start != -1: + function_and_args = line[start:].strip() + else: + # append to existing line and look for closing ')' + start = len(function_and_args) + function_and_args += line.strip() + + if function_and_args: + end = function_and_args.find(")", start) + + if end != -1: + start_args = function_and_args.find("(") + results.append(function_and_args[start_args + 1 : end]) + function_and_args = None + + return results + + def get_latest_op_versions(root_dir): """Find the entries for the latest opset for each operator.""" op_to_opset = {} files = [ os.path.join(root_dir, "onnxruntime/core/providers/cpu/cpu_execution_provider.cc"), - os.path.join(root_dir, "onnxruntime/contrib_ops/cpu_contrib_kernels.cc"), + os.path.join(root_dir, "onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc"), ] for file in files: # e.g. class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Clip); - calls = get_call_args_from_file(file, "ONNX_OPERATOR_KERNEL_CLASS_NAME") + calls = get_multiline_call_args_from_file(file, "ONNX_OPERATOR_KERNEL_CLASS_NAME") for call in calls: args = call.split(",") domain = args[1].strip() @@ -79,7 +116,7 @@ def get_latest_op_versions(root_dir): op_to_opset[domain + "." + op] = opset # e.g. class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, float, ArgMax); - calls = get_call_args_from_file(file, "ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME") + calls = get_multiline_call_args_from_file(file, "ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME") for call in calls: args = call.split(",") domain = args[1].strip()