onnxruntime/tools/python/gen_ort_mobile_pkg_doc.py

98 lines
3.6 KiB
Python
Raw Normal View History

import argparse
import os
import pathlib
from util import reduced_build_config_parser
from util.ort_format_model.operator_type_usage_processors import GloballyAllowedTypesOpTypeImplFilter
def generate_docs(output_file, required_ops, op_type_impl_filter):
with open(output_file, "w") as out:
out.write("# ONNX Runtime Mobile Pre-Built Package Operator and Type Support\n\n")
# Description
out.write("## Supported operators and types\n\n")
out.write(
"The supported operators and types are based on what is required to support float32 and quantized "
"versions of popular models. The full list of input models used to determine this list is available "
"[here](https://github.com/microsoft/onnxruntime/blob/main/tools/ci_build/github/android/mobile_package"
".required_operators.readme.txt)"
)
out.write("\n\n")
# Globally supported types
out.write("## Supported data input types\n\n")
assert op_type_impl_filter.__class__ is GloballyAllowedTypesOpTypeImplFilter
global_types = op_type_impl_filter.global_type_list()
for type in sorted(global_types):
Adopt linrtunner as the linting tool - take 2 (#15085) ### Description `lintrunner` is a linter runner successfully used by pytorch, onnx and onnx-script. It provides a uniform experience running linters locally and in CI. It supports all major dev systems: Windows, Linux and MacOs. The checks are enforced by the `Python format` workflow. This PR adopts `lintrunner` to onnxruntime and fixed ~2000 flake8 errors in Python code. `lintrunner` now runs all required python lints including `ruff`(replacing `flake8`), `black` and `isort`. Future lints like `clang-format` can be added. Most errors are auto-fixed by `ruff` and the fixes should be considered robust. Lints that are more complicated to fix are applied `# noqa` for now and should be fixed in follow up PRs. ### Notable changes 1. This PR **removed some suboptimal patterns**: - `not xxx in` -> `xxx not in` membership checks - bare excepts (`except:` -> `except Exception`) - unused imports The follow up PR will remove: - `import *` - mutable values as default in function definitions (`def func(a=[])`) - more unused imports - unused local variables 2. Use `ruff` to replace `flake8`. `ruff` is much (40x) faster than flake8 and is more robust. We are using it successfully in onnx and onnx-script. It also supports auto-fixing many flake8 errors. 3. Removed the legacy flake8 ci flow and updated docs. 4. The added workflow supports SARIF code scanning reports on github, example snapshot: ![image](https://user-images.githubusercontent.com/11205048/212598953-d60ce8a9-f242-4fa8-8674-8696b704604a.png) 5. Removed `onnxruntime-python-checks-ci-pipeline` as redundant ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. --> Unified linting experience in CI and local. Replacing https://github.com/microsoft/onnxruntime/pull/14306 --------- Signed-off-by: Justin Chu <justinchu@microsoft.com>
2023-03-24 22:29:03 +00:00
out.write(f" - {type}\n")
out.write("\n")
out.write("NOTE: Operators used to manipulate dimensions and indices will support int32 and int64.\n\n")
domain_op_opsets = []
for domain in sorted(required_ops.keys()):
op_opsets = {}
domain_op_opsets.append((domain, op_opsets))
for opset in sorted(required_ops[domain].keys()):
str_opset = str(opset)
for op in required_ops[domain][opset]:
Adopt linrtunner as the linting tool - take 2 (#15085) ### Description `lintrunner` is a linter runner successfully used by pytorch, onnx and onnx-script. It provides a uniform experience running linters locally and in CI. It supports all major dev systems: Windows, Linux and MacOs. The checks are enforced by the `Python format` workflow. This PR adopts `lintrunner` to onnxruntime and fixed ~2000 flake8 errors in Python code. `lintrunner` now runs all required python lints including `ruff`(replacing `flake8`), `black` and `isort`. Future lints like `clang-format` can be added. Most errors are auto-fixed by `ruff` and the fixes should be considered robust. Lints that are more complicated to fix are applied `# noqa` for now and should be fixed in follow up PRs. ### Notable changes 1. This PR **removed some suboptimal patterns**: - `not xxx in` -> `xxx not in` membership checks - bare excepts (`except:` -> `except Exception`) - unused imports The follow up PR will remove: - `import *` - mutable values as default in function definitions (`def func(a=[])`) - more unused imports - unused local variables 2. Use `ruff` to replace `flake8`. `ruff` is much (40x) faster than flake8 and is more robust. We are using it successfully in onnx and onnx-script. It also supports auto-fixing many flake8 errors. 3. Removed the legacy flake8 ci flow and updated docs. 4. The added workflow supports SARIF code scanning reports on github, example snapshot: ![image](https://user-images.githubusercontent.com/11205048/212598953-d60ce8a9-f242-4fa8-8674-8696b704604a.png) 5. Removed `onnxruntime-python-checks-ci-pipeline` as redundant ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. --> Unified linting experience in CI and local. Replacing https://github.com/microsoft/onnxruntime/pull/14306 --------- Signed-off-by: Justin Chu <justinchu@microsoft.com>
2023-03-24 22:29:03 +00:00
op_with_domain = f"{domain}:{op}"
if op_with_domain not in op_opsets:
op_opsets[op_with_domain] = []
op_opsets[op_with_domain].append(str_opset)
out.write("## Supported Operators\n\n")
out.write("|Operator|Opsets|\n")
out.write("|--------|------|\n")
for domain, op_opsets in domain_op_opsets:
Adopt linrtunner as the linting tool - take 2 (#15085) ### Description `lintrunner` is a linter runner successfully used by pytorch, onnx and onnx-script. It provides a uniform experience running linters locally and in CI. It supports all major dev systems: Windows, Linux and MacOs. The checks are enforced by the `Python format` workflow. This PR adopts `lintrunner` to onnxruntime and fixed ~2000 flake8 errors in Python code. `lintrunner` now runs all required python lints including `ruff`(replacing `flake8`), `black` and `isort`. Future lints like `clang-format` can be added. Most errors are auto-fixed by `ruff` and the fixes should be considered robust. Lints that are more complicated to fix are applied `# noqa` for now and should be fixed in follow up PRs. ### Notable changes 1. This PR **removed some suboptimal patterns**: - `not xxx in` -> `xxx not in` membership checks - bare excepts (`except:` -> `except Exception`) - unused imports The follow up PR will remove: - `import *` - mutable values as default in function definitions (`def func(a=[])`) - more unused imports - unused local variables 2. Use `ruff` to replace `flake8`. `ruff` is much (40x) faster than flake8 and is more robust. We are using it successfully in onnx and onnx-script. It also supports auto-fixing many flake8 errors. 3. Removed the legacy flake8 ci flow and updated docs. 4. The added workflow supports SARIF code scanning reports on github, example snapshot: ![image](https://user-images.githubusercontent.com/11205048/212598953-d60ce8a9-f242-4fa8-8674-8696b704604a.png) 5. Removed `onnxruntime-python-checks-ci-pipeline` as redundant ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. --> Unified linting experience in CI and local. Replacing https://github.com/microsoft/onnxruntime/pull/14306 --------- Signed-off-by: Justin Chu <justinchu@microsoft.com>
2023-03-24 22:29:03 +00:00
out.write(f"|**{domain}**||\n")
for op in sorted(op_opsets.keys()):
out.write("|{}|{}|\n".format(op, ", ".join(op_opsets[op])))
out.write("|||\n")
def main():
script_dir = os.path.dirname(os.path.realpath(__file__))
parser = argparse.ArgumentParser(
description="ONNX Runtime Mobile Pre-Built Package Operator and Type Support Documentation Generator",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
default_config_path = pathlib.Path(
os.path.join(script_dir, "../ci_build/github/android/mobile_package.required_operators.config")
).resolve()
default_output_path = pathlib.Path(
os.path.join(script_dir, "../../docs/ORTMobilePackageOperatorTypeSupport.md")
).resolve()
parser.add_argument(
"--config_path",
help="Path to build configuration used to generate package.",
required=False,
type=pathlib.Path,
default=default_config_path,
)
parser.add_argument(
"--output_path",
help="output markdown file path",
required=False,
type=pathlib.Path,
default=default_output_path,
)
args = parser.parse_args()
config_file = args.config_path.resolve(strict=True) # must exist so strict=True
output_path = args.output_path.resolve()
enable_type_reduction = True
required_ops, op_type_impl_filter = reduced_build_config_parser.parse_config(config_file, enable_type_reduction)
generate_docs(output_path, required_ops, op_type_impl_filter)
if __name__ == "__main__":
main()