Pre-commit flake8/clang-tidy (#15102)

Summary:
Provide a pre-commit hook that does flake8 and clang tidy checks. Enables the clang-tidy script to run in parallel to make it fast enough to be used in a pre-commit hook.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/15102

Reviewed By: soumith

Differential Revision: D13429629

Pulled By: zdevito

fbshipit-source-id: bd52fe5652f29b033de8d9926d78350b2da4c2fc

CONTRIBUTING.md

@@ -248,6 +248,7 @@ On the initial build, you can also speed things up with the environment
 variables `DEBUG` and `NO_CUDA`.
 
 - `DEBUG=1` will enable debug builds (-g -O0)
+- `REL_WITH_DEB_INFO=1` will enable debug symbols with optimizations (-g -O3)
 - `NO_CUDA=1` will disable compiling CUDA (in case you are developing on something not CUDA related), to save compile time.
 
 For example:
@@ -463,6 +464,16 @@ root folder if you used `setup.py build`. You can use `-c <clang-tidy-binary>`
 to change the clang-tidy this script uses. Make sure you have PyYaml installed,
 which is in PyTorch's `requirements.txt`.
 
+### Pre-commit Tidy/Linting Hook
+
+We use clang-tidy and flake8 to perform additional formatting and semantic checking
+of code. We provide a pre-commit git hook for performing these checks, before
+a commit is created:
+
+  ```sh
+  $ ln -s ../../tools/git-pre-commit .git/hooks/pre-commit
+  ```
+
 ## Caffe2 notes
 
 In 2018, we merged Caffe2 into the PyTorch source repository.  While the

tools/clang_tidy.py

@@ -21,7 +21,12 @@ import re
 import shlex
 import subprocess
 import sys
+import tempfile
 
+try:
+    from shlex import quote
+except ImportError:
+    from pipes import quote
 
 Patterns = collections.namedtuple("Patterns", "positive, negative")
 
@@ -122,6 +127,34 @@ def get_changed_lines(revision, filename):
 
     return {"name": filename, "lines": changed_lines}
 
+ninja_template = """
+rule do_cmd
+  command = $cmd
+  description = Running clang-tidy
+
+{build_rules}
+"""
+
+build_template = """
+build {i}: do_cmd
+  cmd = {cmd}
+"""
+
+
+def run_shell_commands_in_parallel(commands):
+    """runs all the commands in parallel with ninja, commands is a List[List[str]]"""
+    build_entries = [build_template.format(i=i, cmd=' '.join([quote(s) for s in command]))
+                     for i, command in enumerate(commands)]
+
+    file_contents = ninja_template.format(build_rules='\n'.join(build_entries))
+    f = tempfile.NamedTemporaryFile(delete=False)
+    try:
+        f.write(file_contents)
+        f.close()
+        return run_shell_command(['ninja', '-f', f.name])
+    finally:
+        os.unlink(f.name)
+
 
 def run_clang_tidy(options, line_filters, files):
     """Executes the actual clang-tidy command in the shell."""
@@ -134,16 +167,22 @@ def run_clang_tidy(options, line_filters, files):
         with open(options.config_file) as config:
             # Here we convert the YAML config file to a JSON blob.
             command += ["-config", json.dumps(yaml.load(config))]
+    command += options.extra_args
+
     if line_filters:
         command += ["-line-filter", json.dumps(line_filters)]
-    command += options.extra_args
-    command += files
 
-    if options.dry_run:
-        command = [re.sub(r"^([{[].*[]}])$", r"'\1'", arg) for arg in command]
-        return " ".join(command)
+    if options.parallel:
+        commands = [list(command) + [f] for f in files]
+        output = run_shell_commands_in_parallel(commands)
+    else:
+        command += files
+        if options.dry_run:
+            command = [re.sub(r"^([{[].*[]}])$", r"'\1'", arg) for arg in command]
+            return " ".join(command)
+
+        output = run_shell_command(command)
 
-    output = run_shell_command(command)
     if not options.keep_going and "[clang-diagnostic-error]" in output:
         message = "Found clang-diagnostic-errors in clang-tidy output: {}"
         raise RuntimeError(message.format(output))
@@ -210,6 +249,12 @@ def parse_options():
         action="store_true",
         help="Don't error on compiler errors (clang-diagnostic-error)",
     )
+    parser.add_argument(
+        "-j",
+        "--parallel",
+        action="store_true",
+        help="Run clang tidy in parallel per-file (requires ninja to be installed).",
+    )
     parser.add_argument(
         "extra_args", nargs="*", help="Extra arguments to forward to clang-tidy"
     )

tools/flake8_hook.py

@@ -0,0 +1,11 @@
+import sys
+
+from flake8.main import git
+
+if __name__ == '__main__':
+    sys.exit(
+        git.hook(
+            strict=git.config_for('strict'),
+            lazy=git.config_for('lazy'),
+        )
+    )

tools/git-pre-commit

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+echo "Running pre-commit flake8"
+python tools/flake8_hook.py
+echo "Running pre-commit clang-tidy"
+python tools/clang_tidy.py \
+  --paths torch/csrc \
+  --diff HEAD \
+  -g"-torch/csrc/distributed/Module.cpp" \
+  -g"-torch/csrc/jit/export.cpp" \
+  -g"-torch/csrc/jit/import.cpp" \
+  -j