Added Lint task and parallelized format.py (#161)

styleguide/format.py

@@ -1,19 +1,19 @@
 #!/usr/bin/env python3
 
-# This script runs all formatting tasks on the code base.
-#
-# Passing "-v" as an argument enables verbosity. Otherwise, this script takes no
-# arguments. This should be invoked from either the styleguide directory or the
-# root directory of the project.
-
+import argparse
+import multiprocessing
 import os
+import queue
 import subprocess
 import sys
+from threading import Lock
+from threading import Thread
 
 from clangformat import ClangFormat
 from licenseupdate import LicenseUpdate
 from newline import Newline
 from whitespace import Whitespace
+from lint import Lint
 from task import Task
 
 # Check that the current directory is part of a Git repository
@@ -21,6 +21,28 @@ def inGitRepo(directory):
     ret = subprocess.run(["git", "rev-parse"], stderr = subprocess.DEVNULL)
     return ret.returncode == 0
 
+def threadFunc(workQueue, isVerbose, printLock):
+    # Lint is run last since previous tasks can affect its output
+    tasks = [ClangFormat(), LicenseUpdate(), Newline(), Whitespace(), Lint()]
+
+    try:
+        while True:
+            name = workQueue.get_nowait()
+
+            if isVerbose:
+                printLock.acquire()
+                print("Processing", name,)
+                for task in tasks:
+                    if task.fileMatchesExtension(name):
+                        print("  with " + type(task).__name__)
+                printLock.release()
+
+            for task in tasks:
+                if task.fileMatchesExtension(name):
+                    task.run(name)
+    except queue.Empty:
+        pass
+
 def main():
     if not inGitRepo("."):
         print("Error: not invoked within a Git repository", file = sys.stderr)
@@ -33,6 +55,13 @@ def main():
     else:
         configPath = "."
 
+    # Delete temporary files from previous incomplete run
+    files = [os.path.join(dp, f) for dp, dn, fn in
+             os.walk(os.path.expanduser(configPath)) for f in fn]
+    for f in files:
+        if f.endswith(".tmp"):
+            os.remove(f)
+
     # Recursively create list of files in given directory
     files = [os.path.join(dp, f) for dp, dn, fn in
              os.walk(os.path.expanduser(configPath)) for f in fn]
@@ -44,23 +73,35 @@ def main():
     # Don't format generated files
     files = [name for name in files if Task.notGeneratedFile(name)]
 
-    clangFormat = ClangFormat()
-    licenseUpdate = LicenseUpdate()
-    newline = Newline()
-    whitespace = Whitespace()
+    # Parse command-line arguments
+    parser = argparse.ArgumentParser(description = "Runs all formatting tasks on the code base. This should be invoked from either the styleguide directory or the root directory of the project.")
+    parser.add_argument("-v", dest = "verbose", action = "store_true",
+                        help = "enable output verbosity")
+    parser.add_argument("-j", dest = "jobs", type = int,
+                        default = multiprocessing.cpu_count(),
+                        help = "number of jobs to run (default is number of cores)")
+    args = parser.parse_args()
+    jobs = args.jobs
+    isVerbose = args.verbose
 
-    # Check for verbose flag
-    isVerbose = len(sys.argv) > 1 and sys.argv[1] == "-v"
+    # Add files to work queue
+    workQueue = queue.Queue()
+    for file in files:
+        workQueue.put(file)
 
-    for name in files:
-        if isVerbose:
-            print("Processing", name,)
+    threads = []
+    printLock = Lock()
 
-        for task in [clangFormat, licenseUpdate, newline, whitespace]:
-            if task.fileMatchesExtension(name):
-                if isVerbose:
-                    print("  with " + type(task).__name__)
-                task.run(name)
+    # Start worker threads
+    for i in range(0, jobs):
+        thread = Thread(target = threadFunc,
+                        args = (workQueue, isVerbose, printLock))
+        thread.start()
+        threads.append(thread)
+
+    # Wait for worker threads to finish
+    for i in range(0, jobs):
+        threads[i].join()
 
 if __name__ == "__main__":
     main()

styleguide/lint.py

@@ -0,0 +1,57 @@
+# This task runs cpplint.py on all C++ source files.
+
+"""
+cpplint.py was converted from python 2 to python 3 with the following command
+and a patch was applied:
+
+2to3 -f all -f buffer -f idioms -f set_literal -f ws_comma -nw cpplint.py
+
+diff --git a/styleguide/cpplint.py b/styleguide/cpplint.py
+--- a/styleguide/cpplint.py
++++ b/styleguide/cpplint.py
+@@ -6094,13 +6094,6 @@ def ParseArguments(args):
+ def main():
+   filenames = ParseArguments(sys.argv[1:])
+ 
+-  # Change stderr to write with replacement characters so we don't die
+-  # if we try to print something containing non-ASCII characters.
+-  sys.stderr = codecs.StreamReaderWriter(sys.stderr,
+-                                         codecs.getreader('utf8'),
+-                                         codecs.getwriter('utf8'),
+-                                         'replace')
+-
+   _cpplint_state.ResetErrorCounts()
+   for filename in filenames:
+     ProcessFile(filename, _cpplint_state.verbose_level)
+"""
+
+import os
+import subprocess
+import sys
+from task import Task
+
+class Lint(Task):
+    def getIncludeExtensions(self):
+        return ["cpp", "h", "inc"]
+
+    def run(self, name):
+        # Handle running in either the root or styleguide directories
+        cpplintPrefix = ""
+        if os.getcwd().rpartition(os.sep)[2] != "styleguide":
+            cpplintPrefix = "styleguide/"
+
+        # Run cpplint.py
+        proc = subprocess.Popen(["python", cpplintPrefix + "cpplint.py",
+                                 "--filter="
+                                 "-build/c++11,"
+                                 "-build/header_guard,"
+                                 "-build/include,"
+                                 "-build/namespaces,"
+                                 "-readability/todo,"
+                                 "-runtime/references,"
+                                 "-runtime/string",
+                                 "--extensions=cpp,h,inc",
+                                 name], bufsize = 1, stderr = subprocess.PIPE)
+        for line in proc.stderr:
+            if b"Done processing" not in line and b"Total errors" not in line:
+                print(line.rstrip().decode(sys.stdout.encoding))