Automate fmt update (#3486)

Also refactored upstream_utils to make writing and maintaining new
upstream repo extractions easier.

.styleguide

@@ -9,6 +9,10 @@ cppSrcFileInclude {
   \.cpp$
 }
 
+modifiableFileExclude {
+  \.patch$
+}
+
 generatedFileExclude {
   FRCNetComm\.java$
   simulation/gz_msgs/src/include/simulation/gz_msgs/msgs\.h$

README.md

@@ -147,6 +147,8 @@ The integration test directories for C++ and Java contain test code that runs on
 
 The hal directory contains more C++ code meant to run on the roboRIO. HAL is an acronym for "Hardware Abstraction Layer", and it interfaces with the NI Libraries. The NI Libraries contain the low-level code for controlling devices on your robot. The NI Libraries are found in the ni-libraries folder.
 
+The upstream_utils directory contains scripts for updating copies of thirdparty code in the repository.
+
 The [styleguide repository](https://github.com/wpilibsuite/styleguide) contains our style guides for C++ and Java code. Anything submitted to the WPILib project needs to follow the code style guides outlined in there. For details about the style, please see the contributors document [here](CONTRIBUTING.md#coding-guidelines).
 
 # Contributing to WPILib

upstream_utils/update_drake.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+
+from upstream_utils import setup_upstream_repo, comment_out_invalid_includes, walk_cwd_and_copy_if, apply_patches
+
+
+def main():
+    root, repo = setup_upstream_repo(
+        "https://github.com/RobotLocomotion/drake",
+        "8b72428dce6959d077e17c3c3a7a5ef4a17107ee")
+    wpimath = os.path.join(root, "wpimath")
+
+    # Delete old install
+    for d in [
+            "src/main/native/cpp/drake", "src/main/native/include/drake",
+            "src/test/native/cpp/drake", "src/test/native/include/drake"
+    ]:
+        shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True)
+
+    # Copy drake source files into allwpilib
+    src_files = walk_cwd_and_copy_if(
+        lambda dp, f: f in
+        ["drake_assert_and_throw.cc", "discrete_algebraic_riccati_equation.cc"],
+        os.path.join(wpimath, "src/main/native/cpp/drake"))
+
+    # Copy drake header files into allwpilib
+    include_files = walk_cwd_and_copy_if(
+        lambda dp, f: f in [
+            "drake_assert.h", "drake_assertion_error.h",
+            "is_approx_equal_abstol.h", "never_destroyed.h", "drake_copyable.h",
+            "drake_throw.h", "discrete_algebraic_riccati_equation.h"
+        ], os.path.join(wpimath, "src/main/native/include/drake"))
+
+    # Copy drake test source files into allwpilib
+    os.chdir(os.path.join(repo, "math/test"))
+    test_src_files = walk_cwd_and_copy_if(
+        lambda dp, f: f == "discrete_algebraic_riccati_equation_test.cc",
+        os.path.join(wpimath, "src/test/native/cpp/drake"))
+    os.chdir(repo)
+
+    # Copy drake test header files into allwpilib
+    test_include_files = walk_cwd_and_copy_if(
+        lambda dp, f: f == "eigen_matrix_compare.h",
+        os.path.join(wpimath, "src/test/native/include/drake"))
+
+    for f in src_files:
+        comment_out_invalid_includes(
+            f, [os.path.join(wpimath, "src/main/native/include")])
+    for f in include_files:
+        comment_out_invalid_includes(
+            f, [os.path.join(wpimath, "src/main/native/include")])
+    for f in test_src_files:
+        comment_out_invalid_includes(f, [
+            os.path.join(wpimath, "src/main/native/include"),
+            os.path.join(wpimath, "src/test/native/include")
+        ])
+    for f in test_include_files:
+        comment_out_invalid_includes(f, [
+            os.path.join(wpimath, "src/main/native/include"),
+            os.path.join(wpimath, "src/test/native/include")
+        ])
+
+    apply_patches(root, ["upstream_utils/drake-replace-dense-with-core.patch"])
+
+
+if __name__ == "__main__":
+    main()

upstream_utils/update_eigen.py

@@ -2,9 +2,9 @@
 
 import os
 import re
-import subprocess
+import shutil
 
-from upstream_utils import clone_repo, comment_out_invalid_includes, copy_to, walk_if
+from upstream_utils import setup_upstream_repo, comment_out_invalid_includes, walk_cwd_and_copy_if, apply_patches
 
 
 def eigen_inclusions(dp, f):
@@ -15,6 +15,9 @@ def eigen_inclusions(dp, f):
     dp -- directory path
     f -- filename
     """
+    if not dp.startswith("./Eigen"):
+        return False
+
     abspath = os.path.join(dp, f)
 
     # Exclude build system
@@ -74,6 +77,9 @@ def unsupported_inclusions(dp, f):
     dp -- directory path
     f -- filename
     """
+    if not dp.startswith("./unsupported"):
+        return False
+
     abspath = os.path.join(dp, f)
 
     # Exclude build system and READMEs
@@ -85,39 +91,34 @@ def unsupported_inclusions(dp, f):
 
 
 def main():
-    wpimath = os.getcwd()
-    clone_repo("https://gitlab.com/libeigen/eigen.git", "3.3.9")
-    repo = os.getcwd()
+    root, repo = setup_upstream_repo("https://gitlab.com/libeigen/eigen.git",
+                                     "3.3.9")
+    wpimath = os.path.join(root, "wpimath")
+
+    # Delete old install
+    for d in [
+            "src/main/native/eigeninclude/Eigen",
+            "src/main/native/eigeninclude/unsupported"
+    ]:
+        shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True)
 
     # Copy Eigen headers into allwpilib
-    os.chdir(os.path.join(repo, "Eigen"))
-    files = walk_if(".", eigen_inclusions)
-    dest = os.path.join(wpimath, "src/main/native/eigeninclude/Eigen")
-    copy_to(files, dest)
-
-    # Files moved to dest
-    for i in range(len(files)):
-        files[i] = os.path.join(dest, files[i])
-
-    for f in files:
-        comment_out_invalid_includes(f, [dest])
+    eigen_files = walk_cwd_and_copy_if(
+        eigen_inclusions, os.path.join(wpimath, "src/main/native/eigeninclude"))
 
     # Copy unsupported headers into allwpilib
-    os.chdir(os.path.join(repo, "unsupported"))
-    files = walk_if(".", unsupported_inclusions)
-    dest = os.path.join(wpimath, "src/main/native/eigeninclude/unsupported")
-    copy_to(files, dest)
+    unsupported_files = walk_cwd_and_copy_if(
+        unsupported_inclusions,
+        os.path.join(wpimath, "src/main/native/eigeninclude"))
 
-    # Files moved to dest
-    for i in range(len(files)):
-        files[i] = os.path.join(dest, files[i])
+    for f in eigen_files:
+        comment_out_invalid_includes(
+            f, [os.path.join(wpimath, "src/main/native/eigeninclude")])
+    for f in unsupported_files:
+        comment_out_invalid_includes(
+            f, [os.path.join(wpimath, "src/main/native/eigeninclude")])
 
-    for f in files:
-        comment_out_invalid_includes(f, [dest])
-
-    # Apply patches
-    os.chdir(wpimath)
-    subprocess.check_output(["git", "apply", "eigen-maybe-uninitialized.patch"])
+    apply_patches(root, ["upstream_utils/eigen-maybe-uninitialized.patch"])
 
 
 if __name__ == "__main__":

upstream_utils/update_fmt.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+
+from upstream_utils import setup_upstream_repo, comment_out_invalid_includes, walk_cwd_and_copy_if
+
+
+def main():
+    root, repo = setup_upstream_repo("https://github.com/fmtlib/fmt", "8.0.1")
+    wpiutil = os.path.join(root, "wpiutil")
+
+    # Delete old install
+    for d in ["src/main/native/fmtlib/src", "src/main/native/fmtlib/include"]:
+        shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
+
+    # Copy fmt source files into allwpilib
+    src_files = walk_cwd_and_copy_if(
+        lambda dp, f: dp.endswith("src") and f.endswith(".cc") and f !=
+        "fmt.cc", os.path.join(wpiutil, "src/main/native/fmtlib"))
+
+    # Copy fmt header files into allwpilib
+    include_files = walk_cwd_and_copy_if(
+        lambda dp, f: dp.endswith("include/fmt"),
+        os.path.join(wpiutil, "src/main/native/fmtlib"))
+
+    for f in src_files:
+        comment_out_invalid_includes(
+            f, [os.path.join(wpiutil, "src/main/native/fmtlib/include")])
+    for f in include_files:
+        comment_out_invalid_includes(
+            f, [os.path.join(wpiutil, "src/main/native/fmtlib/include")])
+
+
+if __name__ == "__main__":
+    main()

upstream_utils/upstream_utils.py

@@ -18,7 +18,7 @@ def clone_repo(url, treeish):
     os.chdir(tempfile.gettempdir())
 
     repo = os.path.basename(url)
-    dest = os.path.join(os.getcwd(), repo).rstrip(".git")
+    dest = os.path.join(os.getcwd(), repo).removesuffix(".git")
 
     # Clone Git repository into current directory or update it
     if not os.path.exists(dest):
@@ -45,6 +45,39 @@ def clone_repo(url, treeish):
         subprocess.run(["git", "checkout", treeish])
 
 
+def get_repo_root():
+    """Returns the Git repository root as an absolute path.
+
+    An empty string is returned if no repository root was found.
+    """
+    current_dir = os.path.abspath(os.getcwd())
+    while current_dir != os.path.dirname(current_dir):
+        if os.path.exists(current_dir + os.sep + ".git"):
+            return current_dir
+        current_dir = os.path.dirname(current_dir)
+    return ""
+
+
+def setup_upstream_repo(url, treeish):
+    """Clones the given upstream repository, then returns the root of the
+    destination Git repository as well as the cloned upstream Git repository.
+
+    The current working directory will be set to the cloned upstream repository
+    folder.
+
+    Keyword arguments:
+    url -- The URL of the git repo
+    treeish -- The tree-ish to check out (branch or tag)
+
+    Returns:
+    root -- root directory of destination Git repository
+    repo -- root directory of cloned upstream Git repository
+    """
+    root = get_repo_root()
+    clone_repo(url, treeish)
+    return root, os.getcwd()
+
+
 def walk_if(top, pred):
     """Walks the current directory, then returns a list of files for which the
     given predicate is true.
@@ -63,17 +96,22 @@ def walk_if(top, pred):
 
 
 def copy_to(files, root):
-    if os.path.exists(root):
-        # Delete old install
-        for filename in os.listdir(root):
-            filepath = os.path.join(root, filename)
-            if os.path.isfile(filepath) or os.path.islink(filepath):
-                os.unlink(filepath)
-            elif os.path.isdir(filepath):
-                shutil.rmtree(filepath)
-    else:
+    """Copies list of files to root by appending the relative paths of the files
+    to root.
+
+    The leading directories of root will be created if they don't yet exist.
+
+    Keyword arguments:
+    files -- list of files to copy
+    root -- destination
+
+    Returns:
+    The list of files in their destination.
+    """
+    if not os.path.exists(root):
         os.makedirs(root)
 
+    dest_files = []
     for f in files:
         dest_file = os.path.join(root, f)
 
@@ -87,6 +125,28 @@ def copy_to(files, root):
             os.makedirs(dest_dir)
 
         shutil.copyfile(f, dest_file)
+        dest_files.append(dest_file)
+    return dest_files
+
+
+def walk_cwd_and_copy_if(pred, root):
+    """Walks the current directory, generates a list of files for which the
+    given predicate is true, then copies that list to root by appending the
+    relative paths of the files to root.
+
+    The leading directories of root will be created if they don't yet exist.
+
+    Keyword arguments:
+    pred -- a function that takes a directory path and a filename, then returns
+            True if the file should be included in the output list
+    root -- destination
+
+    Returns:
+    The list of files in their destination.
+    """
+    files = walk_if(".", pred)
+    files = copy_to(files, root)
+    return files
 
 
 def comment_out_invalid_includes(filename, include_roots):
@@ -127,3 +187,15 @@ def comment_out_invalid_includes(filename, include_roots):
     if old_contents != new_contents:
         with open(filename, "w") as f:
             f.write(new_contents)
+
+
+def apply_patches(root, patches):
+    """Apply list of patches to the destination Git repository.
+
+    Keyword arguments:
+    root -- the root directory of the destination Git repository
+    patches -- list of patch files relative to the root
+    """
+    os.chdir(root)
+    for patch in patches:
+        subprocess.check_output(["git", "apply", patch])

wpimath/update_drake.py

@@ -1,137 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import re
-import subprocess
-
-from upstream_utils import clone_repo, comment_out_invalid_includes, copy_to, walk_if
-
-
-def drake_main_cpp_inclusions(dp, f):
-    """Returns true if the given source file in the drake git repo should be
-    copied into allwpilib
-
-    Keyword arguments:
-    dp -- directory path
-    f -- filename
-    """
-    return f in [
-        "drake_assert_and_throw.cc", "discrete_algebraic_riccati_equation.cc"
-    ]
-
-
-def drake_main_h_inclusions(dp, f):
-    """Returns true if the given header file in the drake git repo should be
-    copied into allwpilib
-
-    Keyword arguments:
-    dp -- directory path
-    f -- filename
-    """
-    return f in [
-        "drake_assert.h", "drake_assertion_error.h", "is_approx_equal_abstol.h",
-        "never_destroyed.h", "drake_copyable.h", "drake_throw.h",
-        "discrete_algebraic_riccati_equation.h"
-    ]
-
-
-def drake_test_cpp_inclusions(dp, f):
-    """Returns true if the given source file in the drake git repo should be
-    copied into allwpilib
-
-    Keyword arguments:
-    dp -- directory path
-    f -- filename
-    """
-    return f == "discrete_algebraic_riccati_equation_test.cc"
-
-
-def drake_test_h_inclusions(dp, f):
-    """Returns true if the given header file in the drake git repo should be
-    copied into allwpilib
-
-    Keyword arguments:
-    dp -- directory path
-    f -- filename
-    """
-    return f == "eigen_matrix_compare.h"
-
-
-def main():
-    wpimath = os.getcwd()
-    clone_repo("https://github.com/RobotLocomotion/drake",
-               "8b72428dce6959d077e17c3c3a7a5ef4a17107ee")
-    repo = os.getcwd()
-
-    # Copy drake source files into allwpilib
-    files = walk_if(".", drake_main_cpp_inclusions)
-    dest = os.path.join(wpimath, "src/main/native/cpp/drake")
-    copy_to(files, dest)
-
-    for i in range(len(files)):
-        # Files moved to dest
-        files[i] = os.path.join(dest, files[i])
-
-        # copy_to() renames .cc files to .cpp
-        if files[i].endswith(".cc"):
-            files[i] = os.path.splitext(files[i])[0] + ".cpp"
-
-    for f in files:
-        comment_out_invalid_includes(
-            f, [os.path.join(wpimath, "src/main/native/include")])
-
-    # Copy drake test source files into allwpilib
-    os.chdir(os.path.join(repo, "math/test"))
-    files = walk_if(".", drake_test_cpp_inclusions)
-    dest = os.path.join(wpimath, "src/test/native/cpp/drake")
-    copy_to(files, dest)
-    os.chdir(repo)
-
-    for i in range(len(files)):
-        # Files moved to dest
-        files[i] = os.path.join(dest, files[i])
-
-        # copy_to() renames .cc files to .cpp
-        if files[i].endswith(".cc"):
-            files[i] = os.path.splitext(files[i])[0] + ".cpp"
-
-    for f in files:
-        comment_out_invalid_includes(f, [
-            os.path.join(wpimath, "src/main/native/include"),
-            os.path.join(wpimath, "src/test/native/include")
-        ])
-
-    # Copy drake header files into allwpilib
-    files = walk_if(".", drake_main_h_inclusions)
-    dest = os.path.join(wpimath, "src/main/native/include/drake")
-    copy_to(files, dest)
-
-    # Files moved to dest
-    for i in range(len(files)):
-        files[i] = os.path.join(dest, files[i])
-
-    for f in files:
-        comment_out_invalid_includes(
-            f, [os.path.join(wpimath, "src/main/native/include")])
-
-    # Copy drake test header files into allwpilib
-    files = walk_if(".", drake_test_h_inclusions)
-    dest = os.path.join(wpimath, "src/test/native/include/drake")
-    copy_to(files, dest)
-
-    # Files moved to dest
-    for i in range(len(files)):
-        files[i] = os.path.join(dest, files[i])
-
-    for f in files:
-        comment_out_invalid_includes(
-            f, [os.path.join(wpimath, "src/main/native/include")])
-
-    # Apply patches
-    os.chdir(wpimath)
-    subprocess.check_output(
-        ["git", "apply", "drake-replace-dense-with-core.patch"])
-
-
-if __name__ == "__main__":
-    main()