diff --git a/.styleguide b/.styleguide index 548565111d..5cbd3e79f1 100644 --- a/.styleguide +++ b/.styleguide @@ -9,6 +9,10 @@ cppSrcFileInclude { \.cpp$ } +modifiableFileExclude { + \.patch$ +} + generatedFileExclude { FRCNetComm\.java$ simulation/gz_msgs/src/include/simulation/gz_msgs/msgs\.h$ diff --git a/README.md b/README.md index d8ed01cce5..4aa5c94830 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,8 @@ The integration test directories for C++ and Java contain test code that runs on The hal directory contains more C++ code meant to run on the roboRIO. HAL is an acronym for "Hardware Abstraction Layer", and it interfaces with the NI Libraries. The NI Libraries contain the low-level code for controlling devices on your robot. The NI Libraries are found in the ni-libraries folder. +The upstream_utils directory contains scripts for updating copies of thirdparty code in the repository. + The [styleguide repository](https://github.com/wpilibsuite/styleguide) contains our style guides for C++ and Java code. Anything submitted to the WPILib project needs to follow the code style guides outlined in there. For details about the style, please see the contributors document [here](CONTRIBUTING.md#coding-guidelines). # Contributing to WPILib diff --git a/wpimath/drake-replace-dense-with-core.patch b/upstream_utils/drake-replace-dense-with-core.patch similarity index 100% rename from wpimath/drake-replace-dense-with-core.patch rename to upstream_utils/drake-replace-dense-with-core.patch diff --git a/wpimath/eigen-maybe-uninitialized.patch b/upstream_utils/eigen-maybe-uninitialized.patch similarity index 100% rename from wpimath/eigen-maybe-uninitialized.patch rename to upstream_utils/eigen-maybe-uninitialized.patch diff --git a/upstream_utils/update_drake.py b/upstream_utils/update_drake.py new file mode 100755 index 0000000000..c52c05ffbc --- /dev/null +++ b/upstream_utils/update_drake.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import os +import shutil + +from upstream_utils import setup_upstream_repo, comment_out_invalid_includes, walk_cwd_and_copy_if, apply_patches + + +def main(): + root, repo = setup_upstream_repo( + "https://github.com/RobotLocomotion/drake", + "8b72428dce6959d077e17c3c3a7a5ef4a17107ee") + wpimath = os.path.join(root, "wpimath") + + # Delete old install + for d in [ + "src/main/native/cpp/drake", "src/main/native/include/drake", + "src/test/native/cpp/drake", "src/test/native/include/drake" + ]: + shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) + + # Copy drake source files into allwpilib + src_files = walk_cwd_and_copy_if( + lambda dp, f: f in + ["drake_assert_and_throw.cc", "discrete_algebraic_riccati_equation.cc"], + os.path.join(wpimath, "src/main/native/cpp/drake")) + + # Copy drake header files into allwpilib + include_files = walk_cwd_and_copy_if( + lambda dp, f: f in [ + "drake_assert.h", "drake_assertion_error.h", + "is_approx_equal_abstol.h", "never_destroyed.h", "drake_copyable.h", + "drake_throw.h", "discrete_algebraic_riccati_equation.h" + ], os.path.join(wpimath, "src/main/native/include/drake")) + + # Copy drake test source files into allwpilib + os.chdir(os.path.join(repo, "math/test")) + test_src_files = walk_cwd_and_copy_if( + lambda dp, f: f == "discrete_algebraic_riccati_equation_test.cc", + os.path.join(wpimath, "src/test/native/cpp/drake")) + os.chdir(repo) + + # Copy drake test header files into allwpilib + test_include_files = walk_cwd_and_copy_if( + lambda dp, f: f == "eigen_matrix_compare.h", + os.path.join(wpimath, "src/test/native/include/drake")) + + for f in src_files: + comment_out_invalid_includes( + f, [os.path.join(wpimath, "src/main/native/include")]) + for f in include_files: + comment_out_invalid_includes( + f, [os.path.join(wpimath, "src/main/native/include")]) + for f in test_src_files: + comment_out_invalid_includes(f, [ + os.path.join(wpimath, "src/main/native/include"), + os.path.join(wpimath, "src/test/native/include") + ]) + for f in test_include_files: + comment_out_invalid_includes(f, [ + os.path.join(wpimath, "src/main/native/include"), + os.path.join(wpimath, "src/test/native/include") + ]) + + apply_patches(root, ["upstream_utils/drake-replace-dense-with-core.patch"]) + + +if __name__ == "__main__": + main() diff --git a/wpimath/update_eigen.py b/upstream_utils/update_eigen.py similarity index 63% rename from wpimath/update_eigen.py rename to upstream_utils/update_eigen.py index 905d8805b4..e9be219de5 100755 --- a/wpimath/update_eigen.py +++ b/upstream_utils/update_eigen.py @@ -2,9 +2,9 @@ import os import re -import subprocess +import shutil -from upstream_utils import clone_repo, comment_out_invalid_includes, copy_to, walk_if +from upstream_utils import setup_upstream_repo, comment_out_invalid_includes, walk_cwd_and_copy_if, apply_patches def eigen_inclusions(dp, f): @@ -15,6 +15,9 @@ def eigen_inclusions(dp, f): dp -- directory path f -- filename """ + if not dp.startswith("./Eigen"): + return False + abspath = os.path.join(dp, f) # Exclude build system @@ -74,6 +77,9 @@ def unsupported_inclusions(dp, f): dp -- directory path f -- filename """ + if not dp.startswith("./unsupported"): + return False + abspath = os.path.join(dp, f) # Exclude build system and READMEs @@ -85,39 +91,34 @@ def unsupported_inclusions(dp, f): def main(): - wpimath = os.getcwd() - clone_repo("https://gitlab.com/libeigen/eigen.git", "3.3.9") - repo = os.getcwd() + root, repo = setup_upstream_repo("https://gitlab.com/libeigen/eigen.git", + "3.3.9") + wpimath = os.path.join(root, "wpimath") + + # Delete old install + for d in [ + "src/main/native/eigeninclude/Eigen", + "src/main/native/eigeninclude/unsupported" + ]: + shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) # Copy Eigen headers into allwpilib - os.chdir(os.path.join(repo, "Eigen")) - files = walk_if(".", eigen_inclusions) - dest = os.path.join(wpimath, "src/main/native/eigeninclude/Eigen") - copy_to(files, dest) - - # Files moved to dest - for i in range(len(files)): - files[i] = os.path.join(dest, files[i]) - - for f in files: - comment_out_invalid_includes(f, [dest]) + eigen_files = walk_cwd_and_copy_if( + eigen_inclusions, os.path.join(wpimath, "src/main/native/eigeninclude")) # Copy unsupported headers into allwpilib - os.chdir(os.path.join(repo, "unsupported")) - files = walk_if(".", unsupported_inclusions) - dest = os.path.join(wpimath, "src/main/native/eigeninclude/unsupported") - copy_to(files, dest) + unsupported_files = walk_cwd_and_copy_if( + unsupported_inclusions, + os.path.join(wpimath, "src/main/native/eigeninclude")) - # Files moved to dest - for i in range(len(files)): - files[i] = os.path.join(dest, files[i]) + for f in eigen_files: + comment_out_invalid_includes( + f, [os.path.join(wpimath, "src/main/native/eigeninclude")]) + for f in unsupported_files: + comment_out_invalid_includes( + f, [os.path.join(wpimath, "src/main/native/eigeninclude")]) - for f in files: - comment_out_invalid_includes(f, [dest]) - - # Apply patches - os.chdir(wpimath) - subprocess.check_output(["git", "apply", "eigen-maybe-uninitialized.patch"]) + apply_patches(root, ["upstream_utils/eigen-maybe-uninitialized.patch"]) if __name__ == "__main__": diff --git a/upstream_utils/update_fmt.py b/upstream_utils/update_fmt.py new file mode 100755 index 0000000000..faddbae803 --- /dev/null +++ b/upstream_utils/update_fmt.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import os +import shutil + +from upstream_utils import setup_upstream_repo, comment_out_invalid_includes, walk_cwd_and_copy_if + + +def main(): + root, repo = setup_upstream_repo("https://github.com/fmtlib/fmt", "8.0.1") + wpiutil = os.path.join(root, "wpiutil") + + # Delete old install + for d in ["src/main/native/fmtlib/src", "src/main/native/fmtlib/include"]: + shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) + + # Copy fmt source files into allwpilib + src_files = walk_cwd_and_copy_if( + lambda dp, f: dp.endswith("src") and f.endswith(".cc") and f != + "fmt.cc", os.path.join(wpiutil, "src/main/native/fmtlib")) + + # Copy fmt header files into allwpilib + include_files = walk_cwd_and_copy_if( + lambda dp, f: dp.endswith("include/fmt"), + os.path.join(wpiutil, "src/main/native/fmtlib")) + + for f in src_files: + comment_out_invalid_includes( + f, [os.path.join(wpiutil, "src/main/native/fmtlib/include")]) + for f in include_files: + comment_out_invalid_includes( + f, [os.path.join(wpiutil, "src/main/native/fmtlib/include")]) + + +if __name__ == "__main__": + main() diff --git a/wpimath/upstream_utils.py b/upstream_utils/upstream_utils.py similarity index 59% rename from wpimath/upstream_utils.py rename to upstream_utils/upstream_utils.py index 928dce8edb..c4ee1aeb12 100644 --- a/wpimath/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -18,7 +18,7 @@ def clone_repo(url, treeish): os.chdir(tempfile.gettempdir()) repo = os.path.basename(url) - dest = os.path.join(os.getcwd(), repo).rstrip(".git") + dest = os.path.join(os.getcwd(), repo).removesuffix(".git") # Clone Git repository into current directory or update it if not os.path.exists(dest): @@ -45,6 +45,39 @@ def clone_repo(url, treeish): subprocess.run(["git", "checkout", treeish]) +def get_repo_root(): + """Returns the Git repository root as an absolute path. + + An empty string is returned if no repository root was found. + """ + current_dir = os.path.abspath(os.getcwd()) + while current_dir != os.path.dirname(current_dir): + if os.path.exists(current_dir + os.sep + ".git"): + return current_dir + current_dir = os.path.dirname(current_dir) + return "" + + +def setup_upstream_repo(url, treeish): + """Clones the given upstream repository, then returns the root of the + destination Git repository as well as the cloned upstream Git repository. + + The current working directory will be set to the cloned upstream repository + folder. + + Keyword arguments: + url -- The URL of the git repo + treeish -- The tree-ish to check out (branch or tag) + + Returns: + root -- root directory of destination Git repository + repo -- root directory of cloned upstream Git repository + """ + root = get_repo_root() + clone_repo(url, treeish) + return root, os.getcwd() + + def walk_if(top, pred): """Walks the current directory, then returns a list of files for which the given predicate is true. @@ -63,17 +96,22 @@ def walk_if(top, pred): def copy_to(files, root): - if os.path.exists(root): - # Delete old install - for filename in os.listdir(root): - filepath = os.path.join(root, filename) - if os.path.isfile(filepath) or os.path.islink(filepath): - os.unlink(filepath) - elif os.path.isdir(filepath): - shutil.rmtree(filepath) - else: + """Copies list of files to root by appending the relative paths of the files + to root. + + The leading directories of root will be created if they don't yet exist. + + Keyword arguments: + files -- list of files to copy + root -- destination + + Returns: + The list of files in their destination. + """ + if not os.path.exists(root): os.makedirs(root) + dest_files = [] for f in files: dest_file = os.path.join(root, f) @@ -87,6 +125,28 @@ def copy_to(files, root): os.makedirs(dest_dir) shutil.copyfile(f, dest_file) + dest_files.append(dest_file) + return dest_files + + +def walk_cwd_and_copy_if(pred, root): + """Walks the current directory, generates a list of files for which the + given predicate is true, then copies that list to root by appending the + relative paths of the files to root. + + The leading directories of root will be created if they don't yet exist. + + Keyword arguments: + pred -- a function that takes a directory path and a filename, then returns + True if the file should be included in the output list + root -- destination + + Returns: + The list of files in their destination. + """ + files = walk_if(".", pred) + files = copy_to(files, root) + return files def comment_out_invalid_includes(filename, include_roots): @@ -127,3 +187,15 @@ def comment_out_invalid_includes(filename, include_roots): if old_contents != new_contents: with open(filename, "w") as f: f.write(new_contents) + + +def apply_patches(root, patches): + """Apply list of patches to the destination Git repository. + + Keyword arguments: + root -- the root directory of the destination Git repository + patches -- list of patch files relative to the root + """ + os.chdir(root) + for patch in patches: + subprocess.check_output(["git", "apply", patch]) diff --git a/wpimath/update_drake.py b/wpimath/update_drake.py deleted file mode 100755 index 79e9e90532..0000000000 --- a/wpimath/update_drake.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python3 - -import os -import re -import subprocess - -from upstream_utils import clone_repo, comment_out_invalid_includes, copy_to, walk_if - - -def drake_main_cpp_inclusions(dp, f): - """Returns true if the given source file in the drake git repo should be - copied into allwpilib - - Keyword arguments: - dp -- directory path - f -- filename - """ - return f in [ - "drake_assert_and_throw.cc", "discrete_algebraic_riccati_equation.cc" - ] - - -def drake_main_h_inclusions(dp, f): - """Returns true if the given header file in the drake git repo should be - copied into allwpilib - - Keyword arguments: - dp -- directory path - f -- filename - """ - return f in [ - "drake_assert.h", "drake_assertion_error.h", "is_approx_equal_abstol.h", - "never_destroyed.h", "drake_copyable.h", "drake_throw.h", - "discrete_algebraic_riccati_equation.h" - ] - - -def drake_test_cpp_inclusions(dp, f): - """Returns true if the given source file in the drake git repo should be - copied into allwpilib - - Keyword arguments: - dp -- directory path - f -- filename - """ - return f == "discrete_algebraic_riccati_equation_test.cc" - - -def drake_test_h_inclusions(dp, f): - """Returns true if the given header file in the drake git repo should be - copied into allwpilib - - Keyword arguments: - dp -- directory path - f -- filename - """ - return f == "eigen_matrix_compare.h" - - -def main(): - wpimath = os.getcwd() - clone_repo("https://github.com/RobotLocomotion/drake", - "8b72428dce6959d077e17c3c3a7a5ef4a17107ee") - repo = os.getcwd() - - # Copy drake source files into allwpilib - files = walk_if(".", drake_main_cpp_inclusions) - dest = os.path.join(wpimath, "src/main/native/cpp/drake") - copy_to(files, dest) - - for i in range(len(files)): - # Files moved to dest - files[i] = os.path.join(dest, files[i]) - - # copy_to() renames .cc files to .cpp - if files[i].endswith(".cc"): - files[i] = os.path.splitext(files[i])[0] + ".cpp" - - for f in files: - comment_out_invalid_includes( - f, [os.path.join(wpimath, "src/main/native/include")]) - - # Copy drake test source files into allwpilib - os.chdir(os.path.join(repo, "math/test")) - files = walk_if(".", drake_test_cpp_inclusions) - dest = os.path.join(wpimath, "src/test/native/cpp/drake") - copy_to(files, dest) - os.chdir(repo) - - for i in range(len(files)): - # Files moved to dest - files[i] = os.path.join(dest, files[i]) - - # copy_to() renames .cc files to .cpp - if files[i].endswith(".cc"): - files[i] = os.path.splitext(files[i])[0] + ".cpp" - - for f in files: - comment_out_invalid_includes(f, [ - os.path.join(wpimath, "src/main/native/include"), - os.path.join(wpimath, "src/test/native/include") - ]) - - # Copy drake header files into allwpilib - files = walk_if(".", drake_main_h_inclusions) - dest = os.path.join(wpimath, "src/main/native/include/drake") - copy_to(files, dest) - - # Files moved to dest - for i in range(len(files)): - files[i] = os.path.join(dest, files[i]) - - for f in files: - comment_out_invalid_includes( - f, [os.path.join(wpimath, "src/main/native/include")]) - - # Copy drake test header files into allwpilib - files = walk_if(".", drake_test_h_inclusions) - dest = os.path.join(wpimath, "src/test/native/include/drake") - copy_to(files, dest) - - # Files moved to dest - for i in range(len(files)): - files[i] = os.path.join(dest, files[i]) - - for f in files: - comment_out_invalid_includes( - f, [os.path.join(wpimath, "src/main/native/include")]) - - # Apply patches - os.chdir(wpimath) - subprocess.check_output( - ["git", "apply", "drake-replace-dense-with-core.patch"]) - - -if __name__ == "__main__": - main()