From 5f261a88afd499a7b16ada419ca7fd9faaee2f99 Mon Sep 17 00:00:00 2001 From: Joseph Eng <91924258+KangarooKoala@users.noreply.github.com> Date: Tue, 16 Jul 2024 17:20:07 -0700 Subject: [PATCH] [upstream_utils] Rework upstream_utils scripts (#6829) --- .github/workflows/upstream-utils.yml | 55 ++- upstream_utils/README.md | 83 +--- upstream_utils/{update_eigen.py => eigen.py} | 38 +- .../{update_expected.py => expected.py} | 24 +- upstream_utils/{update_fmt.py => fmt.py} | 21 +- upstream_utils/{update_gcem.py => gcem.py} | 27 +- upstream_utils/{update_json.py => json.py} | 43 +- upstream_utils/{update_libuv.py => libuv.py} | 43 +- upstream_utils/{update_llvm.py => llvm.py} | 32 +- .../{update_memory.py => memory.py} | 17 +- upstream_utils/{update_mpack.py => mpack.py} | 34 +- .../{update_protobuf.py => protobuf.py} | 50 ++- .../{update_sleipnir.py => sleipnir.py} | 42 +- ...update_stack_walker.py => stack_walker.py} | 54 +-- upstream_utils/upstream_utils.py | 410 ++++++++++++++++++ 15 files changed, 698 insertions(+), 275 deletions(-) rename upstream_utils/{update_eigen.py => eigen.py} (87%) rename upstream_utils/{update_expected.py => expected.py} (69%) rename upstream_utils/{update_fmt.py => fmt.py} (80%) rename upstream_utils/{update_gcem.py => gcem.py} (76%) rename upstream_utils/{update_json.py => json.py} (78%) rename upstream_utils/{update_libuv.py => libuv.py} (86%) rename upstream_utils/{update_llvm.py => llvm.py} (96%) rename upstream_utils/{update_memory.py => memory.py} (91%) rename upstream_utils/{update_mpack.py => mpack.py} (80%) rename upstream_utils/{update_protobuf.py => protobuf.py} (97%) rename upstream_utils/{update_sleipnir.py => sleipnir.py} (76%) rename upstream_utils/{update_stack_walker.py => stack_walker.py} (64%) diff --git a/.github/workflows/upstream-utils.yml b/.github/workflows/upstream-utils.yml index 102aaf621b..e9c04a4582 100644 --- a/.github/workflows/upstream-utils.yml +++ b/.github/workflows/upstream-utils.yml @@ -30,50 +30,61 @@ jobs: run: | git config --global user.email "you@example.com" git config --global user.name "Your Name" - - name: Run update_eigen.py + - name: Run eigen.py run: | cd upstream_utils - ./update_eigen.py - - name: Run update_fmt.py + ./eigen.py clone + ./eigen.py copy-upstream-to-thirdparty + - name: Run fmt.py run: | cd upstream_utils - ./update_fmt.py - - name: Run update_gcem.py + ./fmt.py clone + ./fmt.py copy-upstream-to-thirdparty + - name: Run gcem.py run: | cd upstream_utils - ./update_gcem.py - - name: Run update_json.py + ./gcem.py clone + ./gcem.py copy-upstream-to-thirdparty + - name: Run json.py run: | cd upstream_utils - ./update_json.py - - name: Run update_libuv.py + ./json.py clone + ./json.py copy-upstream-to-thirdparty + - name: Run libuv.py run: | cd upstream_utils - ./update_libuv.py - - name: Run update_llvm.py + ./libuv.py clone + ./libuv.py copy-upstream-to-thirdparty + - name: Run llvm.py run: | cd upstream_utils - ./update_llvm.py - - name: Run update_mpack.py + ./llvm.py clone + ./llvm.py copy-upstream-to-thirdparty + - name: Run mpack.py run: | cd upstream_utils - ./update_mpack.py - - name: Run update_stack_walker.py + ./mpack.py clone + ./mpack.py copy-upstream-to-thirdparty + - name: Run stack_walker.py run: | cd upstream_utils - ./update_stack_walker.py - - name: Run update_memory.py + ./stack_walker.py clone + ./stack_walker.py copy-upstream-to-thirdparty + - name: Run memory.py run: | cd upstream_utils - ./update_memory.py - - name: Run update_protobuf.py + ./memory.py clone + ./memory.py copy-upstream-to-thirdparty + - name: Run protobuf.py run: | cd upstream_utils - ./update_protobuf.py - - name: Run update_sleipnir.py + ./protobuf.py clone + ./protobuf.py copy-upstream-to-thirdparty + - name: Run sleipnir.py run: | cd upstream_utils - ./update_sleipnir.py + ./sleipnir.py clone + ./sleipnir.py copy-upstream-to-thirdparty - name: Add untracked files to index so they count as changes run: git add -A - name: Check output diff --git a/upstream_utils/README.md b/upstream_utils/README.md index 979dd0d672..2e200ae5b5 100644 --- a/upstream_utils/README.md +++ b/upstream_utils/README.md @@ -20,59 +20,24 @@ versions. Each library has its own patch directory (e.g., `lib_patches`). The example below will update a hypothetical library called `lib` to the tag `2.0`. -Start in the `upstream_utils` folder. Restore the original repo. +Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exists. ```bash -./update_.py +./.py clone ``` -Navigate to the repo. +Rebase the clone of the upstream repo. ```bash -cd /tmp/lib +./.py rebase 2.0 ``` -Fetch the desired version using one of the following methods. +Update the `upstream_utils` patch files and the tag in the script. ```bash -# Fetch a full branch or tag -git fetch origin 2.0 - -# Fetch just a tag (useful for expensive-to-clone repos) -git fetch --depth 1 origin tag 2.0 +./.py format-patch ``` -Rebase any patches onto the new version. If the old version and new version are -on the same branch, run the following. +Copy the updated upstream files into the thirdparty files within allwpilib. ```bash -git rebase 2.0 -``` - -If the old version and new version are on different branches (e.g., -llvm-project), use interactive rebase instead and remove commits that are common -between the two branches from the list of commits to rebase. In other words, -only commits representing downstream patches should be listed. -```bash -git rebase -i 2.0 -``` - -Generate patch files for the new version. -```bash -git format-patch 2.0..HEAD --zero-commit --abbrev=40 --no-signature -``` - -Move the patch files to `upstream_utils`. -``` -mv *.patch allwpilib/upstream_utils/lib_patches -``` - -Navigate back to `upstream_utils`. -```bash -cd allwpilib/upstream_utils -``` - -Modify the version number in the call to `setup_upstream_repo()` in -`update_.py`, then rerun `update_.py` to reimport the thirdparty -files. -```bash -./update_.py +./.py copy-upstream-to-thirdparty ``` ## Adding patch to thirdparty library @@ -80,12 +45,17 @@ files. The example below will add a new patch file to a hypothetical library called `lib` (Replace `` with `llvm`, `fmt`, `eigen`, ... in the following steps). -Start in the `upstream_utils` folder. Restore the original repo. +Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exists. ```bash -./update_.py +./.py clone ``` -Navigate to the repo. +Update the clone of the upstream repo. +```bash +./.py reset +``` + +Navigate to the repo. If you can't find it, the directory of the clone is printed at the start of the `clone` command. ```bash cd /tmp/ ``` @@ -96,24 +66,17 @@ git add ... git commit -m "..." ``` -Generate patch files. -```bash -git format-patch 2.0..HEAD --zero-commit --abbrev=40 --no-signature -``` -where `2.0` is replaced with the version specified in `update_.py`. - -Move the patch files to `upstream_utils`. -``` -mv *.patch allwpilib/upstream_utils/_patches -``` - Navigate back to `upstream_utils`. ```bash cd allwpilib/upstream_utils ``` -Update the list of patch files in `update_.py`, then rerun -`update_.py` to reimport the thirdparty files. +Update the `upstream_utils` patch files. ```bash -./update_.py +./.py format-patch +``` + +Update the list of patch files in `.py`, then rerun `.py` to reimport the thirdparty files. +```bash +./.py copy-upstream-to-thirdparty ``` diff --git a/upstream_utils/update_eigen.py b/upstream_utils/eigen.py similarity index 87% rename from upstream_utils/update_eigen.py rename to upstream_utils/eigen.py index 66c82385e9..ace408ecb0 100755 --- a/upstream_utils/update_eigen.py +++ b/upstream_utils/eigen.py @@ -5,11 +5,9 @@ import re import shutil from upstream_utils import ( - get_repo_root, - clone_repo, comment_out_invalid_includes, walk_cwd_and_copy_if, - git_am, + Lib, ) @@ -94,25 +92,9 @@ def unsupported_inclusions(dp, f): return "MatrixFunctions" in abspath -def main(): - upstream_root = clone_repo( - "https://gitlab.com/libeigen/eigen.git", - # master on 2024-05-22 - "c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpimath = os.path.join(wpilib_root, "wpimath") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Disable-warnings.patch", - "0002-Intellisense-fix.patch", - "0003-Suppress-has_denorm-and-has_denorm_loss-deprecation-.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/eigen_patches", f)) - # Delete old install for d in ["src/main/native/thirdparty/eigen/include"]: shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) @@ -139,10 +121,24 @@ def main(): ) shutil.copyfile( - os.path.join(upstream_root, ".clang-format"), + ".clang-format", os.path.join(wpimath, "src/main/native/thirdparty/eigen/include/.clang-format"), ) +def main(): + name = "eigen" + url = "https://gitlab.com/libeigen/eigen.git" + tag = "c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198" + patch_list = [ + "0001-Disable-warnings.patch", + "0002-Intellisense-fix.patch", + "0003-Suppress-has_denorm-and-has_denorm_loss-deprecation-.patch", + ] + + eigen = Lib(name, url, tag, patch_list, copy_upstream_src) + eigen.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_expected.py b/upstream_utils/expected.py similarity index 69% rename from upstream_utils/update_expected.py rename to upstream_utils/expected.py index 1b459fe5ac..08569e3561 100755 --- a/upstream_utils/update_expected.py +++ b/upstream_utils/expected.py @@ -10,26 +10,18 @@ from upstream_utils import ( comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo( - "https://github.com/TartanLlama/expected", - # master on 2024-01-25 - "3f0ca7b19253129700a073abfa6d8638d9f7c80c", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") # Copy expected header into allwpilib dest_filename = os.path.join( wpiutil, "src/main/native/thirdparty/expected/include/wpi/expected" ) - shutil.copyfile( - os.path.join(upstream_root, "include/tl/expected.hpp"), dest_filename - ) + shutil.copyfile("include/tl/expected.hpp", dest_filename) # Rename namespace from tl to wpi with open(dest_filename) as f: @@ -41,5 +33,15 @@ def main(): f.write(content) +def main(): + name = "expected" + url = "https://github.com/TartanLlama/expected" + # master on 2024-01-25 + tag = "3f0ca7b19253129700a073abfa6d8638d9f7c80c" + + expected = Lib(name, url, tag, [], copy_upstream_src) + expected.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_fmt.py b/upstream_utils/fmt.py similarity index 80% rename from upstream_utils/update_fmt.py rename to upstream_utils/fmt.py index 94aef193f7..8f03859f54 100755 --- a/upstream_utils/update_fmt.py +++ b/upstream_utils/fmt.py @@ -9,19 +9,13 @@ from upstream_utils import ( comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/fmtlib/fmt", "11.0.1") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in ["0001-Suppress-warnings-we-can-t-fix.patch"]: - git_am(os.path.join(wpilib_root, "upstream_utils/fmt_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/fmtlib/src", @@ -51,5 +45,16 @@ def main(): ) +def main(): + name = "fmt" + url = "https://github.com/fmtlib/fmt" + tag = "11.0.1" + + patch_list = ["0001-Suppress-warnings-we-can-t-fix.patch"] + + fmt = Lib(name, url, tag, patch_list, copy_upstream_src) + fmt.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_gcem.py b/upstream_utils/gcem.py similarity index 76% rename from upstream_utils/update_gcem.py rename to upstream_utils/gcem.py index 20f6242596..b6fd472a72 100755 --- a/upstream_utils/update_gcem.py +++ b/upstream_utils/gcem.py @@ -9,22 +9,13 @@ from upstream_utils import ( comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/kthohr/gcem.git", "v1.18.0") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpimath = os.path.join(wpilib_root, "wpimath") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Call-std-functions-if-not-constant-evaluated.patch", - "0002-Add-hypot-x-y-z.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/gcem_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/gcem/include", @@ -43,5 +34,19 @@ def main(): ) +def main(): + name = "gcem" + url = "https://github.com/kthohr/gcem.git" + tag = "v1.18.0" + + patch_list = [ + "0001-Call-std-functions-if-not-constant-evaluated.patch", + "0002-Add-hypot-x-y-z.patch", + ] + + gcem = Lib(name, url, tag, patch_list, copy_upstream_src) + gcem.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_json.py b/upstream_utils/json.py similarity index 78% rename from upstream_utils/update_json.py rename to upstream_utils/json.py index 9cad98d5fa..78924ef2a1 100755 --- a/upstream_utils/update_json.py +++ b/upstream_utils/json.py @@ -8,27 +8,13 @@ from upstream_utils import ( clone_repo, walk_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/nlohmann/json", "v3.11.3") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Remove-version-from-namespace.patch", - "0002-Make-serializer-public.patch", - "0003-Make-dump_escaped-take-std-string_view.patch", - "0004-Add-llvm-stream-support.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/json_patches", f), - use_threeway=True, - ) - # Delete old install for d in [ "src/main/native/thirdparty/json/include", @@ -36,11 +22,9 @@ def main(): shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) # Create lists of source and destination files - os.chdir(os.path.join(upstream_root, "include/nlohmann")) + os.chdir("include/nlohmann") files = walk_if(".", lambda dp, f: True) - src_include_files = [ - os.path.join(os.path.join(upstream_root, "include/nlohmann"), f) for f in files - ] + src_include_files = [os.path.abspath(f) for f in files] wpiutil_json_root = os.path.join( wpiutil, "src/main/native/thirdparty/json/include/wpi" ) @@ -74,5 +58,24 @@ def main(): f.write(content) +def main(): + name = "json" + url = "https://github.com/nlohmann/json" + tag = "v3.11.3" + + patch_list = [ + "0001-Remove-version-from-namespace.patch", + "0002-Make-serializer-public.patch", + "0003-Make-dump_escaped-take-std-string_view.patch", + "0004-Add-llvm-stream-support.patch", + ] + patch_options = { + "use_threeway": True, + } + + json = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options) + json.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_libuv.py b/upstream_utils/libuv.py similarity index 86% rename from upstream_utils/update_libuv.py rename to upstream_utils/libuv.py index 0290c4d73a..9a2865838c 100755 --- a/upstream_utils/update_libuv.py +++ b/upstream_utils/libuv.py @@ -9,30 +9,13 @@ from upstream_utils import ( comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/libuv/libuv", "v1.48.0") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpinet = os.path.join(wpilib_root, "wpinet") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Revert-win-process-write-minidumps-when-sending-SIGQ.patch", - "0002-Fix-missing-casts.patch", - "0003-Fix-warnings.patch", - "0004-Preprocessor-cleanup.patch", - "0005-Cleanup-problematic-language.patch", - "0006-Fix-Win32-warning-suppression-pragma.patch", - "0007-Use-C-atomics.patch", - "0008-Remove-static-from-array-indices.patch", - "0009-Add-pragmas-for-missing-libraries-and-set-_WIN32_WIN.patch", - "0010-Remove-swearing.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/libuv_patches", f)) - # Delete old install for d in ["src/main/native/thirdparty/libuv"]: shutil.rmtree(os.path.join(wpinet, d), ignore_errors=True) @@ -71,5 +54,27 @@ def main(): ) +def main(): + name = "libuv" + url = "https://github.com/libuv/libuv" + tag = "v1.48.0" + + patch_list = [ + "0001-Revert-win-process-write-minidumps-when-sending-SIGQ.patch", + "0002-Fix-missing-casts.patch", + "0003-Fix-warnings.patch", + "0004-Preprocessor-cleanup.patch", + "0005-Cleanup-problematic-language.patch", + "0006-Fix-Win32-warning-suppression-pragma.patch", + "0007-Use-C-atomics.patch", + "0008-Remove-static-from-array-indices.patch", + "0009-Add-pragmas-for-missing-libraries-and-set-_WIN32_WIN.patch", + "0010-Remove-swearing.patch", + ] + + libuv = Lib(name, url, tag, patch_list, copy_upstream_src) + libuv.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_llvm.py b/upstream_utils/llvm.py similarity index 96% rename from upstream_utils/update_llvm.py rename to upstream_utils/llvm.py index d2d3c379df..abbc077a65 100755 --- a/upstream_utils/update_llvm.py +++ b/upstream_utils/llvm.py @@ -9,6 +9,7 @@ from upstream_utils import ( comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) @@ -170,14 +171,20 @@ def overwrite_tests(wpiutil_root, llvm_root): run_global_replacements(wpi_files) -def main(): - upstream_root = clone_repo("https://github.com/llvm/llvm-project", "llvmorg-18.1.8") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): + upstream_root = os.path.abspath(".") wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ + overwrite_source(wpiutil, upstream_root) + overwrite_tests(wpiutil, upstream_root) + + +def main(): + name = "llvm" + url = "https://github.com/llvm/llvm-project" + tag = "llvmorg-18.1.8" + + patch_list = [ "0001-Remove-StringRef-ArrayRef-and-Optional.patch", "0002-Wrap-std-min-max-calls-in-parens-for-Windows-warning.patch", "0003-Change-unique_function-storage-size.patch", @@ -215,14 +222,13 @@ def main(): "0035-Remove-auto-conversion-from-raw_ostream.patch", "0036-Add-SmallVector-erase_if.patch", "0037-Fix-AlignedCharArrayUnion-for-C-23.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/llvm_patches", f), - use_threeway=True, - ) + ] + patch_options = { + "use_threeway": True, + } - overwrite_source(wpiutil, upstream_root) - overwrite_tests(wpiutil, upstream_root) + llvm = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options) + llvm.main() if __name__ == "__main__": diff --git a/upstream_utils/update_memory.py b/upstream_utils/memory.py similarity index 91% rename from upstream_utils/update_memory.py rename to upstream_utils/memory.py index 6645aaf60c..4a61858422 100755 --- a/upstream_utils/update_memory.py +++ b/upstream_utils/memory.py @@ -9,6 +9,7 @@ from upstream_utils import ( comment_out_invalid_includes, walk_if, copy_to, + Lib, ) @@ -58,9 +59,7 @@ def run_global_replacements(memory_files): f.write(content) -def main(): - upstream_root = clone_repo("https://github.com/foonathan/memory", "v0.7-3") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") # Delete old install @@ -71,7 +70,6 @@ def main(): shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) # Copy sources - os.chdir(upstream_root) src_files = walk_if("src", lambda dp, f: f.endswith(".cpp") or f.endswith(".hpp")) src_files = copy_to( src_files, os.path.join(wpiutil, "src/main/native/thirdparty/memory") @@ -80,7 +78,7 @@ def main(): run_source_replacements(src_files) # Copy headers - os.chdir(os.path.join(upstream_root, "include", "foonathan")) + os.chdir(os.path.join("include", "foonathan")) include_files = walk_if(".", lambda dp, f: f.endswith(".hpp")) include_files = copy_to( include_files, @@ -100,5 +98,14 @@ def main(): ) +def main(): + name = "memory" + url = "https://github.com/foonathan/memory" + tag = "v0.7-3" + + memory = Lib(name, url, tag, [], copy_upstream_src) + memory.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_mpack.py b/upstream_utils/mpack.py similarity index 80% rename from upstream_utils/update_mpack.py rename to upstream_utils/mpack.py index 68315e933a..ad395377a6 100755 --- a/upstream_utils/update_mpack.py +++ b/upstream_utils/mpack.py @@ -9,12 +9,11 @@ from upstream_utils import ( clone_repo, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/ludocode/mpack", "v1.1.1") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") # Delete old install @@ -24,19 +23,6 @@ def main(): ]: shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) - # Apply patches to upstream Git repo - os.chdir(upstream_root) - - for f in [ - "0001-Don-t-emit-inline-defs.patch", - "0002-Update-amalgamation-script.patch", - "0003-Use-namespace-for-C.patch", - "0004-Group-doxygen-into-MPack-module.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/mpack_patches", f), - ) - # Run the amalgmation script subprocess.check_call(["bash", "tools/amalgamate.sh"]) @@ -56,5 +42,21 @@ def main(): ) +def main(): + name = "mpack" + url = "https://github.com/ludocode/mpack" + tag = "v1.1.1" + + patch_list = [ + "0001-Don-t-emit-inline-defs.patch", + "0002-Update-amalgamation-script.patch", + "0003-Use-namespace-for-C.patch", + "0004-Group-doxygen-into-MPack-module.patch", + ] + + mpack = Lib(name, url, tag, patch_list, copy_upstream_src) + mpack.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_protobuf.py b/upstream_utils/protobuf.py similarity index 97% rename from upstream_utils/update_protobuf.py rename to upstream_utils/protobuf.py index 1f004cd0c4..8b2adf5347 100755 --- a/upstream_utils/update_protobuf.py +++ b/upstream_utils/protobuf.py @@ -11,6 +11,7 @@ from upstream_utils import ( walk_cwd_and_copy_if, walk_if, git_am, + Lib, ) protobuf_lite_sources = set( @@ -255,31 +256,10 @@ def matches(dp, f, files): return p in files -def main(): - upstream_root = clone_repo( - "https://github.com/protocolbuffers/protobuf", "v3.21.12" - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): + upstream_root = os.path.abspath(".") wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Fix-sign-compare-warnings.patch", - "0002-Remove-redundant-move.patch", - "0003-Fix-maybe-uninitialized-warnings.patch", - "0004-Fix-coded_stream-WriteRaw.patch", - "0005-Suppress-enum-enum-conversion-warning.patch", - "0006-Fix-noreturn-function-returning.patch", - "0007-Work-around-GCC-12-restrict-warning-compiler-bug.patch", - "0008-Disable-MSVC-switch-warning.patch", - "0009-Disable-unused-function-warning.patch", - "0010-Disable-pedantic-warning.patch", - "0011-Avoid-use-of-sprintf.patch", - "0012-Suppress-stringop-overflow-warning-false-positives.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/protobuf_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/protobuf/src", @@ -304,5 +284,29 @@ def main(): ) +def main(): + name = "protobuf" + url = "https://github.com/protocolbuffers/protobuf" + tag = "v3.21.12" + + patch_list = [ + "0001-Fix-sign-compare-warnings.patch", + "0002-Remove-redundant-move.patch", + "0003-Fix-maybe-uninitialized-warnings.patch", + "0004-Fix-coded_stream-WriteRaw.patch", + "0005-Suppress-enum-enum-conversion-warning.patch", + "0006-Fix-noreturn-function-returning.patch", + "0007-Work-around-GCC-12-restrict-warning-compiler-bug.patch", + "0008-Disable-MSVC-switch-warning.patch", + "0009-Disable-unused-function-warning.patch", + "0010-Disable-pedantic-warning.patch", + "0011-Avoid-use-of-sprintf.patch", + "0012-Suppress-stringop-overflow-warning-false-positives.patch", + ] + + protobuf = Lib(name, url, tag, patch_list, copy_upstream_src) + protobuf.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_sleipnir.py b/upstream_utils/sleipnir.py similarity index 76% rename from upstream_utils/update_sleipnir.py rename to upstream_utils/sleipnir.py index 4b783c82ba..11b20cc29a 100755 --- a/upstream_utils/update_sleipnir.py +++ b/upstream_utils/sleipnir.py @@ -9,30 +9,13 @@ from upstream_utils import ( copy_to, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo( - "https://github.com/SleipnirGroup/Sleipnir", - # main on 2024-07-09 - "b6ffa2d4fdb99cab1bf79491f715a6a9a86633b5", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpimath = os.path.join(wpilib_root, "wpimath") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Remove-using-enum-declarations.patch", - "0002-Use-fmtlib.patch", - "0003-Remove-unsupported-constexpr.patch", - "0004-Use-wpi-SmallVector.patch", - "0005-Suppress-clang-tidy-false-positives.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/sleipnir_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/sleipnir/src", @@ -41,7 +24,6 @@ def main(): shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) # Copy Sleipnir source files into allwpilib - os.chdir(upstream_root) src_files = [os.path.join(dp, f) for dp, dn, fn in os.walk("src") for f in fn] src_files = copy_to( src_files, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir") @@ -65,10 +47,28 @@ def main(): ".styleguide-license", ]: shutil.copyfile( - os.path.join(upstream_root, filename), + filename, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir", filename), ) +def main(): + name = "sleipnir" + url = "https://github.com/SleipnirGroup/Sleipnir" + # main on 2024-07-09 + tag = "b6ffa2d4fdb99cab1bf79491f715a6a9a86633b5" + + patch_list = [ + "0001-Remove-using-enum-declarations.patch", + "0002-Use-fmtlib.patch", + "0003-Remove-unsupported-constexpr.patch", + "0004-Use-wpi-SmallVector.patch", + "0005-Suppress-clang-tidy-false-positives.patch", + ] + + sleipnir = Lib(name, url, tag, patch_list, copy_upstream_src) + sleipnir.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_stack_walker.py b/upstream_utils/stack_walker.py similarity index 64% rename from upstream_utils/update_stack_walker.py rename to upstream_utils/stack_walker.py index a58886ec79..190f12d04d 100755 --- a/upstream_utils/update_stack_walker.py +++ b/upstream_utils/stack_walker.py @@ -10,11 +10,12 @@ from upstream_utils import ( comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def crlf_to_lf(stackwalker_dir): - for root, _, files in os.walk(stackwalker_dir): +def crlf_to_lf(): + for root, _, files in os.walk("."): if ".git" in root: continue @@ -28,35 +29,13 @@ def crlf_to_lf(stackwalker_dir): with open(filename, "wb") as f: f.write(content) - cwd = os.getcwd() - os.chdir(stackwalker_dir) subprocess.check_call(["git", "add", "-A"]) subprocess.check_call(["git", "commit", "-m", "Fix line endings"]) - os.chdir(cwd) -def main(): - upstream_root = clone_repo( - "https://github.com/JochenKalmbach/StackWalker", - "5b0df7a4db8896f6b6dc45d36e383c52577e3c6b", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") - # Run CRLF -> LF before trying any patches - crlf_to_lf(upstream_root) - - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Add-advapi-pragma.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/stack_walker_patches", f), - ignore_whitespace=True, - ) - shutil.copy( os.path.join("Main", "StackWalker", "StackWalker.h"), os.path.join(wpiutil, "src/main/native/windows/StackWalker.h"), @@ -68,5 +47,30 @@ def main(): ) +def main(): + name = "stack_walker" + url = "https://github.com/JochenKalmbach/StackWalker" + tag = "5b0df7a4db8896f6b6dc45d36e383c52577e3c6b" + + patch_list = [ + "0001-Add-advapi-pragma.patch", + ] + patch_options = { + "ignore_whitespace": True, + } + + stack_walker = Lib( + name, + url, + tag, + patch_list, + copy_upstream_src, + patch_options, + pre_patch_hook=crlf_to_lf, + pre_patch_commits=1, + ) + stack_walker.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index ccf668b738..5851df11f9 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -1,7 +1,9 @@ +import argparse import os import re import shutil import subprocess +import sys import tempfile @@ -198,3 +200,411 @@ def git_am(patch, use_threeway=False, ignore_whitespace=False): args.append("--ignore-whitespace") subprocess.check_output(args + [patch]) + + +def has_git_rev(rev): + """Checks whether the Git repository in the current directory has the given + revision. + + Keyword arguments: + rev -- The revision to check + + Returns: + True if the revision exists, otherwise False. + """ + cmd = ["git", "rev-parse", "--verify", "-q", rev] + return subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode == 0 + + +class Lib: + def __init__( + self, + name, + url, + tag, + patch_list, + copy_upstream_src, + patch_options={}, + *, + pre_patch_hook=None, + pre_patch_commits=0, + ): + """Initializes a Lib instance. + + Keyword arguments: + name -- The name of the library. + url -- The URL of the upstream repository. + tag -- The tag in the upstream repository to use. Can be any + (e.g., commit hash or tag). + patch_list -- The list of patches in the patch directory to apply. + copy_upstream_src -- A callable that takes the path to the wpilib root + and copies the files from the clone of the upstream + into the appropriate thirdparty directory. Will + only be called when the current directory is the + upstream clone. + patch_options -- The dictionary of options to use when applying patches. + Corresponds to the parameters of git_am. + + Keyword-only arguments: + pre_patch_hook -- Optional callable taking no parameters that will be + called before applying patches. + pre_patch_commits -- Number of commits added by pre_patch_hook. + """ + self.name = name + self.url = url + self.old_tag = tag + self.patch_list = patch_list + self.copy_upstream_src = copy_upstream_src + self.patch_options = patch_options + self.pre_patch_hook = pre_patch_hook + self.pre_patch_commits = pre_patch_commits + self.wpilib_root = get_repo_root() + + def check_patches(self): + """Checks that the patch list supplied to the constructor matches the + patches in the patch directory. + """ + patch_directory_patches = set() + patch_directory = os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches" + ) + if os.path.exists(patch_directory): + for f in os.listdir(patch_directory): + if f.endswith(".patch"): + patch_directory_patches.add(f) + patches = set(self.patch_list) + patch_directory_only = sorted(patch_directory_patches - patches) + patch_list_only = sorted(patches - patch_directory_patches) + common_patches = sorted(patch_directory_patches & patches) + warning = False + if patch_directory_only: + print( + f"WARNING: The patch directory has patches {patch_directory_only} not in the patch list" + ) + warning = True + if patch_list_only: + print( + f"WARNING: The patch list has patches {patch_list_only} not in the patch directory" + ) + warning = True + if warning and common_patches: + print( + f" Note: The patch directory and the patch list both have patches {common_patches}" + ) + + def get_repo_path(self, tempdir=None): + """Returns the path to the clone of the upstream repository. + + Keyword argument: + tempdir -- The path to the temporary directory to use. If None (the + default), uses tempfile.gettempdir(). + + Returns: + The path to the clone of the upstream repository. Will be absolute if + tempdir is absolute. + """ + if tempdir is None: + tempdir = tempfile.gettempdir() + repo = os.path.basename(self.url) + dest = os.path.join(tempdir, repo) + dest = dest.removesuffix(".git") + return dest + + def open_repo(self, *, err_msg_if_absent): + """Changes the current working directory to the upstream repository. If + err_msg_if_absent is not None and the upstream repository does not + exist, the program exits with return code 1. + + Keyword-only argument: + err_msg_if_absent -- The error message to print to stderr if the + upstream repository does not exist. If None, the upstream repository + will be cloned without emitting any warnings. + """ + os.chdir(tempfile.gettempdir()) + + dest = self.get_repo_path(os.getcwd()) + + print(f"INFO: Opening repository at {dest}") + + if not os.path.exists(dest): + if err_msg_if_absent is None: + subprocess.run(["git", "clone", "--filter=tree:0", self.url]) + else: + print(err_msg_if_absent, file=sys.stderr) + exit(1) + os.chdir(dest) + + def get_root_tags(self): + """Returns a list of potential root tags. + + Returns: + A list of the potential root tags. + """ + root_tag_output = subprocess.run( + ["git", "tag", "--list", "upstream_utils_root-*"], + capture_output=True, + text=True, + ).stdout + return root_tag_output.splitlines() + + def get_root_tag(self): + """Returns the root tag (the default tag to apply the patches relative + to). If there are multiple candidates, prints an error to stderr and the + program exits with return code 1. + + Returns: + The root tag. + """ + root_tags = self.get_root_tags() + if len(root_tags) == 0: + print( + "ERROR: Could not determine root tag: No tags match 'upstream_utils_root-*'", + file=sys.stderr, + ) + exit(1) + if len(root_tags) > 1: + print( + f"ERROR: Could not determine root tag: Multiple candidates: {root_tags}", + file=sys.stderr, + ) + exit(1) + return root_tags[0] + + def set_root_tag(self, tag): + """Sets the root tag, deleting any potential candidates first. + + Keyword argument: + tag -- The tag to set as the root tag. + """ + root_tags = self.get_root_tags() + + if len(root_tags) > 1: + print(f"WARNING: Deleting multiple root tags {root_tags}", file=sys.stderr) + + for root_tag in root_tags: + subprocess.run(["git", "tag", "-d", root_tag]) + + subprocess.run(["git", "tag", f"upstream_utils_root-{tag}", tag]) + + def apply_patches(self): + """Applies the patches listed in the patch list to the current + directory. + """ + if self.pre_patch_hook is not None: + self.pre_patch_hook() + + for f in self.patch_list: + git_am( + os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches", f + ), + **self.patch_options, + ) + + def replace_tag(self, tag): + """Replaces the tag in the script. + + Keyword argument: + tag -- The tag to replace the script tag with. + """ + path = os.path.join(self.wpilib_root, f"upstream_utils/{self.name}.py") + with open(path, "r") as file: + lines = file.readlines() + + previous_text = f'tag = "{self.old_tag}"' + new_text = f'tag = "{tag}"' + for i in range(len(lines)): + if previous_text in lines[i]: + if i - 1 >= 0 and "#" in lines[i - 1]: + print( + f"WARNING: Automatically updating tag in line {i + 1} with a comment above it that may need updating.", + file=sys.stderr, + ) + lines[i] = lines[i].replace(previous_text, new_text) + + with open(path, "w") as file: + file.writelines(lines) + + def info(self): + """Prints info about the library to stdout.""" + print(f"Repository name: {self.name}") + print(f"Upstream URL: {self.url}") + print(f"Upstream tag: {self.old_tag}") + print(f"Path to upstream clone: {self.get_repo_path()}") + print(f"Patches to apply: {self.patch_list}") + print(f"Patch options: {self.patch_options}") + print(f"Pre patch commits: {self.pre_patch_commits}") + print(f"WPILib root: {self.wpilib_root}") + + def clone(self): + """Clones the upstream repository and sets it up.""" + self.open_repo(err_msg_if_absent=None) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + self.set_root_tag(self.old_tag) + + def reset(self): + """Resets the clone of the upstream repository to the state specified by + the script and patches. + """ + self.open_repo( + err_msg_if_absent='There\'s nothing to reset. Run the "clone" command first.' + ) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + self.apply_patches() + + self.set_root_tag(self.old_tag) + + def rebase(self, new_tag): + """Rebases the patches. + + Keyword argument: + new_tag -- The tag to rebase onto. + """ + self.open_repo( + err_msg_if_absent='There\'s nothing to rebase. Run the "clone" command first.' + ) + + subprocess.run(["git", "fetch", "origin", new_tag]) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + self.apply_patches() + + self.set_root_tag(new_tag) + + subprocess.run(["git", "rebase", "--onto", new_tag, self.old_tag]) + + # Detect merge conflict by detecting if we stopped in the middle of a rebase + if has_git_rev("REBASE_HEAD"): + print( + f"Merge conflicts when rebasing onto {new_tag}! You must manually resolve them.", + file=sys.stderr, + ) + + def format_patch(self): + """Generates patch files for the upstream repository and moves them into + the patch directory. + """ + self.open_repo( + err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' + ) + + root_tag = self.get_root_tag() + script_tag = root_tag.removeprefix("upstream_utils_root-") + + start_commit = root_tag + if self.pre_patch_commits > 0: + commits_since_tag_output = subprocess.run( + ["git", "log", "--format=format:%h", f"{start_commit}..HEAD"], + capture_output=True, + ).stdout + commits_since_tag = commits_since_tag_output.count(b"\n") + 1 + start_commit = f"HEAD~{commits_since_tag - self.pre_patch_commits}" + + subprocess.run( + [ + "git", + "format-patch", + f"{start_commit}..HEAD", + "--abbrev=40", + "--zero-commit", + "--no-signature", + ] + ) + + patch_dest = os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches" + ) + + if not os.path.exists(patch_dest): + print( + f"WARNING: Patch directory {patch_dest} does not exist", file=sys.stderr + ) + else: + shutil.rmtree(patch_dest) + + is_first = True + for f in os.listdir(): + if f.endswith(".patch"): + if is_first: + os.mkdir(patch_dest) + is_first = False + shutil.move(f, patch_dest) + + self.replace_tag(script_tag) + + def copy_upstream_to_thirdparty(self): + """Copies files from the upstream repository into the thirdparty + directory. + """ + self.open_repo( + err_msg_if_absent='There\'s no repository to copy from. Run the "clone" command first.' + ) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + self.apply_patches() + + self.copy_upstream_src(self.wpilib_root) + + def main(self, argv=sys.argv[1:]): + """Processes the given arguments. + + Keyword argument: + argv -- The arguments to process. Defaults to the arguments passed to + the program. + """ + parser = argparse.ArgumentParser( + description=f"CLI manager of the {self.name} upstream library" + ) + subparsers = parser.add_subparsers(dest="subcommand", required=True) + + subparsers.add_parser( + "info", help="Displays information about the upstream library" + ) + + subparsers.add_parser( + "clone", help="Clones the upstream repository in a local tempdir" + ) + + subparsers.add_parser( + "reset", + help="Resets the clone of the upstream repository to the tag and applies patches", + ) + + parser_rebase = subparsers.add_parser( + "rebase", help="Rebases the clone of the upstream repository" + ) + parser_rebase.add_argument("new_tag", help="The tag to rebase onto") + + parser_format_patch = subparsers.add_parser( + "format-patch", + help="Generates patch files for the upstream repository and moves them into the upstream_utils patch directory", + ) + + subparsers.add_parser( + "copy-upstream-to-thirdparty", + help="Copies files from the upstream repository into the thirdparty directory in allwpilib", + ) + + args = parser.parse_args(argv) + + self.wpilib_root = get_repo_root() + if args.subcommand == "info": + self.info() + elif args.subcommand == "clone": + self.clone() + elif args.subcommand == "reset": + self.reset() + elif args.subcommand == "rebase": + self.rebase(args.new_tag) + elif args.subcommand == "format-patch": + self.format_patch() + elif args.subcommand == "copy-upstream-to-thirdparty": + self.copy_upstream_to_thirdparty() + + self.check_patches()