From cd39c5e2a14e368dcf692bdcb8176200ab4d1914 Mon Sep 17 00:00:00 2001 From: Joseph Eng <91924258+KangarooKoala@users.noreply.github.com> Date: Tue, 23 Jul 2024 15:58:15 -0700 Subject: [PATCH] [upstream_utils] Determine patch list from patch files (#6869) --- upstream_utils/README.md | 2 +- upstream_utils/apriltag.py | 13 +---- upstream_utils/eigen.py | 8 +--- upstream_utils/expected.py | 2 +- upstream_utils/fmt.py | 4 +- upstream_utils/gcem.py | 7 +-- upstream_utils/gl3w.py | 4 +- upstream_utils/glfw.py | 7 +-- upstream_utils/googletest.py | 2 +- upstream_utils/imgui.py | 4 +- upstream_utils/implot.py | 4 +- upstream_utils/json.py | 8 +--- upstream_utils/libuv.py | 15 +----- upstream_utils/llvm.py | 41 +--------------- upstream_utils/memory.py | 2 +- upstream_utils/mpack.py | 9 +--- upstream_utils/protobuf.py | 17 +------ upstream_utils/sleipnir.py | 10 +--- upstream_utils/stack_walker.py | 4 -- upstream_utils/stb.py | 4 +- upstream_utils/upstream_utils.py | 81 +++++++++++--------------------- 21 files changed, 46 insertions(+), 202 deletions(-) diff --git a/upstream_utils/README.md b/upstream_utils/README.md index 2e200ae5b5..0ac61e372c 100644 --- a/upstream_utils/README.md +++ b/upstream_utils/README.md @@ -76,7 +76,7 @@ Update the `upstream_utils` patch files. ./.py format-patch ``` -Update the list of patch files in `.py`, then rerun `.py` to reimport the thirdparty files. +Rerun `.py` to reimport the thirdparty files. ```bash ./.py copy-upstream-to-thirdparty ``` diff --git a/upstream_utils/apriltag.py b/upstream_utils/apriltag.py index 3992693ab4..e03be24f6c 100755 --- a/upstream_utils/apriltag.py +++ b/upstream_utils/apriltag.py @@ -71,22 +71,11 @@ def main(): url = "https://github.com/AprilRobotics/apriltag.git" tag = "ebdb2017e04b8e36f7d8a12ce60060416a905e12" - patch_list = [ - "0001-apriltag_pose.c-Set-NULL-when-second-solution-could-.patch", - "0002-zmaxheapify-Avoid-return-of-void-expression.patch", - "0003-Avoid-unused-variable-warnings-in-release-builds.patch", - "0004-Make-orthogonal_iteration-exit-early-upon-convergenc.patch", - "0005-Fix-signed-left-shift-warning.patch", - "0006-Avoid-incompatible-pointer-warning.patch", - "0007-Fix-GCC-14-calloc-warning.patch", - "0008-Remove-calls-to-postscript_image.patch", - "0009-Fix-clang-16-warnings.patch", - ] patch_options = { "ignore_whitespace": True, } - apriltag = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options) + apriltag = Lib(name, url, tag, copy_upstream_src, patch_options) apriltag.main() diff --git a/upstream_utils/eigen.py b/upstream_utils/eigen.py index 577e55f50b..c7b72a81ef 100755 --- a/upstream_utils/eigen.py +++ b/upstream_utils/eigen.py @@ -131,13 +131,7 @@ def main(): url = "https://gitlab.com/libeigen/eigen.git" tag = "c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198" - patch_list = [ - "0001-Disable-warnings.patch", - "0002-Intellisense-fix.patch", - "0003-Suppress-has_denorm-and-has_denorm_loss-deprecation-.patch", - ] - - eigen = Lib(name, url, tag, patch_list, copy_upstream_src) + eigen = Lib(name, url, tag, copy_upstream_src) eigen.main() diff --git a/upstream_utils/expected.py b/upstream_utils/expected.py index ed2b9d24eb..bec2a366e6 100755 --- a/upstream_utils/expected.py +++ b/upstream_utils/expected.py @@ -31,7 +31,7 @@ def main(): # master on 2024-01-25 tag = "3f0ca7b19253129700a073abfa6d8638d9f7c80c" - expected = Lib(name, url, tag, [], copy_upstream_src) + expected = Lib(name, url, tag, copy_upstream_src) expected.main() diff --git a/upstream_utils/fmt.py b/upstream_utils/fmt.py index 5c2f5fe787..9380eb1c22 100755 --- a/upstream_utils/fmt.py +++ b/upstream_utils/fmt.py @@ -37,9 +37,7 @@ def main(): url = "https://github.com/fmtlib/fmt" tag = "11.0.1" - patch_list = ["0001-Suppress-warnings-we-can-t-fix.patch"] - - fmt = Lib(name, url, tag, patch_list, copy_upstream_src) + fmt = Lib(name, url, tag, copy_upstream_src) fmt.main() diff --git a/upstream_utils/gcem.py b/upstream_utils/gcem.py index 61dbbe87e3..25fd7d9e95 100755 --- a/upstream_utils/gcem.py +++ b/upstream_utils/gcem.py @@ -30,12 +30,7 @@ def main(): url = "https://github.com/kthohr/gcem.git" tag = "v1.18.0" - patch_list = [ - "0001-Call-std-functions-if-not-constant-evaluated.patch", - "0002-Add-hypot-x-y-z.patch", - ] - - gcem = Lib(name, url, tag, patch_list, copy_upstream_src) + gcem = Lib(name, url, tag, copy_upstream_src) gcem.main() diff --git a/upstream_utils/gl3w.py b/upstream_utils/gl3w.py index c075971491..8855fd9d4c 100755 --- a/upstream_utils/gl3w.py +++ b/upstream_utils/gl3w.py @@ -22,9 +22,7 @@ def main(): url = "https://github.com/skaslev/gl3w" tag = "5f8d7fd191ba22ff2b60c1106d7135bb9a335533" - patch_list = [] - - gl3w = Lib(name, url, tag, patch_list, copy_upstream_src) + gl3w = Lib(name, url, tag, copy_upstream_src) gl3w.main() diff --git a/upstream_utils/glfw.py b/upstream_utils/glfw.py index caf9c14bd7..944b33e2bc 100755 --- a/upstream_utils/glfw.py +++ b/upstream_utils/glfw.py @@ -71,12 +71,7 @@ def main(): url = "https://github.com/glfw/glfw.git" tag = "6b57e08bb0078c9834889eab871bac2368198c15" - patch_list = [ - "0001-Suppress-Compiler-Warnings.patch", - "0002-Disable-docs-build-by-default.patch", - ] - - glfw = Lib(name, url, tag, patch_list, copy_upstream_src) + glfw = Lib(name, url, tag, copy_upstream_src) glfw.main() diff --git a/upstream_utils/googletest.py b/upstream_utils/googletest.py index 9706b8b556..b164228328 100755 --- a/upstream_utils/googletest.py +++ b/upstream_utils/googletest.py @@ -69,7 +69,7 @@ def main(): url = "https://github.com/google/googletest.git" tag = "v1.14.0" - googletest = Lib(name, url, tag, [], copy_upstream_src) + googletest = Lib(name, url, tag, copy_upstream_src) googletest.main() diff --git a/upstream_utils/imgui.py b/upstream_utils/imgui.py index 27e1081c2b..d682848071 100755 --- a/upstream_utils/imgui.py +++ b/upstream_utils/imgui.py @@ -66,9 +66,7 @@ def main(): url = "https://github.com/ocornut/imgui.git" tag = "64b1e448d20c9be9275af731c34b4c7bf14a8e95" - patch_list = [] - - imgui = Lib(name, url, tag, patch_list, copy_upstream_src) + imgui = Lib(name, url, tag, copy_upstream_src) imgui.main() diff --git a/upstream_utils/implot.py b/upstream_utils/implot.py index 0d0ddfda21..00bf796e76 100755 --- a/upstream_utils/implot.py +++ b/upstream_utils/implot.py @@ -43,9 +43,7 @@ def main(): url = "https://github.com/epezent/implot.git" tag = "18c72431f8265e2b0b5378a3a73d8a883b2175ff" - patch_list = ["0001-Supress-compiler-warnings.patch"] - - implot = Lib(name, url, tag, patch_list, copy_upstream_src) + implot = Lib(name, url, tag, copy_upstream_src) implot.main() diff --git a/upstream_utils/json.py b/upstream_utils/json.py index 23766cdff1..46b49a3903 100755 --- a/upstream_utils/json.py +++ b/upstream_utils/json.py @@ -60,17 +60,11 @@ def main(): url = "https://github.com/nlohmann/json" tag = "v3.11.3" - patch_list = [ - "0001-Remove-version-from-namespace.patch", - "0002-Make-serializer-public.patch", - "0003-Make-dump_escaped-take-std-string_view.patch", - "0004-Add-llvm-stream-support.patch", - ] patch_options = { "use_threeway": True, } - json = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options) + json = Lib(name, url, tag, copy_upstream_src, patch_options) json.main() diff --git a/upstream_utils/libuv.py b/upstream_utils/libuv.py index 4cb025f632..c82f2bfdf8 100755 --- a/upstream_utils/libuv.py +++ b/upstream_utils/libuv.py @@ -56,20 +56,7 @@ def main(): url = "https://github.com/libuv/libuv" tag = "v1.48.0" - patch_list = [ - "0001-Revert-win-process-write-minidumps-when-sending-SIGQ.patch", - "0002-Fix-missing-casts.patch", - "0003-Fix-warnings.patch", - "0004-Preprocessor-cleanup.patch", - "0005-Cleanup-problematic-language.patch", - "0006-Fix-Win32-warning-suppression-pragma.patch", - "0007-Use-C-atomics.patch", - "0008-Remove-static-from-array-indices.patch", - "0009-Add-pragmas-for-missing-libraries-and-set-_WIN32_WIN.patch", - "0010-Remove-swearing.patch", - ] - - libuv = Lib(name, url, tag, patch_list, copy_upstream_src) + libuv = Lib(name, url, tag, copy_upstream_src) libuv.main() diff --git a/upstream_utils/llvm.py b/upstream_utils/llvm.py index 67bad0032c..c79ea29698 100755 --- a/upstream_utils/llvm.py +++ b/upstream_utils/llvm.py @@ -177,50 +177,11 @@ def main(): url = "https://github.com/llvm/llvm-project" tag = "llvmorg-18.1.8" - patch_list = [ - "0001-Remove-StringRef-ArrayRef-and-Optional.patch", - "0002-Wrap-std-min-max-calls-in-parens-for-Windows-warning.patch", - "0003-Change-unique_function-storage-size.patch", - "0004-Threading-updates.patch", - "0005-ifdef-guard-safety.patch", - "0006-Explicitly-use-std.patch", - "0007-Remove-format_provider.patch", - "0008-Add-compiler-warning-pragmas.patch", - "0009-Remove-unused-functions.patch", - "0010-Detemplatize-SmallVectorBase.patch", - "0011-Add-vectors-to-raw_ostream.patch", - "0012-Extra-collections-features.patch", - "0013-EpochTracker-ABI-macro.patch", - "0014-Delete-numbers-from-MathExtras.patch", - "0015-Add-lerp-and-sgn.patch", - "0016-Fixup-includes.patch", - "0017-Use-std-is_trivially_copy_constructible.patch", - "0018-Windows-support.patch", - "0019-Prefer-fmtlib.patch", - "0020-Prefer-wpi-s-fs.h.patch", - "0021-Remove-unused-functions.patch", - "0022-OS-specific-changes.patch", - "0023-Use-SmallVector-for-UTF-conversion.patch", - "0024-Prefer-to-use-static-pointers-in-raw_ostream.patch", - "0025-constexpr-endian-byte-swap.patch", - "0026-Copy-type-traits-from-STLExtras.h-into-PointerUnion..patch", - "0027-Remove-StringMap-test-for-llvm-sort.patch", - "0028-Unused-variable-in-release-mode.patch", - "0029-Use-C-20-bit-header.patch", - "0030-Remove-DenseMap-GTest-printer-test.patch", - "0031-Replace-deprecated-std-aligned_storage_t.patch", - "0032-raw_ostream-Add-SetNumBytesInBuffer.patch", - "0033-type_traits.h-Add-is_constexpr.patch", - "0034-Add-back-removed-raw_string_ostream-write_impl.patch", - "0035-Remove-auto-conversion-from-raw_ostream.patch", - "0036-Add-SmallVector-erase_if.patch", - "0037-Fix-AlignedCharArrayUnion-for-C-23.patch", - ] patch_options = { "use_threeway": True, } - llvm = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options) + llvm = Lib(name, url, tag, copy_upstream_src, patch_options) llvm.main() diff --git a/upstream_utils/memory.py b/upstream_utils/memory.py index 2aa1d2de9d..85b69dfaf8 100755 --- a/upstream_utils/memory.py +++ b/upstream_utils/memory.py @@ -100,7 +100,7 @@ def main(): url = "https://github.com/foonathan/memory" tag = "v0.7-3" - memory = Lib(name, url, tag, [], copy_upstream_src) + memory = Lib(name, url, tag, copy_upstream_src) memory.main() diff --git a/upstream_utils/mpack.py b/upstream_utils/mpack.py index f49c3a1db1..103c67be9e 100755 --- a/upstream_utils/mpack.py +++ b/upstream_utils/mpack.py @@ -45,14 +45,7 @@ def main(): url = "https://github.com/ludocode/mpack" tag = "v1.1.1" - patch_list = [ - "0001-Don-t-emit-inline-defs.patch", - "0002-Update-amalgamation-script.patch", - "0003-Use-namespace-for-C.patch", - "0004-Group-doxygen-into-MPack-module.patch", - ] - - mpack = Lib(name, url, tag, patch_list, copy_upstream_src) + mpack = Lib(name, url, tag, copy_upstream_src) mpack.main() diff --git a/upstream_utils/protobuf.py b/upstream_utils/protobuf.py index 1b0cba26de..79a9f53f90 100755 --- a/upstream_utils/protobuf.py +++ b/upstream_utils/protobuf.py @@ -284,22 +284,7 @@ def main(): url = "https://github.com/protocolbuffers/protobuf" tag = "v3.21.12" - patch_list = [ - "0001-Fix-sign-compare-warnings.patch", - "0002-Remove-redundant-move.patch", - "0003-Fix-maybe-uninitialized-warnings.patch", - "0004-Fix-coded_stream-WriteRaw.patch", - "0005-Suppress-enum-enum-conversion-warning.patch", - "0006-Fix-noreturn-function-returning.patch", - "0007-Work-around-GCC-12-restrict-warning-compiler-bug.patch", - "0008-Disable-MSVC-switch-warning.patch", - "0009-Disable-unused-function-warning.patch", - "0010-Disable-pedantic-warning.patch", - "0011-Avoid-use-of-sprintf.patch", - "0012-Suppress-stringop-overflow-warning-false-positives.patch", - ] - - protobuf = Lib(name, url, tag, patch_list, copy_upstream_src) + protobuf = Lib(name, url, tag, copy_upstream_src) protobuf.main() diff --git a/upstream_utils/sleipnir.py b/upstream_utils/sleipnir.py index f4cd1b8b2e..2e2d4003cb 100755 --- a/upstream_utils/sleipnir.py +++ b/upstream_utils/sleipnir.py @@ -54,15 +54,7 @@ def main(): # main on 2024-07-09 tag = "b6ffa2d4fdb99cab1bf79491f715a6a9a86633b5" - patch_list = [ - "0001-Remove-using-enum-declarations.patch", - "0002-Use-fmtlib.patch", - "0003-Remove-unsupported-constexpr.patch", - "0004-Use-wpi-SmallVector.patch", - "0005-Suppress-clang-tidy-false-positives.patch", - ] - - sleipnir = Lib(name, url, tag, patch_list, copy_upstream_src) + sleipnir = Lib(name, url, tag, copy_upstream_src) sleipnir.main() diff --git a/upstream_utils/stack_walker.py b/upstream_utils/stack_walker.py index 5b5c2634db..d0499d20c6 100755 --- a/upstream_utils/stack_walker.py +++ b/upstream_utils/stack_walker.py @@ -45,9 +45,6 @@ def main(): url = "https://github.com/JochenKalmbach/StackWalker" tag = "5b0df7a4db8896f6b6dc45d36e383c52577e3c6b" - patch_list = [ - "0001-Add-advapi-pragma.patch", - ] patch_options = { "ignore_whitespace": True, } @@ -56,7 +53,6 @@ def main(): name, url, tag, - patch_list, copy_upstream_src, patch_options, pre_patch_hook=crlf_to_lf, diff --git a/upstream_utils/stb.py b/upstream_utils/stb.py index e7dc0195fb..72a0b8811c 100755 --- a/upstream_utils/stb.py +++ b/upstream_utils/stb.py @@ -44,9 +44,7 @@ def main(): url = "https://github.com/nothings/stb.git" tag = "c9064e317699d2e495f36ba4f9ac037e88ee371a" - patch_list = [] - - stb = Lib(name, url, tag, patch_list, copy_upstream_src) + stb = Lib(name, url, tag, copy_upstream_src) stb.main() diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index 7fc287ba0b..f715d66277 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -224,7 +224,6 @@ class Lib: name, url, tag, - patch_list, copy_upstream_src, patch_options={}, *, @@ -238,7 +237,6 @@ class Lib: url -- The URL of the upstream repository. tag -- The tag in the upstream repository to use. Can be any (e.g., commit hash or tag). - patch_list -- The list of patches in the patch directory to apply. copy_upstream_src -- A callable that takes the path to the wpilib root and copies the files from the clone of the upstream into the appropriate thirdparty directory. Will @@ -255,45 +253,12 @@ class Lib: self.name = name self.url = url self.old_tag = tag - self.patch_list = patch_list self.copy_upstream_src = copy_upstream_src self.patch_options = patch_options self.pre_patch_hook = pre_patch_hook self.pre_patch_commits = pre_patch_commits self.wpilib_root = get_repo_root() - def check_patches(self): - """Checks that the patch list supplied to the constructor matches the - patches in the patch directory. - """ - patch_directory_patches = set() - patch_directory = os.path.join( - self.wpilib_root, f"upstream_utils/{self.name}_patches" - ) - if os.path.exists(patch_directory): - for f in os.listdir(patch_directory): - if f.endswith(".patch"): - patch_directory_patches.add(f) - patches = set(self.patch_list) - patch_directory_only = sorted(patch_directory_patches - patches) - patch_list_only = sorted(patches - patch_directory_patches) - common_patches = sorted(patch_directory_patches & patches) - warning = False - if patch_directory_only: - print( - f"WARNING: The patch directory has patches {patch_directory_only} not in the patch list" - ) - warning = True - if patch_list_only: - print( - f"WARNING: The patch list has patches {patch_list_only} not in the patch directory" - ) - warning = True - if warning and common_patches: - print( - f" Note: The patch directory and the patch list both have patches {common_patches}" - ) - def get_repo_path(self, tempdir=None): """Returns the path to the clone of the upstream repository. @@ -388,6 +353,26 @@ class Lib: subprocess.run(["git", "tag", f"upstream_utils_root-{tag}", tag]) + def get_patch_directory(self): + """Returns the path to the directory containing the patch files. + + Returns: + The absolute path to the directory containing the patch files. + """ + return os.path.join(self.wpilib_root, f"upstream_utils/{self.name}_patches") + + def get_patch_list(self): + """Returns a list of the filenames of the patches to apply. + + Returns: + A list of the filenames of the patches to apply, sorted in lexicographic + order by the Unicode code points.""" + if not os.path.exists(self.get_patch_directory()): + return [] + return sorted( + f for f in os.listdir(self.get_patch_directory()) if f.endswith(".patch") + ) + def apply_patches(self): """Applies the patches listed in the patch list to the current directory. @@ -395,11 +380,9 @@ class Lib: if self.pre_patch_hook is not None: self.pre_patch_hook() - for f in self.patch_list: + for f in self.get_patch_list(): git_am( - os.path.join( - self.wpilib_root, f"upstream_utils/{self.name}_patches", f - ), + os.path.join(self.get_patch_directory(), f), **self.patch_options, ) @@ -433,7 +416,7 @@ class Lib: print(f"Upstream URL: {self.url}") print(f"Upstream tag: {self.old_tag}") print(f"Path to upstream clone: {self.get_repo_path()}") - print(f"Patches to apply: {self.patch_list}") + print(f"Patches to apply: {self.get_patch_list()}") print(f"Patch options: {self.patch_options}") print(f"Pre patch commits: {self.pre_patch_commits}") print(f"WPILib root: {self.wpilib_root}") @@ -518,24 +501,16 @@ class Lib: ] ) - patch_dest = os.path.join( - self.wpilib_root, f"upstream_utils/{self.name}_patches" - ) - - if not os.path.exists(patch_dest): - print( - f"WARNING: Patch directory {patch_dest} does not exist", file=sys.stderr - ) - else: - shutil.rmtree(patch_dest) + if os.path.exists(self.get_patch_directory()): + shutil.rmtree(self.get_patch_directory()) is_first = True for f in os.listdir(): if f.endswith(".patch"): if is_first: - os.mkdir(patch_dest) + os.mkdir(self.get_patch_directory()) is_first = False - shutil.move(f, patch_dest) + shutil.move(f, self.get_patch_directory()) self.replace_tag(script_tag) @@ -608,5 +583,3 @@ class Lib: self.format_patch() elif args.subcommand == "copy-upstream-to-thirdparty": self.copy_upstream_to_thirdparty() - - self.check_patches()