diff --git a/upstream_utils/apriltag.py b/upstream_utils/apriltag.py index ea95919417..1c3d643848 100755 --- a/upstream_utils/apriltag.py +++ b/upstream_utils/apriltag.py @@ -1,9 +1,14 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path -from upstream_utils import Lib, comment_out_invalid_includes, walk_cwd_and_copy_if +from upstream_utils import ( + Lib, + comment_out_invalid_includes, + has_prefix, + walk_cwd_and_copy_if, +) def remove_tag(f: str): @@ -18,47 +23,45 @@ def remove_tag(f: str): return False -def copy_upstream_src(wpilib_root): - apriltag = os.path.join(wpilib_root, "apriltag") +def copy_upstream_src(wpilib_root: Path): + apriltag = wpilib_root / "apriltag" # Delete old install shutil.rmtree( - os.path.join(apriltag, "src/main/native/thirdparty/apriltag"), + apriltag / "src/main/native/thirdparty/apriltag", ignore_errors=True, ) shutil.rmtree( - os.path.join(apriltag, "src/main/include/thirdparty/apriltag"), + apriltag / "src/main/include/thirdparty/apriltag", ignore_errors=True, ) # Copy apriltag source files into allwpilib src_files = walk_cwd_and_copy_if( lambda dp, f: (f.endswith(".c") or f.endswith(".cpp")) - and not dp.startswith(os.path.join(".", "example")) - and not dp.startswith(os.path.join(".", "test")) - and not f.endswith("getopt.c") + and not has_prefix(dp, Path("example")) + and not has_prefix(dp, Path("test")) + and not f == "getopt.c" and not "py" in f and not remove_tag(f), - os.path.join(apriltag, "src/main/native/thirdparty/apriltag/src"), + apriltag / "src/main/native/thirdparty/apriltag/src", ) # Copy apriltag header files into allwpilib walk_cwd_and_copy_if( lambda dp, f: f.endswith(".h") - and not f.endswith("getopt.h") - and not f.endswith("postscript_utils.h") + and not f == "getopt.h" + and not f == "postscript_utils.h" and not remove_tag(f), - os.path.join(apriltag, "src/main/native/thirdparty/apriltag/include"), + apriltag / "src/main/native/thirdparty/apriltag/include", ) for f in src_files: comment_out_invalid_includes( f, [ - os.path.join(apriltag, "src/main/native/thirdparty/apriltag/include"), - os.path.join( - apriltag, "src/main/native/thirdparty/apriltag/include/common" - ), + apriltag / "src/main/native/thirdparty/apriltag/include", + apriltag / "src/main/native/thirdparty/apriltag/include/common", ], ) diff --git a/upstream_utils/argparse_lib.py b/upstream_utils/argparse_lib.py index 644193701c..2dd3363d0a 100755 --- a/upstream_utils/argparse_lib.py +++ b/upstream_utils/argparse_lib.py @@ -1,18 +1,17 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path from upstream_utils import Lib -def copy_upstream_src(wpilib_root): - wpiutil = os.path.join(wpilib_root, "wpiutil") +def copy_upstream_src(wpilib_root: Path): + wpiutil = wpilib_root / "wpiutil" # Copy header into allwpilib - dest_filename = os.path.join( - wpiutil, - f"src/main/native/thirdparty/argparse/include/wpi/argparse.h", + dest_filename = ( + wpiutil / f"src/main/native/thirdparty/argparse/include/wpi/argparse.h" ) shutil.copyfile("include/argparse/argparse.hpp", dest_filename) # Rename namespace from argparse to wpi diff --git a/upstream_utils/debugging.py b/upstream_utils/debugging.py index 117cf90048..a56348682c 100755 --- a/upstream_utils/debugging.py +++ b/upstream_utils/debugging.py @@ -1,26 +1,25 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path -from upstream_utils import Lib, walk_cwd_and_copy_if +from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if -def copy_upstream_src(wpilib_root): - wpiutil = os.path.join(wpilib_root, "wpiutil") +def copy_upstream_src(wpilib_root: Path): + wpiutil = wpilib_root / "wpiutil" # Delete old install for d in [ "src/main/native/thirdparty/debugging/src", "src/main/native/thirdparty/debugging/include", ]: - shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) + shutil.rmtree(wpiutil / d, ignore_errors=True) # Copy debugging files into allwpilib filenames = walk_cwd_and_copy_if( - lambda dp, f: dp.startswith(os.path.join(".", "src")) - or dp.startswith(os.path.join(".", "include")), - os.path.join(wpiutil, "src/main/native/thirdparty/debugging"), + lambda dp, f: has_prefix(dp, Path("src")) or has_prefix(dp, Path("include")), + wpiutil / "src/main/native/thirdparty/debugging", ) for filename in filenames: diff --git a/upstream_utils/eigen.py b/upstream_utils/eigen.py index 58879b60a0..93f68f79d5 100755 --- a/upstream_utils/eigen.py +++ b/upstream_utils/eigen.py @@ -1,13 +1,17 @@ #!/usr/bin/env python3 -import os -import re import shutil +from pathlib import Path -from upstream_utils import Lib, comment_out_invalid_includes, walk_cwd_and_copy_if +from upstream_utils import ( + Lib, + comment_out_invalid_includes, + has_prefix, + walk_cwd_and_copy_if, +) -def eigen_inclusions(dp, f): +def eigen_inclusions(dp: Path, f: str): """Returns true if the given file in the "Eigen" include directory of the Eigen git repo should be copied into allwpilib @@ -15,11 +19,9 @@ def eigen_inclusions(dp, f): dp -- directory path f -- filename """ - if not dp.startswith(os.path.join(".", "Eigen")): + if not has_prefix(dp, Path("Eigen")): return False - abspath = os.path.join(dp, f) - # Exclude NonMPL2.h since all non-MPL2 code will be excluded anyway if f == "NonMPL2.h": return False @@ -36,13 +38,13 @@ def eigen_inclusions(dp, f): if "MKL" in f: return False - # Include architectures we care about - if "Core/arch/" in abspath: + # Include architectures we care about by filtering for Core/arch + if "Core" in dp.parts and "arch" in dp.parts: return ( - "arch/AVX/" in abspath - or "arch/Default" in abspath - or "arch/NEON" in abspath - or "arch/SSE" in abspath + "AVX" in dp.parts + or "Default" in dp.parts + or "NEON" in dp.parts + or "SSE" in dp.parts ) # Include the following modules @@ -65,10 +67,13 @@ def eigen_inclusions(dp, f): "misc", "plugins", ] - return bool(re.search(r"|".join("/" + m for m in modules), abspath)) + for m in modules: + if m in dp.parts or f == m: + return True + return False -def unsupported_inclusions(dp, f): +def unsupported_inclusions(dp: Path, f: str): """Returns true if the given file in the "unsupported" include directory of the Eigen git repo should be copied into allwpilib @@ -76,50 +81,50 @@ def unsupported_inclusions(dp, f): dp -- directory path f -- filename """ - if not dp.startswith(os.path.join(".", "unsupported")): + if not has_prefix(dp, Path("unsupported")): return False - abspath = os.path.join(dp, f) + abspath = dp / f # Exclude build system and READMEs if f == "CMakeLists.txt" or "README" in f: return False # Include the MatrixFunctions module - return "MatrixFunctions" in abspath + return "MatrixFunctions" in abspath.parts -def copy_upstream_src(wpilib_root): - wpimath = os.path.join(wpilib_root, "wpimath") +def copy_upstream_src(wpilib_root: Path): + wpimath = wpilib_root / "wpimath" # Delete old install for d in ["src/main/native/thirdparty/eigen/include"]: - shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) + shutil.rmtree(wpimath / d, ignore_errors=True) # Copy Eigen headers into allwpilib eigen_files = walk_cwd_and_copy_if( eigen_inclusions, - os.path.join(wpimath, "src/main/native/thirdparty/eigen/include"), + wpimath / "src/main/native/thirdparty/eigen/include", ) # Copy unsupported headers into allwpilib unsupported_files = walk_cwd_and_copy_if( unsupported_inclusions, - os.path.join(wpimath, "src/main/native/thirdparty/eigen/include"), + wpimath / "src/main/native/thirdparty/eigen/include", ) for f in eigen_files: comment_out_invalid_includes( - f, [os.path.join(wpimath, "src/main/native/thirdparty/eigen/include")] + f, [wpimath / "src/main/native/thirdparty/eigen/include"] ) for f in unsupported_files: comment_out_invalid_includes( - f, [os.path.join(wpimath, "src/main/native/thirdparty/eigen/include")] + f, [wpimath / "src/main/native/thirdparty/eigen/include"] ) shutil.copyfile( ".clang-format", - os.path.join(wpimath, "src/main/native/thirdparty/eigen/include/.clang-format"), + wpimath / "src/main/native/thirdparty/eigen/include/.clang-format", ) diff --git a/upstream_utils/expected.py b/upstream_utils/expected.py index cfa323c44c..68d27f9ae6 100755 --- a/upstream_utils/expected.py +++ b/upstream_utils/expected.py @@ -1,18 +1,16 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path from upstream_utils import Lib -def copy_upstream_src(wpilib_root): - wpiutil = os.path.join(wpilib_root, "wpiutil") +def copy_upstream_src(wpilib_root: Path): + wpiutil = wpilib_root / "wpiutil" # Copy expected header into allwpilib - dest_filename = os.path.join( - wpiutil, "src/main/native/thirdparty/expected/include/wpi/expected" - ) + dest_filename = wpiutil / "src/main/native/thirdparty/expected/include/wpi/expected" shutil.copyfile("include/tl/expected.hpp", dest_filename) # Rename namespace from tl to wpi, and detail to detail_expected diff --git a/upstream_utils/fmt.py b/upstream_utils/fmt.py index 0345b401c9..ed515e5b82 100755 --- a/upstream_utils/fmt.py +++ b/upstream_utils/fmt.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path -from upstream_utils import Lib, walk_cwd_and_copy_if +from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if -def copy_upstream_src(wpilib_root): - wpiutil = os.path.join(wpilib_root, "wpiutil") +def copy_upstream_src(wpilib_root: Path): + wpiutil = wpilib_root / "wpiutil" # Delete old install for d in [ "src/main/native/thirdparty/fmtlib/src", "src/main/native/thirdparty/fmtlib/include", ]: - shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) + shutil.rmtree(wpiutil / d, ignore_errors=True) # Copy fmt source files into allwpilib walk_cwd_and_copy_if( - lambda dp, f: dp.startswith(os.path.join(".", "src")) + lambda dp, f: has_prefix(dp, Path("src")) and f.endswith(".cc") and f != "fmt.cc", - os.path.join(wpiutil, "src/main/native/thirdparty/fmtlib"), + wpiutil / "src/main/native/thirdparty/fmtlib", ) # Copy fmt header files into allwpilib walk_cwd_and_copy_if( - lambda dp, f: dp.startswith(os.path.join(".", "include", "fmt")), - os.path.join(wpiutil, "src/main/native/thirdparty/fmtlib"), + lambda dp, f: has_prefix(dp, Path("include/fmt")), + wpiutil / "src/main/native/thirdparty/fmtlib", ) diff --git a/upstream_utils/gcem.py b/upstream_utils/gcem.py index 1b4f2d1e44..0b9dc4fab6 100755 --- a/upstream_utils/gcem.py +++ b/upstream_utils/gcem.py @@ -1,24 +1,24 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path -from upstream_utils import Lib, walk_cwd_and_copy_if +from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if -def copy_upstream_src(wpilib_root): - wpimath = os.path.join(wpilib_root, "wpimath") +def copy_upstream_src(wpilib_root: Path): + wpimath = wpilib_root / "wpimath" # Delete old install for d in [ "src/main/native/thirdparty/gcem/include", ]: - shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) + shutil.rmtree(wpimath / d, ignore_errors=True) # Copy gcem include files into allwpilib walk_cwd_and_copy_if( - lambda dp, f: dp.startswith(os.path.join(".", "include")), - os.path.join(wpimath, "src/main/native/thirdparty/gcem"), + lambda dp, f: has_prefix(dp, Path("include")), + wpimath / "src/main/native/thirdparty/gcem", ) diff --git a/upstream_utils/gl3w.py b/upstream_utils/gl3w.py index 1858d675a6..9b9065877a 100755 --- a/upstream_utils/gl3w.py +++ b/upstream_utils/gl3w.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 -import os +from pathlib import Path from upstream_utils import Lib, walk_cwd_and_copy_if -def copy_upstream_src(wpilib_root): - gl3w = os.path.join(wpilib_root, "thirdparty", "imgui_suite", "gl3w") +def copy_upstream_src(wpilib_root: Path): + gl3w = wpilib_root / "thirdparty/imgui_suite/gl3w" walk_cwd_and_copy_if( lambda dp, f: f == "gl3w_gen.py", - os.path.join(gl3w), + gl3w, ) diff --git a/upstream_utils/glfw.py b/upstream_utils/glfw.py index a2d7e84540..30cba25bf8 100755 --- a/upstream_utils/glfw.py +++ b/upstream_utils/glfw.py @@ -1,57 +1,47 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path -from upstream_utils import Lib, walk_cwd_and_copy_if +from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if -def matches(dp, f, allowed_files): - path = os.path.join(dp, f) - return path in allowed_files - - -def walk_and_copy_if_matches(allowed_files, output_directory): - walk_cwd_and_copy_if( - lambda dp, f: matches(dp, f, allowed_files), - output_directory, - ) - - -def copy_upstream_src(wpilib_root): - glfw = os.path.join(wpilib_root, "thirdparty", "imgui_suite", "glfw") +def copy_upstream_src(wpilib_root: Path): + glfw = wpilib_root / "thirdparty/imgui_suite/glfw" # Delete old install for d in ["include", "src", "CMake"]: - shutil.rmtree(os.path.join(glfw, d), ignore_errors=True) + shutil.rmtree(glfw / d, ignore_errors=True) hdr_allow_list = [ - "./include/GLFW/glfw3.h", - "./include/GLFW/glfw3native.h", + Path("include/GLFW/glfw3.h"), + Path("include/GLFW/glfw3native.h"), ] - walk_and_copy_if_matches(hdr_allow_list, glfw) + walk_cwd_and_copy_if( + lambda dp, f: dp / f in hdr_allow_list, + glfw, + ) - def src_filter(dp, f): - if f.endswith("CMakeLists.txt"): + def src_filter(dp: Path, f: str): + if f == "CMakeLists.txt": return False - if dp.startswith(os.path.join(".", "src")): + if has_prefix(dp, Path("src")): return True return False src_files = walk_cwd_and_copy_if( src_filter, - os.path.join(glfw), + glfw, ) - def cmake_filter(dp, f): - if dp.startswith(os.path.join(".", "CMake")): + def cmake_filter(dp: Path, f: str): + if has_prefix(dp, Path("CMake")): return True - path = os.path.join(dp, f) - if path in ["./src/CMakeLists.txt", "./CMakeLists.txt"]: + if dp / f in [Path("src/CMakeLists.txt"), Path("CMakeLists.txt")]: return True return False @@ -59,7 +49,7 @@ def copy_upstream_src(wpilib_root): # Copy CMAKE files walk_cwd_and_copy_if( cmake_filter, - os.path.join(glfw), + glfw, ) diff --git a/upstream_utils/googletest.py b/upstream_utils/googletest.py index 60613ee535..a58583b17b 100755 --- a/upstream_utils/googletest.py +++ b/upstream_utils/googletest.py @@ -2,8 +2,9 @@ import os import shutil +from pathlib import Path -from upstream_utils import Lib, walk_cwd_and_copy_if, walk_if +from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if EXCLUDED_FILES = [ "gtest_main.cc", @@ -14,49 +15,39 @@ EXCLUDED_FILES = [ ] -def walk_and_remap_copy(third_party_root, include_prefix): - gmock_files = walk_if( - ".", lambda dp, f: include_prefix in dp and f not in EXCLUDED_FILES - ) - - for f in gmock_files: - dst_file = os.path.join( - third_party_root, "include", f[len(include_prefix) + 1 :] - ) - dest_dir = os.path.dirname(dst_file) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - shutil.copyfile(f, dst_file) - - -def copy_upstream_src(wpilib_root): - third_party_root = os.path.join(wpilib_root, "thirdparty/googletest") +def copy_upstream_src(wpilib_root: Path): + upstream_root = Path(".").absolute() + third_party_root = wpilib_root / "thirdparty/googletest" # Delete old install for d in [ "include", "src", ]: - shutil.rmtree(os.path.join(third_party_root, d), ignore_errors=True) + shutil.rmtree(third_party_root / d, ignore_errors=True) walk_cwd_and_copy_if( - lambda dp, f: "googlemock/src" in dp and f not in EXCLUDED_FILES, - os.path.join(third_party_root, "src"), + lambda dp, f: has_prefix(dp, Path("googlemock/src")) + and f not in EXCLUDED_FILES, + third_party_root / "src", ) walk_cwd_and_copy_if( - lambda dp, f: "googletest/src" in dp and f not in EXCLUDED_FILES, - os.path.join(third_party_root, "src"), + lambda dp, f: has_prefix(dp, Path("googletest/src")) + and f not in EXCLUDED_FILES, + third_party_root / "src", ) - walk_and_remap_copy( - third_party_root, - "./googlemock/include", + os.chdir(upstream_root / "googlemock/include") + walk_cwd_and_copy_if( + lambda dp, f: f not in EXCLUDED_FILES, + third_party_root / "include", ) - walk_and_remap_copy( - third_party_root, - "./googletest/include", + os.chdir(upstream_root / "googletest/include") + walk_cwd_and_copy_if( + lambda dp, f: f not in EXCLUDED_FILES, + third_party_root / "include", ) diff --git a/upstream_utils/imgui.py b/upstream_utils/imgui.py index 76dce5e94d..0fd28ef8b7 100755 --- a/upstream_utils/imgui.py +++ b/upstream_utils/imgui.py @@ -1,62 +1,57 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path from upstream_utils import Lib, walk_cwd_and_copy_if -def matches(dp, f, allowed_files): - path = os.path.join(dp, f) - return path in allowed_files - - -def walk_and_copy_if_matches(allowed_files, output_directory): +def walk_and_copy_if_matches(allowed_files: list[Path], output_directory: Path): walk_cwd_and_copy_if( - lambda dp, f: matches(dp, f, allowed_files), + lambda dp, f: dp / f in allowed_files, output_directory, ) -def copy_upstream_src(wpilib_root): - imgui = os.path.join(wpilib_root, "thirdparty", "imgui_suite", "imgui") +def copy_upstream_src(wpilib_root: Path): + imgui = wpilib_root / "thirdparty/imgui_suite/imgui" # Delete old install for d in ["include", "cpp"]: - shutil.rmtree(os.path.join(imgui, d), ignore_errors=True) + shutil.rmtree(imgui / d, ignore_errors=True) hdr_allow_list = [ - "./imgui.h", - "./imstb_truetype.h", - "./imgui_internal.h", - "./imstb_rectpack.h", - "./imconfig.h", - "./imstb_textedit.h", - "./backends/imgui_impl_glfw.h", - "./backends/imgui_impl_metal.h", - "./backends/imgui_impl_opengl3.h", - "./backends/imgui_impl_dx11.h", - "./backends/imgui_impl_opengl3_loader.h", - "./backends/imgui_impl_opengl2.h", - "./misc/cpp/imgui_stdlib.h", + Path("imgui.h"), + Path("imstb_truetype.h"), + Path("imgui_internal.h"), + Path("imstb_rectpack.h"), + Path("imconfig.h"), + Path("imstb_textedit.h"), + Path("backends/imgui_impl_glfw.h"), + Path("backends/imgui_impl_metal.h"), + Path("backends/imgui_impl_opengl3.h"), + Path("backends/imgui_impl_dx11.h"), + Path("backends/imgui_impl_opengl3_loader.h"), + Path("backends/imgui_impl_opengl2.h"), + Path("misc/cpp/imgui_stdlib.h"), ] src_allow_list = [ - "./backends/imgui_impl_dx11.cpp", - "./backends/imgui_impl_glfw.cpp", - "./backends/imgui_impl_metal.mm", - "./backends/imgui_impl_opengl2.cpp", - "./backends/imgui_impl_opengl3.cpp", - "./imgui.cpp", - "./imgui_demo.cpp", - "./imgui_draw.cpp", - "./imgui_tables.cpp", - "./imgui_widgets.cpp", - "./misc/cpp/imgui_stdlib.cpp", + Path("backends/imgui_impl_dx11.cpp"), + Path("backends/imgui_impl_glfw.cpp"), + Path("backends/imgui_impl_metal.mm"), + Path("backends/imgui_impl_opengl2.cpp"), + Path("backends/imgui_impl_opengl3.cpp"), + Path("imgui.cpp"), + Path("imgui_demo.cpp"), + Path("imgui_draw.cpp"), + Path("imgui_tables.cpp"), + Path("imgui_widgets.cpp"), + Path("misc/cpp/imgui_stdlib.cpp"), ] - walk_and_copy_if_matches(hdr_allow_list, os.path.join(imgui, "include")) - walk_and_copy_if_matches(src_allow_list, os.path.join(imgui, "cpp")) + walk_and_copy_if_matches(hdr_allow_list, imgui / "include") + walk_and_copy_if_matches(src_allow_list, imgui / "cpp") def main(): diff --git a/upstream_utils/implot.py b/upstream_utils/implot.py index 73101e84d9..c43d3dfef5 100755 --- a/upstream_utils/implot.py +++ b/upstream_utils/implot.py @@ -1,37 +1,27 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path from upstream_utils import Lib, walk_cwd_and_copy_if -def matches(dp, f, allowed_files): - path = os.path.join(dp, f) - return path in allowed_files - - -def walk_and_copy_if_matches(allowed_files, output_directory): - walk_cwd_and_copy_if( - lambda dp, f: matches(dp, f, allowed_files), - output_directory, - ) - - -def copy_upstream_src(wpilib_root): - implot = os.path.join(wpilib_root, "thirdparty", "imgui_suite", "implot") +def copy_upstream_src(wpilib_root: Path): + implot = wpilib_root / "thirdparty/imgui_suite/implot" # Delete old install for d in ["include", "cpp"]: - shutil.rmtree(os.path.join(implot, d), ignore_errors=True) + shutil.rmtree(implot / d, ignore_errors=True) # Copy files walk_cwd_and_copy_if( lambda dp, f: f.endswith(".h"), - os.path.join(implot, "include"), + implot / "include", ) - walk_and_copy_if_matches( - ["./implot_items.cpp", "./implot.cpp"], os.path.join(implot, "cpp") + implot_files = [Path("implot_items.cpp"), Path("implot.cpp")] + walk_cwd_and_copy_if( + lambda dp, f: dp / f in implot_files, + implot / "cpp", ) diff --git a/upstream_utils/json.py b/upstream_utils/json.py index 050e4dd66e..2b83d44486 100755 --- a/upstream_utils/json.py +++ b/upstream_utils/json.py @@ -2,35 +2,32 @@ import os import shutil +from pathlib import Path from upstream_utils import Lib, walk_if -def copy_upstream_src(wpilib_root): - wpiutil = os.path.join(wpilib_root, "wpiutil") +def copy_upstream_src(wpilib_root: Path): + wpiutil = wpilib_root / "wpiutil" # Delete old install for d in [ "src/main/native/thirdparty/json/include", ]: - shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) + shutil.rmtree(wpiutil / d, ignore_errors=True) # Create lists of source and destination files os.chdir("include/nlohmann") - files = walk_if(".", lambda dp, f: True) - src_include_files = [os.path.abspath(f) for f in files] - wpiutil_json_root = os.path.join( - wpiutil, "src/main/native/thirdparty/json/include/wpi" - ) - dest_include_files = [ - os.path.join(wpiutil_json_root, f.replace(".hpp", ".h")) for f in files - ] + files = walk_if(Path("."), lambda dp, f: True) + src_include_files = [f.absolute() for f in files] + wpiutil_json_root = wpiutil / "src/main/native/thirdparty/json/include/wpi" + dest_include_files = [wpiutil_json_root / f.with_suffix(".h") for f in files] # Copy json header files into allwpilib for i in range(len(src_include_files)): - dest_dir = os.path.dirname(dest_include_files[i]) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) + dest_dir = dest_include_files[i].parent + if not dest_dir.exists(): + dest_dir.mkdir(parents=True) shutil.copyfile(src_include_files[i], dest_include_files[i]) for include_file in dest_include_files: diff --git a/upstream_utils/libdogleg.py b/upstream_utils/libdogleg.py index 68ab38326a..590df443fa 100755 --- a/upstream_utils/libdogleg.py +++ b/upstream_utils/libdogleg.py @@ -1,24 +1,24 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path from upstream_utils import Lib, walk_cwd_and_copy_if -def copy_upstream_src(wpilib_root): - wpical = os.path.join(wpilib_root, "wpical") +def copy_upstream_src(wpilib_root: Path): + wpical = wpilib_root / "wpical" # Delete old install for d in [ "src/main/native/thirdparty/libdogleg/src", "src/main/native/thirdparty/libdogleg/include", ]: - shutil.rmtree(os.path.join(wpical, d), ignore_errors=True) + shutil.rmtree(wpical / d, ignore_errors=True) files = walk_cwd_and_copy_if( - lambda dp, f: f.endswith("dogleg.h"), - os.path.join(wpical, "src/main/native/thirdparty/libdogleg/include"), + lambda dp, f: f == "dogleg.h", + wpical / "src/main/native/thirdparty/libdogleg/include", ) for f in files: with open(f) as file: @@ -30,8 +30,8 @@ def copy_upstream_src(wpilib_root): file.write(content) files = walk_cwd_and_copy_if( - lambda dp, f: f.endswith("dogleg.cpp"), - os.path.join(wpical, "src/main/native/thirdparty/libdogleg/src"), + lambda dp, f: f == "dogleg.cpp", + wpical / "src/main/native/thirdparty/libdogleg/src", ) for f in files: with open(f) as file: diff --git a/upstream_utils/libuv.py b/upstream_utils/libuv.py index c837fe96be..469a996833 100755 --- a/upstream_utils/libuv.py +++ b/upstream_utils/libuv.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path -from upstream_utils import Lib, walk_cwd_and_copy_if +from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if -def copy_upstream_src(wpilib_root): - wpinet = os.path.join(wpilib_root, "wpinet") +def copy_upstream_src(wpilib_root: Path): + wpinet = wpilib_root / "wpinet" # Delete old install for d in ["src/main/native/thirdparty/libuv"]: - shutil.rmtree(os.path.join(wpinet, d), ignore_errors=True) + shutil.rmtree(wpinet / d, ignore_errors=True) include_ignorelist = [ "aix.h", @@ -21,9 +21,8 @@ def copy_upstream_src(wpilib_root): ] walk_cwd_and_copy_if( - lambda dp, f: dp.startswith(os.path.join(".", "include")) - and f not in include_ignorelist, - os.path.join(wpinet, "src/main/native/thirdparty/libuv"), + lambda dp, f: has_prefix(dp, Path("include")) and f not in include_ignorelist, + wpinet / "src/main/native/thirdparty/libuv", ) src_ignorelist = [ @@ -43,9 +42,8 @@ def copy_upstream_src(wpilib_root): "sysinfo-memory.c", ] walk_cwd_and_copy_if( - lambda dp, f: dp.startswith(os.path.join(".", "src")) - and f not in src_ignorelist, - os.path.join(wpinet, "src/main/native/thirdparty/libuv"), + lambda dp, f: has_prefix(dp, Path("src")) and f not in src_ignorelist, + wpinet / "src/main/native/thirdparty/libuv", rename_c_to_cpp=True, ) diff --git a/upstream_utils/llvm.py b/upstream_utils/llvm.py index 0098c1be45..a3c198606f 100755 --- a/upstream_utils/llvm.py +++ b/upstream_utils/llvm.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path from upstream_utils import Lib -def run_global_replacements(wpiutil_llvm_files): +def run_global_replacements(wpiutil_llvm_files: list[Path]): for wpi_file in wpiutil_llvm_files: with open(wpi_file) as f: content = f.read() @@ -23,7 +23,7 @@ def run_global_replacements(wpiutil_llvm_files): # Fix uses of span content = content.replace("span", "std::span") content = content.replace("include ", "include ") - if wpi_file.endswith("ConvertUTFWrapper.cpp"): + if wpi_file.name == "ConvertUTFWrapper.cpp": content = content.replace( "const UTF16 *Src = reinterpret_cast(SrcBytes.begin());", "const UTF16 *Src = reinterpret_cast(&*SrcBytes.begin());", @@ -88,19 +88,19 @@ def run_global_replacements(wpiutil_llvm_files): f.write(content) -def flattened_llvm_files(llvm, dirs_to_keep): - file_lookup = {} +def flattened_llvm_files(llvm: Path, dirs_to_keep: list[Path]): + file_lookup: dict[str, Path] = {} for dir_to_keep in dirs_to_keep: - dir_to_crawl = os.path.join(llvm, dir_to_keep) - for root, _, files in os.walk(dir_to_crawl): + dir_to_crawl = llvm / dir_to_keep + for root, _, files in dir_to_crawl.walk(): for f in files: - file_lookup[f] = os.path.join(root, f) + file_lookup[f] = root / f return file_lookup -def find_wpiutil_llvm_files(wpiutil_root, subfolder): +def find_wpiutil_llvm_files(wpiutil_root: Path, subfolder: str): # These files have substantial changes, not worth managing with the patching process ignore_list = [ "StringExtras.h", @@ -110,22 +110,22 @@ def find_wpiutil_llvm_files(wpiutil_root, subfolder): "SmallVectorMemoryBuffer.h", ] - wpiutil_files = [] - for root, _, files in os.walk(os.path.join(wpiutil_root, subfolder)): + wpiutil_files: list[Path] = [] + for root, _, files in (wpiutil_root / subfolder).walk(): for f in files: if f not in ignore_list: - full_file = os.path.join(root, f) + full_file = root / f wpiutil_files.append(full_file) return wpiutil_files -def overwrite_files(wpiutil_files, llvm_files): +def overwrite_files(wpiutil_files: list[Path], llvm_files: dict[str, Path]): # Very sparse rips from LLVM sources. Not worth tyring to make match upstream unmatched_files_whitelist = ["fs.h", "fs.cpp", "function_ref.h"] for wpi_file in wpiutil_files: - wpi_base_name = os.path.basename(wpi_file) + wpi_base_name = wpi_file.name if wpi_base_name in llvm_files: shutil.copyfile(llvm_files[wpi_base_name], wpi_file) @@ -133,14 +133,14 @@ def overwrite_files(wpiutil_files, llvm_files): print(f"No file match for {wpi_file}, check if LLVM deleted it") -def overwrite_source(wpiutil_root, llvm_root): +def overwrite_source(wpiutil_root: Path, llvm_root: Path): llvm_files = flattened_llvm_files( llvm_root, [ - "llvm/include/llvm/ADT/", - "llvm/include/llvm/Config", - "llvm/include/llvm/Support/", - "llvm/lib/Support/", + Path("llvm/include/llvm/ADT/"), + Path("llvm/include/llvm/Config/"), + Path("llvm/include/llvm/Support/"), + Path("llvm/lib/Support/"), ], ) wpi_files = find_wpiutil_llvm_files( @@ -153,10 +153,14 @@ def overwrite_source(wpiutil_root, llvm_root): run_global_replacements(wpi_files) -def overwrite_tests(wpiutil_root, llvm_root): +def overwrite_tests(wpiutil_root: Path, llvm_root: Path): llvm_files = flattened_llvm_files( llvm_root, - ["llvm/unittests/ADT/", "llvm/unittests/Config", "llvm/unittests/Support/"], + [ + Path("llvm/unittests/ADT/"), + Path("llvm/unittests/Config/"), + Path("llvm/unittests/Support/"), + ], ) wpi_files = find_wpiutil_llvm_files(wpiutil_root, "src/test/native/cpp/llvm") @@ -164,9 +168,9 @@ def overwrite_tests(wpiutil_root, llvm_root): run_global_replacements(wpi_files) -def copy_upstream_src(wpilib_root): - upstream_root = os.path.abspath(".") - wpiutil = os.path.join(wpilib_root, "wpiutil") +def copy_upstream_src(wpilib_root: Path): + upstream_root = Path(".").absolute() + wpiutil = wpilib_root / "wpiutil" overwrite_source(wpiutil, upstream_root) overwrite_tests(wpiutil, upstream_root) diff --git a/upstream_utils/memory.py b/upstream_utils/memory.py index 0099de5ea9..99ecb11a89 100755 --- a/upstream_utils/memory.py +++ b/upstream_utils/memory.py @@ -2,11 +2,12 @@ import os import shutil +from pathlib import Path -from upstream_utils import Lib, copy_to, walk_if +from upstream_utils import Lib, walk_cwd_and_copy_if -def run_source_replacements(memory_files): +def run_source_replacements(memory_files: list[Path]): for wpi_file in memory_files: with open(wpi_file) as f: content = f.read() @@ -21,9 +22,9 @@ def run_source_replacements(memory_files): f.write(content) -def run_header_replacements(memory_files): +def run_header_replacements(memory_files: list[Path]): for wpi_file in memory_files: - if "detail" not in wpi_file: + if "detail" not in wpi_file.parts: continue with open(wpi_file) as f: content = f.read() @@ -35,7 +36,7 @@ def run_header_replacements(memory_files): f.write(content) -def run_global_replacements(memory_files): +def run_global_replacements(memory_files: list[Path]): for wpi_file in memory_files: with open(wpi_file) as f: content = f.read() @@ -52,42 +53,41 @@ def run_global_replacements(memory_files): f.write(content) -def copy_upstream_src(wpilib_root): - wpiutil = os.path.join(wpilib_root, "wpiutil") +def copy_upstream_src(wpilib_root: Path): + upstream_root = Path(".").absolute() + wpiutil = wpilib_root / "wpiutil" # Delete old install for d in [ "src/main/native/thirdparty/memory/src", "src/main/native/thirdparty/memory/include", ]: - shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) + shutil.rmtree(wpiutil / d, ignore_errors=True) # Copy sources - src_files = walk_if("src", lambda dp, f: f.endswith(".cpp") or f.endswith(".hpp")) - src_files = copy_to( - src_files, os.path.join(wpiutil, "src/main/native/thirdparty/memory") + os.chdir(upstream_root / "src") + src_files = walk_cwd_and_copy_if( + lambda dp, f: f.endswith(".cpp") or f.endswith(".hpp"), + wpiutil / "src/main/native/thirdparty/memory/src", ) run_global_replacements(src_files) run_source_replacements(src_files) # Copy headers - os.chdir(os.path.join("include", "foonathan")) - include_files = walk_if(".", lambda dp, f: f.endswith(".hpp")) - include_files = copy_to( - include_files, - os.path.join(wpiutil, "src/main/native/thirdparty/memory/include/wpi"), + os.chdir(upstream_root / "include/foonathan") + include_files = walk_cwd_and_copy_if( + lambda dp, f: f.endswith(".hpp"), + wpiutil / "src/main/native/thirdparty/memory/include/wpi", ) - os.chdir(os.path.join("..", "..")) + os.chdir(upstream_root) run_global_replacements(include_files) run_header_replacements(include_files) # Copy config_impl.hpp shutil.copyfile( - os.path.join(wpilib_root, "upstream_utils/memory_files/config_impl.hpp"), - os.path.join( - wpiutil, - "src/main/native/thirdparty/memory/include/wpi/memory/config_impl.hpp", - ), + wpilib_root / "upstream_utils/memory_files/config_impl.hpp", + wpiutil + / "src/main/native/thirdparty/memory/include/wpi/memory/config_impl.hpp", ) diff --git a/upstream_utils/mpack.py b/upstream_utils/mpack.py index 1acff50a37..e4fca5c39b 100755 --- a/upstream_utils/mpack.py +++ b/upstream_utils/mpack.py @@ -3,36 +3,34 @@ import os import shutil import subprocess +from pathlib import Path from upstream_utils import Lib, walk_cwd_and_copy_if -def copy_upstream_src(wpilib_root): - wpiutil = os.path.join(wpilib_root, "wpiutil") +def copy_upstream_src(wpilib_root: Path): + wpiutil = wpilib_root / "wpiutil" # Delete old install for d in [ "src/main/native/thirdparty/mpack/src", "src/main/native/thirdparty/mpack/include", ]: - shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) + shutil.rmtree(wpiutil / d, ignore_errors=True) # Run the amalgmation script subprocess.check_call(["bash", "tools/amalgamate.sh"]) # Copy the files - amalgamation_source_dir = os.path.join( - ".", ".build", "amalgamation", "src", "mpack" - ) - os.chdir(amalgamation_source_dir) + os.chdir(Path(".build/amalgamation/src/mpack")) walk_cwd_and_copy_if( lambda dp, f: f.endswith(".h"), - os.path.join(wpiutil, "src/main/native/thirdparty/mpack/include/wpi"), + wpiutil / "src/main/native/thirdparty/mpack/include/wpi", ) walk_cwd_and_copy_if( lambda dp, f: f.endswith(".c"), - os.path.join(wpiutil, "src/main/native/thirdparty/mpack/src"), + wpiutil / "src/main/native/thirdparty/mpack/src", rename_c_to_cpp=True, ) diff --git a/upstream_utils/mrcal.py b/upstream_utils/mrcal.py index be6f57d7c6..4af10d908e 100755 --- a/upstream_utils/mrcal.py +++ b/upstream_utils/mrcal.py @@ -1,27 +1,27 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path -from upstream_utils import Lib, walk_cwd_and_copy_if +from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if -def copy_upstream_src(wpilib_root): - wpical = os.path.join(wpilib_root, "wpical") +def copy_upstream_src(wpilib_root: Path): + wpical = wpilib_root / "wpical" # Delete old install for d in [ "src/main/native/thirdparty/mrcal/src", "src/main/native/thirdparty/mrcal/include", ]: - shutil.rmtree(os.path.join(wpical, d), ignore_errors=True) + shutil.rmtree(wpical / d, ignore_errors=True) files = walk_cwd_and_copy_if( lambda dp, f: (f.endswith(".h") or f.endswith(".hh")) - and not f.endswith("heap.h") - and not f.endswith("stereo-matching-libelas.h") - and not dp.startswith(os.path.join(".", "test")), - os.path.join(wpical, "src/main/native/thirdparty/mrcal/include"), + and not f == "heap.h" + and not f == "stereo-matching-libelas.h" + and not has_prefix(dp, Path("test")), + wpical / "src/main/native/thirdparty/mrcal/include", ) files = files + walk_cwd_and_copy_if( lambda dp, f: ( @@ -30,16 +30,16 @@ def copy_upstream_src(wpilib_root): or f.endswith(".cpp") or f.endswith(".pl") ) - and not f.endswith("heap.cc") - and not f.endswith("mrcal-pywrap.c") - and not f.endswith("image.c") - and not f.endswith("stereo.c") - and not f.endswith("stereo-matching-libelas.cc") - and not f.endswith("uncertainty.c") - and not f.endswith("traverse-sensor-links.c") - and not dp.startswith(os.path.join(".", "doc")) - and not dp.startswith(os.path.join(".", "test")), - os.path.join(wpical, "src/main/native/thirdparty/mrcal/src"), + and not f == "heap.cc" + and not f == "mrcal-pywrap.c" + and not f == "image.c" + and not f == "stereo.c" + and not f == "stereo-matching-libelas.cc" + and not f == "uncertainty.c" + and not f == "traverse-sensor-links.c" + and not has_prefix(dp, Path("doc")) + and not has_prefix(dp, Path("test")), + wpical / "src/main/native/thirdparty/mrcal/src", ) for f in files: diff --git a/upstream_utils/mrcal_java.py b/upstream_utils/mrcal_java.py index 5fdd639b16..489df1f78c 100755 --- a/upstream_utils/mrcal_java.py +++ b/upstream_utils/mrcal_java.py @@ -2,46 +2,30 @@ import os import shutil +from pathlib import Path from upstream_utils import Lib, walk_cwd_and_copy_if -def delete_lines_by_range(file_path, start_line, end_line): - # Read all lines from the file - with open(file_path, "r") as file: - lines = file.readlines() - - # Filter out lines that are within the specified range - filtered_lines = [ - line - for i, line in enumerate(lines, start=1) - if not (start_line <= i <= end_line) - ] - - # Write the remaining lines back to the file - with open(file_path, "w") as file: - file.writelines(filtered_lines) - - -def copy_upstream_src(wpilib_root): - wpical = os.path.join(wpilib_root, "wpical") +def copy_upstream_src(wpilib_root: Path): + wpical = wpilib_root / "wpical" # Delete old install for d in [ "src/main/native/thirdparty/mrcal_java/src", "src/main/native/thirdparty/mrcal_java/include", ]: - shutil.rmtree(os.path.join(wpical, d), ignore_errors=True) + shutil.rmtree(wpical / d, ignore_errors=True) os.chdir("src") files = walk_cwd_and_copy_if( - lambda dp, f: f.endswith("mrcal_wrapper.h"), - os.path.join(wpical, "src/main/native/thirdparty/mrcal_java/include"), + lambda dp, f: f == "mrcal_wrapper.h", + wpical / "src/main/native/thirdparty/mrcal_java/include", ) files = walk_cwd_and_copy_if( - lambda dp, f: f.endswith("mrcal_wrapper.cpp"), - os.path.join(wpical, "src/main/native/thirdparty/mrcal_java/src"), + lambda dp, f: f == "mrcal_wrapper.cpp", + wpical / "src/main/native/thirdparty/mrcal_java/src", ) for f in files: diff --git a/upstream_utils/nanopb.py b/upstream_utils/nanopb.py index 1da46a8c19..543b19befa 100755 --- a/upstream_utils/nanopb.py +++ b/upstream_utils/nanopb.py @@ -1,39 +1,33 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path -from upstream_utils import Lib, copy_to, walk_if +from upstream_utils import Lib, copy_to, has_prefix, walk_cwd_and_copy_if -nanopb_sources = set( - [ - "pb_encode.c", - "pb_decode.c", - "pb_common.c", - ] -) +nanopb_sources = [ + Path("pb_encode.c"), + Path("pb_decode.c"), + Path("pb_common.c"), +] -nanopb_headers = set( - [ - "pb.h", - "pb_encode.h", - "pb_decode.h", - "pb_common.h", - ] -) +nanopb_headers = [ + Path("pb.h"), + Path("pb_encode.h"), + Path("pb_decode.h"), + Path("pb_common.h"), +] -nanopb_generator = set( - [ - "pb.h", - "pb_encode.h", - "pb_decode.h", - "pb_common.h", - ] -) +nanopb_generator = [ + Path("pb.h"), + Path("pb_encode.h"), + Path("pb_decode.h"), + Path("pb_common.h"), +] -def copy_upstream_src(wpilib_root): - wpiutil = os.path.join(wpilib_root, "wpiutil") +def copy_upstream_src(wpilib_root: Path): + wpiutil = wpilib_root / "wpiutil" # Delete old install for d in [ @@ -41,25 +35,24 @@ def copy_upstream_src(wpilib_root): "src/main/native/thirdparty/nanopb/include", "src/main/native/thirdparty/nanopb/generator", ]: - shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) + shutil.rmtree(wpiutil / d, ignore_errors=True) # Copy nanopb source files into allwpilib copy_to( nanopb_sources, - os.path.join(wpiutil, "src/main/native/thirdparty/nanopb/src"), + wpiutil / "src/main/native/thirdparty/nanopb/src", rename_c_to_cpp=True, ) # Copy nanopb header files into allwpilib copy_to( nanopb_headers, - os.path.join(wpiutil, "src/main/native/thirdparty/nanopb/include"), + wpiutil / "src/main/native/thirdparty/nanopb/include", ) - generator_files = walk_if("generator", lambda dp, f: True) - copy_to( - generator_files, - os.path.join(wpiutil, "src/main/native/thirdparty/nanopb"), + generator_files = walk_cwd_and_copy_if( + lambda dp, f: has_prefix(dp, Path("generator")), + wpiutil / "src/main/native/thirdparty/nanopb", ) diff --git a/upstream_utils/protobuf.py b/upstream_utils/protobuf.py index fc7055f524..9882c735b1 100755 --- a/upstream_utils/protobuf.py +++ b/upstream_utils/protobuf.py @@ -2,237 +2,227 @@ import os import shutil +from pathlib import Path -from upstream_utils import Lib, copy_to, walk_if +from upstream_utils import Lib, walk_cwd_and_copy_if -protobuf_lite_sources = set( - [ - "google/protobuf/any_lite.cc", - "google/protobuf/arena.cc", - "google/protobuf/arenastring.cc", - "google/protobuf/arenaz_sampler.cc", - "google/protobuf/extension_set.cc", - "google/protobuf/generated_enum_util.cc", - "google/protobuf/generated_message_tctable_lite.cc", - "google/protobuf/generated_message_util.cc", - "google/protobuf/implicit_weak_message.cc", - "google/protobuf/inlined_string_field.cc", - "google/protobuf/io/coded_stream.cc", - "google/protobuf/io/io_win32.cc", - "google/protobuf/io/strtod.cc", - "google/protobuf/io/zero_copy_stream.cc", - "google/protobuf/io/zero_copy_stream_impl.cc", - "google/protobuf/io/zero_copy_stream_impl_lite.cc", - "google/protobuf/map.cc", - "google/protobuf/message_lite.cc", - "google/protobuf/parse_context.cc", - "google/protobuf/repeated_field.cc", - "google/protobuf/repeated_ptr_field.cc", - "google/protobuf/stubs/bytestream.cc", - "google/protobuf/stubs/common.cc", - "google/protobuf/stubs/int128.cc", - "google/protobuf/stubs/status.cc", - "google/protobuf/stubs/statusor.cc", - "google/protobuf/stubs/stringpiece.cc", - "google/protobuf/stubs/stringprintf.cc", - "google/protobuf/stubs/structurally_valid.cc", - "google/protobuf/stubs/strutil.cc", - "google/protobuf/stubs/time.cc", - "google/protobuf/wire_format_lite.cc", - ] -) +protobuf_lite_sources = { + Path("any_lite.cc"), + Path("arena.cc"), + Path("arenastring.cc"), + Path("arenaz_sampler.cc"), + Path("extension_set.cc"), + Path("generated_enum_util.cc"), + Path("generated_message_tctable_lite.cc"), + Path("generated_message_util.cc"), + Path("implicit_weak_message.cc"), + Path("inlined_string_field.cc"), + Path("io/coded_stream.cc"), + Path("io/io_win32.cc"), + Path("io/strtod.cc"), + Path("io/zero_copy_stream.cc"), + Path("io/zero_copy_stream_impl.cc"), + Path("io/zero_copy_stream_impl_lite.cc"), + Path("map.cc"), + Path("message_lite.cc"), + Path("parse_context.cc"), + Path("repeated_field.cc"), + Path("repeated_ptr_field.cc"), + Path("stubs/bytestream.cc"), + Path("stubs/common.cc"), + Path("stubs/int128.cc"), + Path("stubs/status.cc"), + Path("stubs/statusor.cc"), + Path("stubs/stringpiece.cc"), + Path("stubs/stringprintf.cc"), + Path("stubs/structurally_valid.cc"), + Path("stubs/strutil.cc"), + Path("stubs/time.cc"), + Path("wire_format_lite.cc"), +} -protobuf_lite_includes = set( - [ - "google/protobuf/any.h", - "google/protobuf/arena.h", - "google/protobuf/arena_impl.h", - "google/protobuf/arenastring.h", - "google/protobuf/arenaz_sampler.h", - "google/protobuf/endian.h", - "google/protobuf/explicitly_constructed.h", - "google/protobuf/extension_set.h", - "google/protobuf/extension_set_inl.h", - "google/protobuf/generated_enum_util.h", - "google/protobuf/generated_message_tctable_decl.h", - "google/protobuf/generated_message_tctable_impl.h", - "google/protobuf/generated_message_util.h", - "google/protobuf/has_bits.h", - "google/protobuf/implicit_weak_message.h", - "google/protobuf/inlined_string_field.h", - "google/protobuf/io/coded_stream.h", - "google/protobuf/io/io_win32.h", - "google/protobuf/io/strtod.h", - "google/protobuf/io/zero_copy_stream.h", - "google/protobuf/io/zero_copy_stream_impl.h", - "google/protobuf/io/zero_copy_stream_impl_lite.h", - "google/protobuf/map.h", - "google/protobuf/map_entry_lite.h", - "google/protobuf/map_field_lite.h", - "google/protobuf/map_type_handler.h", - "google/protobuf/message_lite.h", - "google/protobuf/metadata_lite.h", - "google/protobuf/parse_context.h", - "google/protobuf/port.h", - "google/protobuf/repeated_field.h", - "google/protobuf/repeated_ptr_field.h", - "google/protobuf/stubs/bytestream.h", - "google/protobuf/stubs/callback.h", - "google/protobuf/stubs/casts.h", - "google/protobuf/stubs/common.h", - "google/protobuf/stubs/hash.h", - "google/protobuf/stubs/logging.h", - "google/protobuf/stubs/macros.h", - "google/protobuf/stubs/map_util.h", - "google/protobuf/stubs/mutex.h", - "google/protobuf/stubs/once.h", - "google/protobuf/stubs/platform_macros.h", - "google/protobuf/stubs/port.h", - "google/protobuf/stubs/status.h", - "google/protobuf/stubs/stl_util.h", - "google/protobuf/stubs/stringpiece.h", - "google/protobuf/stubs/strutil.h", - "google/protobuf/stubs/template_util.h", - "google/protobuf/wire_format_lite.h", - ] -) +protobuf_lite_includes = { + Path("google/protobuf/any.h"), + Path("google/protobuf/arena.h"), + Path("google/protobuf/arena_impl.h"), + Path("google/protobuf/arenastring.h"), + Path("google/protobuf/arenaz_sampler.h"), + Path("google/protobuf/endian.h"), + Path("google/protobuf/explicitly_constructed.h"), + Path("google/protobuf/extension_set.h"), + Path("google/protobuf/extension_set_inl.h"), + Path("google/protobuf/generated_enum_util.h"), + Path("google/protobuf/generated_message_tctable_decl.h"), + Path("google/protobuf/generated_message_tctable_impl.h"), + Path("google/protobuf/generated_message_util.h"), + Path("google/protobuf/has_bits.h"), + Path("google/protobuf/implicit_weak_message.h"), + Path("google/protobuf/inlined_string_field.h"), + Path("google/protobuf/io/coded_stream.h"), + Path("google/protobuf/io/io_win32.h"), + Path("google/protobuf/io/strtod.h"), + Path("google/protobuf/io/zero_copy_stream.h"), + Path("google/protobuf/io/zero_copy_stream_impl.h"), + Path("google/protobuf/io/zero_copy_stream_impl_lite.h"), + Path("google/protobuf/map.h"), + Path("google/protobuf/map_entry_lite.h"), + Path("google/protobuf/map_field_lite.h"), + Path("google/protobuf/map_type_handler.h"), + Path("google/protobuf/message_lite.h"), + Path("google/protobuf/metadata_lite.h"), + Path("google/protobuf/parse_context.h"), + Path("google/protobuf/port.h"), + Path("google/protobuf/repeated_field.h"), + Path("google/protobuf/repeated_ptr_field.h"), + Path("google/protobuf/stubs/bytestream.h"), + Path("google/protobuf/stubs/callback.h"), + Path("google/protobuf/stubs/casts.h"), + Path("google/protobuf/stubs/common.h"), + Path("google/protobuf/stubs/hash.h"), + Path("google/protobuf/stubs/logging.h"), + Path("google/protobuf/stubs/macros.h"), + Path("google/protobuf/stubs/map_util.h"), + Path("google/protobuf/stubs/mutex.h"), + Path("google/protobuf/stubs/once.h"), + Path("google/protobuf/stubs/platform_macros.h"), + Path("google/protobuf/stubs/port.h"), + Path("google/protobuf/stubs/status.h"), + Path("google/protobuf/stubs/stl_util.h"), + Path("google/protobuf/stubs/stringpiece.h"), + Path("google/protobuf/stubs/strutil.h"), + Path("google/protobuf/stubs/template_util.h"), + Path("google/protobuf/wire_format_lite.h"), +} +protobuf_sources = { + Path("any.cc"), + Path("any.pb.cc"), + Path("api.pb.cc"), + Path("compiler/importer.cc"), + Path("compiler/parser.cc"), + Path("descriptor.cc"), + Path("descriptor.pb.cc"), + Path("descriptor_database.cc"), + Path("duration.pb.cc"), + Path("dynamic_message.cc"), + Path("empty.pb.cc"), + Path("extension_set_heavy.cc"), + Path("field_mask.pb.cc"), + Path("generated_message_bases.cc"), + Path("generated_message_reflection.cc"), + Path("generated_message_tctable_full.cc"), + Path("io/gzip_stream.cc"), + Path("io/printer.cc"), + Path("io/tokenizer.cc"), + Path("map_field.cc"), + Path("message.cc"), + Path("reflection_ops.cc"), + Path("service.cc"), + Path("source_context.pb.cc"), + Path("struct.pb.cc"), + Path("stubs/substitute.cc"), + Path("text_format.cc"), + Path("timestamp.pb.cc"), + Path("type.pb.cc"), + Path("unknown_field_set.cc"), + Path("util/delimited_message_util.cc"), + Path("util/field_comparator.cc"), + Path("util/field_mask_util.cc"), + Path("util/internal/datapiece.cc"), + Path("util/internal/default_value_objectwriter.cc"), + Path("util/internal/error_listener.cc"), + Path("util/internal/field_mask_utility.cc"), + Path("util/internal/json_escaping.cc"), + Path("util/internal/json_objectwriter.cc"), + Path("util/internal/json_stream_parser.cc"), + Path("util/internal/object_writer.cc"), + Path("util/internal/proto_writer.cc"), + Path("util/internal/protostream_objectsource.cc"), + Path("util/internal/protostream_objectwriter.cc"), + Path("util/internal/type_info.cc"), + Path("util/internal/utility.cc"), + Path("util/json_util.cc"), + Path("util/message_differencer.cc"), + Path("util/time_util.cc"), + Path("util/type_resolver_util.cc"), + Path("wire_format.cc"), + Path("wrappers.pb.cc"), +} -protobuf_sources = set( - [ - "google/protobuf/any.cc", - "google/protobuf/any.pb.cc", - "google/protobuf/api.pb.cc", - "google/protobuf/compiler/importer.cc", - "google/protobuf/compiler/parser.cc", - "google/protobuf/descriptor.cc", - "google/protobuf/descriptor.pb.cc", - "google/protobuf/descriptor_database.cc", - "google/protobuf/duration.pb.cc", - "google/protobuf/dynamic_message.cc", - "google/protobuf/empty.pb.cc", - "google/protobuf/extension_set_heavy.cc", - "google/protobuf/field_mask.pb.cc", - "google/protobuf/generated_message_bases.cc", - "google/protobuf/generated_message_reflection.cc", - "google/protobuf/generated_message_tctable_full.cc", - "google/protobuf/io/gzip_stream.cc", - "google/protobuf/io/printer.cc", - "google/protobuf/io/tokenizer.cc", - "google/protobuf/map_field.cc", - "google/protobuf/message.cc", - "google/protobuf/reflection_ops.cc", - "google/protobuf/service.cc", - "google/protobuf/source_context.pb.cc", - "google/protobuf/struct.pb.cc", - "google/protobuf/stubs/substitute.cc", - "google/protobuf/text_format.cc", - "google/protobuf/timestamp.pb.cc", - "google/protobuf/type.pb.cc", - "google/protobuf/unknown_field_set.cc", - "google/protobuf/util/delimited_message_util.cc", - "google/protobuf/util/field_comparator.cc", - "google/protobuf/util/field_mask_util.cc", - "google/protobuf/util/internal/datapiece.cc", - "google/protobuf/util/internal/default_value_objectwriter.cc", - "google/protobuf/util/internal/error_listener.cc", - "google/protobuf/util/internal/field_mask_utility.cc", - "google/protobuf/util/internal/json_escaping.cc", - "google/protobuf/util/internal/json_objectwriter.cc", - "google/protobuf/util/internal/json_stream_parser.cc", - "google/protobuf/util/internal/object_writer.cc", - "google/protobuf/util/internal/proto_writer.cc", - "google/protobuf/util/internal/protostream_objectsource.cc", - "google/protobuf/util/internal/protostream_objectwriter.cc", - "google/protobuf/util/internal/type_info.cc", - "google/protobuf/util/internal/utility.cc", - "google/protobuf/util/json_util.cc", - "google/protobuf/util/message_differencer.cc", - "google/protobuf/util/time_util.cc", - "google/protobuf/util/type_resolver_util.cc", - "google/protobuf/wire_format.cc", - "google/protobuf/wrappers.pb.cc", - ] -) +protobuf_includes = { + Path("google/protobuf/any.pb.h"), + Path("google/protobuf/api.pb.h"), + Path("google/protobuf/compiler/importer.h"), + Path("google/protobuf/compiler/parser.h"), + Path("google/protobuf/descriptor.h"), + Path("google/protobuf/descriptor.pb.h"), + Path("google/protobuf/descriptor_database.h"), + Path("google/protobuf/duration.pb.h"), + Path("google/protobuf/dynamic_message.h"), + Path("google/protobuf/empty.pb.h"), + Path("google/protobuf/field_access_listener.h"), + Path("google/protobuf/field_mask.pb.h"), + Path("google/protobuf/generated_enum_reflection.h"), + Path("google/protobuf/generated_message_bases.h"), + Path("google/protobuf/generated_message_reflection.h"), + Path("google/protobuf/io/gzip_stream.h"), + Path("google/protobuf/io/printer.h"), + Path("google/protobuf/io/tokenizer.h"), + Path("google/protobuf/map_entry.h"), + Path("google/protobuf/map_field.h"), + Path("google/protobuf/map_field_inl.h"), + Path("google/protobuf/message.h"), + Path("google/protobuf/metadata.h"), + Path("google/protobuf/reflection.h"), + Path("google/protobuf/reflection_internal.h"), + Path("google/protobuf/reflection_ops.h"), + Path("google/protobuf/service.h"), + Path("google/protobuf/source_context.pb.h"), + Path("google/protobuf/struct.pb.h"), + Path("google/protobuf/text_format.h"), + Path("google/protobuf/timestamp.pb.h"), + Path("google/protobuf/type.pb.h"), + Path("google/protobuf/unknown_field_set.h"), + Path("google/protobuf/util/delimited_message_util.h"), + Path("google/protobuf/util/field_comparator.h"), + Path("google/protobuf/util/field_mask_util.h"), + Path("google/protobuf/util/json_util.h"), + Path("google/protobuf/util/message_differencer.h"), + Path("google/protobuf/util/time_util.h"), + Path("google/protobuf/util/type_resolver.h"), + Path("google/protobuf/util/type_resolver_util.h"), + Path("google/protobuf/wire_format.h"), + Path("google/protobuf/wrappers.pb.h"), +} -protobuf_includes = set( - [ - "google/protobuf/any.pb.h", - "google/protobuf/api.pb.h", - "google/protobuf/compiler/importer.h", - "google/protobuf/compiler/parser.h", - "google/protobuf/descriptor.h", - "google/protobuf/descriptor.pb.h", - "google/protobuf/descriptor_database.h", - "google/protobuf/duration.pb.h", - "google/protobuf/dynamic_message.h", - "google/protobuf/empty.pb.h", - "google/protobuf/field_access_listener.h", - "google/protobuf/field_mask.pb.h", - "google/protobuf/generated_enum_reflection.h", - "google/protobuf/generated_message_bases.h", - "google/protobuf/generated_message_reflection.h", - "google/protobuf/io/gzip_stream.h", - "google/protobuf/io/printer.h", - "google/protobuf/io/tokenizer.h", - "google/protobuf/map_entry.h", - "google/protobuf/map_field.h", - "google/protobuf/map_field_inl.h", - "google/protobuf/message.h", - "google/protobuf/metadata.h", - "google/protobuf/reflection.h", - "google/protobuf/reflection_internal.h", - "google/protobuf/reflection_ops.h", - "google/protobuf/service.h", - "google/protobuf/source_context.pb.h", - "google/protobuf/struct.pb.h", - "google/protobuf/text_format.h", - "google/protobuf/timestamp.pb.h", - "google/protobuf/type.pb.h", - "google/protobuf/unknown_field_set.h", - "google/protobuf/util/delimited_message_util.h", - "google/protobuf/util/field_comparator.h", - "google/protobuf/util/field_mask_util.h", - "google/protobuf/util/json_util.h", - "google/protobuf/util/message_differencer.h", - "google/protobuf/util/time_util.h", - "google/protobuf/util/type_resolver.h", - "google/protobuf/util/type_resolver_util.h", - "google/protobuf/wire_format.h", - "google/protobuf/wrappers.pb.h", - ] -) - -protobuf_internal_includes = set( - [ - "google/protobuf/port_def.inc", - "google/protobuf/port_undef.inc", - "google/protobuf/stubs/int128.h", - "google/protobuf/stubs/mathutil.h", - "google/protobuf/stubs/statusor.h", - "google/protobuf/stubs/status_macros.h", - "google/protobuf/stubs/stringprintf.h", - "google/protobuf/stubs/substitute.h", - "google/protobuf/stubs/time.h", - "google/protobuf/util/internal/constants.h", - "google/protobuf/util/internal/datapiece.h", - "google/protobuf/util/internal/default_value_objectwriter.h", - "google/protobuf/util/internal/error_listener.h", - "google/protobuf/util/internal/field_mask_utility.h", - "google/protobuf/util/internal/json_escaping.h", - "google/protobuf/util/internal/json_objectwriter.h", - "google/protobuf/util/internal/json_stream_parser.h", - "google/protobuf/util/internal/location_tracker.h", - "google/protobuf/util/internal/object_location_tracker.h", - "google/protobuf/util/internal/object_source.h", - "google/protobuf/util/internal/object_writer.h", - "google/protobuf/util/internal/proto_writer.h", - "google/protobuf/util/internal/protostream_objectsource.h", - "google/protobuf/util/internal/protostream_objectwriter.h", - "google/protobuf/util/internal/structured_objectwriter.h", - "google/protobuf/util/internal/type_info.h", - "google/protobuf/util/internal/utility.h", - ] -) +protobuf_internal_includes = { + Path("google/protobuf/port_def.inc"), + Path("google/protobuf/port_undef.inc"), + Path("google/protobuf/stubs/int128.h"), + Path("google/protobuf/stubs/mathutil.h"), + Path("google/protobuf/stubs/statusor.h"), + Path("google/protobuf/stubs/status_macros.h"), + Path("google/protobuf/stubs/stringprintf.h"), + Path("google/protobuf/stubs/substitute.h"), + Path("google/protobuf/stubs/time.h"), + Path("google/protobuf/util/internal/constants.h"), + Path("google/protobuf/util/internal/datapiece.h"), + Path("google/protobuf/util/internal/default_value_objectwriter.h"), + Path("google/protobuf/util/internal/error_listener.h"), + Path("google/protobuf/util/internal/field_mask_utility.h"), + Path("google/protobuf/util/internal/json_escaping.h"), + Path("google/protobuf/util/internal/json_objectwriter.h"), + Path("google/protobuf/util/internal/json_stream_parser.h"), + Path("google/protobuf/util/internal/location_tracker.h"), + Path("google/protobuf/util/internal/object_location_tracker.h"), + Path("google/protobuf/util/internal/object_source.h"), + Path("google/protobuf/util/internal/object_writer.h"), + Path("google/protobuf/util/internal/proto_writer.h"), + Path("google/protobuf/util/internal/protostream_objectsource.h"), + Path("google/protobuf/util/internal/protostream_objectwriter.h"), + Path("google/protobuf/util/internal/structured_objectwriter.h"), + Path("google/protobuf/util/internal/type_info.h"), + Path("google/protobuf/util/internal/utility.h"), +} use_src_files = protobuf_lite_sources | protobuf_sources use_include_files = ( @@ -240,38 +230,29 @@ use_include_files = ( ) -def matches(dp, f, files): - if not dp.startswith(os.path.join(".", "src")): - return False - p = dp[6:] + "/" + f - return p in files - - -def copy_upstream_src(wpilib_root): - upstream_root = os.path.abspath(".") - wpiutil = os.path.join(wpilib_root, "wpiutil") +def copy_upstream_src(wpilib_root: Path): + upstream_root = Path(".").absolute() + wpiutil = wpilib_root / "wpiutil" # Delete old install for d in [ "src/main/native/thirdparty/protobuf/src", "src/main/native/thirdparty/protobuf/include", ]: - shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) + shutil.rmtree(wpiutil / d, ignore_errors=True) # Copy protobuf source files into allwpilib - src_files = walk_if(".", lambda dp, f: matches(dp, f, use_src_files)) - src_files = [f[22:] for f in src_files] - os.chdir(os.path.join(upstream_root, "src/google/protobuf")) - copy_to(src_files, os.path.join(wpiutil, "src/main/native/thirdparty/protobuf/src")) + os.chdir(upstream_root / "src/google/protobuf") + walk_cwd_and_copy_if( + lambda dp, f: dp / f in use_src_files, + wpiutil / "src/main/native/thirdparty/protobuf/src", + ) # Copy protobuf header files into allwpilib - os.chdir(upstream_root) - include_files = walk_if(".", lambda dp, f: matches(dp, f, use_include_files)) - include_files = [f[6:] for f in include_files] - os.chdir(os.path.join(upstream_root, "src")) - copy_to( - include_files, - os.path.join(wpiutil, "src/main/native/thirdparty/protobuf/include"), + os.chdir(upstream_root / "src") + walk_cwd_and_copy_if( + lambda dp, f: dp / f in use_include_files, + wpiutil / "src/main/native/thirdparty/protobuf/include", ) diff --git a/upstream_utils/sleipnir.py b/upstream_utils/sleipnir.py index ba42c3fd78..c1b0de9a87 100755 --- a/upstream_utils/sleipnir.py +++ b/upstream_utils/sleipnir.py @@ -1,58 +1,35 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path -from upstream_utils import Lib, copy_to +from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if -def copy_upstream_src(wpilib_root): - wpimath = os.path.join(wpilib_root, "wpimath") +def copy_upstream_src(wpilib_root: Path): + wpimath = wpilib_root / "wpimath" # Delete old install for d in [ "src/main/native/thirdparty/sleipnir/src", "src/main/native/thirdparty/sleipnir/include", ]: - shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) + shutil.rmtree(wpimath / d, ignore_errors=True) - # Copy Sleipnir source files into allwpilib - src_files = [os.path.join(dp, f) for dp, dn, fn in os.walk("src") for f in fn] - src_files = copy_to( - src_files, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir") + # Copy Sleipnir files into allwpilib + walk_cwd_and_copy_if( + lambda dp, f: (has_prefix(dp, Path("include")) or has_prefix(dp, Path("src"))) + or f == ".clang-format" + or f == ".clang-tidy" + or f == ".styleguide" + or f == ".styleguide-license", + wpimath / "src/main/native/thirdparty/sleipnir", ) - # Copy Sleipnir header files into allwpilib - include_files = [ - os.path.join(dp, f) for dp, dn, fn in os.walk("include") for f in fn - ] - include_files = copy_to( - include_files, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir") - ) - - for filename in [ - ".clang-format", - ".clang-tidy", - ".styleguide", - ".styleguide-license", - ]: - shutil.copyfile( - filename, - os.path.join(wpimath, "src/main/native/thirdparty/sleipnir", filename), - ) - # Write shim for wpi::SmallVector - try: - os.mkdir( - os.path.join(wpimath, "src/main/native/thirdparty/sleipnir/include/gch") - ) - except: - pass + (wpimath / "src/main/native/thirdparty/sleipnir/include/gch").mkdir() with open( - os.path.join( - wpimath, - "src/main/native/thirdparty/sleipnir/include/gch/small_vector.hpp", - ), + wpimath / "src/main/native/thirdparty/sleipnir/include/gch/small_vector.hpp", "w", ) as f: f.write( diff --git a/upstream_utils/stb.py b/upstream_utils/stb.py index 06fb69a396..3dee40e5cd 100755 --- a/upstream_utils/stb.py +++ b/upstream_utils/stb.py @@ -1,27 +1,27 @@ #!/usr/bin/env python3 -import os import shutil +from pathlib import Path from upstream_utils import Lib, walk_cwd_and_copy_if -def copy_upstream_src(wpilib_root): - stb = os.path.join(wpilib_root, "thirdparty", "imgui_suite", "stb") +def copy_upstream_src(wpilib_root: Path): + stb = wpilib_root / "thirdparty/imgui_suite/stb" # Delete old install for d in ["include", "cpp"]: - shutil.rmtree(os.path.join(stb, d), ignore_errors=True) + shutil.rmtree(stb / d, ignore_errors=True) # Copy files - files = walk_cwd_and_copy_if( + walk_cwd_and_copy_if( lambda dp, f: f == "stb_image.h", - os.path.join(stb, "include"), + stb / "include", ) - os.makedirs(os.path.join(stb, "cpp")) + (stb / "cpp").mkdir(parents=True) - with open(os.path.join(stb, "cpp", "stb_image.cpp"), "w") as f: + with open(stb / "cpp/stb_image.cpp", "w") as f: f.write( """#define STBI_WINDOWS_UTF8 #define STB_IMAGE_IMPLEMENTATION diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index bb5e901ae9..b621ccce01 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -5,6 +5,8 @@ import shutil import subprocess import sys import tempfile +from collections.abc import Callable +from pathlib import Path def get_repo_root(): @@ -12,15 +14,17 @@ def get_repo_root(): An empty string is returned if no repository root was found. """ - return subprocess.run( - ["git", "rev-parse", "--show-toplevel"], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - encoding="ascii", - ).stdout.rstrip() + return Path( + subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + encoding="ascii", + ).stdout.rstrip() + ) -def walk_if(top, pred): +def walk_if(top: Path, pred: Callable[[Path, str], bool]): """Walks the current directory, then returns a list of files for which the given predicate is true. @@ -29,12 +33,10 @@ def walk_if(top, pred): pred -- a function that takes a directory path and a filename, then returns True if the file should be included in the output list """ - return [ - os.path.join(dp, f) for dp, dn, fn in os.walk(top) for f in fn if pred(dp, f) - ] + return [dp / f for dp, dn, fn in top.walk() for f in fn if pred(dp, f)] -def copy_to(files, root, rename_c_to_cpp=False): +def copy_to(files: list[Path], root: Path, rename_c_to_cpp=False): """Copies list of files to root by appending the relative paths of the files to root. @@ -48,31 +50,33 @@ def copy_to(files, root, rename_c_to_cpp=False): Returns: The list of files in their destination. """ - if not os.path.exists(root): - os.makedirs(root) + if not root.exists(): + root.mkdir(parents=True) - dest_files = [] + dest_files: list[Path] = [] for f in files: - dest_file = os.path.join(root, f) + dest_file = root / f # Rename .cc or .cxx file to .cpp - if dest_file.endswith(".cc") or dest_file.endswith(".cxx"): - dest_file = os.path.splitext(dest_file)[0] + ".cpp" + if dest_file.match("*.cc") or dest_file.match("*.cxx"): + dest_file = dest_file.with_suffix(".cpp") - if rename_c_to_cpp and dest_file.endswith(".c"): - dest_file = os.path.splitext(dest_file)[0] + ".cpp" + if rename_c_to_cpp and dest_file.match("*.c"): + dest_file = dest_file.with_suffix(".cpp") # Make leading directory - dest_dir = os.path.dirname(dest_file) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) + dest_dir = dest_file.parent + if not dest_dir.exists(): + dest_dir.mkdir(parents=True) shutil.copyfile(f, dest_file) dest_files.append(dest_file) return dest_files -def walk_cwd_and_copy_if(pred, root, rename_c_to_cpp=False): +def walk_cwd_and_copy_if( + pred: Callable[[Path, str], bool], root: Path, rename_c_to_cpp=False +): """Walks the current directory, generates a list of files for which the given predicate is true, then copies that list to root by appending the relative paths of the files to root. @@ -88,12 +92,12 @@ def walk_cwd_and_copy_if(pred, root, rename_c_to_cpp=False): Returns: The list of files in their destination. """ - files = walk_if(".", pred) + files = walk_if(Path("."), pred) files = copy_to(files, root, rename_c_to_cpp) return files -def comment_out_invalid_includes(filename, include_roots): +def comment_out_invalid_includes(filename: Path, include_roots: list[Path]): """Comment out #include directives that include a nonexistent file Keyword arguments: @@ -114,11 +118,8 @@ def comment_out_invalid_includes(filename, include_roots): # Comment out #include if the file doesn't exist in current directory or # include root - if not os.path.exists( - os.path.join(os.path.dirname(filename), include) - ) and not any( - os.path.exists(os.path.join(include_root, include)) - for include_root in include_roots + if not (filename.parent / include).exists() and not any( + (include_root / include).exists() for include_root in include_roots ): new_contents += "// " @@ -135,7 +136,22 @@ def comment_out_invalid_includes(filename, include_roots): f.write(new_contents) -def git_am(patch, use_threeway=False, ignore_whitespace=False): +def has_prefix(path: Path, prefix: Path): + """Checks if a path has a certain prefix. + + Keyword arguments: + path -- path to check if it begins with the prefix + prefix -- prefix to use + + Returns: + True if the path begins with the prefix, False otherwise. + """ + return len(path.parts) >= len(prefix.parts) and all( + p1 == p2 for p1, p2 in zip(path.parts, prefix.parts) + ) + + +def git_am(patch: Path, use_threeway=False, ignore_whitespace=False): """Apply patch to a Git repository in the current directory using "git am". Keyword arguments: @@ -152,7 +168,7 @@ def git_am(patch, use_threeway=False, ignore_whitespace=False): subprocess.check_output(args + [patch]) -def has_git_rev(rev): +def has_git_rev(rev: str): """Checks whether the Git repository in the current directory has the given revision. @@ -169,10 +185,10 @@ def has_git_rev(rev): class Lib: def __init__( self, - name, - url, - tag, - copy_upstream_src, + name: str, + url: str, + tag: str, + copy_upstream_src: Callable[[Path], None], patch_options={}, *, pre_patch_hook=None, @@ -207,7 +223,7 @@ class Lib: self.pre_patch_commits = pre_patch_commits self.wpilib_root = get_repo_root() - def get_repo_path(self, tempdir=None): + def get_repo_path(self, tempdir: str | None = None): """Returns the path to the clone of the upstream repository. Keyword argument: @@ -221,11 +237,11 @@ class Lib: if tempdir is None: tempdir = tempfile.gettempdir() repo = os.path.basename(self.url) - dest = os.path.join(tempdir, repo) - dest = dest.removesuffix(".git") + dest = Path(tempdir, repo) + dest = dest.with_suffix("") return dest - def open_repo(self, *, err_msg_if_absent): + def open_repo(self, *, err_msg_if_absent: str | None): """Changes the current working directory to the upstream repository. If err_msg_if_absent is not None and the upstream repository does not exist, the program exits with return code 1. @@ -241,7 +257,7 @@ class Lib: print(f"INFO: Opening repository at {dest}") - if not os.path.exists(dest): + if not dest.exists(): if err_msg_if_absent is None: subprocess.run(["git", "clone", "--filter=tree:0", self.url]) else: @@ -285,7 +301,7 @@ class Lib: exit(1) return root_tags[0] - def set_root_tag(self, tag): + def set_root_tag(self, tag: str): """Sets the root tag, deleting any potential candidates first. Keyword argument: @@ -307,19 +323,17 @@ class Lib: Returns: The absolute path to the directory containing the patch files. """ - return os.path.join(self.wpilib_root, f"upstream_utils/{self.name}_patches") + return self.wpilib_root / f"upstream_utils/{self.name}_patches" - def get_patch_list(self): + def get_patch_list(self) -> list[Path]: """Returns a list of the filenames of the patches to apply. Returns: A list of the filenames of the patches to apply, sorted in lexicographic order by the Unicode code points.""" - if not os.path.exists(self.get_patch_directory()): + if not self.get_patch_directory().exists(): return [] - return sorted( - f for f in os.listdir(self.get_patch_directory()) if f.endswith(".patch") - ) + return sorted(self.get_patch_directory().glob("*.patch")) def apply_patches(self): """Applies the patches listed in the patch list to the current @@ -330,17 +344,17 @@ class Lib: for f in self.get_patch_list(): git_am( - os.path.join(self.get_patch_directory(), f), + self.get_patch_directory() / f, **self.patch_options, ) - def replace_tag(self, tag): + def replace_tag(self, tag: str): """Replaces the tag in the script. Keyword argument: tag -- The tag to replace the script tag with. """ - path = os.path.join(self.wpilib_root, f"upstream_utils/{self.name}.py") + path = self.wpilib_root / f"upstream_utils/{self.name}.py" with open(path, "r") as file: lines = file.readlines() @@ -391,7 +405,7 @@ class Lib: self.set_root_tag(self.old_tag) - def rebase(self, new_tag): + def rebase(self, new_tag: str): """Rebases the patches. Keyword argument: @@ -449,7 +463,7 @@ class Lib: ] ) - if os.path.exists(self.get_patch_directory()): + if self.get_patch_directory().exists(): shutil.rmtree(self.get_patch_directory()) is_first = True diff --git a/wpimath/src/main/native/thirdparty/eigen/include/Eigen/src/ThreadPool/CoreThreadPoolDevice.h b/wpimath/src/main/native/thirdparty/eigen/include/Eigen/src/ThreadPool/CoreThreadPoolDevice.h deleted file mode 100644 index c603a38a2f..0000000000 --- a/wpimath/src/main/native/thirdparty/eigen/include/Eigen/src/ThreadPool/CoreThreadPoolDevice.h +++ /dev/null @@ -1,336 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2023 Charlie Schlosser -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_CORE_THREAD_POOL_DEVICE_H -#define EIGEN_CORE_THREAD_POOL_DEVICE_H - -namespace Eigen { - -// CoreThreadPoolDevice provides an easy-to-understand Device for parallelizing Eigen Core expressions with -// Threadpool. Expressions are recursively split evenly until the evaluation cost is less than the threshold for -// delegating the task to a thread. -/* - a - / \ - / \ - / \ - / \ - / \ - / \ - / \ - a e - / \ / \ - / \ / \ - / \ / \ - a c e g - / \ / \ / \ / \ - / \ / \ / \ / \ - a b c d e f g h -*/ -// Each task descends the binary tree to the left, delegates the right task to a new thread, and continues to the -// left. This ensures that work is evenly distributed to the thread pool as quickly as possible and minimizes the number -// of tasks created during the evaluation. Consider an expression that is divided into 8 chunks. The -// primary task 'a' creates tasks 'e' 'c' and 'b', and executes its portion of the expression at the bottom of the -// tree. Likewise, task 'e' creates tasks 'g' and 'f', and executes its portion of the expression. - -struct CoreThreadPoolDevice { - using Task = std::function; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoreThreadPoolDevice(ThreadPool& pool, float threadCostThreshold = 3e-5f) - : m_pool(pool) { - eigen_assert(threadCostThreshold >= 0.0f && "threadCostThreshold must be non-negative"); - m_costFactor = threadCostThreshold; - } - - template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int calculateLevels(Index size, float cost) const { - eigen_assert(cost >= 0.0f && "cost must be non-negative"); - Index numOps = size / PacketSize; - int actualThreads = numOps < m_pool.NumThreads() ? static_cast(numOps) : m_pool.NumThreads(); - float totalCost = static_cast(numOps) * cost; - float idealThreads = totalCost * m_costFactor; - if (idealThreads < static_cast(actualThreads)) { - idealThreads = numext::maxi(idealThreads, 1.0f); - actualThreads = numext::mini(actualThreads, static_cast(idealThreads)); - } - int maxLevel = internal::log2_ceil(actualThreads); - return maxLevel; - } - -// MSVC does not like inlining parallelForImpl -#if EIGEN_COMP_MSVC && !EIGEN_COMP_CLANG -#define EIGEN_PARALLEL_FOR_INLINE -#else -#define EIGEN_PARALLEL_FOR_INLINE EIGEN_STRONG_INLINE -#endif - - template - EIGEN_DEVICE_FUNC EIGEN_PARALLEL_FOR_INLINE void parallelForImpl(Index begin, Index end, UnaryFunctor& f, - Barrier& barrier, int level) { - while (level > 0) { - level--; - Index size = end - begin; - eigen_assert(size % PacketSize == 0 && "this function assumes size is a multiple of PacketSize"); - Index mid = begin + numext::round_down(size >> 1, PacketSize); - Task right = [this, mid, end, &f, &barrier, level]() { - parallelForImpl(mid, end, f, barrier, level); - }; - m_pool.Schedule(std::move(right)); - end = mid; - } - for (Index i = begin; i < end; i += PacketSize) f(i); - barrier.Notify(); - } - - template - EIGEN_DEVICE_FUNC EIGEN_PARALLEL_FOR_INLINE void parallelForImpl(Index outerBegin, Index outerEnd, Index innerBegin, - Index innerEnd, BinaryFunctor& f, Barrier& barrier, - int level) { - while (level > 0) { - level--; - Index outerSize = outerEnd - outerBegin; - if (outerSize > 1) { - Index outerMid = outerBegin + (outerSize >> 1); - Task right = [this, &f, &barrier, outerMid, outerEnd, innerBegin, innerEnd, level]() { - parallelForImpl(outerMid, outerEnd, innerBegin, innerEnd, f, barrier, level); - }; - m_pool.Schedule(std::move(right)); - outerEnd = outerMid; - } else { - Index innerSize = innerEnd - innerBegin; - eigen_assert(innerSize % PacketSize == 0 && "this function assumes innerSize is a multiple of PacketSize"); - Index innerMid = innerBegin + numext::round_down(innerSize >> 1, PacketSize); - Task right = [this, &f, &barrier, outerBegin, outerEnd, innerMid, innerEnd, level]() { - parallelForImpl(outerBegin, outerEnd, innerMid, innerEnd, f, barrier, level); - }; - m_pool.Schedule(std::move(right)); - innerEnd = innerMid; - } - } - for (Index outer = outerBegin; outer < outerEnd; outer++) - for (Index inner = innerBegin; inner < innerEnd; inner += PacketSize) f(outer, inner); - barrier.Notify(); - } - -#undef EIGEN_PARALLEL_FOR_INLINE - - template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void parallelFor(Index begin, Index end, UnaryFunctor& f, float cost) { - Index size = end - begin; - int maxLevel = calculateLevels(size, cost); - Barrier barrier(1 << maxLevel); - parallelForImpl(begin, end, f, barrier, maxLevel); - barrier.Wait(); - } - - template - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void parallelFor(Index outerBegin, Index outerEnd, Index innerBegin, - Index innerEnd, BinaryFunctor& f, float cost) { - Index outerSize = outerEnd - outerBegin; - Index innerSize = innerEnd - innerBegin; - Index size = outerSize * innerSize; - int maxLevel = calculateLevels(size, cost); - Barrier barrier(1 << maxLevel); - parallelForImpl(outerBegin, outerEnd, innerBegin, innerEnd, f, barrier, maxLevel); - barrier.Wait(); - } - - ThreadPool& m_pool; - // costFactor is the cost of delegating a task to a thread - // the inverse is used to avoid a floating point division - float m_costFactor; -}; - -// specialization of coefficient-wise assignment loops for CoreThreadPoolDevice - -namespace internal { - -#ifdef EIGEN_PARSED_BY_DOXYGEN -struct Kernel; -#endif - -template -struct cost_helper { - using SrcEvaluatorType = typename Kernel::SrcEvaluatorType; - using DstEvaluatorType = typename Kernel::DstEvaluatorType; - using SrcXprType = typename SrcEvaluatorType::XprType; - using DstXprType = typename DstEvaluatorType::XprType; - static constexpr Index Cost = functor_cost::Cost + functor_cost::Cost; -}; - -template -struct dense_assignment_loop_with_device { - static constexpr Index XprEvaluationCost = cost_helper::Cost; - struct AssignmentFunctor : public Kernel { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {} - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer, Index inner) { - this->assignCoeffByOuterInner(outer, inner); - } - }; - - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) { - const Index innerSize = kernel.innerSize(); - const Index outerSize = kernel.outerSize(); - constexpr float cost = static_cast(XprEvaluationCost); - AssignmentFunctor functor(kernel); - device.template parallelFor(0, outerSize, 0, innerSize, functor, cost); - } -}; - -template -struct dense_assignment_loop_with_device { - using DstXprType = typename Kernel::DstEvaluatorType::XprType; - static constexpr Index XprEvaluationCost = cost_helper::Cost, InnerSize = DstXprType::InnerSizeAtCompileTime; - struct AssignmentFunctor : public Kernel { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {} - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer) { - copy_using_evaluator_DefaultTraversal_InnerUnrolling::run(*this, outer); - } - }; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) { - const Index outerSize = kernel.outerSize(); - AssignmentFunctor functor(kernel); - constexpr float cost = static_cast(XprEvaluationCost) * static_cast(InnerSize); - device.template parallelFor(0, outerSize, functor, cost); - } -}; - -template -struct dense_assignment_loop_with_device { - using PacketType = typename Kernel::PacketType; - static constexpr Index XprEvaluationCost = cost_helper::Cost, PacketSize = unpacket_traits::size, - SrcAlignment = Kernel::AssignmentTraits::SrcAlignment, - DstAlignment = Kernel::AssignmentTraits::DstAlignment; - struct AssignmentFunctor : public Kernel { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {} - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer, Index inner) { - this->template assignPacketByOuterInner(outer, inner); - } - }; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) { - const Index innerSize = kernel.innerSize(); - const Index outerSize = kernel.outerSize(); - const float cost = static_cast(XprEvaluationCost) * static_cast(innerSize); - AssignmentFunctor functor(kernel); - device.template parallelFor(0, outerSize, 0, innerSize, functor, cost); - } -}; - -template -struct dense_assignment_loop_with_device { - using PacketType = typename Kernel::PacketType; - using DstXprType = typename Kernel::DstEvaluatorType::XprType; - static constexpr Index XprEvaluationCost = cost_helper::Cost, PacketSize = unpacket_traits::size, - SrcAlignment = Kernel::AssignmentTraits::SrcAlignment, - DstAlignment = Kernel::AssignmentTraits::DstAlignment, - InnerSize = DstXprType::InnerSizeAtCompileTime; - struct AssignmentFunctor : public Kernel { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {} - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer) { - copy_using_evaluator_innervec_InnerUnrolling::run(*this, outer); - } - }; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) { - const Index outerSize = kernel.outerSize(); - constexpr float cost = static_cast(XprEvaluationCost) * static_cast(InnerSize); - AssignmentFunctor functor(kernel); - device.template parallelFor(0, outerSize, functor, cost); - } -}; - -template -struct dense_assignment_loop_with_device { - using Scalar = typename Kernel::Scalar; - using PacketType = typename Kernel::PacketType; - static constexpr Index XprEvaluationCost = cost_helper::Cost, PacketSize = unpacket_traits::size; - struct PacketAssignmentFunctor : public Kernel { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketAssignmentFunctor(Kernel& kernel) : Kernel(kernel) {} - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer, Index inner) { - this->template assignPacketByOuterInner(outer, inner); - } - }; - struct ScalarAssignmentFunctor : public Kernel { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ScalarAssignmentFunctor(Kernel& kernel) : Kernel(kernel) {} - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer) { - const Index innerSize = this->innerSize(); - const Index packetAccessSize = numext::round_down(innerSize, PacketSize); - for (Index inner = packetAccessSize; inner < innerSize; inner++) this->assignCoeffByOuterInner(outer, inner); - } - }; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) { - const Index outerSize = kernel.outerSize(); - const Index innerSize = kernel.innerSize(); - const Index packetAccessSize = numext::round_down(innerSize, PacketSize); - constexpr float packetCost = static_cast(XprEvaluationCost); - const float scalarCost = static_cast(XprEvaluationCost) * static_cast(innerSize - packetAccessSize); - PacketAssignmentFunctor packetFunctor(kernel); - ScalarAssignmentFunctor scalarFunctor(kernel); - device.template parallelFor(0, outerSize, 0, packetAccessSize, packetFunctor, - packetCost); - device.template parallelFor(0, outerSize, scalarFunctor, scalarCost); - }; -}; - -template -struct dense_assignment_loop_with_device { - static constexpr Index XprEvaluationCost = cost_helper::Cost; - struct AssignmentFunctor : public Kernel { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {} - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index index) { this->assignCoeff(index); } - }; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) { - const Index size = kernel.size(); - constexpr float cost = static_cast(XprEvaluationCost); - AssignmentFunctor functor(kernel); - device.template parallelFor(0, size, functor, cost); - } -}; - -template -struct dense_assignment_loop_with_device { - using Scalar = typename Kernel::Scalar; - using PacketType = typename Kernel::PacketType; - static constexpr Index XprEvaluationCost = cost_helper::Cost, - RequestedAlignment = Kernel::AssignmentTraits::LinearRequiredAlignment, - PacketSize = unpacket_traits::size, - DstIsAligned = Kernel::AssignmentTraits::DstAlignment >= RequestedAlignment, - DstAlignment = packet_traits::AlignedOnScalar ? RequestedAlignment - : Kernel::AssignmentTraits::DstAlignment, - SrcAlignment = Kernel::AssignmentTraits::JointAlignment; - struct AssignmentFunctor : public Kernel { - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {} - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index index) { - this->template assignPacket(index); - } - }; - static constexpr bool UsePacketSegment = Kernel::AssignmentTraits::UsePacketSegment; - using head_loop = - unaligned_dense_assignment_loop; - using tail_loop = unaligned_dense_assignment_loop; - - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) { - const Index size = kernel.size(); - const Index alignedStart = - DstIsAligned ? 0 : internal::first_aligned(kernel.dstDataPtr(), size); - const Index alignedEnd = alignedStart + numext::round_down(size - alignedStart, PacketSize); - - head_loop::run(kernel, 0, alignedStart); - - constexpr float cost = static_cast(XprEvaluationCost); - AssignmentFunctor functor(kernel); - device.template parallelFor(alignedStart, alignedEnd, functor, cost); - - tail_loop::run(kernel, alignedEnd, size); - } -}; - -} // namespace internal - -} // namespace Eigen - -#endif // EIGEN_CORE_THREAD_POOL_DEVICE_H