Team2890/allwpilib
4
0
Fork 0
mirror of https://github.com/wpilibsuite/allwpilib synced 2026-06-19 00:41:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

[upstream_utils] Use pathlib instead of os.path (#7983)

Browse Source 
A noteworthy change is the replacement of the `dp.startswith(os.path.join(".", "subdir"))` pattern. pathlib doesn't offer something with similar semantics besides `match` and `full_match`, so there's now a helper function that replicates the behavior.

Other notable changes include the addition of type annotations to ensure code correctness, using == to check file names instead of `endswith` for clarity (`endswith` is still used to check extensions), manual walking and copying being refactored in googletest, json, memory, nanopb, protobuf, and sleipnir to use `walk_cwd_and_copy_if`, and matching functions being shortened to the point where they can just be inlined into the lambda.

Co-authored-by: Tyler Veness <calcmogul@gmail.com>
Co-authored-by: David Vo <auscompgeek@users.noreply.github.com>
This commit is contained in:
Gold856
2025-05-29 22:05:22 +00:00
committed by GitHub
parent de718f7ae5
commit ca05ffa1b9
26 changed files with 625 additions and 1045 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
upstream_utils/apriltag.py
View File

@@ -1,9 +1,14 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, comment_out_invalid_includes, walk_cwd_and_copy_if
from upstream_utils import (
Lib,
comment_out_invalid_includes,
has_prefix,
walk_cwd_and_copy_if,
)
def remove_tag(f: str):
@@ -18,47 +23,45 @@ def remove_tag(f: str):
return False
def copy_upstream_src(wpilib_root):
apriltag = os.path.join(wpilib_root, "apriltag")
def copy_upstream_src(wpilib_root: Path):
apriltag = wpilib_root / "apriltag"
# Delete old install
shutil.rmtree(
os.path.join(apriltag, "src/main/native/thirdparty/apriltag"),
apriltag / "src/main/native/thirdparty/apriltag",
ignore_errors=True,
)
shutil.rmtree(
os.path.join(apriltag, "src/main/include/thirdparty/apriltag"),
apriltag / "src/main/include/thirdparty/apriltag",
ignore_errors=True,
)
# Copy apriltag source files into allwpilib
src_files = walk_cwd_and_copy_if(
lambda dp, f: (f.endswith(".c") or f.endswith(".cpp"))
and not dp.startswith(os.path.join(".", "example"))
and not dp.startswith(os.path.join(".", "test"))
and not f.endswith("getopt.c")
and not has_prefix(dp, Path("example"))
and not has_prefix(dp, Path("test"))
and not f == "getopt.c"
and not "py" in f
and not remove_tag(f),
os.path.join(apriltag, "src/main/native/thirdparty/apriltag/src"),
apriltag / "src/main/native/thirdparty/apriltag/src",
)
# Copy apriltag header files into allwpilib
walk_cwd_and_copy_if(
lambda dp, f: f.endswith(".h")
and not f.endswith("getopt.h")
and not f.endswith("postscript_utils.h")
and not f == "getopt.h"
and not f == "postscript_utils.h"
and not remove_tag(f),
os.path.join(apriltag, "src/main/native/thirdparty/apriltag/include"),
apriltag / "src/main/native/thirdparty/apriltag/include",
)
for f in src_files:
comment_out_invalid_includes(
f,
[
os.path.join(apriltag, "src/main/native/thirdparty/apriltag/include"),
os.path.join(
apriltag, "src/main/native/thirdparty/apriltag/include/common"
),
apriltag / "src/main/native/thirdparty/apriltag/include",
apriltag / "src/main/native/thirdparty/apriltag/include/common",
],
)

11
upstream_utils/argparse_lib.py
View File

@@ -1,18 +1,17 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
def copy_upstream_src(wpilib_root: Path):
wpiutil = wpilib_root / "wpiutil"
# Copy header into allwpilib
dest_filename = os.path.join(
wpiutil,
f"src/main/native/thirdparty/argparse/include/wpi/argparse.h",
dest_filename = (
wpiutil / f"src/main/native/thirdparty/argparse/include/wpi/argparse.h"
)
shutil.copyfile("include/argparse/argparse.hpp", dest_filename)
# Rename namespace from argparse to wpi

15
upstream_utils/debugging.py
View File

@@ -1,26 +1,25 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
def copy_upstream_src(wpilib_root: Path):
wpiutil = wpilib_root / "wpiutil"
# Delete old install
for d in [
"src/main/native/thirdparty/debugging/src",
"src/main/native/thirdparty/debugging/include",
]:
shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
shutil.rmtree(wpiutil / d, ignore_errors=True)
# Copy debugging files into allwpilib
filenames = walk_cwd_and_copy_if(
lambda dp, f: dp.startswith(os.path.join(".", "src"))
or dp.startswith(os.path.join(".", "include")),
os.path.join(wpiutil, "src/main/native/thirdparty/debugging"),
lambda dp, f: has_prefix(dp, Path("src")) or has_prefix(dp, Path("include")),
wpiutil / "src/main/native/thirdparty/debugging",
)
for filename in filenames:

57
upstream_utils/eigen.py
View File

@@ -1,13 +1,17 @@
#!/usr/bin/env python3
import os
import re
import shutil
from pathlib import Path
from upstream_utils import Lib, comment_out_invalid_includes, walk_cwd_and_copy_if
from upstream_utils import (
Lib,
comment_out_invalid_includes,
has_prefix,
walk_cwd_and_copy_if,
)
def eigen_inclusions(dp, f):
def eigen_inclusions(dp: Path, f: str):
"""Returns true if the given file in the "Eigen" include directory of the
Eigen git repo should be copied into allwpilib
@@ -15,11 +19,9 @@ def eigen_inclusions(dp, f):
dp -- directory path
f -- filename
"""
if not dp.startswith(os.path.join(".", "Eigen")):
if not has_prefix(dp, Path("Eigen")):
return False
abspath = os.path.join(dp, f)
# Exclude NonMPL2.h since all non-MPL2 code will be excluded anyway
if f == "NonMPL2.h":
return False
@@ -36,13 +38,13 @@ def eigen_inclusions(dp, f):
if "MKL" in f:
return False
# Include architectures we care about
if "Core/arch/" in abspath:
# Include architectures we care about by filtering for Core/arch
if "Core" in dp.parts and "arch" in dp.parts:
return (
"arch/AVX/" in abspath
or "arch/Default" in abspath
or "arch/NEON" in abspath
or "arch/SSE" in abspath
"AVX" in dp.parts
or "Default" in dp.parts
or "NEON" in dp.parts
or "SSE" in dp.parts
)
# Include the following modules
@@ -65,10 +67,13 @@ def eigen_inclusions(dp, f):
"misc",
"plugins",
]
return bool(re.search(r"|".join("/" + m for m in modules), abspath))
for m in modules:
if m in dp.parts or f == m:
return True
return False
def unsupported_inclusions(dp, f):
def unsupported_inclusions(dp: Path, f: str):
"""Returns true if the given file in the "unsupported" include directory of
the Eigen git repo should be copied into allwpilib
@@ -76,50 +81,50 @@ def unsupported_inclusions(dp, f):
dp -- directory path
f -- filename
"""
if not dp.startswith(os.path.join(".", "unsupported")):
if not has_prefix(dp, Path("unsupported")):
return False
abspath = os.path.join(dp, f)
abspath = dp / f
# Exclude build system and READMEs
if f == "CMakeLists.txt" or "README" in f:
return False
# Include the MatrixFunctions module
return "MatrixFunctions" in abspath
return "MatrixFunctions" in abspath.parts
def copy_upstream_src(wpilib_root):
wpimath = os.path.join(wpilib_root, "wpimath")
def copy_upstream_src(wpilib_root: Path):
wpimath = wpilib_root / "wpimath"
# Delete old install
for d in ["src/main/native/thirdparty/eigen/include"]:
shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True)
shutil.rmtree(wpimath / d, ignore_errors=True)
# Copy Eigen headers into allwpilib
eigen_files = walk_cwd_and_copy_if(
eigen_inclusions,
os.path.join(wpimath, "src/main/native/thirdparty/eigen/include"),
wpimath / "src/main/native/thirdparty/eigen/include",
)
# Copy unsupported headers into allwpilib
unsupported_files = walk_cwd_and_copy_if(
unsupported_inclusions,
os.path.join(wpimath, "src/main/native/thirdparty/eigen/include"),
wpimath / "src/main/native/thirdparty/eigen/include",
)
for f in eigen_files:
comment_out_invalid_includes(
f, [os.path.join(wpimath, "src/main/native/thirdparty/eigen/include")]
f, [wpimath / "src/main/native/thirdparty/eigen/include"]
)
for f in unsupported_files:
comment_out_invalid_includes(
f, [os.path.join(wpimath, "src/main/native/thirdparty/eigen/include")]
f, [wpimath / "src/main/native/thirdparty/eigen/include"]
)
shutil.copyfile(
".clang-format",
os.path.join(wpimath, "src/main/native/thirdparty/eigen/include/.clang-format"),
wpimath / "src/main/native/thirdparty/eigen/include/.clang-format",
)

10
upstream_utils/expected.py
View File

@@ -1,18 +1,16 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
def copy_upstream_src(wpilib_root: Path):
wpiutil = wpilib_root / "wpiutil"
# Copy expected header into allwpilib
dest_filename = os.path.join(
wpiutil, "src/main/native/thirdparty/expected/include/wpi/expected"
)
dest_filename = wpiutil / "src/main/native/thirdparty/expected/include/wpi/expected"
shutil.copyfile("include/tl/expected.hpp", dest_filename)
# Rename namespace from tl to wpi, and detail to detail_expected

18
upstream_utils/fmt.py
View File

@@ -1,33 +1,33 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
def copy_upstream_src(wpilib_root: Path):
wpiutil = wpilib_root / "wpiutil"
# Delete old install
for d in [
"src/main/native/thirdparty/fmtlib/src",
"src/main/native/thirdparty/fmtlib/include",
]:
shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
shutil.rmtree(wpiutil / d, ignore_errors=True)
# Copy fmt source files into allwpilib
walk_cwd_and_copy_if(
lambda dp, f: dp.startswith(os.path.join(".", "src"))
lambda dp, f: has_prefix(dp, Path("src"))
and f.endswith(".cc")
and f != "fmt.cc",
os.path.join(wpiutil, "src/main/native/thirdparty/fmtlib"),
wpiutil / "src/main/native/thirdparty/fmtlib",
)
# Copy fmt header files into allwpilib
walk_cwd_and_copy_if(
lambda dp, f: dp.startswith(os.path.join(".", "include", "fmt")),
os.path.join(wpiutil, "src/main/native/thirdparty/fmtlib"),
lambda dp, f: has_prefix(dp, Path("include/fmt")),
wpiutil / "src/main/native/thirdparty/fmtlib",
)

14
upstream_utils/gcem.py
View File

@@ -1,24 +1,24 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if
def copy_upstream_src(wpilib_root):
wpimath = os.path.join(wpilib_root, "wpimath")
def copy_upstream_src(wpilib_root: Path):
wpimath = wpilib_root / "wpimath"
# Delete old install
for d in [
"src/main/native/thirdparty/gcem/include",
]:
shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True)
shutil.rmtree(wpimath / d, ignore_errors=True)
# Copy gcem include files into allwpilib
walk_cwd_and_copy_if(
lambda dp, f: dp.startswith(os.path.join(".", "include")),
os.path.join(wpimath, "src/main/native/thirdparty/gcem"),
lambda dp, f: has_prefix(dp, Path("include")),
wpimath / "src/main/native/thirdparty/gcem",
)

8
upstream_utils/gl3w.py
View File

@@ -1,16 +1,16 @@
#!/usr/bin/env python3
import os
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
def copy_upstream_src(wpilib_root):
gl3w = os.path.join(wpilib_root, "thirdparty", "imgui_suite", "gl3w")
def copy_upstream_src(wpilib_root: Path):
gl3w = wpilib_root / "thirdparty/imgui_suite/gl3w"
walk_cwd_and_copy_if(
lambda dp, f: f == "gl3w_gen.py",
os.path.join(gl3w),
gl3w,
)

48
upstream_utils/glfw.py
View File

@@ -1,57 +1,47 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if
def matches(dp, f, allowed_files):
path = os.path.join(dp, f)
return path in allowed_files
def walk_and_copy_if_matches(allowed_files, output_directory):
walk_cwd_and_copy_if(
lambda dp, f: matches(dp, f, allowed_files),
output_directory,
)
def copy_upstream_src(wpilib_root):
glfw = os.path.join(wpilib_root, "thirdparty", "imgui_suite", "glfw")
def copy_upstream_src(wpilib_root: Path):
glfw = wpilib_root / "thirdparty/imgui_suite/glfw"
# Delete old install
for d in ["include", "src", "CMake"]:
shutil.rmtree(os.path.join(glfw, d), ignore_errors=True)
shutil.rmtree(glfw / d, ignore_errors=True)
hdr_allow_list = [
"./include/GLFW/glfw3.h",
"./include/GLFW/glfw3native.h",
Path("include/GLFW/glfw3.h"),
Path("include/GLFW/glfw3native.h"),
]
walk_and_copy_if_matches(hdr_allow_list, glfw)
walk_cwd_and_copy_if(
lambda dp, f: dp / f in hdr_allow_list,
glfw,
)
def src_filter(dp, f):
if f.endswith("CMakeLists.txt"):
def src_filter(dp: Path, f: str):
if f == "CMakeLists.txt":
return False
if dp.startswith(os.path.join(".", "src")):
if has_prefix(dp, Path("src")):
return True
return False
src_files = walk_cwd_and_copy_if(
src_filter,
os.path.join(glfw),
glfw,
)
def cmake_filter(dp, f):
if dp.startswith(os.path.join(".", "CMake")):
def cmake_filter(dp: Path, f: str):
if has_prefix(dp, Path("CMake")):
return True
path = os.path.join(dp, f)
if path in ["./src/CMakeLists.txt", "./CMakeLists.txt"]:
if dp / f in [Path("src/CMakeLists.txt"), Path("CMakeLists.txt")]:
return True
return False
@@ -59,7 +49,7 @@ def copy_upstream_src(wpilib_root):
# Copy CMAKE files
walk_cwd_and_copy_if(
cmake_filter,
os.path.join(glfw),
glfw,
)

49
upstream_utils/googletest.py
View File

@@ -2,8 +2,9 @@
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if, walk_if
from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if
EXCLUDED_FILES = [
"gtest_main.cc",
@@ -14,49 +15,39 @@ EXCLUDED_FILES = [
]
def walk_and_remap_copy(third_party_root, include_prefix):
gmock_files = walk_if(
".", lambda dp, f: include_prefix in dp and f not in EXCLUDED_FILES
)
for f in gmock_files:
dst_file = os.path.join(
third_party_root, "include", f[len(include_prefix) + 1 :]
)
dest_dir = os.path.dirname(dst_file)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
shutil.copyfile(f, dst_file)
def copy_upstream_src(wpilib_root):
third_party_root = os.path.join(wpilib_root, "thirdparty/googletest")
def copy_upstream_src(wpilib_root: Path):
upstream_root = Path(".").absolute()
third_party_root = wpilib_root / "thirdparty/googletest"
# Delete old install
for d in [
"include",
"src",
]:
shutil.rmtree(os.path.join(third_party_root, d), ignore_errors=True)
shutil.rmtree(third_party_root / d, ignore_errors=True)
walk_cwd_and_copy_if(
lambda dp, f: "googlemock/src" in dp and f not in EXCLUDED_FILES,
os.path.join(third_party_root, "src"),
lambda dp, f: has_prefix(dp, Path("googlemock/src"))
and f not in EXCLUDED_FILES,
third_party_root / "src",
)
walk_cwd_and_copy_if(
lambda dp, f: "googletest/src" in dp and f not in EXCLUDED_FILES,
os.path.join(third_party_root, "src"),
lambda dp, f: has_prefix(dp, Path("googletest/src"))
and f not in EXCLUDED_FILES,
third_party_root / "src",
)
walk_and_remap_copy(
third_party_root,
"./googlemock/include",
os.chdir(upstream_root / "googlemock/include")
walk_cwd_and_copy_if(
lambda dp, f: f not in EXCLUDED_FILES,
third_party_root / "include",
)
walk_and_remap_copy(
third_party_root,
"./googletest/include",
os.chdir(upstream_root / "googletest/include")
walk_cwd_and_copy_if(
lambda dp, f: f not in EXCLUDED_FILES,
third_party_root / "include",
)

69
upstream_utils/imgui.py
View File

@@ -1,62 +1,57 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
def matches(dp, f, allowed_files):
path = os.path.join(dp, f)
return path in allowed_files
def walk_and_copy_if_matches(allowed_files, output_directory):
def walk_and_copy_if_matches(allowed_files: list[Path], output_directory: Path):
walk_cwd_and_copy_if(
lambda dp, f: matches(dp, f, allowed_files),
lambda dp, f: dp / f in allowed_files,
output_directory,
)
def copy_upstream_src(wpilib_root):
imgui = os.path.join(wpilib_root, "thirdparty", "imgui_suite", "imgui")
def copy_upstream_src(wpilib_root: Path):
imgui = wpilib_root / "thirdparty/imgui_suite/imgui"
# Delete old install
for d in ["include", "cpp"]:
shutil.rmtree(os.path.join(imgui, d), ignore_errors=True)
shutil.rmtree(imgui / d, ignore_errors=True)
hdr_allow_list = [
"./imgui.h",
"./imstb_truetype.h",
"./imgui_internal.h",
"./imstb_rectpack.h",
"./imconfig.h",
"./imstb_textedit.h",
"./backends/imgui_impl_glfw.h",
"./backends/imgui_impl_metal.h",
"./backends/imgui_impl_opengl3.h",
"./backends/imgui_impl_dx11.h",
"./backends/imgui_impl_opengl3_loader.h",
"./backends/imgui_impl_opengl2.h",
"./misc/cpp/imgui_stdlib.h",
Path("imgui.h"),
Path("imstb_truetype.h"),
Path("imgui_internal.h"),
Path("imstb_rectpack.h"),
Path("imconfig.h"),
Path("imstb_textedit.h"),
Path("backends/imgui_impl_glfw.h"),
Path("backends/imgui_impl_metal.h"),
Path("backends/imgui_impl_opengl3.h"),
Path("backends/imgui_impl_dx11.h"),
Path("backends/imgui_impl_opengl3_loader.h"),
Path("backends/imgui_impl_opengl2.h"),
Path("misc/cpp/imgui_stdlib.h"),
]
src_allow_list = [
"./backends/imgui_impl_dx11.cpp",
"./backends/imgui_impl_glfw.cpp",
"./backends/imgui_impl_metal.mm",
"./backends/imgui_impl_opengl2.cpp",
"./backends/imgui_impl_opengl3.cpp",
"./imgui.cpp",
"./imgui_demo.cpp",
"./imgui_draw.cpp",
"./imgui_tables.cpp",
"./imgui_widgets.cpp",
"./misc/cpp/imgui_stdlib.cpp",
Path("backends/imgui_impl_dx11.cpp"),
Path("backends/imgui_impl_glfw.cpp"),
Path("backends/imgui_impl_metal.mm"),
Path("backends/imgui_impl_opengl2.cpp"),
Path("backends/imgui_impl_opengl3.cpp"),
Path("imgui.cpp"),
Path("imgui_demo.cpp"),
Path("imgui_draw.cpp"),
Path("imgui_tables.cpp"),
Path("imgui_widgets.cpp"),
Path("misc/cpp/imgui_stdlib.cpp"),
]
walk_and_copy_if_matches(hdr_allow_list, os.path.join(imgui, "include"))
walk_and_copy_if_matches(src_allow_list, os.path.join(imgui, "cpp"))
walk_and_copy_if_matches(hdr_allow_list, imgui / "include")
walk_and_copy_if_matches(src_allow_list, imgui / "cpp")
def main():

28
upstream_utils/implot.py
View File

@@ -1,37 +1,27 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
def matches(dp, f, allowed_files):
path = os.path.join(dp, f)
return path in allowed_files
def walk_and_copy_if_matches(allowed_files, output_directory):
walk_cwd_and_copy_if(
lambda dp, f: matches(dp, f, allowed_files),
output_directory,
)
def copy_upstream_src(wpilib_root):
implot = os.path.join(wpilib_root, "thirdparty", "imgui_suite", "implot")
def copy_upstream_src(wpilib_root: Path):
implot = wpilib_root / "thirdparty/imgui_suite/implot"
# Delete old install
for d in ["include", "cpp"]:
shutil.rmtree(os.path.join(implot, d), ignore_errors=True)
shutil.rmtree(implot / d, ignore_errors=True)
# Copy files
walk_cwd_and_copy_if(
lambda dp, f: f.endswith(".h"),
os.path.join(implot, "include"),
implot / "include",
)
walk_and_copy_if_matches(
["./implot_items.cpp", "./implot.cpp"], os.path.join(implot, "cpp")
implot_files = [Path("implot_items.cpp"), Path("implot.cpp")]
walk_cwd_and_copy_if(
lambda dp, f: dp / f in implot_files,
implot / "cpp",
)

25
upstream_utils/json.py
View File

@@ -2,35 +2,32 @@
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_if
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
def copy_upstream_src(wpilib_root: Path):
wpiutil = wpilib_root / "wpiutil"
# Delete old install
for d in [
"src/main/native/thirdparty/json/include",
]:
shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
shutil.rmtree(wpiutil / d, ignore_errors=True)
# Create lists of source and destination files
os.chdir("include/nlohmann")
files = walk_if(".", lambda dp, f: True)
src_include_files = [os.path.abspath(f) for f in files]
wpiutil_json_root = os.path.join(
wpiutil, "src/main/native/thirdparty/json/include/wpi"
)
dest_include_files = [
os.path.join(wpiutil_json_root, f.replace(".hpp", ".h")) for f in files
]
files = walk_if(Path("."), lambda dp, f: True)
src_include_files = [f.absolute() for f in files]
wpiutil_json_root = wpiutil / "src/main/native/thirdparty/json/include/wpi"
dest_include_files = [wpiutil_json_root / f.with_suffix(".h") for f in files]
# Copy json header files into allwpilib
for i in range(len(src_include_files)):
dest_dir = os.path.dirname(dest_include_files[i])
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
dest_dir = dest_include_files[i].parent
if not dest_dir.exists():
dest_dir.mkdir(parents=True)
shutil.copyfile(src_include_files[i], dest_include_files[i])
for include_file in dest_include_files:

16
upstream_utils/libdogleg.py
View File

@@ -1,24 +1,24 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
def copy_upstream_src(wpilib_root):
wpical = os.path.join(wpilib_root, "wpical")
def copy_upstream_src(wpilib_root: Path):
wpical = wpilib_root / "wpical"
# Delete old install
for d in [
"src/main/native/thirdparty/libdogleg/src",
"src/main/native/thirdparty/libdogleg/include",
]:
shutil.rmtree(os.path.join(wpical, d), ignore_errors=True)
shutil.rmtree(wpical / d, ignore_errors=True)
files = walk_cwd_and_copy_if(
lambda dp, f: f.endswith("dogleg.h"),
os.path.join(wpical, "src/main/native/thirdparty/libdogleg/include"),
lambda dp, f: f == "dogleg.h",
wpical / "src/main/native/thirdparty/libdogleg/include",
)
for f in files:
with open(f) as file:
@@ -30,8 +30,8 @@ def copy_upstream_src(wpilib_root):
file.write(content)
files = walk_cwd_and_copy_if(
lambda dp, f: f.endswith("dogleg.cpp"),
os.path.join(wpical, "src/main/native/thirdparty/libdogleg/src"),
lambda dp, f: f == "dogleg.cpp",
wpical / "src/main/native/thirdparty/libdogleg/src",
)
for f in files:
with open(f) as file:

20
upstream_utils/libuv.py
View File

@@ -1,17 +1,17 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if
def copy_upstream_src(wpilib_root):
wpinet = os.path.join(wpilib_root, "wpinet")
def copy_upstream_src(wpilib_root: Path):
wpinet = wpilib_root / "wpinet"
# Delete old install
for d in ["src/main/native/thirdparty/libuv"]:
shutil.rmtree(os.path.join(wpinet, d), ignore_errors=True)
shutil.rmtree(wpinet / d, ignore_errors=True)
include_ignorelist = [
"aix.h",
@@ -21,9 +21,8 @@ def copy_upstream_src(wpilib_root):
]
walk_cwd_and_copy_if(
lambda dp, f: dp.startswith(os.path.join(".", "include"))
and f not in include_ignorelist,
os.path.join(wpinet, "src/main/native/thirdparty/libuv"),
lambda dp, f: has_prefix(dp, Path("include")) and f not in include_ignorelist,
wpinet / "src/main/native/thirdparty/libuv",
)
src_ignorelist = [
@@ -43,9 +42,8 @@ def copy_upstream_src(wpilib_root):
"sysinfo-memory.c",
]
walk_cwd_and_copy_if(
lambda dp, f: dp.startswith(os.path.join(".", "src"))
and f not in src_ignorelist,
os.path.join(wpinet, "src/main/native/thirdparty/libuv"),
lambda dp, f: has_prefix(dp, Path("src")) and f not in src_ignorelist,
wpinet / "src/main/native/thirdparty/libuv",
rename_c_to_cpp=True,
)

52
upstream_utils/llvm.py
View File

@@ -1,12 +1,12 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib
def run_global_replacements(wpiutil_llvm_files):
def run_global_replacements(wpiutil_llvm_files: list[Path]):
for wpi_file in wpiutil_llvm_files:
with open(wpi_file) as f:
content = f.read()
@@ -23,7 +23,7 @@ def run_global_replacements(wpiutil_llvm_files):
# Fix uses of span
content = content.replace("span", "std::span")
content = content.replace("include <std::span>", "include <span>")
if wpi_file.endswith("ConvertUTFWrapper.cpp"):
if wpi_file.name == "ConvertUTFWrapper.cpp":
content = content.replace(
"const UTF16 *Src = reinterpret_cast<const UTF16 *>(SrcBytes.begin());",
"const UTF16 *Src = reinterpret_cast<const UTF16 *>(&*SrcBytes.begin());",
@@ -88,19 +88,19 @@ def run_global_replacements(wpiutil_llvm_files):
f.write(content)
def flattened_llvm_files(llvm, dirs_to_keep):
file_lookup = {}
def flattened_llvm_files(llvm: Path, dirs_to_keep: list[Path]):
file_lookup: dict[str, Path] = {}
for dir_to_keep in dirs_to_keep:
dir_to_crawl = os.path.join(llvm, dir_to_keep)
for root, _, files in os.walk(dir_to_crawl):
dir_to_crawl = llvm / dir_to_keep
for root, _, files in dir_to_crawl.walk():
for f in files:
file_lookup[f] = os.path.join(root, f)
file_lookup[f] = root / f
return file_lookup
def find_wpiutil_llvm_files(wpiutil_root, subfolder):
def find_wpiutil_llvm_files(wpiutil_root: Path, subfolder: str):
# These files have substantial changes, not worth managing with the patching process
ignore_list = [
"StringExtras.h",
@@ -110,22 +110,22 @@ def find_wpiutil_llvm_files(wpiutil_root, subfolder):
"SmallVectorMemoryBuffer.h",
]
wpiutil_files = []
for root, _, files in os.walk(os.path.join(wpiutil_root, subfolder)):
wpiutil_files: list[Path] = []
for root, _, files in (wpiutil_root / subfolder).walk():
for f in files:
if f not in ignore_list:
full_file = os.path.join(root, f)
full_file = root / f
wpiutil_files.append(full_file)
return wpiutil_files
def overwrite_files(wpiutil_files, llvm_files):
def overwrite_files(wpiutil_files: list[Path], llvm_files: dict[str, Path]):
# Very sparse rips from LLVM sources. Not worth tyring to make match upstream
unmatched_files_whitelist = ["fs.h", "fs.cpp", "function_ref.h"]
for wpi_file in wpiutil_files:
wpi_base_name = os.path.basename(wpi_file)
wpi_base_name = wpi_file.name
if wpi_base_name in llvm_files:
shutil.copyfile(llvm_files[wpi_base_name], wpi_file)
@@ -133,14 +133,14 @@ def overwrite_files(wpiutil_files, llvm_files):
print(f"No file match for {wpi_file}, check if LLVM deleted it")
def overwrite_source(wpiutil_root, llvm_root):
def overwrite_source(wpiutil_root: Path, llvm_root: Path):
llvm_files = flattened_llvm_files(
llvm_root,
[
"llvm/include/llvm/ADT/",
"llvm/include/llvm/Config",
"llvm/include/llvm/Support/",
"llvm/lib/Support/",
Path("llvm/include/llvm/ADT/"),
Path("llvm/include/llvm/Config/"),
Path("llvm/include/llvm/Support/"),
Path("llvm/lib/Support/"),
],
)
wpi_files = find_wpiutil_llvm_files(
@@ -153,10 +153,14 @@ def overwrite_source(wpiutil_root, llvm_root):
run_global_replacements(wpi_files)
def overwrite_tests(wpiutil_root, llvm_root):
def overwrite_tests(wpiutil_root: Path, llvm_root: Path):
llvm_files = flattened_llvm_files(
llvm_root,
["llvm/unittests/ADT/", "llvm/unittests/Config", "llvm/unittests/Support/"],
[
Path("llvm/unittests/ADT/"),
Path("llvm/unittests/Config/"),
Path("llvm/unittests/Support/"),
],
)
wpi_files = find_wpiutil_llvm_files(wpiutil_root, "src/test/native/cpp/llvm")
@@ -164,9 +168,9 @@ def overwrite_tests(wpiutil_root, llvm_root):
run_global_replacements(wpi_files)
def copy_upstream_src(wpilib_root):
upstream_root = os.path.abspath(".")
wpiutil = os.path.join(wpilib_root, "wpiutil")
def copy_upstream_src(wpilib_root: Path):
upstream_root = Path(".").absolute()
wpiutil = wpilib_root / "wpiutil"
overwrite_source(wpiutil, upstream_root)
overwrite_tests(wpiutil, upstream_root)

44
upstream_utils/memory.py
View File

@@ -2,11 +2,12 @@
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, copy_to, walk_if
from upstream_utils import Lib, walk_cwd_and_copy_if
def run_source_replacements(memory_files):
def run_source_replacements(memory_files: list[Path]):
for wpi_file in memory_files:
with open(wpi_file) as f:
content = f.read()
@@ -21,9 +22,9 @@ def run_source_replacements(memory_files):
f.write(content)
def run_header_replacements(memory_files):
def run_header_replacements(memory_files: list[Path]):
for wpi_file in memory_files:
if "detail" not in wpi_file:
if "detail" not in wpi_file.parts:
continue
with open(wpi_file) as f:
content = f.read()
@@ -35,7 +36,7 @@ def run_header_replacements(memory_files):
f.write(content)
def run_global_replacements(memory_files):
def run_global_replacements(memory_files: list[Path]):
for wpi_file in memory_files:
with open(wpi_file) as f:
content = f.read()
@@ -52,42 +53,41 @@ def run_global_replacements(memory_files):
f.write(content)
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
def copy_upstream_src(wpilib_root: Path):
upstream_root = Path(".").absolute()
wpiutil = wpilib_root / "wpiutil"
# Delete old install
for d in [
"src/main/native/thirdparty/memory/src",
"src/main/native/thirdparty/memory/include",
]:
shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
shutil.rmtree(wpiutil / d, ignore_errors=True)
# Copy sources
src_files = walk_if("src", lambda dp, f: f.endswith(".cpp") or f.endswith(".hpp"))
src_files = copy_to(
src_files, os.path.join(wpiutil, "src/main/native/thirdparty/memory")
os.chdir(upstream_root / "src")
src_files = walk_cwd_and_copy_if(
lambda dp, f: f.endswith(".cpp") or f.endswith(".hpp"),
wpiutil / "src/main/native/thirdparty/memory/src",
)
run_global_replacements(src_files)
run_source_replacements(src_files)
# Copy headers
os.chdir(os.path.join("include", "foonathan"))
include_files = walk_if(".", lambda dp, f: f.endswith(".hpp"))
include_files = copy_to(
include_files,
os.path.join(wpiutil, "src/main/native/thirdparty/memory/include/wpi"),
os.chdir(upstream_root / "include/foonathan")
include_files = walk_cwd_and_copy_if(
lambda dp, f: f.endswith(".hpp"),
wpiutil / "src/main/native/thirdparty/memory/include/wpi",
)
os.chdir(os.path.join("..", ".."))
os.chdir(upstream_root)
run_global_replacements(include_files)
run_header_replacements(include_files)
# Copy config_impl.hpp
shutil.copyfile(
os.path.join(wpilib_root, "upstream_utils/memory_files/config_impl.hpp"),
os.path.join(
wpiutil,
"src/main/native/thirdparty/memory/include/wpi/memory/config_impl.hpp",
),
wpilib_root / "upstream_utils/memory_files/config_impl.hpp",
wpiutil
/ "src/main/native/thirdparty/memory/include/wpi/memory/config_impl.hpp",
)

16
upstream_utils/mpack.py
View File

@@ -3,36 +3,34 @@
import os
import shutil
import subprocess
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
def copy_upstream_src(wpilib_root: Path):
wpiutil = wpilib_root / "wpiutil"
# Delete old install
for d in [
"src/main/native/thirdparty/mpack/src",
"src/main/native/thirdparty/mpack/include",
]:
shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
shutil.rmtree(wpiutil / d, ignore_errors=True)
# Run the amalgmation script
subprocess.check_call(["bash", "tools/amalgamate.sh"])
# Copy the files
amalgamation_source_dir = os.path.join(
".", ".build", "amalgamation", "src", "mpack"
)
os.chdir(amalgamation_source_dir)
os.chdir(Path(".build/amalgamation/src/mpack"))
walk_cwd_and_copy_if(
lambda dp, f: f.endswith(".h"),
os.path.join(wpiutil, "src/main/native/thirdparty/mpack/include/wpi"),
wpiutil / "src/main/native/thirdparty/mpack/include/wpi",
)
walk_cwd_and_copy_if(
lambda dp, f: f.endswith(".c"),
os.path.join(wpiutil, "src/main/native/thirdparty/mpack/src"),
wpiutil / "src/main/native/thirdparty/mpack/src",
rename_c_to_cpp=True,
)

38
upstream_utils/mrcal.py
View File

@@ -1,27 +1,27 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if
def copy_upstream_src(wpilib_root):
wpical = os.path.join(wpilib_root, "wpical")
def copy_upstream_src(wpilib_root: Path):
wpical = wpilib_root / "wpical"
# Delete old install
for d in [
"src/main/native/thirdparty/mrcal/src",
"src/main/native/thirdparty/mrcal/include",
]:
shutil.rmtree(os.path.join(wpical, d), ignore_errors=True)
shutil.rmtree(wpical / d, ignore_errors=True)
files = walk_cwd_and_copy_if(
lambda dp, f: (f.endswith(".h") or f.endswith(".hh"))
and not f.endswith("heap.h")
and not f.endswith("stereo-matching-libelas.h")
and not dp.startswith(os.path.join(".", "test")),
os.path.join(wpical, "src/main/native/thirdparty/mrcal/include"),
and not f == "heap.h"
and not f == "stereo-matching-libelas.h"
and not has_prefix(dp, Path("test")),
wpical / "src/main/native/thirdparty/mrcal/include",
)
files = files + walk_cwd_and_copy_if(
lambda dp, f: (
@@ -30,16 +30,16 @@ def copy_upstream_src(wpilib_root):
or f.endswith(".cpp")
or f.endswith(".pl")
)
and not f.endswith("heap.cc")
and not f.endswith("mrcal-pywrap.c")
and not f.endswith("image.c")
and not f.endswith("stereo.c")
and not f.endswith("stereo-matching-libelas.cc")
and not f.endswith("uncertainty.c")
and not f.endswith("traverse-sensor-links.c")
and not dp.startswith(os.path.join(".", "doc"))
and not dp.startswith(os.path.join(".", "test")),
os.path.join(wpical, "src/main/native/thirdparty/mrcal/src"),
and not f == "heap.cc"
and not f == "mrcal-pywrap.c"
and not f == "image.c"
and not f == "stereo.c"
and not f == "stereo-matching-libelas.cc"
and not f == "uncertainty.c"
and not f == "traverse-sensor-links.c"
and not has_prefix(dp, Path("doc"))
and not has_prefix(dp, Path("test")),
wpical / "src/main/native/thirdparty/mrcal/src",
)
for f in files:

32
upstream_utils/mrcal_java.py
View File

@@ -2,46 +2,30 @@
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
def delete_lines_by_range(file_path, start_line, end_line):
# Read all lines from the file
with open(file_path, "r") as file:
lines = file.readlines()
# Filter out lines that are within the specified range
filtered_lines = [
line
for i, line in enumerate(lines, start=1)
if not (start_line <= i <= end_line)
]
# Write the remaining lines back to the file
with open(file_path, "w") as file:
file.writelines(filtered_lines)
def copy_upstream_src(wpilib_root):
wpical = os.path.join(wpilib_root, "wpical")
def copy_upstream_src(wpilib_root: Path):
wpical = wpilib_root / "wpical"
# Delete old install
for d in [
"src/main/native/thirdparty/mrcal_java/src",
"src/main/native/thirdparty/mrcal_java/include",
]:
shutil.rmtree(os.path.join(wpical, d), ignore_errors=True)
shutil.rmtree(wpical / d, ignore_errors=True)
os.chdir("src")
files = walk_cwd_and_copy_if(
lambda dp, f: f.endswith("mrcal_wrapper.h"),
os.path.join(wpical, "src/main/native/thirdparty/mrcal_java/include"),
lambda dp, f: f == "mrcal_wrapper.h",
wpical / "src/main/native/thirdparty/mrcal_java/include",
)
files = walk_cwd_and_copy_if(
lambda dp, f: f.endswith("mrcal_wrapper.cpp"),
os.path.join(wpical, "src/main/native/thirdparty/mrcal_java/src"),
lambda dp, f: f == "mrcal_wrapper.cpp",
wpical / "src/main/native/thirdparty/mrcal_java/src",
)
for f in files:

61
upstream_utils/nanopb.py
View File

@@ -1,39 +1,33 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, copy_to, walk_if
from upstream_utils import Lib, copy_to, has_prefix, walk_cwd_and_copy_if
nanopb_sources = set(
[
"pb_encode.c",
"pb_decode.c",
"pb_common.c",
]
)
nanopb_sources = [
Path("pb_encode.c"),
Path("pb_decode.c"),
Path("pb_common.c"),
]
nanopb_headers = set(
[
"pb.h",
"pb_encode.h",
"pb_decode.h",
"pb_common.h",
]
)
nanopb_headers = [
Path("pb.h"),
Path("pb_encode.h"),
Path("pb_decode.h"),
Path("pb_common.h"),
]
nanopb_generator = set(
[
"pb.h",
"pb_encode.h",
"pb_decode.h",
"pb_common.h",
]
)
nanopb_generator = [
Path("pb.h"),
Path("pb_encode.h"),
Path("pb_decode.h"),
Path("pb_common.h"),
]
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
def copy_upstream_src(wpilib_root: Path):
wpiutil = wpilib_root / "wpiutil"
# Delete old install
for d in [
@@ -41,25 +35,24 @@ def copy_upstream_src(wpilib_root):
"src/main/native/thirdparty/nanopb/include",
"src/main/native/thirdparty/nanopb/generator",
]:
shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
shutil.rmtree(wpiutil / d, ignore_errors=True)
# Copy nanopb source files into allwpilib
copy_to(
nanopb_sources,
os.path.join(wpiutil, "src/main/native/thirdparty/nanopb/src"),
wpiutil / "src/main/native/thirdparty/nanopb/src",
rename_c_to_cpp=True,
)
# Copy nanopb header files into allwpilib
copy_to(
nanopb_headers,
os.path.join(wpiutil, "src/main/native/thirdparty/nanopb/include"),
wpiutil / "src/main/native/thirdparty/nanopb/include",
)
generator_files = walk_if("generator", lambda dp, f: True)
copy_to(
generator_files,
os.path.join(wpiutil, "src/main/native/thirdparty/nanopb"),
generator_files = walk_cwd_and_copy_if(
lambda dp, f: has_prefix(dp, Path("generator")),
wpiutil / "src/main/native/thirdparty/nanopb",
)

477
upstream_utils/protobuf.py
View File

@@ -2,237 +2,227 @@
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, copy_to, walk_if
from upstream_utils import Lib, walk_cwd_and_copy_if
protobuf_lite_sources = set(
[
"google/protobuf/any_lite.cc",
"google/protobuf/arena.cc",
"google/protobuf/arenastring.cc",
"google/protobuf/arenaz_sampler.cc",
"google/protobuf/extension_set.cc",
"google/protobuf/generated_enum_util.cc",
"google/protobuf/generated_message_tctable_lite.cc",
"google/protobuf/generated_message_util.cc",
"google/protobuf/implicit_weak_message.cc",
"google/protobuf/inlined_string_field.cc",
"google/protobuf/io/coded_stream.cc",
"google/protobuf/io/io_win32.cc",
"google/protobuf/io/strtod.cc",
"google/protobuf/io/zero_copy_stream.cc",
"google/protobuf/io/zero_copy_stream_impl.cc",
"google/protobuf/io/zero_copy_stream_impl_lite.cc",
"google/protobuf/map.cc",
"google/protobuf/message_lite.cc",
"google/protobuf/parse_context.cc",
"google/protobuf/repeated_field.cc",
"google/protobuf/repeated_ptr_field.cc",
"google/protobuf/stubs/bytestream.cc",
"google/protobuf/stubs/common.cc",
"google/protobuf/stubs/int128.cc",
"google/protobuf/stubs/status.cc",
"google/protobuf/stubs/statusor.cc",
"google/protobuf/stubs/stringpiece.cc",
"google/protobuf/stubs/stringprintf.cc",
"google/protobuf/stubs/structurally_valid.cc",
"google/protobuf/stubs/strutil.cc",
"google/protobuf/stubs/time.cc",
"google/protobuf/wire_format_lite.cc",
]
)
protobuf_lite_sources = {
Path("any_lite.cc"),
Path("arena.cc"),
Path("arenastring.cc"),
Path("arenaz_sampler.cc"),
Path("extension_set.cc"),
Path("generated_enum_util.cc"),
Path("generated_message_tctable_lite.cc"),
Path("generated_message_util.cc"),
Path("implicit_weak_message.cc"),
Path("inlined_string_field.cc"),
Path("io/coded_stream.cc"),
Path("io/io_win32.cc"),
Path("io/strtod.cc"),
Path("io/zero_copy_stream.cc"),
Path("io/zero_copy_stream_impl.cc"),
Path("io/zero_copy_stream_impl_lite.cc"),
Path("map.cc"),
Path("message_lite.cc"),
Path("parse_context.cc"),
Path("repeated_field.cc"),
Path("repeated_ptr_field.cc"),
Path("stubs/bytestream.cc"),
Path("stubs/common.cc"),
Path("stubs/int128.cc"),
Path("stubs/status.cc"),
Path("stubs/statusor.cc"),
Path("stubs/stringpiece.cc"),
Path("stubs/stringprintf.cc"),
Path("stubs/structurally_valid.cc"),
Path("stubs/strutil.cc"),
Path("stubs/time.cc"),
Path("wire_format_lite.cc"),
}
protobuf_lite_includes = set(
[
"google/protobuf/any.h",
"google/protobuf/arena.h",
"google/protobuf/arena_impl.h",
"google/protobuf/arenastring.h",
"google/protobuf/arenaz_sampler.h",
"google/protobuf/endian.h",
"google/protobuf/explicitly_constructed.h",
"google/protobuf/extension_set.h",
"google/protobuf/extension_set_inl.h",
"google/protobuf/generated_enum_util.h",
"google/protobuf/generated_message_tctable_decl.h",
"google/protobuf/generated_message_tctable_impl.h",
"google/protobuf/generated_message_util.h",
"google/protobuf/has_bits.h",
"google/protobuf/implicit_weak_message.h",
"google/protobuf/inlined_string_field.h",
"google/protobuf/io/coded_stream.h",
"google/protobuf/io/io_win32.h",
"google/protobuf/io/strtod.h",
"google/protobuf/io/zero_copy_stream.h",
"google/protobuf/io/zero_copy_stream_impl.h",
"google/protobuf/io/zero_copy_stream_impl_lite.h",
"google/protobuf/map.h",
"google/protobuf/map_entry_lite.h",
"google/protobuf/map_field_lite.h",
"google/protobuf/map_type_handler.h",
"google/protobuf/message_lite.h",
"google/protobuf/metadata_lite.h",
"google/protobuf/parse_context.h",
"google/protobuf/port.h",
"google/protobuf/repeated_field.h",
"google/protobuf/repeated_ptr_field.h",
"google/protobuf/stubs/bytestream.h",
"google/protobuf/stubs/callback.h",
"google/protobuf/stubs/casts.h",
"google/protobuf/stubs/common.h",
"google/protobuf/stubs/hash.h",
"google/protobuf/stubs/logging.h",
"google/protobuf/stubs/macros.h",
"google/protobuf/stubs/map_util.h",
"google/protobuf/stubs/mutex.h",
"google/protobuf/stubs/once.h",
"google/protobuf/stubs/platform_macros.h",
"google/protobuf/stubs/port.h",
"google/protobuf/stubs/status.h",
"google/protobuf/stubs/stl_util.h",
"google/protobuf/stubs/stringpiece.h",
"google/protobuf/stubs/strutil.h",
"google/protobuf/stubs/template_util.h",
"google/protobuf/wire_format_lite.h",
]
)
protobuf_lite_includes = {
Path("google/protobuf/any.h"),
Path("google/protobuf/arena.h"),
Path("google/protobuf/arena_impl.h"),
Path("google/protobuf/arenastring.h"),
Path("google/protobuf/arenaz_sampler.h"),
Path("google/protobuf/endian.h"),
Path("google/protobuf/explicitly_constructed.h"),
Path("google/protobuf/extension_set.h"),
Path("google/protobuf/extension_set_inl.h"),
Path("google/protobuf/generated_enum_util.h"),
Path("google/protobuf/generated_message_tctable_decl.h"),
Path("google/protobuf/generated_message_tctable_impl.h"),
Path("google/protobuf/generated_message_util.h"),
Path("google/protobuf/has_bits.h"),
Path("google/protobuf/implicit_weak_message.h"),
Path("google/protobuf/inlined_string_field.h"),
Path("google/protobuf/io/coded_stream.h"),
Path("google/protobuf/io/io_win32.h"),
Path("google/protobuf/io/strtod.h"),
Path("google/protobuf/io/zero_copy_stream.h"),
Path("google/protobuf/io/zero_copy_stream_impl.h"),
Path("google/protobuf/io/zero_copy_stream_impl_lite.h"),
Path("google/protobuf/map.h"),
Path("google/protobuf/map_entry_lite.h"),
Path("google/protobuf/map_field_lite.h"),
Path("google/protobuf/map_type_handler.h"),
Path("google/protobuf/message_lite.h"),
Path("google/protobuf/metadata_lite.h"),
Path("google/protobuf/parse_context.h"),
Path("google/protobuf/port.h"),
Path("google/protobuf/repeated_field.h"),
Path("google/protobuf/repeated_ptr_field.h"),
Path("google/protobuf/stubs/bytestream.h"),
Path("google/protobuf/stubs/callback.h"),
Path("google/protobuf/stubs/casts.h"),
Path("google/protobuf/stubs/common.h"),
Path("google/protobuf/stubs/hash.h"),
Path("google/protobuf/stubs/logging.h"),
Path("google/protobuf/stubs/macros.h"),
Path("google/protobuf/stubs/map_util.h"),
Path("google/protobuf/stubs/mutex.h"),
Path("google/protobuf/stubs/once.h"),
Path("google/protobuf/stubs/platform_macros.h"),
Path("google/protobuf/stubs/port.h"),
Path("google/protobuf/stubs/status.h"),
Path("google/protobuf/stubs/stl_util.h"),
Path("google/protobuf/stubs/stringpiece.h"),
Path("google/protobuf/stubs/strutil.h"),
Path("google/protobuf/stubs/template_util.h"),
Path("google/protobuf/wire_format_lite.h"),
}
protobuf_sources = {
Path("any.cc"),
Path("any.pb.cc"),
Path("api.pb.cc"),
Path("compiler/importer.cc"),
Path("compiler/parser.cc"),
Path("descriptor.cc"),
Path("descriptor.pb.cc"),
Path("descriptor_database.cc"),
Path("duration.pb.cc"),
Path("dynamic_message.cc"),
Path("empty.pb.cc"),
Path("extension_set_heavy.cc"),
Path("field_mask.pb.cc"),
Path("generated_message_bases.cc"),
Path("generated_message_reflection.cc"),
Path("generated_message_tctable_full.cc"),
Path("io/gzip_stream.cc"),
Path("io/printer.cc"),
Path("io/tokenizer.cc"),
Path("map_field.cc"),
Path("message.cc"),
Path("reflection_ops.cc"),
Path("service.cc"),
Path("source_context.pb.cc"),
Path("struct.pb.cc"),
Path("stubs/substitute.cc"),
Path("text_format.cc"),
Path("timestamp.pb.cc"),
Path("type.pb.cc"),
Path("unknown_field_set.cc"),
Path("util/delimited_message_util.cc"),
Path("util/field_comparator.cc"),
Path("util/field_mask_util.cc"),
Path("util/internal/datapiece.cc"),
Path("util/internal/default_value_objectwriter.cc"),
Path("util/internal/error_listener.cc"),
Path("util/internal/field_mask_utility.cc"),
Path("util/internal/json_escaping.cc"),
Path("util/internal/json_objectwriter.cc"),
Path("util/internal/json_stream_parser.cc"),
Path("util/internal/object_writer.cc"),
Path("util/internal/proto_writer.cc"),
Path("util/internal/protostream_objectsource.cc"),
Path("util/internal/protostream_objectwriter.cc"),
Path("util/internal/type_info.cc"),
Path("util/internal/utility.cc"),
Path("util/json_util.cc"),
Path("util/message_differencer.cc"),
Path("util/time_util.cc"),
Path("util/type_resolver_util.cc"),
Path("wire_format.cc"),
Path("wrappers.pb.cc"),
}
protobuf_sources = set(
[
"google/protobuf/any.cc",
"google/protobuf/any.pb.cc",
"google/protobuf/api.pb.cc",
"google/protobuf/compiler/importer.cc",
"google/protobuf/compiler/parser.cc",
"google/protobuf/descriptor.cc",
"google/protobuf/descriptor.pb.cc",
"google/protobuf/descriptor_database.cc",
"google/protobuf/duration.pb.cc",
"google/protobuf/dynamic_message.cc",
"google/protobuf/empty.pb.cc",
"google/protobuf/extension_set_heavy.cc",
"google/protobuf/field_mask.pb.cc",
"google/protobuf/generated_message_bases.cc",
"google/protobuf/generated_message_reflection.cc",
"google/protobuf/generated_message_tctable_full.cc",
"google/protobuf/io/gzip_stream.cc",
"google/protobuf/io/printer.cc",
"google/protobuf/io/tokenizer.cc",
"google/protobuf/map_field.cc",
"google/protobuf/message.cc",
"google/protobuf/reflection_ops.cc",
"google/protobuf/service.cc",
"google/protobuf/source_context.pb.cc",
"google/protobuf/struct.pb.cc",
"google/protobuf/stubs/substitute.cc",
"google/protobuf/text_format.cc",
"google/protobuf/timestamp.pb.cc",
"google/protobuf/type.pb.cc",
"google/protobuf/unknown_field_set.cc",
"google/protobuf/util/delimited_message_util.cc",
"google/protobuf/util/field_comparator.cc",
"google/protobuf/util/field_mask_util.cc",
"google/protobuf/util/internal/datapiece.cc",
"google/protobuf/util/internal/default_value_objectwriter.cc",
"google/protobuf/util/internal/error_listener.cc",
"google/protobuf/util/internal/field_mask_utility.cc",
"google/protobuf/util/internal/json_escaping.cc",
"google/protobuf/util/internal/json_objectwriter.cc",
"google/protobuf/util/internal/json_stream_parser.cc",
"google/protobuf/util/internal/object_writer.cc",
"google/protobuf/util/internal/proto_writer.cc",
"google/protobuf/util/internal/protostream_objectsource.cc",
"google/protobuf/util/internal/protostream_objectwriter.cc",
"google/protobuf/util/internal/type_info.cc",
"google/protobuf/util/internal/utility.cc",
"google/protobuf/util/json_util.cc",
"google/protobuf/util/message_differencer.cc",
"google/protobuf/util/time_util.cc",
"google/protobuf/util/type_resolver_util.cc",
"google/protobuf/wire_format.cc",
"google/protobuf/wrappers.pb.cc",
]
)
protobuf_includes = {
Path("google/protobuf/any.pb.h"),
Path("google/protobuf/api.pb.h"),
Path("google/protobuf/compiler/importer.h"),
Path("google/protobuf/compiler/parser.h"),
Path("google/protobuf/descriptor.h"),
Path("google/protobuf/descriptor.pb.h"),
Path("google/protobuf/descriptor_database.h"),
Path("google/protobuf/duration.pb.h"),
Path("google/protobuf/dynamic_message.h"),
Path("google/protobuf/empty.pb.h"),
Path("google/protobuf/field_access_listener.h"),
Path("google/protobuf/field_mask.pb.h"),
Path("google/protobuf/generated_enum_reflection.h"),
Path("google/protobuf/generated_message_bases.h"),
Path("google/protobuf/generated_message_reflection.h"),
Path("google/protobuf/io/gzip_stream.h"),
Path("google/protobuf/io/printer.h"),
Path("google/protobuf/io/tokenizer.h"),
Path("google/protobuf/map_entry.h"),
Path("google/protobuf/map_field.h"),
Path("google/protobuf/map_field_inl.h"),
Path("google/protobuf/message.h"),
Path("google/protobuf/metadata.h"),
Path("google/protobuf/reflection.h"),
Path("google/protobuf/reflection_internal.h"),
Path("google/protobuf/reflection_ops.h"),
Path("google/protobuf/service.h"),
Path("google/protobuf/source_context.pb.h"),
Path("google/protobuf/struct.pb.h"),
Path("google/protobuf/text_format.h"),
Path("google/protobuf/timestamp.pb.h"),
Path("google/protobuf/type.pb.h"),
Path("google/protobuf/unknown_field_set.h"),
Path("google/protobuf/util/delimited_message_util.h"),
Path("google/protobuf/util/field_comparator.h"),
Path("google/protobuf/util/field_mask_util.h"),
Path("google/protobuf/util/json_util.h"),
Path("google/protobuf/util/message_differencer.h"),
Path("google/protobuf/util/time_util.h"),
Path("google/protobuf/util/type_resolver.h"),
Path("google/protobuf/util/type_resolver_util.h"),
Path("google/protobuf/wire_format.h"),
Path("google/protobuf/wrappers.pb.h"),
}
protobuf_includes = set(
[
"google/protobuf/any.pb.h",
"google/protobuf/api.pb.h",
"google/protobuf/compiler/importer.h",
"google/protobuf/compiler/parser.h",
"google/protobuf/descriptor.h",
"google/protobuf/descriptor.pb.h",
"google/protobuf/descriptor_database.h",
"google/protobuf/duration.pb.h",
"google/protobuf/dynamic_message.h",
"google/protobuf/empty.pb.h",
"google/protobuf/field_access_listener.h",
"google/protobuf/field_mask.pb.h",
"google/protobuf/generated_enum_reflection.h",
"google/protobuf/generated_message_bases.h",
"google/protobuf/generated_message_reflection.h",
"google/protobuf/io/gzip_stream.h",
"google/protobuf/io/printer.h",
"google/protobuf/io/tokenizer.h",
"google/protobuf/map_entry.h",
"google/protobuf/map_field.h",
"google/protobuf/map_field_inl.h",
"google/protobuf/message.h",
"google/protobuf/metadata.h",
"google/protobuf/reflection.h",
"google/protobuf/reflection_internal.h",
"google/protobuf/reflection_ops.h",
"google/protobuf/service.h",
"google/protobuf/source_context.pb.h",
"google/protobuf/struct.pb.h",
"google/protobuf/text_format.h",
"google/protobuf/timestamp.pb.h",
"google/protobuf/type.pb.h",
"google/protobuf/unknown_field_set.h",
"google/protobuf/util/delimited_message_util.h",
"google/protobuf/util/field_comparator.h",
"google/protobuf/util/field_mask_util.h",
"google/protobuf/util/json_util.h",
"google/protobuf/util/message_differencer.h",
"google/protobuf/util/time_util.h",
"google/protobuf/util/type_resolver.h",
"google/protobuf/util/type_resolver_util.h",
"google/protobuf/wire_format.h",
"google/protobuf/wrappers.pb.h",
]
)
protobuf_internal_includes = set(
[
"google/protobuf/port_def.inc",
"google/protobuf/port_undef.inc",
"google/protobuf/stubs/int128.h",
"google/protobuf/stubs/mathutil.h",
"google/protobuf/stubs/statusor.h",
"google/protobuf/stubs/status_macros.h",
"google/protobuf/stubs/stringprintf.h",
"google/protobuf/stubs/substitute.h",
"google/protobuf/stubs/time.h",
"google/protobuf/util/internal/constants.h",
"google/protobuf/util/internal/datapiece.h",
"google/protobuf/util/internal/default_value_objectwriter.h",
"google/protobuf/util/internal/error_listener.h",
"google/protobuf/util/internal/field_mask_utility.h",
"google/protobuf/util/internal/json_escaping.h",
"google/protobuf/util/internal/json_objectwriter.h",
"google/protobuf/util/internal/json_stream_parser.h",
"google/protobuf/util/internal/location_tracker.h",
"google/protobuf/util/internal/object_location_tracker.h",
"google/protobuf/util/internal/object_source.h",
"google/protobuf/util/internal/object_writer.h",
"google/protobuf/util/internal/proto_writer.h",
"google/protobuf/util/internal/protostream_objectsource.h",
"google/protobuf/util/internal/protostream_objectwriter.h",
"google/protobuf/util/internal/structured_objectwriter.h",
"google/protobuf/util/internal/type_info.h",
"google/protobuf/util/internal/utility.h",
]
)
protobuf_internal_includes = {
Path("google/protobuf/port_def.inc"),
Path("google/protobuf/port_undef.inc"),
Path("google/protobuf/stubs/int128.h"),
Path("google/protobuf/stubs/mathutil.h"),
Path("google/protobuf/stubs/statusor.h"),
Path("google/protobuf/stubs/status_macros.h"),
Path("google/protobuf/stubs/stringprintf.h"),
Path("google/protobuf/stubs/substitute.h"),
Path("google/protobuf/stubs/time.h"),
Path("google/protobuf/util/internal/constants.h"),
Path("google/protobuf/util/internal/datapiece.h"),
Path("google/protobuf/util/internal/default_value_objectwriter.h"),
Path("google/protobuf/util/internal/error_listener.h"),
Path("google/protobuf/util/internal/field_mask_utility.h"),
Path("google/protobuf/util/internal/json_escaping.h"),
Path("google/protobuf/util/internal/json_objectwriter.h"),
Path("google/protobuf/util/internal/json_stream_parser.h"),
Path("google/protobuf/util/internal/location_tracker.h"),
Path("google/protobuf/util/internal/object_location_tracker.h"),
Path("google/protobuf/util/internal/object_source.h"),
Path("google/protobuf/util/internal/object_writer.h"),
Path("google/protobuf/util/internal/proto_writer.h"),
Path("google/protobuf/util/internal/protostream_objectsource.h"),
Path("google/protobuf/util/internal/protostream_objectwriter.h"),
Path("google/protobuf/util/internal/structured_objectwriter.h"),
Path("google/protobuf/util/internal/type_info.h"),
Path("google/protobuf/util/internal/utility.h"),
}
use_src_files = protobuf_lite_sources | protobuf_sources
use_include_files = (
@@ -240,38 +230,29 @@ use_include_files = (
)
def matches(dp, f, files):
if not dp.startswith(os.path.join(".", "src")):
return False
p = dp[6:] + "/" + f
return p in files
def copy_upstream_src(wpilib_root):
upstream_root = os.path.abspath(".")
wpiutil = os.path.join(wpilib_root, "wpiutil")
def copy_upstream_src(wpilib_root: Path):
upstream_root = Path(".").absolute()
wpiutil = wpilib_root / "wpiutil"
# Delete old install
for d in [
"src/main/native/thirdparty/protobuf/src",
"src/main/native/thirdparty/protobuf/include",
]:
shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
shutil.rmtree(wpiutil / d, ignore_errors=True)
# Copy protobuf source files into allwpilib
src_files = walk_if(".", lambda dp, f: matches(dp, f, use_src_files))
src_files = [f[22:] for f in src_files]
os.chdir(os.path.join(upstream_root, "src/google/protobuf"))
copy_to(src_files, os.path.join(wpiutil, "src/main/native/thirdparty/protobuf/src"))
os.chdir(upstream_root / "src/google/protobuf")
walk_cwd_and_copy_if(
lambda dp, f: dp / f in use_src_files,
wpiutil / "src/main/native/thirdparty/protobuf/src",
)
# Copy protobuf header files into allwpilib
os.chdir(upstream_root)
include_files = walk_if(".", lambda dp, f: matches(dp, f, use_include_files))
include_files = [f[6:] for f in include_files]
os.chdir(os.path.join(upstream_root, "src"))
copy_to(
include_files,
os.path.join(wpiutil, "src/main/native/thirdparty/protobuf/include"),
os.chdir(upstream_root / "src")
walk_cwd_and_copy_if(
lambda dp, f: dp / f in use_include_files,
wpiutil / "src/main/native/thirdparty/protobuf/include",
)

53
upstream_utils/sleipnir.py
View File

@@ -1,58 +1,35 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, copy_to
from upstream_utils import Lib, has_prefix, walk_cwd_and_copy_if
def copy_upstream_src(wpilib_root):
wpimath = os.path.join(wpilib_root, "wpimath")
def copy_upstream_src(wpilib_root: Path):
wpimath = wpilib_root / "wpimath"
# Delete old install
for d in [
"src/main/native/thirdparty/sleipnir/src",
"src/main/native/thirdparty/sleipnir/include",
]:
shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True)
shutil.rmtree(wpimath / d, ignore_errors=True)
# Copy Sleipnir source files into allwpilib
src_files = [os.path.join(dp, f) for dp, dn, fn in os.walk("src") for f in fn]
src_files = copy_to(
src_files, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir")
)
# Copy Sleipnir header files into allwpilib
include_files = [
os.path.join(dp, f) for dp, dn, fn in os.walk("include") for f in fn
]
include_files = copy_to(
include_files, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir")
)
for filename in [
".clang-format",
".clang-tidy",
".styleguide",
".styleguide-license",
]:
shutil.copyfile(
filename,
os.path.join(wpimath, "src/main/native/thirdparty/sleipnir", filename),
# Copy Sleipnir files into allwpilib
walk_cwd_and_copy_if(
lambda dp, f: (has_prefix(dp, Path("include")) or has_prefix(dp, Path("src")))
or f == ".clang-format"
or f == ".clang-tidy"
or f == ".styleguide"
or f == ".styleguide-license",
wpimath / "src/main/native/thirdparty/sleipnir",
)
# Write shim for wpi::SmallVector
try:
os.mkdir(
os.path.join(wpimath, "src/main/native/thirdparty/sleipnir/include/gch")
)
except:
pass
(wpimath / "src/main/native/thirdparty/sleipnir/include/gch").mkdir()
with open(
os.path.join(
wpimath,
"src/main/native/thirdparty/sleipnir/include/gch/small_vector.hpp",
),
wpimath / "src/main/native/thirdparty/sleipnir/include/gch/small_vector.hpp",
"w",
) as f:
f.write(

16
upstream_utils/stb.py
View File

@@ -1,27 +1,27 @@
#!/usr/bin/env python3
import os
import shutil
from pathlib import Path
from upstream_utils import Lib, walk_cwd_and_copy_if
def copy_upstream_src(wpilib_root):
stb = os.path.join(wpilib_root, "thirdparty", "imgui_suite", "stb")
def copy_upstream_src(wpilib_root: Path):
stb = wpilib_root / "thirdparty/imgui_suite/stb"
# Delete old install
for d in ["include", "cpp"]:
shutil.rmtree(os.path.join(stb, d), ignore_errors=True)
shutil.rmtree(stb / d, ignore_errors=True)
# Copy files
files = walk_cwd_and_copy_if(
walk_cwd_and_copy_if(
lambda dp, f: f == "stb_image.h",
os.path.join(stb, "include"),
stb / "include",
)
os.makedirs(os.path.join(stb, "cpp"))
(stb / "cpp").mkdir(parents=True)
with open(os.path.join(stb, "cpp", "stb_image.cpp"), "w") as f:
with open(stb / "cpp/stb_image.cpp", "w") as f:
f.write(
"""#define STBI_WINDOWS_UTF8
#define STB_IMAGE_IMPLEMENTATION

110
upstream_utils/upstream_utils.py
View File

@@ -5,6 +5,8 @@ import shutil
import subprocess
import sys
import tempfile
from collections.abc import Callable
from pathlib import Path
def get_repo_root():
@@ -12,15 +14,17 @@ def get_repo_root():
An empty string is returned if no repository root was found.
"""
return subprocess.run(
return Path(
subprocess.run(
["git", "rev-parse", "--show-toplevel"],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
encoding="ascii",
).stdout.rstrip()
)
def walk_if(top, pred):
def walk_if(top: Path, pred: Callable[[Path, str], bool]):
"""Walks the current directory, then returns a list of files for which the
given predicate is true.
@@ -29,12 +33,10 @@ def walk_if(top, pred):
pred -- a function that takes a directory path and a filename, then returns
True if the file should be included in the output list
"""
return [
os.path.join(dp, f) for dp, dn, fn in os.walk(top) for f in fn if pred(dp, f)
]
return [dp / f for dp, dn, fn in top.walk() for f in fn if pred(dp, f)]
def copy_to(files, root, rename_c_to_cpp=False):
def copy_to(files: list[Path], root: Path, rename_c_to_cpp=False):
"""Copies list of files to root by appending the relative paths of the files
to root.
@@ -48,31 +50,33 @@ def copy_to(files, root, rename_c_to_cpp=False):
Returns:
The list of files in their destination.
"""
if not os.path.exists(root):
os.makedirs(root)
if not root.exists():
root.mkdir(parents=True)
dest_files = []
dest_files: list[Path] = []
for f in files:
dest_file = os.path.join(root, f)
dest_file = root / f
# Rename .cc or .cxx file to .cpp
if dest_file.endswith(".cc") or dest_file.endswith(".cxx"):
dest_file = os.path.splitext(dest_file)[0] + ".cpp"
if dest_file.match("*.cc") or dest_file.match("*.cxx"):
dest_file = dest_file.with_suffix(".cpp")
if rename_c_to_cpp and dest_file.endswith(".c"):
dest_file = os.path.splitext(dest_file)[0] + ".cpp"
if rename_c_to_cpp and dest_file.match("*.c"):
dest_file = dest_file.with_suffix(".cpp")
# Make leading directory
dest_dir = os.path.dirname(dest_file)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
dest_dir = dest_file.parent
if not dest_dir.exists():
dest_dir.mkdir(parents=True)
shutil.copyfile(f, dest_file)
dest_files.append(dest_file)
return dest_files
def walk_cwd_and_copy_if(pred, root, rename_c_to_cpp=False):
def walk_cwd_and_copy_if(
pred: Callable[[Path, str], bool], root: Path, rename_c_to_cpp=False
):
"""Walks the current directory, generates a list of files for which the
given predicate is true, then copies that list to root by appending the
relative paths of the files to root.
@@ -88,12 +92,12 @@ def walk_cwd_and_copy_if(pred, root, rename_c_to_cpp=False):
Returns:
The list of files in their destination.
"""
files = walk_if(".", pred)
files = walk_if(Path("."), pred)
files = copy_to(files, root, rename_c_to_cpp)
return files
def comment_out_invalid_includes(filename, include_roots):
def comment_out_invalid_includes(filename: Path, include_roots: list[Path]):
"""Comment out #include directives that include a nonexistent file
Keyword arguments:
@@ -114,11 +118,8 @@ def comment_out_invalid_includes(filename, include_roots):
# Comment out #include if the file doesn't exist in current directory or
# include root
if not os.path.exists(
os.path.join(os.path.dirname(filename), include)
) and not any(
os.path.exists(os.path.join(include_root, include))
for include_root in include_roots
if not (filename.parent / include).exists() and not any(
(include_root / include).exists() for include_root in include_roots
):
new_contents += "// "
@@ -135,7 +136,22 @@ def comment_out_invalid_includes(filename, include_roots):
f.write(new_contents)
def git_am(patch, use_threeway=False, ignore_whitespace=False):
def has_prefix(path: Path, prefix: Path):
"""Checks if a path has a certain prefix.
Keyword arguments:
path -- path to check if it begins with the prefix
prefix -- prefix to use
Returns:
True if the path begins with the prefix, False otherwise.
"""
return len(path.parts) >= len(prefix.parts) and all(
p1 == p2 for p1, p2 in zip(path.parts, prefix.parts)
)
def git_am(patch: Path, use_threeway=False, ignore_whitespace=False):
"""Apply patch to a Git repository in the current directory using "git am".
Keyword arguments:
@@ -152,7 +168,7 @@ def git_am(patch, use_threeway=False, ignore_whitespace=False):
subprocess.check_output(args + [patch])
def has_git_rev(rev):
def has_git_rev(rev: str):
"""Checks whether the Git repository in the current directory has the given
revision.
@@ -169,10 +185,10 @@ def has_git_rev(rev):
class Lib:
def __init__(
self,
name,
url,
tag,
copy_upstream_src,
name: str,
url: str,
tag: str,
copy_upstream_src: Callable[[Path], None],
patch_options={},
*,
pre_patch_hook=None,
@@ -207,7 +223,7 @@ class Lib:
self.pre_patch_commits = pre_patch_commits
self.wpilib_root = get_repo_root()
def get_repo_path(self, tempdir=None):
def get_repo_path(self, tempdir: str | None = None):
"""Returns the path to the clone of the upstream repository.
Keyword argument:
@@ -221,11 +237,11 @@ class Lib:
if tempdir is None:
tempdir = tempfile.gettempdir()
repo = os.path.basename(self.url)
dest = os.path.join(tempdir, repo)
dest = dest.removesuffix(".git")
dest = Path(tempdir, repo)
dest = dest.with_suffix("")
return dest
def open_repo(self, *, err_msg_if_absent):
def open_repo(self, *, err_msg_if_absent: str | None):
"""Changes the current working directory to the upstream repository. If
err_msg_if_absent is not None and the upstream repository does not
exist, the program exits with return code 1.
@@ -241,7 +257,7 @@ class Lib:
print(f"INFO: Opening repository at {dest}")
if not os.path.exists(dest):
if not dest.exists():
if err_msg_if_absent is None:
subprocess.run(["git", "clone", "--filter=tree:0", self.url])
else:
@@ -285,7 +301,7 @@ class Lib:
exit(1)
return root_tags[0]
def set_root_tag(self, tag):
def set_root_tag(self, tag: str):
"""Sets the root tag, deleting any potential candidates first.
Keyword argument:
@@ -307,19 +323,17 @@ class Lib:
Returns:
The absolute path to the directory containing the patch files.
"""
return os.path.join(self.wpilib_root, f"upstream_utils/{self.name}_patches")
return self.wpilib_root / f"upstream_utils/{self.name}_patches"
def get_patch_list(self):
def get_patch_list(self) -> list[Path]:
"""Returns a list of the filenames of the patches to apply.
Returns:
A list of the filenames of the patches to apply, sorted in lexicographic
order by the Unicode code points."""
if not os.path.exists(self.get_patch_directory()):
if not self.get_patch_directory().exists():
return []
return sorted(
f for f in os.listdir(self.get_patch_directory()) if f.endswith(".patch")
)
return sorted(self.get_patch_directory().glob("*.patch"))
def apply_patches(self):
"""Applies the patches listed in the patch list to the current
@@ -330,17 +344,17 @@ class Lib:
for f in self.get_patch_list():
git_am(
os.path.join(self.get_patch_directory(), f),
self.get_patch_directory() / f,
**self.patch_options,
)
def replace_tag(self, tag):
def replace_tag(self, tag: str):
"""Replaces the tag in the script.
Keyword argument:
tag -- The tag to replace the script tag with.
"""
path = os.path.join(self.wpilib_root, f"upstream_utils/{self.name}.py")
path = self.wpilib_root / f"upstream_utils/{self.name}.py"
with open(path, "r") as file:
lines = file.readlines()
@@ -391,7 +405,7 @@ class Lib:
self.set_root_tag(self.old_tag)
def rebase(self, new_tag):
def rebase(self, new_tag: str):
"""Rebases the patches.
Keyword argument:
@@ -449,7 +463,7 @@ class Lib:
]
)
if os.path.exists(self.get_patch_directory()):
if self.get_patch_directory().exists():
shutil.rmtree(self.get_patch_directory())
is_first = True

336
wpimath/src/main/native/thirdparty/eigen/include/Eigen/src/ThreadPool/CoreThreadPoolDevice.h vendored
View File

@@ -1,336 +0,0 @@
// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// Copyright (C) 2023 Charlie Schlosser <cs.schlosser@gmail.com>
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
#ifndef EIGEN_CORE_THREAD_POOL_DEVICE_H
#define EIGEN_CORE_THREAD_POOL_DEVICE_H
namespace Eigen {
// CoreThreadPoolDevice provides an easy-to-understand Device for parallelizing Eigen Core expressions with
// Threadpool. Expressions are recursively split evenly until the evaluation cost is less than the threshold for
// delegating the task to a thread.
/*
a
/ \
/ \
/ \
/ \
/ \
/ \
/ \
a e
/ \ / \
/ \ / \
/ \ / \
a c e g
/ \ / \ / \ / \
/ \ / \ / \ / \
a b c d e f g h
*/
// Each task descends the binary tree to the left, delegates the right task to a new thread, and continues to the
// left. This ensures that work is evenly distributed to the thread pool as quickly as possible and minimizes the number
// of tasks created during the evaluation. Consider an expression that is divided into 8 chunks. The
// primary task 'a' creates tasks 'e' 'c' and 'b', and executes its portion of the expression at the bottom of the
// tree. Likewise, task 'e' creates tasks 'g' and 'f', and executes its portion of the expression.
struct CoreThreadPoolDevice {
using Task = std::function<void()>;
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoreThreadPoolDevice(ThreadPool& pool, float threadCostThreshold = 3e-5f)
: m_pool(pool) {
eigen_assert(threadCostThreshold >= 0.0f && "threadCostThreshold must be non-negative");
m_costFactor = threadCostThreshold;
}
template <int PacketSize>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int calculateLevels(Index size, float cost) const {
eigen_assert(cost >= 0.0f && "cost must be non-negative");
Index numOps = size / PacketSize;
int actualThreads = numOps < m_pool.NumThreads() ? static_cast<int>(numOps) : m_pool.NumThreads();
float totalCost = static_cast<float>(numOps) * cost;
float idealThreads = totalCost * m_costFactor;
if (idealThreads < static_cast<float>(actualThreads)) {
idealThreads = numext::maxi(idealThreads, 1.0f);
actualThreads = numext::mini(actualThreads, static_cast<int>(idealThreads));
}
int maxLevel = internal::log2_ceil(actualThreads);
return maxLevel;
}
// MSVC does not like inlining parallelForImpl
#if EIGEN_COMP_MSVC && !EIGEN_COMP_CLANG
#define EIGEN_PARALLEL_FOR_INLINE
#else
#define EIGEN_PARALLEL_FOR_INLINE EIGEN_STRONG_INLINE
#endif
template <typename UnaryFunctor, int PacketSize>
EIGEN_DEVICE_FUNC EIGEN_PARALLEL_FOR_INLINE void parallelForImpl(Index begin, Index end, UnaryFunctor& f,
Barrier& barrier, int level) {
while (level > 0) {
level--;
Index size = end - begin;
eigen_assert(size % PacketSize == 0 && "this function assumes size is a multiple of PacketSize");
Index mid = begin + numext::round_down(size >> 1, PacketSize);
Task right = [this, mid, end, &f, &barrier, level]() {
parallelForImpl<UnaryFunctor, PacketSize>(mid, end, f, barrier, level);
};
m_pool.Schedule(std::move(right));
end = mid;
}
for (Index i = begin; i < end; i += PacketSize) f(i);
barrier.Notify();
}
template <typename BinaryFunctor, int PacketSize>
EIGEN_DEVICE_FUNC EIGEN_PARALLEL_FOR_INLINE void parallelForImpl(Index outerBegin, Index outerEnd, Index innerBegin,
Index innerEnd, BinaryFunctor& f, Barrier& barrier,
int level) {
while (level > 0) {
level--;
Index outerSize = outerEnd - outerBegin;
if (outerSize > 1) {
Index outerMid = outerBegin + (outerSize >> 1);
Task right = [this, &f, &barrier, outerMid, outerEnd, innerBegin, innerEnd, level]() {
parallelForImpl<BinaryFunctor, PacketSize>(outerMid, outerEnd, innerBegin, innerEnd, f, barrier, level);
};
m_pool.Schedule(std::move(right));
outerEnd = outerMid;
} else {
Index innerSize = innerEnd - innerBegin;
eigen_assert(innerSize % PacketSize == 0 && "this function assumes innerSize is a multiple of PacketSize");
Index innerMid = innerBegin + numext::round_down(innerSize >> 1, PacketSize);
Task right = [this, &f, &barrier, outerBegin, outerEnd, innerMid, innerEnd, level]() {
parallelForImpl<BinaryFunctor, PacketSize>(outerBegin, outerEnd, innerMid, innerEnd, f, barrier, level);
};
m_pool.Schedule(std::move(right));
innerEnd = innerMid;
}
}
for (Index outer = outerBegin; outer < outerEnd; outer++)
for (Index inner = innerBegin; inner < innerEnd; inner += PacketSize) f(outer, inner);
barrier.Notify();
}
#undef EIGEN_PARALLEL_FOR_INLINE
template <typename UnaryFunctor, int PacketSize>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void parallelFor(Index begin, Index end, UnaryFunctor& f, float cost) {
Index size = end - begin;
int maxLevel = calculateLevels<PacketSize>(size, cost);
Barrier barrier(1 << maxLevel);
parallelForImpl<UnaryFunctor, PacketSize>(begin, end, f, barrier, maxLevel);
barrier.Wait();
}
template <typename BinaryFunctor, int PacketSize>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void parallelFor(Index outerBegin, Index outerEnd, Index innerBegin,
Index innerEnd, BinaryFunctor& f, float cost) {
Index outerSize = outerEnd - outerBegin;
Index innerSize = innerEnd - innerBegin;
Index size = outerSize * innerSize;
int maxLevel = calculateLevels<PacketSize>(size, cost);
Barrier barrier(1 << maxLevel);
parallelForImpl<BinaryFunctor, PacketSize>(outerBegin, outerEnd, innerBegin, innerEnd, f, barrier, maxLevel);
barrier.Wait();
}
ThreadPool& m_pool;
// costFactor is the cost of delegating a task to a thread
// the inverse is used to avoid a floating point division
float m_costFactor;
};
// specialization of coefficient-wise assignment loops for CoreThreadPoolDevice
namespace internal {
#ifdef EIGEN_PARSED_BY_DOXYGEN
struct Kernel;
#endif
template <typename Kernel>
struct cost_helper {
using SrcEvaluatorType = typename Kernel::SrcEvaluatorType;
using DstEvaluatorType = typename Kernel::DstEvaluatorType;
using SrcXprType = typename SrcEvaluatorType::XprType;
using DstXprType = typename DstEvaluatorType::XprType;
static constexpr Index Cost = functor_cost<SrcXprType>::Cost + functor_cost<DstXprType>::Cost;
};
template <typename Kernel>
struct dense_assignment_loop_with_device<Kernel, CoreThreadPoolDevice, DefaultTraversal, NoUnrolling> {
static constexpr Index XprEvaluationCost = cost_helper<Kernel>::Cost;
struct AssignmentFunctor : public Kernel {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer, Index inner) {
this->assignCoeffByOuterInner(outer, inner);
}
};
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) {
const Index innerSize = kernel.innerSize();
const Index outerSize = kernel.outerSize();
constexpr float cost = static_cast<float>(XprEvaluationCost);
AssignmentFunctor functor(kernel);
device.template parallelFor<AssignmentFunctor, 1>(0, outerSize, 0, innerSize, functor, cost);
}
};
template <typename Kernel>
struct dense_assignment_loop_with_device<Kernel, CoreThreadPoolDevice, DefaultTraversal, InnerUnrolling> {
using DstXprType = typename Kernel::DstEvaluatorType::XprType;
static constexpr Index XprEvaluationCost = cost_helper<Kernel>::Cost, InnerSize = DstXprType::InnerSizeAtCompileTime;
struct AssignmentFunctor : public Kernel {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer) {
copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, 0, InnerSize>::run(*this, outer);
}
};
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) {
const Index outerSize = kernel.outerSize();
AssignmentFunctor functor(kernel);
constexpr float cost = static_cast<float>(XprEvaluationCost) * static_cast<float>(InnerSize);
device.template parallelFor<AssignmentFunctor, 1>(0, outerSize, functor, cost);
}
};
template <typename Kernel>
struct dense_assignment_loop_with_device<Kernel, CoreThreadPoolDevice, InnerVectorizedTraversal, NoUnrolling> {
using PacketType = typename Kernel::PacketType;
static constexpr Index XprEvaluationCost = cost_helper<Kernel>::Cost, PacketSize = unpacket_traits<PacketType>::size,
SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
DstAlignment = Kernel::AssignmentTraits::DstAlignment;
struct AssignmentFunctor : public Kernel {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer, Index inner) {
this->template assignPacketByOuterInner<Unaligned, Unaligned, PacketType>(outer, inner);
}
};
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) {
const Index innerSize = kernel.innerSize();
const Index outerSize = kernel.outerSize();
const float cost = static_cast<float>(XprEvaluationCost) * static_cast<float>(innerSize);
AssignmentFunctor functor(kernel);
device.template parallelFor<AssignmentFunctor, PacketSize>(0, outerSize, 0, innerSize, functor, cost);
}
};
template <typename Kernel>
struct dense_assignment_loop_with_device<Kernel, CoreThreadPoolDevice, InnerVectorizedTraversal, InnerUnrolling> {
using PacketType = typename Kernel::PacketType;
using DstXprType = typename Kernel::DstEvaluatorType::XprType;
static constexpr Index XprEvaluationCost = cost_helper<Kernel>::Cost, PacketSize = unpacket_traits<PacketType>::size,
SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
DstAlignment = Kernel::AssignmentTraits::DstAlignment,
InnerSize = DstXprType::InnerSizeAtCompileTime;
struct AssignmentFunctor : public Kernel {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer) {
copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, InnerSize, SrcAlignment, DstAlignment>::run(*this, outer);
}
};
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) {
const Index outerSize = kernel.outerSize();
constexpr float cost = static_cast<float>(XprEvaluationCost) * static_cast<float>(InnerSize);
AssignmentFunctor functor(kernel);
device.template parallelFor<AssignmentFunctor, PacketSize>(0, outerSize, functor, cost);
}
};
template <typename Kernel>
struct dense_assignment_loop_with_device<Kernel, CoreThreadPoolDevice, SliceVectorizedTraversal, NoUnrolling> {
using Scalar = typename Kernel::Scalar;
using PacketType = typename Kernel::PacketType;
static constexpr Index XprEvaluationCost = cost_helper<Kernel>::Cost, PacketSize = unpacket_traits<PacketType>::size;
struct PacketAssignmentFunctor : public Kernel {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketAssignmentFunctor(Kernel& kernel) : Kernel(kernel) {}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer, Index inner) {
this->template assignPacketByOuterInner<Unaligned, Unaligned, PacketType>(outer, inner);
}
};
struct ScalarAssignmentFunctor : public Kernel {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ScalarAssignmentFunctor(Kernel& kernel) : Kernel(kernel) {}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index outer) {
const Index innerSize = this->innerSize();
const Index packetAccessSize = numext::round_down(innerSize, PacketSize);
for (Index inner = packetAccessSize; inner < innerSize; inner++) this->assignCoeffByOuterInner(outer, inner);
}
};
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) {
const Index outerSize = kernel.outerSize();
const Index innerSize = kernel.innerSize();
const Index packetAccessSize = numext::round_down(innerSize, PacketSize);
constexpr float packetCost = static_cast<float>(XprEvaluationCost);
const float scalarCost = static_cast<float>(XprEvaluationCost) * static_cast<float>(innerSize - packetAccessSize);
PacketAssignmentFunctor packetFunctor(kernel);
ScalarAssignmentFunctor scalarFunctor(kernel);
device.template parallelFor<PacketAssignmentFunctor, PacketSize>(0, outerSize, 0, packetAccessSize, packetFunctor,
packetCost);
device.template parallelFor<ScalarAssignmentFunctor, 1>(0, outerSize, scalarFunctor, scalarCost);
};
};
template <typename Kernel>
struct dense_assignment_loop_with_device<Kernel, CoreThreadPoolDevice, LinearTraversal, NoUnrolling> {
static constexpr Index XprEvaluationCost = cost_helper<Kernel>::Cost;
struct AssignmentFunctor : public Kernel {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index index) { this->assignCoeff(index); }
};
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) {
const Index size = kernel.size();
constexpr float cost = static_cast<float>(XprEvaluationCost);
AssignmentFunctor functor(kernel);
device.template parallelFor<AssignmentFunctor, 1>(0, size, functor, cost);
}
};
template <typename Kernel>
struct dense_assignment_loop_with_device<Kernel, CoreThreadPoolDevice, LinearVectorizedTraversal, NoUnrolling> {
using Scalar = typename Kernel::Scalar;
using PacketType = typename Kernel::PacketType;
static constexpr Index XprEvaluationCost = cost_helper<Kernel>::Cost,
RequestedAlignment = Kernel::AssignmentTraits::LinearRequiredAlignment,
PacketSize = unpacket_traits<PacketType>::size,
DstIsAligned = Kernel::AssignmentTraits::DstAlignment >= RequestedAlignment,
DstAlignment = packet_traits<Scalar>::AlignedOnScalar ? RequestedAlignment
: Kernel::AssignmentTraits::DstAlignment,
SrcAlignment = Kernel::AssignmentTraits::JointAlignment;
struct AssignmentFunctor : public Kernel {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AssignmentFunctor(Kernel& kernel) : Kernel(kernel) {}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(Index index) {
this->template assignPacket<DstAlignment, SrcAlignment, PacketType>(index);
}
};
static constexpr bool UsePacketSegment = Kernel::AssignmentTraits::UsePacketSegment;
using head_loop =
unaligned_dense_assignment_loop<PacketType, DstAlignment, SrcAlignment, UsePacketSegment, DstIsAligned>;
using tail_loop = unaligned_dense_assignment_loop<PacketType, DstAlignment, SrcAlignment, UsePacketSegment, false>;
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel& kernel, CoreThreadPoolDevice& device) {
const Index size = kernel.size();
const Index alignedStart =
DstIsAligned ? 0 : internal::first_aligned<RequestedAlignment>(kernel.dstDataPtr(), size);
const Index alignedEnd = alignedStart + numext::round_down(size - alignedStart, PacketSize);
head_loop::run(kernel, 0, alignedStart);
constexpr float cost = static_cast<float>(XprEvaluationCost);
AssignmentFunctor functor(kernel);
device.template parallelFor<AssignmentFunctor, PacketSize>(alignedStart, alignedEnd, functor, cost);
tail_loop::run(kernel, alignedEnd, size);
}
};
} // namespace internal
} // namespace Eigen
#endif // EIGEN_CORE_THREAD_POOL_DEVICE_H