Team2890/allwpilib
4
0
Fork 0
mirror of https://github.com/wpilibsuite/allwpilib synced 2026-06-24 01:31:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

[upstream_utils] Rework upstream_utils scripts (#6829)

Browse Source
This commit is contained in:
Joseph Eng
2024-07-16 17:20:07 -07:00
committed by GitHub
parent f9d32ad706
commit 5f261a88af
15 changed files with 698 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
.github/workflows/upstream-utils.yml vendored
View File

@@ -30,50 +30,61 @@ jobs:
run: |
git config --global user.email "you@example.com"
git config --global user.name "Your Name"
- name: Run update_eigen.py
- name: Run eigen.py
run: |
cd upstream_utils
./update_eigen.py
- name: Run update_fmt.py
./eigen.py clone
./eigen.py copy-upstream-to-thirdparty
- name: Run fmt.py
run: |
cd upstream_utils
./update_fmt.py
- name: Run update_gcem.py
./fmt.py clone
./fmt.py copy-upstream-to-thirdparty
- name: Run gcem.py
run: |
cd upstream_utils
./update_gcem.py
- name: Run update_json.py
./gcem.py clone
./gcem.py copy-upstream-to-thirdparty
- name: Run json.py
run: |
cd upstream_utils
./update_json.py
- name: Run update_libuv.py
./json.py clone
./json.py copy-upstream-to-thirdparty
- name: Run libuv.py
run: |
cd upstream_utils
./update_libuv.py
- name: Run update_llvm.py
./libuv.py clone
./libuv.py copy-upstream-to-thirdparty
- name: Run llvm.py
run: |
cd upstream_utils
./update_llvm.py
- name: Run update_mpack.py
./llvm.py clone
./llvm.py copy-upstream-to-thirdparty
- name: Run mpack.py
run: |
cd upstream_utils
./update_mpack.py
- name: Run update_stack_walker.py
./mpack.py clone
./mpack.py copy-upstream-to-thirdparty
- name: Run stack_walker.py
run: |
cd upstream_utils
./update_stack_walker.py
- name: Run update_memory.py
./stack_walker.py clone
./stack_walker.py copy-upstream-to-thirdparty
- name: Run memory.py
run: |
cd upstream_utils
./update_memory.py
- name: Run update_protobuf.py
./memory.py clone
./memory.py copy-upstream-to-thirdparty
- name: Run protobuf.py
run: |
cd upstream_utils
./update_protobuf.py
- name: Run update_sleipnir.py
./protobuf.py clone
./protobuf.py copy-upstream-to-thirdparty
- name: Run sleipnir.py
run: |
cd upstream_utils
./update_sleipnir.py
./sleipnir.py clone
./sleipnir.py copy-upstream-to-thirdparty
- name: Add untracked files to index so they count as changes
run: git add -A
- name: Check output

83
upstream_utils/README.md
View File

@@ -20,59 +20,24 @@ versions. Each library has its own patch directory (e.g., `lib_patches`).
The example below will update a hypothetical library called `lib` to the tag
`2.0`.
Start in the `upstream_utils` folder. Restore the original repo.
Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exists.
```bash
./update_<lib>.py
./<lib>.py clone
```
Navigate to the repo.
Rebase the clone of the upstream repo.
```bash
cd /tmp/lib
./<lib>.py rebase 2.0
```
Fetch the desired version using one of the following methods.
Update the `upstream_utils` patch files and the tag in the script.
```bash
# Fetch a full branch or tag
git fetch origin 2.0
# Fetch just a tag (useful for expensive-to-clone repos)
git fetch --depth 1 origin tag 2.0
./<lib>.py format-patch
```
Rebase any patches onto the new version. If the old version and new version are
on the same branch, run the following.
Copy the updated upstream files into the thirdparty files within allwpilib.
```bash
git rebase 2.0
```
If the old version and new version are on different branches (e.g.,
llvm-project), use interactive rebase instead and remove commits that are common
between the two branches from the list of commits to rebase. In other words,
only commits representing downstream patches should be listed.
```bash
git rebase -i 2.0
```
Generate patch files for the new version.
```bash
git format-patch 2.0..HEAD --zero-commit --abbrev=40 --no-signature
```
Move the patch files to `upstream_utils`.
```
mv *.patch allwpilib/upstream_utils/lib_patches
```
Navigate back to `upstream_utils`.
```bash
cd allwpilib/upstream_utils
```
Modify the version number in the call to `setup_upstream_repo()` in
`update_<lib>.py`, then rerun `update_<lib>.py` to reimport the thirdparty
files.
```bash
./update_<lib>.py
./<lib>.py copy-upstream-to-thirdparty
```
## Adding patch to thirdparty library
@@ -80,12 +45,17 @@ files.
The example below will add a new patch file to a hypothetical library called
`lib` (Replace `<lib>` with `llvm`, `fmt`, `eigen`, ... in the following steps).
Start in the `upstream_utils` folder. Restore the original repo.
Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exists.
```bash
./update_<lib>.py
./<lib>.py clone
```
Navigate to the repo.
Update the clone of the upstream repo.
```bash
./<lib>.py reset
```
Navigate to the repo. If you can't find it, the directory of the clone is printed at the start of the `clone` command.
```bash
cd /tmp/<lib>
```
@@ -96,24 +66,17 @@ git add ...
git commit -m "..."
```
Generate patch files.
```bash
git format-patch 2.0..HEAD --zero-commit --abbrev=40 --no-signature
```
where `2.0` is replaced with the version specified in `update_<lib>.py`.
Move the patch files to `upstream_utils`.
```
mv *.patch allwpilib/upstream_utils/<lib>_patches
```
Navigate back to `upstream_utils`.
```bash
cd allwpilib/upstream_utils
```
Update the list of patch files in `update_<lib>.py`, then rerun
`update_<lib>.py` to reimport the thirdparty files.
Update the `upstream_utils` patch files.
```bash
./update_<lib>.py
./<lib>.py format-patch
```
Update the list of patch files in `<lib>.py`, then rerun `<lib>.py` to reimport the thirdparty files.
```bash
./<lib>.py copy-upstream-to-thirdparty
```

38
upstream_utils/update_eigen.py → upstream_utils/eigen.py
View File

@@ -5,11 +5,9 @@ import re
import shutil
from upstream_utils import (
get_repo_root,
clone_repo,
comment_out_invalid_includes,
walk_cwd_and_copy_if,
git_am,
Lib,
)
@@ -94,25 +92,9 @@ def unsupported_inclusions(dp, f):
return "MatrixFunctions" in abspath
def main():
upstream_root = clone_repo(
"https://gitlab.com/libeigen/eigen.git",
# master on 2024-05-22
"c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198",
shallow=False,
)
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
wpimath = os.path.join(wpilib_root, "wpimath")
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Disable-warnings.patch",
"0002-Intellisense-fix.patch",
"0003-Suppress-has_denorm-and-has_denorm_loss-deprecation-.patch",
]:
git_am(os.path.join(wpilib_root, "upstream_utils/eigen_patches", f))
# Delete old install
for d in ["src/main/native/thirdparty/eigen/include"]:
shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True)
@@ -139,10 +121,24 @@ def main():
)
shutil.copyfile(
os.path.join(upstream_root, ".clang-format"),
".clang-format",
os.path.join(wpimath, "src/main/native/thirdparty/eigen/include/.clang-format"),
)
def main():
name = "eigen"
url = "https://gitlab.com/libeigen/eigen.git"
tag = "c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198"
patch_list = [
"0001-Disable-warnings.patch",
"0002-Intellisense-fix.patch",
"0003-Suppress-has_denorm-and-has_denorm_loss-deprecation-.patch",
]
eigen = Lib(name, url, tag, patch_list, copy_upstream_src)
eigen.main()
if __name__ == "__main__":
main()

24
upstream_utils/update_expected.py → upstream_utils/expected.py
View File

@@ -10,26 +10,18 @@ from upstream_utils import (
comment_out_invalid_includes,
walk_cwd_and_copy_if,
git_am,
Lib,
)
def main():
upstream_root = clone_repo(
"https://github.com/TartanLlama/expected",
# master on 2024-01-25
"3f0ca7b19253129700a073abfa6d8638d9f7c80c",
shallow=False,
)
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
# Copy expected header into allwpilib
dest_filename = os.path.join(
wpiutil, "src/main/native/thirdparty/expected/include/wpi/expected"
)
shutil.copyfile(
os.path.join(upstream_root, "include/tl/expected.hpp"), dest_filename
)
shutil.copyfile("include/tl/expected.hpp", dest_filename)
# Rename namespace from tl to wpi
with open(dest_filename) as f:
@@ -41,5 +33,15 @@ def main():
f.write(content)
def main():
name = "expected"
url = "https://github.com/TartanLlama/expected"
# master on 2024-01-25
tag = "3f0ca7b19253129700a073abfa6d8638d9f7c80c"
expected = Lib(name, url, tag, [], copy_upstream_src)
expected.main()
if __name__ == "__main__":
main()

21
upstream_utils/update_fmt.py → upstream_utils/fmt.py
View File

@@ -9,19 +9,13 @@ from upstream_utils import (
comment_out_invalid_includes,
walk_cwd_and_copy_if,
git_am,
Lib,
)
def main():
upstream_root = clone_repo("https://github.com/fmtlib/fmt", "11.0.1")
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in ["0001-Suppress-warnings-we-can-t-fix.patch"]:
git_am(os.path.join(wpilib_root, "upstream_utils/fmt_patches", f))
# Delete old install
for d in [
"src/main/native/thirdparty/fmtlib/src",
@@ -51,5 +45,16 @@ def main():
)
def main():
name = "fmt"
url = "https://github.com/fmtlib/fmt"
tag = "11.0.1"
patch_list = ["0001-Suppress-warnings-we-can-t-fix.patch"]
fmt = Lib(name, url, tag, patch_list, copy_upstream_src)
fmt.main()
if __name__ == "__main__":
main()

27
upstream_utils/update_gcem.py → upstream_utils/gcem.py
View File

@@ -9,22 +9,13 @@ from upstream_utils import (
comment_out_invalid_includes,
walk_cwd_and_copy_if,
git_am,
Lib,
)
def main():
upstream_root = clone_repo("https://github.com/kthohr/gcem.git", "v1.18.0")
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
wpimath = os.path.join(wpilib_root, "wpimath")
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Call-std-functions-if-not-constant-evaluated.patch",
"0002-Add-hypot-x-y-z.patch",
]:
git_am(os.path.join(wpilib_root, "upstream_utils/gcem_patches", f))
# Delete old install
for d in [
"src/main/native/thirdparty/gcem/include",
@@ -43,5 +34,19 @@ def main():
)
def main():
name = "gcem"
url = "https://github.com/kthohr/gcem.git"
tag = "v1.18.0"
patch_list = [
"0001-Call-std-functions-if-not-constant-evaluated.patch",
"0002-Add-hypot-x-y-z.patch",
]
gcem = Lib(name, url, tag, patch_list, copy_upstream_src)
gcem.main()
if __name__ == "__main__":
main()

43
upstream_utils/update_json.py → upstream_utils/json.py
View File

@@ -8,27 +8,13 @@ from upstream_utils import (
clone_repo,
walk_if,
git_am,
Lib,
)
def main():
upstream_root = clone_repo("https://github.com/nlohmann/json", "v3.11.3")
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Remove-version-from-namespace.patch",
"0002-Make-serializer-public.patch",
"0003-Make-dump_escaped-take-std-string_view.patch",
"0004-Add-llvm-stream-support.patch",
]:
git_am(
os.path.join(wpilib_root, "upstream_utils/json_patches", f),
use_threeway=True,
)
# Delete old install
for d in [
"src/main/native/thirdparty/json/include",
@@ -36,11 +22,9 @@ def main():
shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
# Create lists of source and destination files
os.chdir(os.path.join(upstream_root, "include/nlohmann"))
os.chdir("include/nlohmann")
files = walk_if(".", lambda dp, f: True)
src_include_files = [
os.path.join(os.path.join(upstream_root, "include/nlohmann"), f) for f in files
]
src_include_files = [os.path.abspath(f) for f in files]
wpiutil_json_root = os.path.join(
wpiutil, "src/main/native/thirdparty/json/include/wpi"
)
@@ -74,5 +58,24 @@ def main():
f.write(content)
def main():
name = "json"
url = "https://github.com/nlohmann/json"
tag = "v3.11.3"
patch_list = [
"0001-Remove-version-from-namespace.patch",
"0002-Make-serializer-public.patch",
"0003-Make-dump_escaped-take-std-string_view.patch",
"0004-Add-llvm-stream-support.patch",
]
patch_options = {
"use_threeway": True,
}
json = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options)
json.main()
if __name__ == "__main__":
main()

43
upstream_utils/update_libuv.py → upstream_utils/libuv.py
View File

@@ -9,30 +9,13 @@ from upstream_utils import (
comment_out_invalid_includes,
walk_cwd_and_copy_if,
git_am,
Lib,
)
def main():
upstream_root = clone_repo("https://github.com/libuv/libuv", "v1.48.0")
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
wpinet = os.path.join(wpilib_root, "wpinet")
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Revert-win-process-write-minidumps-when-sending-SIGQ.patch",
"0002-Fix-missing-casts.patch",
"0003-Fix-warnings.patch",
"0004-Preprocessor-cleanup.patch",
"0005-Cleanup-problematic-language.patch",
"0006-Fix-Win32-warning-suppression-pragma.patch",
"0007-Use-C-atomics.patch",
"0008-Remove-static-from-array-indices.patch",
"0009-Add-pragmas-for-missing-libraries-and-set-_WIN32_WIN.patch",
"0010-Remove-swearing.patch",
]:
git_am(os.path.join(wpilib_root, "upstream_utils/libuv_patches", f))
# Delete old install
for d in ["src/main/native/thirdparty/libuv"]:
shutil.rmtree(os.path.join(wpinet, d), ignore_errors=True)
@@ -71,5 +54,27 @@ def main():
)
def main():
name = "libuv"
url = "https://github.com/libuv/libuv"
tag = "v1.48.0"
patch_list = [
"0001-Revert-win-process-write-minidumps-when-sending-SIGQ.patch",
"0002-Fix-missing-casts.patch",
"0003-Fix-warnings.patch",
"0004-Preprocessor-cleanup.patch",
"0005-Cleanup-problematic-language.patch",
"0006-Fix-Win32-warning-suppression-pragma.patch",
"0007-Use-C-atomics.patch",
"0008-Remove-static-from-array-indices.patch",
"0009-Add-pragmas-for-missing-libraries-and-set-_WIN32_WIN.patch",
"0010-Remove-swearing.patch",
]
libuv = Lib(name, url, tag, patch_list, copy_upstream_src)
libuv.main()
if __name__ == "__main__":
main()

32
upstream_utils/update_llvm.py → upstream_utils/llvm.py
View File

@@ -9,6 +9,7 @@ from upstream_utils import (
comment_out_invalid_includes,
walk_cwd_and_copy_if,
git_am,
Lib,
)
@@ -170,14 +171,20 @@ def overwrite_tests(wpiutil_root, llvm_root):
run_global_replacements(wpi_files)
def main():
upstream_root = clone_repo("https://github.com/llvm/llvm-project", "llvmorg-18.1.8")
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
upstream_root = os.path.abspath(".")
wpiutil = os.path.join(wpilib_root, "wpiutil")
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
overwrite_source(wpiutil, upstream_root)
overwrite_tests(wpiutil, upstream_root)
def main():
name = "llvm"
url = "https://github.com/llvm/llvm-project"
tag = "llvmorg-18.1.8"
patch_list = [
"0001-Remove-StringRef-ArrayRef-and-Optional.patch",
"0002-Wrap-std-min-max-calls-in-parens-for-Windows-warning.patch",
"0003-Change-unique_function-storage-size.patch",
@@ -215,14 +222,13 @@ def main():
"0035-Remove-auto-conversion-from-raw_ostream.patch",
"0036-Add-SmallVector-erase_if.patch",
"0037-Fix-AlignedCharArrayUnion-for-C-23.patch",
]:
git_am(
os.path.join(wpilib_root, "upstream_utils/llvm_patches", f),
use_threeway=True,
)
]
patch_options = {
"use_threeway": True,
}
overwrite_source(wpiutil, upstream_root)
overwrite_tests(wpiutil, upstream_root)
llvm = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options)
llvm.main()
if __name__ == "__main__":

17
upstream_utils/update_memory.py → upstream_utils/memory.py
View File

@@ -9,6 +9,7 @@ from upstream_utils import (
comment_out_invalid_includes,
walk_if,
copy_to,
Lib,
)
@@ -58,9 +59,7 @@ def run_global_replacements(memory_files):
f.write(content)
def main():
upstream_root = clone_repo("https://github.com/foonathan/memory", "v0.7-3")
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
# Delete old install
@@ -71,7 +70,6 @@ def main():
shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
# Copy sources
os.chdir(upstream_root)
src_files = walk_if("src", lambda dp, f: f.endswith(".cpp") or f.endswith(".hpp"))
src_files = copy_to(
src_files, os.path.join(wpiutil, "src/main/native/thirdparty/memory")
@@ -80,7 +78,7 @@ def main():
run_source_replacements(src_files)
# Copy headers
os.chdir(os.path.join(upstream_root, "include", "foonathan"))
os.chdir(os.path.join("include", "foonathan"))
include_files = walk_if(".", lambda dp, f: f.endswith(".hpp"))
include_files = copy_to(
include_files,
@@ -100,5 +98,14 @@ def main():
)
def main():
name = "memory"
url = "https://github.com/foonathan/memory"
tag = "v0.7-3"
memory = Lib(name, url, tag, [], copy_upstream_src)
memory.main()
if __name__ == "__main__":
main()

34
upstream_utils/update_mpack.py → upstream_utils/mpack.py
View File

@@ -9,12 +9,11 @@ from upstream_utils import (
clone_repo,
walk_cwd_and_copy_if,
git_am,
Lib,
)
def main():
upstream_root = clone_repo("https://github.com/ludocode/mpack", "v1.1.1")
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
# Delete old install
@@ -24,19 +23,6 @@ def main():
]:
shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True)
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Don-t-emit-inline-defs.patch",
"0002-Update-amalgamation-script.patch",
"0003-Use-namespace-for-C.patch",
"0004-Group-doxygen-into-MPack-module.patch",
]:
git_am(
os.path.join(wpilib_root, "upstream_utils/mpack_patches", f),
)
# Run the amalgmation script
subprocess.check_call(["bash", "tools/amalgamate.sh"])
@@ -56,5 +42,21 @@ def main():
)
def main():
name = "mpack"
url = "https://github.com/ludocode/mpack"
tag = "v1.1.1"
patch_list = [
"0001-Don-t-emit-inline-defs.patch",
"0002-Update-amalgamation-script.patch",
"0003-Use-namespace-for-C.patch",
"0004-Group-doxygen-into-MPack-module.patch",
]
mpack = Lib(name, url, tag, patch_list, copy_upstream_src)
mpack.main()
if __name__ == "__main__":
main()

50
upstream_utils/update_protobuf.py → upstream_utils/protobuf.py
View File

@@ -11,6 +11,7 @@ from upstream_utils import (
walk_cwd_and_copy_if,
walk_if,
git_am,
Lib,
)
protobuf_lite_sources = set(
@@ -255,31 +256,10 @@ def matches(dp, f, files):
return p in files
def main():
upstream_root = clone_repo(
"https://github.com/protocolbuffers/protobuf", "v3.21.12"
)
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
upstream_root = os.path.abspath(".")
wpiutil = os.path.join(wpilib_root, "wpiutil")
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Fix-sign-compare-warnings.patch",
"0002-Remove-redundant-move.patch",
"0003-Fix-maybe-uninitialized-warnings.patch",
"0004-Fix-coded_stream-WriteRaw.patch",
"0005-Suppress-enum-enum-conversion-warning.patch",
"0006-Fix-noreturn-function-returning.patch",
"0007-Work-around-GCC-12-restrict-warning-compiler-bug.patch",
"0008-Disable-MSVC-switch-warning.patch",
"0009-Disable-unused-function-warning.patch",
"0010-Disable-pedantic-warning.patch",
"0011-Avoid-use-of-sprintf.patch",
"0012-Suppress-stringop-overflow-warning-false-positives.patch",
]:
git_am(os.path.join(wpilib_root, "upstream_utils/protobuf_patches", f))
# Delete old install
for d in [
"src/main/native/thirdparty/protobuf/src",
@@ -304,5 +284,29 @@ def main():
)
def main():
name = "protobuf"
url = "https://github.com/protocolbuffers/protobuf"
tag = "v3.21.12"
patch_list = [
"0001-Fix-sign-compare-warnings.patch",
"0002-Remove-redundant-move.patch",
"0003-Fix-maybe-uninitialized-warnings.patch",
"0004-Fix-coded_stream-WriteRaw.patch",
"0005-Suppress-enum-enum-conversion-warning.patch",
"0006-Fix-noreturn-function-returning.patch",
"0007-Work-around-GCC-12-restrict-warning-compiler-bug.patch",
"0008-Disable-MSVC-switch-warning.patch",
"0009-Disable-unused-function-warning.patch",
"0010-Disable-pedantic-warning.patch",
"0011-Avoid-use-of-sprintf.patch",
"0012-Suppress-stringop-overflow-warning-false-positives.patch",
]
protobuf = Lib(name, url, tag, patch_list, copy_upstream_src)
protobuf.main()
if __name__ == "__main__":
main()

42
upstream_utils/update_sleipnir.py → upstream_utils/sleipnir.py
View File

@@ -9,30 +9,13 @@ from upstream_utils import (
copy_to,
walk_cwd_and_copy_if,
git_am,
Lib,
)
def main():
upstream_root = clone_repo(
"https://github.com/SleipnirGroup/Sleipnir",
# main on 2024-07-09
"b6ffa2d4fdb99cab1bf79491f715a6a9a86633b5",
shallow=False,
)
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
wpimath = os.path.join(wpilib_root, "wpimath")
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Remove-using-enum-declarations.patch",
"0002-Use-fmtlib.patch",
"0003-Remove-unsupported-constexpr.patch",
"0004-Use-wpi-SmallVector.patch",
"0005-Suppress-clang-tidy-false-positives.patch",
]:
git_am(os.path.join(wpilib_root, "upstream_utils/sleipnir_patches", f))
# Delete old install
for d in [
"src/main/native/thirdparty/sleipnir/src",
@@ -41,7 +24,6 @@ def main():
shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True)
# Copy Sleipnir source files into allwpilib
os.chdir(upstream_root)
src_files = [os.path.join(dp, f) for dp, dn, fn in os.walk("src") for f in fn]
src_files = copy_to(
src_files, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir")
@@ -65,10 +47,28 @@ def main():
".styleguide-license",
]:
shutil.copyfile(
os.path.join(upstream_root, filename),
filename,
os.path.join(wpimath, "src/main/native/thirdparty/sleipnir", filename),
)
def main():
name = "sleipnir"
url = "https://github.com/SleipnirGroup/Sleipnir"
# main on 2024-07-09
tag = "b6ffa2d4fdb99cab1bf79491f715a6a9a86633b5"
patch_list = [
"0001-Remove-using-enum-declarations.patch",
"0002-Use-fmtlib.patch",
"0003-Remove-unsupported-constexpr.patch",
"0004-Use-wpi-SmallVector.patch",
"0005-Suppress-clang-tidy-false-positives.patch",
]
sleipnir = Lib(name, url, tag, patch_list, copy_upstream_src)
sleipnir.main()
if __name__ == "__main__":
main()

54
upstream_utils/update_stack_walker.py → upstream_utils/stack_walker.py
View File

@@ -10,11 +10,12 @@ from upstream_utils import (
comment_out_invalid_includes,
walk_cwd_and_copy_if,
git_am,
Lib,
)
def crlf_to_lf(stackwalker_dir):
for root, _, files in os.walk(stackwalker_dir):
def crlf_to_lf():
for root, _, files in os.walk("."):
if ".git" in root:
continue
@@ -28,35 +29,13 @@ def crlf_to_lf(stackwalker_dir):
with open(filename, "wb") as f:
f.write(content)
cwd = os.getcwd()
os.chdir(stackwalker_dir)
subprocess.check_call(["git", "add", "-A"])
subprocess.check_call(["git", "commit", "-m", "Fix line endings"])
os.chdir(cwd)
def main():
upstream_root = clone_repo(
"https://github.com/JochenKalmbach/StackWalker",
"5b0df7a4db8896f6b6dc45d36e383c52577e3c6b",
shallow=False,
)
wpilib_root = get_repo_root()
def copy_upstream_src(wpilib_root):
wpiutil = os.path.join(wpilib_root, "wpiutil")
# Run CRLF -> LF before trying any patches
crlf_to_lf(upstream_root)
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Add-advapi-pragma.patch",
]:
git_am(
os.path.join(wpilib_root, "upstream_utils/stack_walker_patches", f),
ignore_whitespace=True,
)
shutil.copy(
os.path.join("Main", "StackWalker", "StackWalker.h"),
os.path.join(wpiutil, "src/main/native/windows/StackWalker.h"),
@@ -68,5 +47,30 @@ def main():
)
def main():
name = "stack_walker"
url = "https://github.com/JochenKalmbach/StackWalker"
tag = "5b0df7a4db8896f6b6dc45d36e383c52577e3c6b"
patch_list = [
"0001-Add-advapi-pragma.patch",
]
patch_options = {
"ignore_whitespace": True,
}
stack_walker = Lib(
name,
url,
tag,
patch_list,
copy_upstream_src,
patch_options,
pre_patch_hook=crlf_to_lf,
pre_patch_commits=1,
)
stack_walker.main()
if __name__ == "__main__":
main()

410
upstream_utils/upstream_utils.py
View File

@@ -1,7 +1,9 @@
import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
@@ -198,3 +200,411 @@ def git_am(patch, use_threeway=False, ignore_whitespace=False):
args.append("--ignore-whitespace")
subprocess.check_output(args + [patch])
def has_git_rev(rev):
"""Checks whether the Git repository in the current directory has the given
revision.
Keyword arguments:
rev -- The revision to check
Returns:
True if the revision exists, otherwise False.
"""
cmd = ["git", "rev-parse", "--verify", "-q", rev]
return subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode == 0
class Lib:
def __init__(
self,
name,
url,
tag,
patch_list,
copy_upstream_src,
patch_options={},
*,
pre_patch_hook=None,
pre_patch_commits=0,
):
"""Initializes a Lib instance.
Keyword arguments:
name -- The name of the library.
url -- The URL of the upstream repository.
tag -- The tag in the upstream repository to use. Can be any
<commit-ish> (e.g., commit hash or tag).
patch_list -- The list of patches in the patch directory to apply.
copy_upstream_src -- A callable that takes the path to the wpilib root
and copies the files from the clone of the upstream
into the appropriate thirdparty directory. Will
only be called when the current directory is the
upstream clone.
patch_options -- The dictionary of options to use when applying patches.
Corresponds to the parameters of git_am.
Keyword-only arguments:
pre_patch_hook -- Optional callable taking no parameters that will be
called before applying patches.
pre_patch_commits -- Number of commits added by pre_patch_hook.
"""
self.name = name
self.url = url
self.old_tag = tag
self.patch_list = patch_list
self.copy_upstream_src = copy_upstream_src
self.patch_options = patch_options
self.pre_patch_hook = pre_patch_hook
self.pre_patch_commits = pre_patch_commits
self.wpilib_root = get_repo_root()
def check_patches(self):
"""Checks that the patch list supplied to the constructor matches the
patches in the patch directory.
"""
patch_directory_patches = set()
patch_directory = os.path.join(
self.wpilib_root, f"upstream_utils/{self.name}_patches"
)
if os.path.exists(patch_directory):
for f in os.listdir(patch_directory):
if f.endswith(".patch"):
patch_directory_patches.add(f)
patches = set(self.patch_list)
patch_directory_only = sorted(patch_directory_patches - patches)
patch_list_only = sorted(patches - patch_directory_patches)
common_patches = sorted(patch_directory_patches & patches)
warning = False
if patch_directory_only:
print(
f"WARNING: The patch directory has patches {patch_directory_only} not in the patch list"
)
warning = True
if patch_list_only:
print(
f"WARNING: The patch list has patches {patch_list_only} not in the patch directory"
)
warning = True
if warning and common_patches:
print(
f" Note: The patch directory and the patch list both have patches {common_patches}"
)
def get_repo_path(self, tempdir=None):
"""Returns the path to the clone of the upstream repository.
Keyword argument:
tempdir -- The path to the temporary directory to use. If None (the
default), uses tempfile.gettempdir().
Returns:
The path to the clone of the upstream repository. Will be absolute if
tempdir is absolute.
"""
if tempdir is None:
tempdir = tempfile.gettempdir()
repo = os.path.basename(self.url)
dest = os.path.join(tempdir, repo)
dest = dest.removesuffix(".git")
return dest
def open_repo(self, *, err_msg_if_absent):
"""Changes the current working directory to the upstream repository. If
err_msg_if_absent is not None and the upstream repository does not
exist, the program exits with return code 1.
Keyword-only argument:
err_msg_if_absent -- The error message to print to stderr if the
upstream repository does not exist. If None, the upstream repository
will be cloned without emitting any warnings.
"""
os.chdir(tempfile.gettempdir())
dest = self.get_repo_path(os.getcwd())
print(f"INFO: Opening repository at {dest}")
if not os.path.exists(dest):
if err_msg_if_absent is None:
subprocess.run(["git", "clone", "--filter=tree:0", self.url])
else:
print(err_msg_if_absent, file=sys.stderr)
exit(1)
os.chdir(dest)
def get_root_tags(self):
"""Returns a list of potential root tags.
Returns:
A list of the potential root tags.
"""
root_tag_output = subprocess.run(
["git", "tag", "--list", "upstream_utils_root-*"],
capture_output=True,
text=True,
).stdout
return root_tag_output.splitlines()
def get_root_tag(self):
"""Returns the root tag (the default tag to apply the patches relative
to). If there are multiple candidates, prints an error to stderr and the
program exits with return code 1.
Returns:
The root tag.
"""
root_tags = self.get_root_tags()
if len(root_tags) == 0:
print(
"ERROR: Could not determine root tag: No tags match 'upstream_utils_root-*'",
file=sys.stderr,
)
exit(1)
if len(root_tags) > 1:
print(
f"ERROR: Could not determine root tag: Multiple candidates: {root_tags}",
file=sys.stderr,
)
exit(1)
return root_tags[0]
def set_root_tag(self, tag):
"""Sets the root tag, deleting any potential candidates first.
Keyword argument:
tag -- The tag to set as the root tag.
"""
root_tags = self.get_root_tags()
if len(root_tags) > 1:
print(f"WARNING: Deleting multiple root tags {root_tags}", file=sys.stderr)
for root_tag in root_tags:
subprocess.run(["git", "tag", "-d", root_tag])
subprocess.run(["git", "tag", f"upstream_utils_root-{tag}", tag])
def apply_patches(self):
"""Applies the patches listed in the patch list to the current
directory.
"""
if self.pre_patch_hook is not None:
self.pre_patch_hook()
for f in self.patch_list:
git_am(
os.path.join(
self.wpilib_root, f"upstream_utils/{self.name}_patches", f
),
**self.patch_options,
)
def replace_tag(self, tag):
"""Replaces the tag in the script.
Keyword argument:
tag -- The tag to replace the script tag with.
"""
path = os.path.join(self.wpilib_root, f"upstream_utils/{self.name}.py")
with open(path, "r") as file:
lines = file.readlines()
previous_text = f'tag = "{self.old_tag}"'
new_text = f'tag = "{tag}"'
for i in range(len(lines)):
if previous_text in lines[i]:
if i - 1 >= 0 and "#" in lines[i - 1]:
print(
f"WARNING: Automatically updating tag in line {i + 1} with a comment above it that may need updating.",
file=sys.stderr,
)
lines[i] = lines[i].replace(previous_text, new_text)
with open(path, "w") as file:
file.writelines(lines)
def info(self):
"""Prints info about the library to stdout."""
print(f"Repository name: {self.name}")
print(f"Upstream URL: {self.url}")
print(f"Upstream tag: {self.old_tag}")
print(f"Path to upstream clone: {self.get_repo_path()}")
print(f"Patches to apply: {self.patch_list}")
print(f"Patch options: {self.patch_options}")
print(f"Pre patch commits: {self.pre_patch_commits}")
print(f"WPILib root: {self.wpilib_root}")
def clone(self):
"""Clones the upstream repository and sets it up."""
self.open_repo(err_msg_if_absent=None)
subprocess.run(["git", "switch", "--detach", self.old_tag])
self.set_root_tag(self.old_tag)
def reset(self):
"""Resets the clone of the upstream repository to the state specified by
the script and patches.
"""
self.open_repo(
err_msg_if_absent='There\'s nothing to reset. Run the "clone" command first.'
)
subprocess.run(["git", "switch", "--detach", self.old_tag])
self.apply_patches()
self.set_root_tag(self.old_tag)
def rebase(self, new_tag):
"""Rebases the patches.
Keyword argument:
new_tag -- The tag to rebase onto.
"""
self.open_repo(
err_msg_if_absent='There\'s nothing to rebase. Run the "clone" command first.'
)
subprocess.run(["git", "fetch", "origin", new_tag])
subprocess.run(["git", "switch", "--detach", self.old_tag])
self.apply_patches()
self.set_root_tag(new_tag)
subprocess.run(["git", "rebase", "--onto", new_tag, self.old_tag])
# Detect merge conflict by detecting if we stopped in the middle of a rebase
if has_git_rev("REBASE_HEAD"):
print(
f"Merge conflicts when rebasing onto {new_tag}! You must manually resolve them.",
file=sys.stderr,
)
def format_patch(self):
"""Generates patch files for the upstream repository and moves them into
the patch directory.
"""
self.open_repo(
err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.'
)
root_tag = self.get_root_tag()
script_tag = root_tag.removeprefix("upstream_utils_root-")
start_commit = root_tag
if self.pre_patch_commits > 0:
commits_since_tag_output = subprocess.run(
["git", "log", "--format=format:%h", f"{start_commit}..HEAD"],
capture_output=True,
).stdout
commits_since_tag = commits_since_tag_output.count(b"\n") + 1
start_commit = f"HEAD~{commits_since_tag - self.pre_patch_commits}"
subprocess.run(
[
"git",
"format-patch",
f"{start_commit}..HEAD",
"--abbrev=40",
"--zero-commit",
"--no-signature",
]
)
patch_dest = os.path.join(
self.wpilib_root, f"upstream_utils/{self.name}_patches"
)
if not os.path.exists(patch_dest):
print(
f"WARNING: Patch directory {patch_dest} does not exist", file=sys.stderr
)
else:
shutil.rmtree(patch_dest)
is_first = True
for f in os.listdir():
if f.endswith(".patch"):
if is_first:
os.mkdir(patch_dest)
is_first = False
shutil.move(f, patch_dest)
self.replace_tag(script_tag)
def copy_upstream_to_thirdparty(self):
"""Copies files from the upstream repository into the thirdparty
directory.
"""
self.open_repo(
err_msg_if_absent='There\'s no repository to copy from. Run the "clone" command first.'
)
subprocess.run(["git", "switch", "--detach", self.old_tag])
self.apply_patches()
self.copy_upstream_src(self.wpilib_root)
def main(self, argv=sys.argv[1:]):
"""Processes the given arguments.
Keyword argument:
argv -- The arguments to process. Defaults to the arguments passed to
the program.
"""
parser = argparse.ArgumentParser(
description=f"CLI manager of the {self.name} upstream library"
)
subparsers = parser.add_subparsers(dest="subcommand", required=True)
subparsers.add_parser(
"info", help="Displays information about the upstream library"
)
subparsers.add_parser(
"clone", help="Clones the upstream repository in a local tempdir"
)
subparsers.add_parser(
"reset",
help="Resets the clone of the upstream repository to the tag and applies patches",
)
parser_rebase = subparsers.add_parser(
"rebase", help="Rebases the clone of the upstream repository"
)
parser_rebase.add_argument("new_tag", help="The tag to rebase onto")
parser_format_patch = subparsers.add_parser(
"format-patch",
help="Generates patch files for the upstream repository and moves them into the upstream_utils patch directory",
)
subparsers.add_parser(
"copy-upstream-to-thirdparty",
help="Copies files from the upstream repository into the thirdparty directory in allwpilib",
)
args = parser.parse_args(argv)
self.wpilib_root = get_repo_root()
if args.subcommand == "info":
self.info()
elif args.subcommand == "clone":
self.clone()
elif args.subcommand == "reset":
self.reset()
elif args.subcommand == "rebase":
self.rebase(args.new_tag)
elif args.subcommand == "format-patch":
self.format_patch()
elif args.subcommand == "copy-upstream-to-thirdparty":
self.copy_upstream_to_thirdparty()
self.check_patches()