Team2890/allwpilib
4
0
Fork 0
mirror of https://github.com/wpilibsuite/allwpilib synced 2026-06-19 00:41:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

[upstream_utils] Refactor upstream_utils scripts (#4367)

Browse Source 
* Root folder variable names are now more descriptive
* clone_repo() now restores the current working directory
* Removed setup_upstream_repo() since it's now identical to clone_repo()
* Moved am_patches()'s for loop into user scripts so the filename prefix
  doesn't need to be included in every patch filename
* Renamed am_patches() to git_am() since its only job now is to run "git am"
* Removed unused apply_patches() function
* Fixed typo in git_am()'s ignore_whitespace arg name
This commit is contained in:
Tyler Veness
2022-08-20 07:26:34 -07:00
committed by GitHub
parent d80e8039d7
commit 5adf50d93c
7 changed files with 140 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
upstream_utils/update_drake.py
View File

@@ -4,29 +4,26 @@ import os
import shutil
from upstream_utils import (
setup_upstream_repo,
get_repo_root,
clone_repo,
comment_out_invalid_includes,
walk_cwd_and_copy_if,
am_patches,
git_am,
)
def main():
root, repo = setup_upstream_repo(
"https://github.com/RobotLocomotion/drake", "v1.6.0"
)
wpimath = os.path.join(root, "wpimath")
upstream_root = clone_repo("https://github.com/RobotLocomotion/drake", "v1.6.0")
wpilib_root = get_repo_root()
wpimath = os.path.join(wpilib_root, "wpimath")
prefix = os.path.join(root, "upstream_utils/drake_patches")
am_patches(
repo,
[
os.path.join(prefix, "0001-Replace-Eigen-Dense-with-Eigen-Core.patch"),
os.path.join(
prefix, "0002-Add-WPILIB_DLLEXPORT-to-DARE-function-declarations.patch"
),
],
)
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Replace-Eigen-Dense-with-Eigen-Core.patch",
"0002-Add-WPILIB_DLLEXPORT-to-DARE-function-declarations.patch",
]:
git_am(os.path.join(wpilib_root, "upstream_utils/drake_patches", f))
# Delete old install
for d in [
@@ -60,12 +57,12 @@ def main():
)
# Copy drake test source files into allwpilib
os.chdir(os.path.join(repo, "math/test"))
os.chdir(os.path.join(upstream_root, "math/test"))
test_src_files = walk_cwd_and_copy_if(
lambda dp, f: f == "discrete_algebraic_riccati_equation_test.cc",
os.path.join(wpimath, "src/test/native/cpp/drake"),
)
os.chdir(repo)
os.chdir(upstream_root)
# Copy drake test header files into allwpilib
test_include_files = walk_cwd_and_copy_if(

17
upstream_utils/update_eigen.py
View File

@@ -5,10 +5,11 @@ import re
import shutil
from upstream_utils import (
setup_upstream_repo,
get_repo_root,
clone_repo,
comment_out_invalid_includes,
walk_cwd_and_copy_if,
am_patches,
git_am,
)
@@ -102,12 +103,14 @@ def unsupported_inclusions(dp, f):
def main():
root, repo = setup_upstream_repo("https://gitlab.com/libeigen/eigen.git", "3.4.0")
wpimath = os.path.join(root, "wpimath")
upstream_root = clone_repo("https://gitlab.com/libeigen/eigen.git", "3.4.0")
wpilib_root = get_repo_root()
wpimath = os.path.join(wpilib_root, "wpimath")
# Apply patches to original git repo
prefix = os.path.join(root, "upstream_utils/eigen_patches")
am_patches(repo, [os.path.join(prefix, "0001-Disable-warnings.patch")])
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in ["0001-Disable-warnings.patch"]:
git_am(os.path.join(wpilib_root, "upstream_utils/eigen_patches", f))
# Delete old install
for d in [

28
upstream_utils/update_fmt.py
View File

@@ -4,28 +4,26 @@ import os
import shutil
from upstream_utils import (
setup_upstream_repo,
get_repo_root,
clone_repo,
comment_out_invalid_includes,
walk_cwd_and_copy_if,
am_patches,
git_am,
)
def main():
root, repo = setup_upstream_repo("https://github.com/fmtlib/fmt", "9.0.0")
wpiutil = os.path.join(root, "wpiutil")
upstream_root = clone_repo("https://github.com/fmtlib/fmt", "9.0.0")
wpilib_root = get_repo_root()
wpiutil = os.path.join(wpilib_root, "wpiutil")
# Apply patches to original git repo
prefix = os.path.join(root, "upstream_utils/fmt_patches")
am_patches(
repo,
[
os.path.join(prefix, "0001-Don-t-throw-on-write-failure.patch"),
os.path.join(
prefix, "0002-Suppress-clang-tidy-warning-false-positive.patch"
),
],
)
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Don-t-throw-on-write-failure.patch",
"0002-Suppress-clang-tidy-warning-false-positive.patch",
]:
git_am(os.path.join(wpilib_root, "upstream_utils/fmt_patches", f))
# Delete old install
for d in [

43
upstream_utils/update_libuv.py
View File

@@ -2,38 +2,35 @@
import os
import shutil
import re
from upstream_utils import (
setup_upstream_repo,
get_repo_root,
clone_repo,
comment_out_invalid_includes,
walk_cwd_and_copy_if,
am_patches,
walk_if,
copy_to,
git_am,
)
def main():
root, repo = setup_upstream_repo("https://github.com/libuv/libuv", "v1.44.2")
wpinet = os.path.join(root, "wpinet")
upstream_root = clone_repo("https://github.com/libuv/libuv", "v1.44.2")
wpilib_root = get_repo_root()
wpinet = os.path.join(wpilib_root, "wpinet")
# Apply patches to original git repo
prefix = os.path.join(root, "upstream_utils/libuv_patches")
am_patches(
repo,
[
os.path.join(prefix, "0001-Fix-missing-casts.patch"),
os.path.join(prefix, "0002-Fix-warnings.patch"),
os.path.join(prefix, "0003-Preprocessor-cleanup.patch"),
os.path.join(prefix, "0004-Cleanup-problematic-language.patch"),
os.path.join(prefix, "0005-Use-roborio-time.patch"),
os.path.join(prefix, "0006-Style-comments-cleanup.patch"),
os.path.join(prefix, "0007-Squelch-GCC-12.1-warnings.patch"),
os.path.join(prefix, "0008-Fix-Win32-warning-suppression-pragma.patch"),
os.path.join(prefix, "0009-Avoid-unused-variable-warning-on-Mac.patch"),
],
)
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Fix-missing-casts.patch",
"0002-Fix-warnings.patch",
"0003-Preprocessor-cleanup.patch",
"0004-Cleanup-problematic-language.patch",
"0005-Use-roborio-time.patch",
"0006-Style-comments-cleanup.patch",
"0007-Squelch-GCC-12.1-warnings.patch",
"0008-Fix-Win32-warning-suppression-pragma.patch",
"0009-Avoid-unused-variable-warning-on-Mac.patch",
]:
git_am(os.path.join(wpilib_root, "upstream_utils/libuv_patches", f))
# Delete old install
for d in ["src/main/native/thirdparty/libuv"]:

94
upstream_utils/update_llvm.py
View File

@@ -4,12 +4,11 @@ import os
import shutil
from upstream_utils import (
setup_upstream_repo,
get_repo_root,
clone_repo,
comment_out_invalid_includes,
walk_cwd_and_copy_if,
am_patches,
walk_if,
copy_to,
git_am,
)
@@ -153,54 +152,49 @@ def overwrite_tests(wpiutil_root, llvm_root):
def main():
root, repo = setup_upstream_repo(
"https://github.com/llvm/llvm-project", "llvmorg-14.0.6"
)
wpiutil = os.path.join(root, "wpiutil")
upstream_root = clone_repo("https://github.com/llvm/llvm-project", "llvmorg-14.0.6")
wpilib_root = get_repo_root()
wpiutil = os.path.join(wpilib_root, "wpiutil")
patch_root = os.path.join(root, "upstream_utils/llvm_patches")
frontend_patches = [
os.path.join(patch_root, "0001-Fix-spelling-language-errors.patch"),
os.path.join(patch_root, "0002-Remove-StringRef-ArrayRef-and-Optional.patch"),
os.path.join(
patch_root,
"0003-Wrap-std-min-max-calls-in-parens-for-Windows-warning.patch",
),
os.path.join(patch_root, "0004-Change-unique_function-storage-size.patch"),
os.path.join(patch_root, "0005-Threading-updates.patch"),
os.path.join(patch_root, "0006-ifdef-guard-safety.patch"),
os.path.join(patch_root, "0007-Explicitly-use-std.patch"),
os.path.join(patch_root, "0008-Remove-format_provider.patch"),
os.path.join(patch_root, "0009-Add-compiler-warning-pragmas.patch"),
os.path.join(patch_root, "0010-Remove-unused-functions.patch"),
os.path.join(patch_root, "0011-Detemplatize-SmallVectorBase.patch"),
os.path.join(patch_root, "0012-Add-vectors-to-raw_ostream.patch"),
os.path.join(patch_root, "0013-Extra-collections-features.patch"),
os.path.join(patch_root, "0014-EpochTracker-ABI-macro.patch"),
os.path.join(patch_root, "0015-Delete-numbers-from-MathExtras.patch"),
os.path.join(patch_root, "0016-Add-lerp-and-sgn.patch"),
os.path.join(patch_root, "0017-Fixup-includes.patch"),
os.path.join(patch_root, "0018-Use-std-is_trivially_copy_constructible.patch"),
os.path.join(patch_root, "0019-Windows-support.patch"),
os.path.join(patch_root, "0020-Prefer-fmtlib.patch"),
os.path.join(patch_root, "0021-Prefer-wpi-s-fs.h.patch"),
os.path.join(patch_root, "0022-Remove-unused-functions.patch"),
os.path.join(patch_root, "0023-OS-specific-changes.patch"),
os.path.join(patch_root, "0024-Use-SmallVector-for-UTF-conversion.patch"),
os.path.join(
patch_root, "0025-Prefer-to-use-static-pointers-in-raw_ostream.patch"
),
os.path.join(patch_root, "0026-constexpr-endian-byte-swap.patch"),
os.path.join(
patch_root,
"0027-Copy-type-traits-from-STLExtras.h-into-PointerUnion..patch",
),
os.path.join(patch_root, "0028-Remove-StringMap-test-for-llvm-sort.patch"),
]
am_patches(repo, frontend_patches, use_threeway=True)
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Fix-spelling-language-errors.patch",
"0002-Remove-StringRef-ArrayRef-and-Optional.patch",
"0003-Wrap-std-min-max-calls-in-parens-for-Windows-warning.patch",
"0004-Change-unique_function-storage-size.patch",
"0005-Threading-updates.patch",
"0006-ifdef-guard-safety.patch",
"0007-Explicitly-use-std.patch",
"0008-Remove-format_provider.patch",
"0009-Add-compiler-warning-pragmas.patch",
"0010-Remove-unused-functions.patch",
"0011-Detemplatize-SmallVectorBase.patch",
"0012-Add-vectors-to-raw_ostream.patch",
"0013-Extra-collections-features.patch",
"0014-EpochTracker-ABI-macro.patch",
"0015-Delete-numbers-from-MathExtras.patch",
"0016-Add-lerp-and-sgn.patch",
"0017-Fixup-includes.patch",
"0018-Use-std-is_trivially_copy_constructible.patch",
"0019-Windows-support.patch",
"0020-Prefer-fmtlib.patch",
"0021-Prefer-wpi-s-fs.h.patch",
"0022-Remove-unused-functions.patch",
"0023-OS-specific-changes.patch",
"0024-Use-SmallVector-for-UTF-conversion.patch",
"0025-Prefer-to-use-static-pointers-in-raw_ostream.patch",
"0026-constexpr-endian-byte-swap.patch",
"0027-Copy-type-traits-from-STLExtras.h-into-PointerUnion..patch",
"0028-Remove-StringMap-test-for-llvm-sort.patch",
]:
git_am(
os.path.join(wpilib_root, "upstream_utils/llvm_patches", f),
use_threeway=True,
)
overwrite_source(wpiutil, repo)
overwrite_tests(wpiutil, repo)
overwrite_source(wpiutil, upstream_root)
overwrite_tests(wpiutil, upstream_root)
if __name__ == "__main__":

39
upstream_utils/update_stack_walker.py
View File

@@ -5,12 +5,11 @@ import shutil
import subprocess
from upstream_utils import (
setup_upstream_repo,
get_repo_root,
clone_repo,
comment_out_invalid_includes,
walk_cwd_and_copy_if,
am_patches,
walk_if,
copy_to,
git_am,
)
@@ -29,32 +28,36 @@ def crlf_to_lf(stackwalker_dir):
with open(filename, "wb") as f:
f.write(content)
cwd = os.getcwd()
os.chdir(stackwalker_dir)
subprocess.check_call(["git", "add", "-A"])
subprocess.check_call(["git", "commit", "-m", "Fix line endings"])
os.chdir(cwd)
def main():
root, repo = setup_upstream_repo(
upstream_root = clone_repo(
"https://github.com/JochenKalmbach/StackWalker",
"42e7a6e056a9e7aca911a7e9e54e2e4f90bc2652",
shallow=False,
)
wpiutil = os.path.join(root, "wpiutil")
wpilib_root = get_repo_root()
wpiutil = os.path.join(wpilib_root, "wpiutil")
# Run CRLF -> LF before trying any patches
crlf_to_lf(repo)
crlf_to_lf(upstream_root)
# Apply patches to original git repo
patch_dir = os.path.join(root, "upstream_utils/stack_walker_patches")
am_patches(
repo,
[
os.path.join(patch_dir, "0001-Apply-PR-35.patch"),
os.path.join(patch_dir, "0002-Remove-_M_IX86-checks.patch"),
os.path.join(patch_dir, "0003-Add-advapi-pragma.patch"),
],
ignore_whitespce=True,
)
# Apply patches to upstream Git repo
os.chdir(upstream_root)
for f in [
"0001-Apply-PR-35.patch",
"0002-Remove-_M_IX86-checks.patch",
"0003-Add-advapi-pragma.patch",
]:
git_am(
os.path.join(wpilib_root, "upstream_utils/stack_walker_patches", f),
ignore_whitespace=True,
)
shutil.copy(
os.path.join("Main", "StackWalker", "StackWalker.h"),

64
upstream_utils/upstream_utils.py
View File

@@ -6,16 +6,18 @@ import tempfile
def clone_repo(url, treeish, shallow=True):
"""Clones a git repo at the given URL into a temp folder and checks out the
given tree-ish (either branch or tag).
The current working directory will be set to the repository folder.
"""Clones a Git repo at the given URL into a temp folder, checks out the
given tree-ish (either branch or tag), then returns the repo root.
Keyword argument:
url -- The URL of the git repo
url -- The URL of the Git repo
treeish -- The tree-ish to check out (branch or tag)
shallow -- Whether to do a shallow clone
Returns:
root -- root directory of the cloned Git repository
"""
cwd = os.getcwd()
os.chdir(tempfile.gettempdir())
repo = os.path.basename(url)
@@ -49,6 +51,9 @@ def clone_repo(url, treeish, shallow=True):
else:
subprocess.run(["git", "checkout", treeish])
os.chdir(cwd)
return dest
def get_repo_root():
"""Returns the Git repository root as an absolute path.
@@ -63,27 +68,6 @@ def get_repo_root():
return ""
def setup_upstream_repo(url, treeish, shallow=True):
"""Clones the given upstream repository, then returns the root of the
destination Git repository as well as the cloned upstream Git repository.
The current working directory will be set to the cloned upstream repository
folder.
Keyword arguments:
url -- The URL of the git repo
treeish -- The tree-ish to check out (branch or tag)
shallow -- Whether to do a shallow clone
Returns:
root -- root directory of destination Git repository
repo -- root directory of cloned upstream Git repository
"""
root = get_repo_root()
clone_repo(url, treeish, shallow=shallow)
return root, os.getcwd()
def walk_if(top, pred):
"""Walks the current directory, then returns a list of files for which the
given predicate is true.
@@ -196,32 +180,18 @@ def comment_out_invalid_includes(filename, include_roots):
f.write(new_contents)
def apply_patches(root, patches):
"""Apply list of patches to the destination Git repository using "git
apply".
def git_am(patch, use_threeway=False, ignore_whitespace=False):
"""Apply patch to a Git repository in the current directory using "git am".
Keyword arguments:
root -- the root directory of the destination Git repository
patches -- list of patch files relative to the root
patch -- patch file relative to the root
use_threeway -- use a three-way merge when applying the patch
ignore_whitespace -- ignore whitespace in the patch file
"""
os.chdir(root)
for patch in patches:
subprocess.check_output(["git", "apply", patch])
def am_patches(root, patches, use_threeway=False, ignore_whitespce=False):
"""Apply list of patches to the destination Git repository using "git am".
Keyword arguments:
root -- the root directory of the destination Git repository
patches -- list of patch files relative to the root
"""
os.chdir(root)
args = ["git", "am"]
if use_threeway:
args.append("-3")
if ignore_whitespce:
if ignore_whitespace:
args.append("--ignore-whitespace")
for patch in patches:
subprocess.check_output(args + [patch])
subprocess.check_output(args + [patch])