mirror of
https://github.com/wpilibsuite/allwpilib
synced 2026-06-19 00:41:43 +00:00
68d24bb29e
The initial build file generation for robotpy projects was relatively naive and purpose built to get `allwpilib` compiling, without supporting all the available features. This modifies the generation scripts to be able to support multiple embedded libraries, which will be necessary for #8858, since `mrclib.so` will need to be bundled along with the hal libraries. In addition some cleanup was done to get the wheels looking more like what is in pypi.
481 lines
17 KiB
Python
481 lines
17 KiB
Python
import argparse
|
|
import collections
|
|
import json
|
|
import pathlib
|
|
import re
|
|
from typing import Dict, List, Union
|
|
|
|
import jinja2
|
|
import tomli
|
|
from jinja2 import BaseLoader, Environment
|
|
from semiwrap.makeplan import (
|
|
BuildTarget,
|
|
BuildTargetOutput,
|
|
CppMacroValue,
|
|
Entrypoint,
|
|
ExtensionModule,
|
|
LocalDependency,
|
|
makeplan,
|
|
)
|
|
from semiwrap.pkgconf_cache import PkgconfCache
|
|
from semiwrap.pyproject import PyProject
|
|
|
|
from shared.bazel.rules.robotpy.generation_utils import (
|
|
fixup_native_lib_name,
|
|
fixup_python_dep_name,
|
|
fixup_root_package_name,
|
|
fixup_shared_lib_name,
|
|
)
|
|
from shared.bazel.rules.robotpy.hack_pkgcfgs import hack_pkgconfig
|
|
|
|
|
|
class HeaderToDatConfig:
|
|
def __init__(self, header_to_dat_args: BuildTarget):
|
|
includes = []
|
|
defines = []
|
|
|
|
idx = 0
|
|
while True:
|
|
if header_to_dat_args.args[idx] == "-I":
|
|
includes.append(header_to_dat_args.args[idx + 1])
|
|
elif header_to_dat_args.args[idx] == "-D":
|
|
defines.append(header_to_dat_args.args[idx + 1])
|
|
else:
|
|
break
|
|
idx += 2
|
|
if header_to_dat_args.args[idx] == "--cpp":
|
|
idx += 2
|
|
|
|
args = header_to_dat_args.args[idx:]
|
|
self.class_name = args[0]
|
|
self.yml_file = args[1].path
|
|
self.defines = defines
|
|
|
|
def find_root_dir(include_root):
|
|
"""
|
|
Somewhat naive attempt to find the "root" directory of the repository,
|
|
as specified from the runfiles path
|
|
"""
|
|
if "__main__/" in include_root:
|
|
return pathlib.Path(
|
|
include_root[: include_root.find("__main__/") + len("__main__/")]
|
|
)
|
|
elif "_main/" in include_root:
|
|
return pathlib.Path(
|
|
include_root[: include_root.find("_main/") + len("_main/")]
|
|
)
|
|
else:
|
|
return pathlib.Path(include_root)
|
|
|
|
include_root = str(args[3]).replace("\\", "/")
|
|
root_dir = find_root_dir(include_root)
|
|
if "native" in include_root:
|
|
base_include_root = pathlib.Path(*args[3].relative_to(root_dir).parts[3:])
|
|
base_include_file = args[2].relative_to(include_root)
|
|
base_library = re.search("native/(.*?)/", include_root).groups(1)[0]
|
|
|
|
self.include_file = f"$(execpath :{fixup_native_lib_name('robotpy-native-' + base_library)}.copy_headers)/{base_include_file}"
|
|
self.include_root = f"$(execpath :{fixup_native_lib_name('robotpy-native-' + base_library)}.copy_headers)"
|
|
else:
|
|
if root_dir.is_absolute():
|
|
self.include_file = args[2].relative_to(root_dir)
|
|
self.include_root = args[3].relative_to(root_dir)
|
|
else:
|
|
self.include_file = args[2]
|
|
self.include_root = args[3]
|
|
# type casters = 4
|
|
# dat file = 5
|
|
# d file = 6
|
|
# compiler info = 7
|
|
|
|
self.templates = []
|
|
self.trampolines = []
|
|
|
|
args = args[8:]
|
|
assert 0 == len(args)
|
|
|
|
|
|
class ResolveCastersConfig:
|
|
def __init__(self, item: BuildTarget):
|
|
self.pkl_file = item.args[0].name
|
|
self.dep_file = item.args[1].name
|
|
# semiwrap casters = 2
|
|
self.caster_files = []
|
|
caster_deps = set()
|
|
|
|
for dep_path in item.args[3:]:
|
|
if isinstance(dep_path, BuildTargetOutput):
|
|
output_file = dep_path.target.args[2]
|
|
caster_deps.add(
|
|
f":src/main/python/{dep_path.target.install_path}/{output_file.name}"
|
|
)
|
|
else:
|
|
relevant_parts = dep_path.parts[3:]
|
|
caster_deps.add(
|
|
f"//{relevant_parts[0]}:" + "/".join(relevant_parts[1:])
|
|
)
|
|
|
|
self.caster_deps = sorted(caster_deps)
|
|
|
|
|
|
class GenLibInitPyConfig:
|
|
def __init__(self, item: BuildTarget):
|
|
self.output_file = item.args[0].name
|
|
self.modules = item.args[1:]
|
|
self.install_path = item.install_path
|
|
|
|
|
|
class GenPkgConfConfig:
|
|
def __init__(self, item: BuildTarget):
|
|
self.module_pkg_name = item.args[0]
|
|
self.pkg_name = item.args[1]
|
|
self.project_file = item.args[2].path
|
|
self.output_file = item.args[3].name
|
|
# --libinit-py = 4
|
|
self.libinit_py = item.args[5]
|
|
|
|
assert 0 == len(item.args[6:])
|
|
|
|
self.install_path = item.install_path
|
|
|
|
|
|
class GenModInitHpp:
|
|
def __init__(self, item: BuildTarget):
|
|
self.lib_name = item.args[0]
|
|
self.output_file = item.args[1].name
|
|
idx = 2
|
|
while idx < len(item.args):
|
|
if item.args[idx].command != "header2dat":
|
|
break
|
|
idx += 1
|
|
|
|
assert 0 == len(item.args[idx:])
|
|
|
|
|
|
class PublishCastersConfig:
|
|
def __init__(self, projectcfg, item: BuildTarget):
|
|
self.project_file = item.args[0].path
|
|
self.casters_name = item.args[1]
|
|
self.json_output = item.args[2].name
|
|
self.pc_output = item.args[3].name
|
|
assert 0 == len(item.args[4:])
|
|
|
|
self.install_path = item.install_path
|
|
|
|
self.include_paths = []
|
|
caster_cfg = projectcfg.export_type_casters[self.casters_name]
|
|
|
|
for inc_dir in caster_cfg.includedir:
|
|
self.include_paths.append(f"src/main/python/{inc_dir}")
|
|
|
|
|
|
class BazelExtensionModule:
|
|
def __init__(
|
|
self,
|
|
extension_module: ExtensionModule,
|
|
additional_extension_targets: Dict[str, BuildTarget],
|
|
):
|
|
self.name = extension_module.name
|
|
self.package_name = extension_module.package_name
|
|
self.install_path = extension_module.install_path
|
|
|
|
self.generation_data = self._extract_header_generation(extension_module.sources)
|
|
self.resolve_casters = ResolveCastersConfig(
|
|
additional_extension_targets["resolve-casters"]
|
|
)
|
|
self.gen_libinit = GenLibInitPyConfig(
|
|
additional_extension_targets["gen-libinit-py"]
|
|
)
|
|
self.gen_pkgconf = GenPkgConfConfig(additional_extension_targets["gen-pkgconf"])
|
|
self.gen_modinit = GenModInitHpp(
|
|
additional_extension_targets["gen-modinit-hpp"]
|
|
)
|
|
|
|
self.pkgcache = PkgconfCache()
|
|
|
|
all_dependencies = set()
|
|
|
|
for d in extension_module.depends:
|
|
if isinstance(d, LocalDependency):
|
|
all_dependencies.add(d.name)
|
|
self._collect_local_dependency_names(d, all_dependencies)
|
|
|
|
native_wrapper_dependencies = set()
|
|
local_extension_dependencies = set()
|
|
dynamic_dependencies = set()
|
|
for dep_name in all_dependencies:
|
|
if "native" in dep_name:
|
|
|
|
transitive_deps = set()
|
|
self._get_transitive_native_dependencies(dep_name, transitive_deps)
|
|
for d in transitive_deps:
|
|
if d == "robotpy-native-mrclib":
|
|
continue
|
|
base_library = fixup_root_package_name(
|
|
d.replace("robotpy-native-", "")
|
|
)
|
|
native_wrapper_dependencies.add(
|
|
f"//{base_library}:{fixup_native_lib_name(d)}.copy_headers"
|
|
)
|
|
elif "-casters" in dep_name:
|
|
base_library = dep_name.split("-")[0]
|
|
local_extension_dependencies.add(f"//{base_library}:{dep_name}")
|
|
else:
|
|
base_library = fixup_root_package_name(dep_name.split("_")[0])
|
|
local_extension_dependencies.add(
|
|
f"//{base_library}:{fixup_shared_lib_name(base_library)}"
|
|
)
|
|
dynamic_dependencies.add(
|
|
f"//{base_library}:shared/{fixup_shared_lib_name(base_library)}"
|
|
)
|
|
if dep_name != self.name:
|
|
local_extension_dependencies.add(
|
|
f"//{base_library}:{dep_name}_pybind_library"
|
|
)
|
|
|
|
self.native_wrapper_dependencies = sorted(native_wrapper_dependencies)
|
|
self.local_extension_dependencies = sorted(local_extension_dependencies)
|
|
self.dynamic_dependencies = sorted(dynamic_dependencies)
|
|
|
|
def get_defines(self):
|
|
defines = set()
|
|
for h2d_def in self.generation_data.values():
|
|
defines.update(h2d_def.defines)
|
|
return sorted(defines)
|
|
|
|
def _get_transitive_native_dependencies(self, dep_name, transitive_deps):
|
|
entry = self.pkgcache.get(dep_name)
|
|
transitive_deps.add(dep_name)
|
|
for req in entry.requires:
|
|
if req not in transitive_deps:
|
|
transitive_deps.add(req)
|
|
self._get_transitive_native_dependencies(req, transitive_deps)
|
|
|
|
def _collect_local_dependency_names(self, dep, all_dependencies):
|
|
for child_dep in dep.depends:
|
|
if isinstance(child_dep, str):
|
|
if child_dep != "semiwrap":
|
|
all_dependencies.add(child_dep)
|
|
elif isinstance(child_dep, LocalDependency):
|
|
all_dependencies.add(child_dep.name)
|
|
self._collect_local_dependency_names(child_dep, all_dependencies)
|
|
else:
|
|
raise
|
|
|
|
def _extract_header_generation(self, sources) -> Dict[str, HeaderToDatConfig]:
|
|
generation_data: Dict[str, HeaderToDatConfig] = {}
|
|
|
|
def get_h2d_config(target_info: BuildTarget) -> HeaderToDatConfig:
|
|
config = HeaderToDatConfig(target_info)
|
|
if config.class_name not in generation_data:
|
|
generation_data[config.class_name] = config
|
|
return generation_data[config.class_name]
|
|
|
|
for source in sources:
|
|
if source.command == "dat2cpp":
|
|
h2d_config = get_h2d_config(source.args[0])
|
|
elif source.command == "dat2trampoline":
|
|
h2d_config = get_h2d_config(source.args[0])
|
|
name, out_file = source.args[1:]
|
|
h2d_config.trampolines.append((name, out_file.name))
|
|
elif source.command == "dat2tmplcpp":
|
|
h2d_config = get_h2d_config(source.args[0])
|
|
name, out_file = source.args[1:]
|
|
h2d_config.templates.append((out_file.name[:-4], name))
|
|
elif source.command == "dat2tmplhpp":
|
|
# Handled by dat2tmplcpp
|
|
continue
|
|
elif source.command == "gen-modinit-hpp":
|
|
# Handled elsewhere
|
|
continue
|
|
else:
|
|
raise Exception("Unknown command", source.command)
|
|
|
|
return generation_data
|
|
|
|
|
|
def generate_pybind_build_file(
|
|
pkgcfgs: List[pathlib.Path],
|
|
project_file: pathlib.Path,
|
|
package_root_file: str,
|
|
stripped_include_prefix: str,
|
|
yml_prefix: Union[str, None],
|
|
output_file: pathlib.Path,
|
|
):
|
|
project_dir = project_file.parent
|
|
plan = makeplan(project_dir)
|
|
|
|
hack_pkgconfig(pkgcfgs)
|
|
|
|
extension_modules = []
|
|
entry_points = collections.defaultdict(list)
|
|
|
|
pyproject = PyProject(project_file)
|
|
projectcfg = pyproject.project
|
|
|
|
# Cache built up for an extension module. Gets reset when an ExtensionModule is encountered
|
|
additional_extension_targets: Dict[str, BuildTarget] = {}
|
|
publish_casters_targets = []
|
|
|
|
for item in plan:
|
|
if isinstance(item, ExtensionModule):
|
|
extension_modules.append(
|
|
BazelExtensionModule(item, additional_extension_targets)
|
|
)
|
|
additional_extension_targets = {}
|
|
elif isinstance(item, BuildTarget):
|
|
if item.command in [
|
|
"resolve-casters",
|
|
"gen-libinit-py",
|
|
"gen-pkgconf",
|
|
"gen-modinit-hpp",
|
|
]:
|
|
if item.command in additional_extension_targets:
|
|
raise Exception(f"Repeated target {item.command}")
|
|
additional_extension_targets[item.command] = item
|
|
elif item.command in [
|
|
"header2dat",
|
|
"dat2cpp",
|
|
"dat2tmplcpp",
|
|
"dat2tmplhpp",
|
|
"dat2trampoline",
|
|
"make-pyi",
|
|
]:
|
|
pass
|
|
elif item.command == "publish-casters":
|
|
publish_casters_targets.append(PublishCastersConfig(projectcfg, item))
|
|
else:
|
|
raise Exception(f"Unhandled build target {item.command}")
|
|
elif isinstance(item, Entrypoint):
|
|
entry_points[item.group].append(f"{item.name} = {item.package}")
|
|
elif isinstance(item, LocalDependency):
|
|
pass
|
|
elif isinstance(item, CppMacroValue):
|
|
pass
|
|
else:
|
|
raise Exception(f"Unknown item {type(item)}")
|
|
|
|
with open(project_file, "rb") as fp:
|
|
raw_config = tomli.load(fp)
|
|
|
|
try:
|
|
top_level_name = raw_config["tool"]["hatch"]["build"]["targets"]["wheel"][
|
|
"packages"
|
|
]
|
|
except KeyError:
|
|
top_level_name = [raw_config["project"]["name"]]
|
|
assert len(top_level_name) == 1
|
|
top_level_name = top_level_name[0]
|
|
|
|
template_file = "shared/bazel/rules/robotpy/pybind_build_file_template.jinja2"
|
|
with open(template_file, "r") as f:
|
|
template_contents = f.read()
|
|
|
|
def jsonify(item):
|
|
if isinstance(item, jinja2.runtime.Undefined):
|
|
return "None"
|
|
return json.dumps(item)
|
|
|
|
def target_from_python_dep(python_dep):
|
|
if "native" in python_dep:
|
|
base_library = python_dep.replace("robotpy-native-", "")
|
|
return f"//{fixup_root_package_name(base_library)}:{fixup_python_dep_name(python_dep)}"
|
|
else:
|
|
base_library = python_dep.replace("robotpy-", "")
|
|
return f"//{fixup_root_package_name(base_library)}:{fixup_python_dep_name(python_dep)}"
|
|
|
|
EXTERNAL_PYPI_DEPS = [
|
|
"robotpy-cli",
|
|
"pytest-reraise",
|
|
"pytest",
|
|
]
|
|
|
|
python_deps = []
|
|
has_external_python_deps = False
|
|
if "dependencies" in raw_config["project"]:
|
|
for d in raw_config["project"]["dependencies"]:
|
|
for external_dep in EXTERNAL_PYPI_DEPS:
|
|
if external_dep in d:
|
|
has_external_python_deps = True
|
|
python_deps.append(f'requirement("{external_dep}")')
|
|
break
|
|
else:
|
|
pd = target_from_python_dep(d.split("==")[0])
|
|
python_deps.append(pd)
|
|
|
|
env = Environment(loader=BaseLoader)
|
|
env.filters["jsonify"] = jsonify
|
|
template = env.from_string(template_contents)
|
|
|
|
all_local_native_deps = set()
|
|
for em in extension_modules:
|
|
all_local_native_deps.update(em.native_wrapper_dependencies)
|
|
all_local_native_deps = sorted(all_local_native_deps)
|
|
|
|
try:
|
|
version_file = raw_config["tool"]["hatch"]["build"]["hooks"]["robotpy"][
|
|
"version_file"
|
|
]
|
|
except:
|
|
version_file = None
|
|
|
|
# The entry points defined above are implicit to how the project is broken down in the toml files.
|
|
# This adds potentially extra explicitly declared entry points
|
|
if "entry-points" in raw_config["project"]:
|
|
explicit_entry_points = raw_config["project"]["entry-points"]
|
|
for entry_point_type in explicit_entry_points:
|
|
for ep_key, ep_value in explicit_entry_points[entry_point_type].items():
|
|
entry_points[entry_point_type].append(f"{ep_key} = {ep_value}")
|
|
|
|
strip_path_prefixes = [
|
|
f"{fixup_root_package_name(top_level_name)}/{stripped_include_prefix}",
|
|
f"{fixup_root_package_name(top_level_name)}",
|
|
]
|
|
|
|
with open(output_file, "w") as f:
|
|
f.write(
|
|
template.render(
|
|
extension_modules=extension_modules,
|
|
top_level_name=top_level_name,
|
|
publish_casters_targets=publish_casters_targets,
|
|
python_deps=sorted(python_deps),
|
|
all_local_native_deps=all_local_native_deps,
|
|
stripped_include_prefix=stripped_include_prefix,
|
|
strip_path_prefixes=strip_path_prefixes,
|
|
yml_prefix=yml_prefix,
|
|
package_root_file=package_root_file,
|
|
raw_project_config=raw_config["project"],
|
|
entry_points=entry_points,
|
|
version_file=version_file,
|
|
has_external_python_deps=has_external_python_deps,
|
|
)
|
|
+ "\n"
|
|
)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--project_file", type=pathlib.Path, required=True)
|
|
parser.add_argument("--output_file", type=pathlib.Path, required=True)
|
|
parser.add_argument(
|
|
"--stripped_include_prefix", type=str, default="src/main/python"
|
|
)
|
|
parser.add_argument("--yml_prefix", type=str)
|
|
parser.add_argument("--package_root_file", type=str)
|
|
parser.add_argument("--pkgcfgs", type=pathlib.Path, nargs="+")
|
|
|
|
args = parser.parse_args()
|
|
|
|
generate_pybind_build_file(
|
|
args.pkgcfgs,
|
|
args.project_file,
|
|
args.package_root_file,
|
|
args.stripped_include_prefix,
|
|
args.yml_prefix,
|
|
args.output_file,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|