Team2890/allwpilib
4
0
Fork 0
mirror of https://github.com/wpilibsuite/allwpilib synced 2026-06-19 00:41:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

[bazel][robotpy] Add mirror for robotpy's wpiuil and wpinet libraries (#8062)

Browse Source 
Project import generated by Copybara.

GitOrigin-RevId: 92ea93d1b47a82667044bd0af05f7fdb34d2c2c2
This commit is contained in:
PJ Reiniger
2025-08-30 14:55:11 -04:00
committed by GitHub
parent 96004f9bb5
commit bd1dcc4358
96 changed files with 7271 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
shared/bazel/compiler_flags/linux_flags.rc
View File

@@ -29,3 +29,7 @@ build:linux --host_cxxopt=-Wno-missing-requires
build:linux --host_cxxopt=-Wno-implicit-fallthrough
build:linux --host_per_file_copt=external/zlib/.*\.c@-Wno-deprecated-non-prototype
# Set soname. Needed for robotpy
build:linux --features=set_soname
build:linux --host_features=set_soname

85
shared/bazel/rules/robotpy/BUILD.bazel Normal file
View File

@@ -0,0 +1,85 @@
load("@allwpilib_pip_deps//:requirements.bzl", "requirement", "whl_requirement")
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("@rules_python//python:defs.bzl", "py_binary", "py_library")
load("@rules_python//python:pip.bzl", "whl_filegroup")
exports_files(["wrapper.py"])
py_library(
name = "hack_pkgcfgs",
srcs = ["hack_pkgcfgs.py"],
visibility = ["//visibility:public"],
)
py_library(
name = "generation_utils",
srcs = ["generation_utils.py"],
visibility = ["//visibility:public"],
deps = [
requirement("semiwrap"),
requirement("jinja2"),
],
)
py_binary(
name = "generate_native_build_file",
srcs = ["generate_native_build_file.py"],
visibility = ["//visibility:public"],
deps = [
":generation_utils",
":hack_pkgcfgs",
requirement("semiwrap"),
requirement("jinja2"),
],
)
filegroup(
name = "jinja_templates",
srcs = glob(["*.jinja2"]),
visibility = ["//visibility:public"],
)
py_binary(
name = "generate_pybind_build_file",
srcs = ["generate_pybind_build_file.py"],
data = [
":jinja_templates",
],
visibility = ["//visibility:public"],
deps = [
":generation_utils",
":hack_pkgcfgs",
requirement("semiwrap"),
requirement("jinja2"),
],
)
py_binary(
name = "wrapper",
srcs = ["wrapper.py"],
visibility = ["//visibility:public"],
deps = [
"//shared/bazel/rules/robotpy:hack_pkgcfgs",
requirement("semiwrap"),
],
)
whl_filegroup(
name = "semiwrap_header_files",
pattern = "semiwrap/include",
whl = whl_requirement("semiwrap"),
)
cc_library(
name = "semiwrap_headers",
hdrs = [":semiwrap_header_files"],
includes = ["semiwrap_header_files/semiwrap/include"],
visibility = ["//visibility:public"],
)
whl_filegroup(
name = "semiwrap_casters_files",
pattern = "semiwrap/semiwrap.pybind11.json",
visibility = ["//visibility:public"],
whl = whl_requirement("semiwrap"),
)

94
shared/bazel/rules/robotpy/build_info_gen.bzl Normal file
View File

@@ -0,0 +1,94 @@
load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files")
load("//shared/bazel/rules/robotpy:compatibility_select.bzl", "robotpy_compatibility_select")
def generate_robotpy_native_wrapper_build_info(name, pyproject_toml, third_party_dirs = []):
"""
This function will generate the bazel file necessary to declare a library that wraps a standard allwpilib library.
Params:
pyproject_toml - Path to the native library wrappers definition file
third_party_dirs - Any directories under src/main/native/thirdparty that should be used by semiwrap
"""
cmd = "$(location //shared/bazel/rules/robotpy:generate_native_build_file) --output_file=$(OUTS)"
cmd += " --project_cfg=$(location " + pyproject_toml + ")"
if third_party_dirs:
cmd += " --third_party_dirs "
for d in third_party_dirs:
cmd += " " + d
native.genrule(
name = "{}.gen_build_info".format(name),
tools = ["//shared/bazel/rules/robotpy:generate_native_build_file"],
srcs = [pyproject_toml],
outs = ["{}-generated_build_info.bzl".format(name)],
cmd = cmd,
tags = ["robotpy"],
target_compatible_with = robotpy_compatibility_select(),
)
write_source_files(
name = "{}.generate_build_info".format(name),
files = {
"robotpy_native_build_info.bzl": "{}-generated_build_info.bzl".format(name),
},
visibility = ["//visibility:public"],
suggested_update_target = "//:write_robotpy_generated_native_files",
tags = ["robotpy"],
target_compatible_with = robotpy_compatibility_select(),
)
def generate_robotpy_pybind_build_info(
name,
package_root_file,
yaml_files = [],
pkgcfgs = [],
additional_srcs = [],
generated_file_name = "robotpy_pybind_build_info.bzl",
pyproject_toml = "src/main/python/pyproject.toml",
stripped_include_prefix = None,
yml_prefix = None):
"""
This function will generate the bazel file necessary to build a pybind library with all of its extensions.
Params:
package_root_file - An __init__.py file used to key the semiwrap wrappers on the project root.
yaml_files - All of the yaml files used by semi wrap to run library wrapping
pkgcfgs - Local files used to trick semiwrap into thinking a library is installed
additional_srcs - Any additional sources needed by the semiwrap process
generated_file_name - Indicates the path of the auto-generated file
pyproject_toml - Location of the pyproject.toml file that defines this project
yml_prefix - Optional. Used in the event that the yml files are in a non-standard location
"""
cmd = "$(location //shared/bazel/rules/robotpy:generate_pybind_build_file) --project_file=$(location " + pyproject_toml + ") --output_file=$(OUTS)"
cmd += " --package_root_file=" + package_root_file
if stripped_include_prefix:
cmd += " --stripped_include_prefix=" + stripped_include_prefix
if yml_prefix:
cmd += " --yml_prefix=" + yml_prefix
if pkgcfgs:
cmd += " --pkgcfgs "
for x in pkgcfgs:
cmd += " $(locations " + x + ")"
native.genrule(
name = "{}.gen_build_info".format(name),
tools = ["//shared/bazel/rules/robotpy:generate_pybind_build_file"],
srcs = [pyproject_toml, package_root_file] + yaml_files + pkgcfgs + additional_srcs + ["//shared/bazel/rules/robotpy:jinja_templates"],
outs = ["{}-generated_build_info.bzl".format(name)],
cmd = cmd,
tags = ["robotpy"],
target_compatible_with = robotpy_compatibility_select(),
)
write_source_files(
name = "{}.generate_build_info".format(name),
files = {
generated_file_name: "{}-generated_build_info.bzl".format(name),
},
suggested_update_target = "//:write_robotpy_generated_pybind_files",
visibility = ["//visibility:public"],
tags = ["robotpy"],
target_compatible_with = robotpy_compatibility_select(),
)

6
shared/bazel/rules/robotpy/compatibility_select.bzl Normal file
View File

@@ -0,0 +1,6 @@
def robotpy_compatibility_select():
return select({
"@bazel_tools//src/conditions:windows": ["@platforms//:incompatible"],
"@rules_bzlmodrio_toolchains//constraints/is_systemcore:systemcore": ["@platforms//:incompatible"],
"//conditions:default": [],
})

110
shared/bazel/rules/robotpy/generate_native_build_file.py Normal file
View File

@@ -0,0 +1,110 @@
import argparse
import json
import tomli
from jinja2 import BaseLoader, Environment
from shared.bazel.rules.robotpy.generation_utils import (
fixup_python_dep_name,
fixup_root_package_name,
fixup_shared_lib_name,
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--project_cfg")
parser.add_argument("--output_file")
parser.add_argument("--third_party_dirs", nargs="+")
args = parser.parse_args()
with open(args.project_cfg, "rb") as fp:
raw_config = tomli.load(fp)
def double_quotes(data):
if data:
return json.dumps(data)
return None
def get_pc_dep(library):
base_project = library.replace("robotpy-native-", "")
wpilib_project = fixup_root_package_name(base_project)
return f"//{wpilib_project}:native/{base_project}/{library}.pc"
def get_python_dep(library):
base_project = library.replace("robotpy-native-", "")
wpilib_project = fixup_root_package_name(base_project)
return f"//{fixup_root_package_name(wpilib_project)}:{fixup_python_dep_name(library)}"
env = Environment(loader=BaseLoader)
env.filters["double_quotes"] = double_quotes
env.filters["get_pc_dep"] = get_pc_dep
env.filters["get_python_dep"] = get_python_dep
template = env.from_string(BUILD_FILE_TEMPLATE)
nativelib_config = raw_config["tool"]["hatch"]["build"]["hooks"]["nativelib"]
project_name = nativelib_config["pcfile"][0]["name"]
root_package = fixup_root_package_name(project_name)
shared_library_name = fixup_shared_lib_name(project_name)
with open(args.output_file, "w") as f:
f.write(
template.render(
raw_project_config=raw_config["project"],
nativelib_config=nativelib_config,
root_package=root_package,
shared_library_name=shared_library_name,
third_party_dirs=args.third_party_dirs or [],
)
)
BUILD_FILE_TEMPLATE = """# THIS FILE IS AUTO GENERATED
load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
load("//shared/bazel/rules/robotpy:pybind_rules.bzl", "native_wrappery_library")
def define_native_wrapper(name, pyproject_toml = None):
copy_to_directory(
name = "{}.copy_headers".format(name),
srcs = native.glob(["src/main/native/include/**"]) + native.glob(["src/generated/main/native/include/**"], allow_empty = True){% if third_party_dirs %} + native.glob([
{%- for dir in third_party_dirs %}
"src/main/native/thirdparty/{{dir}}/include/**",
{%- endfor %}
]){%- endif %},
out = "native/{{nativelib_config.pcfile[0].name}}/include",
root_paths = ["src/main/native/include/"],
replace_prefixes = {
"{{root_package}}/src/generated/main/native/include": "",
"{{root_package}}/src/main/native/include": "",
{%- for dir in third_party_dirs %}
"{{root_package}}/src/main/native/thirdparty/{{dir}}/include": "",
{%- endfor %}
},
verbose = False,
visibility = ["//visibility:public"],
)
native_wrappery_library(
name = name,
pyproject_toml = pyproject_toml or "src/main/python/native-pyproject.toml",
libinit_file = "native/{{nativelib_config.pcfile[0].name}}/_init_{{raw_project_config.name.replace("-", "_")}}.py",
pc_file = "native/{{nativelib_config.pcfile[0].name}}/{{raw_project_config.name}}.pc",
pc_deps = [
{%- for dep in nativelib_config.pcfile[0].requires | sort %}
"{{dep | get_pc_dep}}",
{%- endfor %}
],
deps = [
{%- for dep in nativelib_config.pcfile[0].requires | sort %}
"{{dep | get_python_dep}}",
{%- endfor %}
],
headers = "{}.copy_headers".format(name),
native_shared_library = "shared/{{shared_library_name}}",
install_path = "native/{{nativelib_config.pcfile[0].name}}/",
)
"""
if __name__ == "__main__":
main()

428
shared/bazel/rules/robotpy/generate_pybind_build_file.py Normal file
View File

@@ -0,0 +1,428 @@
import argparse
import collections
import json
import pathlib
import re
from typing import Dict, List, Union
import jinja2
import tomli
from jinja2 import BaseLoader, Environment
from semiwrap.makeplan import (
BuildTarget,
BuildTargetOutput,
CppMacroValue,
Entrypoint,
ExtensionModule,
LocalDependency,
makeplan,
)
from semiwrap.pkgconf_cache import PkgconfCache
from semiwrap.pyproject import PyProject
from shared.bazel.rules.robotpy.generation_utils import (
fixup_native_lib_name,
fixup_python_dep_name,
fixup_root_package_name,
fixup_shared_lib_name,
)
from shared.bazel.rules.robotpy.hack_pkgcfgs import hack_pkgconfig
class HeaderToDatConfig:
def __init__(self, header_to_dat_args: BuildTarget):
includes = []
defines = []
idx = 0
while True:
if header_to_dat_args.args[idx] == "-I":
includes.append(header_to_dat_args.args[idx + 1])
elif header_to_dat_args.args[idx] == "-D":
defines.append(header_to_dat_args.args[idx + 1])
else:
break
idx += 2
if header_to_dat_args.args[idx] == "--cpp":
idx += 2
args = header_to_dat_args.args[idx:]
self.class_name = args[0]
self.yml_file = args[1].path
self.defines = defines
include_root = str(args[3])
if "native" in include_root:
root_dir = pathlib.Path(
include_root[: include_root.find("__main__/") + len("__main__/")]
)
base_include_root = pathlib.Path(*args[3].relative_to(root_dir).parts[3:])
base_include_file = args[2].relative_to(include_root)
base_library = re.search("native/(.*?)/", include_root).groups(1)[0]
self.include_file = f"$(execpath :{fixup_native_lib_name('robotpy-native-' + base_library)}.copy_headers)/{base_include_file}"
self.include_root = f"$(execpath :{fixup_native_lib_name('robotpy-native-' + base_library)}.copy_headers)"
else:
root_dir = pathlib.Path(
include_root[: include_root.find("__main__/") + len("__main__/")]
)
if root_dir.is_absolute():
self.include_file = args[2].relative_to(root_dir)
self.include_root = args[3].relative_to(root_dir)
else:
self.include_file = args[2]
self.include_root = args[3]
# type casters = 4
# dat file = 5
# d file = 6
# compiler info = 7
self.templates = []
self.trampolines = []
args = args[8:]
assert 0 == len(args)
class ResolveCastersConfig:
def __init__(self, item: BuildTarget):
self.pkl_file = item.args[0].name
self.dep_file = item.args[1].name
# semiwrap casters = 2
self.caster_files = []
caster_deps = set()
for dep_path in item.args[3:]:
if isinstance(dep_path, BuildTargetOutput):
output_file = dep_path.target.args[2]
caster_deps.add(
f":src/main/python/{dep_path.target.install_path}/{output_file.name}"
)
else:
relevant_parts = dep_path.parts[3:]
caster_deps.add(
f"//{relevant_parts[0]}:" + "/".join(relevant_parts[1:])
)
self.caster_deps = sorted(caster_deps)
class GenLibInitPyConfig:
def __init__(self, item: BuildTarget):
self.output_file = item.args[0].name
self.modules = item.args[1:]
self.install_path = item.install_path
class GenPkgConfConfig:
def __init__(self, item: BuildTarget):
self.module_pkg_name = item.args[0]
self.pkg_name = item.args[1]
self.project_file = item.args[2].path
self.output_file = item.args[3].name
# --libinit-py = 4
self.libinit_py = item.args[5]
assert 0 == len(item.args[6:])
self.install_path = item.install_path
class GenModInitHpp:
def __init__(self, item: BuildTarget):
self.lib_name = item.args[0]
self.output_file = item.args[1].name
idx = 2
while idx < len(item.args):
if item.args[idx].command != "header2dat":
break
idx += 1
assert 0 == len(item.args[idx:])
class PublishCastersConfig:
def __init__(self, projectcfg, item: BuildTarget):
self.project_file = item.args[0].path
self.casters_name = item.args[1]
self.json_output = item.args[2].name
self.pc_output = item.args[3].name
assert 0 == len(item.args[4:])
self.install_path = item.install_path
self.include_paths = []
caster_cfg = projectcfg.export_type_casters[self.casters_name]
for inc_dir in caster_cfg.includedir:
self.include_paths.append(f"src/main/python/{inc_dir}")
class BazelExtensionModule:
def __init__(
self,
extension_module: ExtensionModule,
additional_extension_targets: Dict[str, BuildTarget],
):
self.name = extension_module.name
self.package_name = extension_module.package_name
self.install_path = extension_module.install_path
self.generation_data = self._extract_header_generation(extension_module.sources)
self.resolve_casters = ResolveCastersConfig(
additional_extension_targets["resolve-casters"]
)
self.gen_libinit = GenLibInitPyConfig(
additional_extension_targets["gen-libinit-py"]
)
self.gen_pkgconf = GenPkgConfConfig(additional_extension_targets["gen-pkgconf"])
self.gen_modinit = GenModInitHpp(
additional_extension_targets["gen-modinit-hpp"]
)
self.pkgcache = PkgconfCache()
all_dependencies = set()
for d in extension_module.depends:
if isinstance(d, LocalDependency):
all_dependencies.add(d.name)
self._collect_local_dependency_names(d, all_dependencies)
native_wrapper_dependencies = set()
local_extension_dependencies = set()
dynamic_dependencies = set()
for dep_name in all_dependencies:
if "native" in dep_name:
transative_deps = set()
self._get_transative_native_dependencies(dep_name, transative_deps)
for d in transative_deps:
base_library = fixup_root_package_name(
d.replace("robotpy-native-", "")
)
native_wrapper_dependencies.add(
f"//{base_library}:{fixup_native_lib_name(d)}.copy_headers"
)
elif "-casters" in dep_name:
base_library = dep_name.split("-")[0]
local_extension_dependencies.add(f"//{base_library}:{dep_name}")
else:
base_library = fixup_root_package_name(dep_name.split("_")[0])
local_extension_dependencies.add(
f"//{base_library}:{fixup_shared_lib_name(base_library)}"
)
dynamic_dependencies.add(
f"//{base_library}:shared/{fixup_shared_lib_name(base_library)}"
)
if dep_name != self.name:
local_extension_dependencies.add(
f"//{base_library}:{dep_name}_pybind_library"
)
self.native_wrapper_dependencies = sorted(native_wrapper_dependencies)
self.local_extension_dependencies = sorted(local_extension_dependencies)
self.dynamic_dependencies = sorted(dynamic_dependencies)
def get_defines(self):
defines = set()
for h2d_def in self.generation_data.values():
defines.update(h2d_def.defines)
return sorted(defines)
def _get_transative_native_dependencies(self, dep_name, transative_deps):
entry = self.pkgcache.get(dep_name)
transative_deps.add(dep_name)
for req in entry.requires:
if req not in transative_deps:
transative_deps.add(req)
self._get_transative_native_dependencies(req, transative_deps)
def _collect_local_dependency_names(self, dep, all_dependencies):
for child_dep in dep.depends:
if isinstance(child_dep, str):
if child_dep != "semiwrap":
all_dependencies.add(child_dep)
elif isinstance(child_dep, LocalDependency):
all_dependencies.add(child_dep.name)
self._collect_local_dependency_names(child_dep, all_dependencies)
else:
raise
def _extract_header_generation(self, sources) -> Dict[str, HeaderToDatConfig]:
generation_data: Dict[str, HeaderToDatConfig] = {}
def get_h2d_config(target_info: BuildTarget) -> HeaderToDatConfig:
config = HeaderToDatConfig(target_info)
if config.class_name not in generation_data:
generation_data[config.class_name] = config
return generation_data[config.class_name]
for source in sources:
if source.command == "dat2cpp":
h2d_config = get_h2d_config(source.args[0])
elif source.command == "dat2trampoline":
h2d_config = get_h2d_config(source.args[0])
name, out_file = source.args[1:]
h2d_config.trampolines.append((name, out_file.name))
elif source.command == "dat2tmplcpp":
h2d_config = get_h2d_config(source.args[0])
name, out_file = source.args[1:]
h2d_config.templates.append((out_file.name[:-4], name))
elif source.command == "dat2tmplhpp":
# Handled by dat2tmplcpp
continue
elif source.command == "gen-modinit-hpp":
# Handled elsewhere
continue
else:
raise Exception("Unknown command", source.command)
return generation_data
def generate_pybind_build_file(
pkgcfgs: List[pathlib.Path],
project_file: pathlib.Path,
package_root_file: str,
stripped_include_prefix: str,
yml_prefix: Union[str, None],
output_file: pathlib.Path,
):
project_dir = project_file.parent
plan = makeplan(project_dir)
hack_pkgconfig(pkgcfgs)
extension_modules = []
entry_points = collections.defaultdict(list)
pyproject = PyProject(project_file)
projectcfg = pyproject.project
# Cache built up for an extension module. Gets reset when an ExtensionModule is encountered
additional_extension_targets: Dict[str, BuildTarget] = {}
publish_casters_targets = []
for item in plan:
if isinstance(item, ExtensionModule):
extension_modules.append(
BazelExtensionModule(item, additional_extension_targets)
)
additional_extension_targets = {}
elif isinstance(item, BuildTarget):
if item.command in [
"resolve-casters",
"gen-libinit-py",
"gen-pkgconf",
"gen-modinit-hpp",
]:
if item.command in additional_extension_targets:
raise Exception(f"Repeated target {item.command}")
additional_extension_targets[item.command] = item
elif item.command in [
"header2dat",
"dat2cpp",
"dat2tmplcpp",
"dat2tmplhpp",
"dat2trampoline",
"make-pyi",
]:
pass
elif item.command == "publish-casters":
publish_casters_targets.append(PublishCastersConfig(projectcfg, item))
else:
raise Exception(f"Unhandled build target {item.command}")
elif isinstance(item, Entrypoint):
entry_points[item.group].append(f"{item.name} = {item.package}")
elif isinstance(item, LocalDependency):
pass
elif isinstance(item, CppMacroValue):
pass
else:
raise Exception(f"Unknown item {type(item)}")
with open(project_file, "rb") as fp:
raw_config = tomli.load(fp)
try:
top_level_name = raw_config["tool"]["hatch"]["build"]["targets"]["wheel"][
"packages"
]
except KeyError:
top_level_name = [raw_config["project"]["name"]]
assert len(top_level_name) == 1
top_level_name = top_level_name[0]
template_file = "shared/bazel/rules/robotpy/pybind_build_file_template.jinja2"
with open(template_file, "r") as f:
template_contents = f.read()
def jsonify(item):
if isinstance(item, jinja2.runtime.Undefined):
return "None"
return json.dumps(item)
def target_from_python_dep(python_dep):
if "native" in python_dep:
base_library = python_dep.replace("robotpy-native-", "")
return f"//{fixup_root_package_name(base_library)}:{fixup_python_dep_name(python_dep)}"
else:
base_library = python_dep.replace("robotpy-", "")
return f"//{fixup_root_package_name(base_library)}:{fixup_python_dep_name(python_dep)}"
python_deps = []
if "dependencies" in raw_config["project"]:
for d in raw_config["project"]["dependencies"]:
if "robotpy-cli" in d:
continue
pd = target_from_python_dep(d.split("==")[0])
python_deps.append(pd)
env = Environment(loader=BaseLoader)
env.filters["jsonify"] = jsonify
template = env.from_string(template_contents)
with open(output_file, "w") as f:
f.write(
template.render(
extension_modules=extension_modules,
top_level_name=top_level_name,
publish_casters_targets=publish_casters_targets,
python_deps=sorted(python_deps),
stripped_include_prefix=stripped_include_prefix,
yml_prefix=yml_prefix,
package_root_file=package_root_file,
raw_project_config=raw_config["project"],
entry_points=entry_points,
)
+ "\n"
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--project_file", type=pathlib.Path, required=True)
parser.add_argument("--output_file", type=pathlib.Path, required=True)
parser.add_argument(
"--stripped_include_prefix", type=str, default="src/main/python"
)
parser.add_argument("--yml_prefix", type=str)
parser.add_argument("--package_root_file", type=str)
parser.add_argument("--pkgcfgs", type=pathlib.Path, nargs="+")
args = parser.parse_args()
generate_pybind_build_file(
args.pkgcfgs,
args.project_file,
args.package_root_file,
args.stripped_include_prefix,
args.yml_prefix,
args.output_file,
)
if __name__ == "__main__":
main()

42
shared/bazel/rules/robotpy/generation_utils.py Normal file
View File

@@ -0,0 +1,42 @@
def fixup_root_package_name(name):
if name == "wpihal":
return "hal"
if name == "wpilib":
return "wpilibc"
if name == "wpilog":
return "datalog"
if name == "xrp":
return "xrpVendordep"
if name == "romi":
return "romiVendordep"
if name == "pyntcore":
return "ntcore"
return name
def fixup_native_lib_name(name):
return name
def fixup_shared_lib_name(name):
if name == "wpihal":
return "wpiHal"
if name == "hal":
return "wpiHal"
if name == "wpilib":
return "wpilibc"
if name == "xrp":
return "xrpVendordep"
if name == "romi":
return "romiVendordep"
return name
def fixup_python_dep_name(name):
if name == "robotpy-datalog":
return "robotpy-wpilog"
if name == "robotpy-ntcore":
return "pyntcore"
if name == "wpilib":
return "robotpy-wpilib"
return name

19
shared/bazel/rules/robotpy/hack_pkgcfgs.py Normal file
View File

@@ -0,0 +1,19 @@
import os
import pathlib
from typing import List
def hack_pkgconfig(pkgcfgs: List[pathlib.Path]):
"""
This will place the given files in the PKG_CONFIG_PATH in such a way that will trick
semiwrap into thinking the libraries have been installed
"""
pkg_config_paths = os.environ.get("PKG_CONFIG_PATH", "").split(os.pathsep)
if pkgcfgs:
for pc in pkgcfgs:
# pkg_config_paths.append(str(pc.parent.absolute()))
pkg_config_paths.append(str(pc.parent))
os.environ["PKG_CONFIG_PATH"] = os.pathsep.join(pkg_config_paths)

12
shared/bazel/rules/robotpy/hatchlib_native_port/BUILD.bazel Normal file
View File

@@ -0,0 +1,12 @@
load("@allwpilib_pip_deps//:requirements.bzl", "requirement")
load("@rules_python//python:defs.bzl", "py_binary")
py_binary(
name = "generate_native_lib_files",
srcs = glob(["*.py"]),
visibility = ["//visibility:public"],
deps = [
"//shared/bazel/rules/robotpy:hack_pkgcfgs",
requirement("semiwrap"),
],
)

1
shared/bazel/rules/robotpy/hatchlib_native_port/README.md Normal file
View File

@@ -0,0 +1 @@
This is a port of [hatch_nativelib](https://github.com/robotpy/hatch-nativelib/tree/main/src/hatch_nativelib). That tool is not librar-icized and required a fork.

106
shared/bazel/rules/robotpy/hatchlib_native_port/config.py Normal file
View File

@@ -0,0 +1,106 @@
import dataclasses
import pathlib
import typing as T
@dataclasses.dataclass
class PcFileConfig:
"""
Contents of [[tool.hatch.build.hooks.nativelib.pcfile]] items
"""
pcfile: str
"""
File to write pkgconf file to (relative to pyproject.toml)
"""
description: T.Optional[str] = None
"""Description of this package. If not specified, uses the first line of the package description."""
name: T.Optional[str] = None
"""Name of this package. If not specified, is basename of pcfile without extension"""
version: T.Optional[str] = None
"""If not specified, set to package version"""
includedir: T.Optional[str] = None
"""Where include files can be found (relative to pyproject.toml)"""
libdir: T.Optional[str] = None
"""Where the library is located. If not specified, it is next to pcfile"""
shared_libraries: T.Optional[T.List[str]] = None
"""Name of shared libraries located in libdir (without extension)"""
libs_private: T.Optional[str] = None
"""The link flags for private libraries not exposed to applications"""
requires: T.Optional[T.List[str]] = None
"""
Names of other packages this package requires. They must be installed
at build time.
"""
requires_private: T.Optional[T.List[str]] = None
"""
Names of private packages this package requires. They must be installed
at build time.
"""
extra_cflags: T.Optional[str] = None
"""A list of extra compiler flags to be added to Cflags after header search path"""
extra_link_flags: T.Optional[str] = None
"""A list of extra link flags to be added to Libs"""
variables: T.Optional[T.Dict[str, str]] = None
"""
Custom variables to add to the generated file. Prefix, libdir, includedir must not be specified."""
init_module: str = "auto"
"""
If specified, the name of the python module that will be written next to
the .pc file which will load the shared_libraries
"""
enable_if: T.Optional[str] = None
"""
This is a PEP 508 environment marker specification.
This pcfile will only be generated if the environment marker matches the current
build environment
"""
def get_name(self) -> str:
if self.name:
return self.name
return self.get_pc_path().name[:-3]
def get_out_path(self) -> pathlib.Path:
return self.get_pc_path().parent
def get_pc_path(self) -> pathlib.Path:
pc_path = pathlib.PurePosixPath(self.pcfile)
if pc_path.is_absolute():
raise ValueError(f"pcfile must not be absolute (is {pc_path})")
if not pc_path.name.endswith(".pc"):
raise ValueError(f"pcfile must end with .pc (is {pc_path})")
return pathlib.Path(pc_path)
def get_init_module(self) -> str:
if self.init_module == "auto":
name = self.get_pc_path().name[:-3]
name = name.replace("-", "_").replace(".", "_")
module = f"_init_{name}"
else:
module = self.init_module
if not module.isidentifier():
raise ValueError(
f"init_module must be a valid python identifier (got {module})"
)
return module
def get_init_module_path(self) -> pathlib.Path:
module = self.get_init_module()
return self.get_out_path() / f"{module}.py"

318
shared/bazel/rules/robotpy/hatchlib_native_port/generate_native_lib_files.py Normal file
View File

@@ -0,0 +1,318 @@
import functools
import pathlib
import platform
import sys
import typing as T
import pkgconf
import tomli
from packaging.markers import Marker
from shared.bazel.rules.robotpy.hack_pkgcfgs import hack_pkgconfig
from shared.bazel.rules.robotpy.hatchlib_native_port.config import PcFileConfig
from shared.bazel.rules.robotpy.hatchlib_native_port.validate import parse_input
# Port of https://github.com/robotpy/hatch-nativelib/blob/main/src/hatch_nativelib/plugin.py
INITPY_VARNAME = "pkgconf_pypi_initpy"
platform_sys = platform.system()
is_windows = platform_sys == "Windows"
is_macos = platform_sys == "Darwin"
class NativelibHook:
def __init__(self, output_pcfile, output_libinit, config, metadata):
self.output_pcfile = output_pcfile
self.output_libinit = output_libinit
self.config = config
self.root_pth = output_pcfile.parent.parent.parent
self.metadata = metadata
def initialize(self):
for pcfg in self._pcfiles:
self._generate_pcfile(pcfg, {})
def _get_pkg_from_path(self, path: pathlib.Path) -> str:
rel = path.relative_to(self.root_pth)
return str(rel).replace("/", ".").replace("\\", ".")
def _generate_pcfile(
self, pcfg: PcFileConfig, build_data: T.Dict[str, T.Any]
) -> pathlib.Path:
pcfile_rel = pcfg.get_pc_path()
pcfile = self.output_pcfile
prefix_rel = pcfile_rel.parent
prefix_path = pcfile.parent
prefix = "${pcfiledir}"
# variables first
variables = {}
variables["prefix"] = prefix
if pcfg.includedir:
increl = pathlib.PurePosixPath(pcfg.includedir).relative_to(
prefix_rel.as_posix()
)
variables["includedir"] = f"${{prefix}}/{increl}"
if pcfg.shared_libraries:
if pcfg.libdir:
librel = pathlib.PurePosixPath(pcfg.libdir).relative_to(
prefix_rel.as_posix()
)
variables["libdir"] = f"${{prefix}}/{librel}"
else:
variables["libdir"] = "${prefix}"
if pcfg.variables:
for n in ("prefix", "includedir", "libdir", INITPY_VARNAME):
if n in pcfg.variables:
raise ValueError(f"variables may not contain {n}")
variables.update(variables)
# If there are libraries, generate _init_NAME.py for each
if pcfg.shared_libraries:
package = self._get_pkg_from_path(prefix_path)
variables[INITPY_VARNAME] = f"{package}.{pcfg.get_init_module()}"
self._generate_init_py(pcfg, prefix_path, build_data)
# .. not documented but it works?
# eps = self.metadata.core.entry_points.setdefault("pkg_config", {})
# eps[pcfg.get_name()] = package
contents = [f"{k}={v}" for k, v in variables.items()]
contents.append("")
description = pcfg.description
if description is None:
description = self.metadata["description"]
if not description:
raise ValueError(
f"tool.hatch.build.hooks.nativelib.pcfile: description not provided for {pcfg.get_name()}"
)
contents += [
f"Name: {pcfg.get_name()}",
f"Description: {description}",
]
version = pcfg.version or self.metadata["version"]
if version:
contents.append(f"Version: {version}")
libs = []
if pcfg.shared_libraries:
libs.append("-L${libdir}")
libs.extend(f"-l{lib}" for lib in pcfg.shared_libraries)
cflags = []
if pcfg.includedir:
cflags.append("-I${includedir}")
if pcfg.extra_cflags:
cflags.append(pcfg.extra_cflags)
if pcfg.requires:
contents.append(f"Requires: {' '.join(pcfg.requires)}")
if pcfg.requires_private:
contents.append(f"Requires.private: {' '.join(pcfg.requires_private)}")
if libs:
contents.append(f"Libs: {' '.join(libs)}")
if pcfg.libs_private:
contents.append(f"Libs.private: {pcfg.libs_private}")
if cflags:
contents.append(f"Cflags: {' '.join(cflags)}")
content = ("\n".join(contents)) + "\n"
pcfile.parent.mkdir(parents=True, exist_ok=True)
with open(pcfile, "w") as fp:
fp.write(content)
return pcfile
def _generate_init_py(
self,
pcfg: PcFileConfig,
prefix_path: pathlib.Path,
build_data: T.Dict[str, T.Any],
):
libinit_py_rel = pcfg.get_init_module_path()
self.root_pth / libinit_py_rel
libdir = prefix_path
if pcfg.libdir:
libdir = self.root_pth / pathlib.PurePosixPath(pcfg.libdir)
libdir = pathlib.Path(str(libdir).replace("src/", "").replace("src\\", ""))
lib_paths = []
assert pcfg.shared_libraries is not None
for lib in pcfg.shared_libraries:
lib_path = libdir / self._make_shared_lib_fname(lib)
lib_paths.append(lib_path)
if pcfg.requires:
requires = pcfg.requires
else:
requires = []
_write_libinit_py(self.output_libinit, lib_paths, requires)
def _make_shared_lib_fname(self, lib: str):
if is_windows:
return f"{lib}.dll"
elif is_macos:
return f"lib{lib}.dylib"
else:
return f"lib{lib}.so"
@functools.cached_property
def _pcfiles(self) -> T.List[PcFileConfig]:
pcfiles = []
for i, raw_pc in enumerate(self.config.get("pcfile", [])):
pcfile = parse_input(
raw_pc,
PcFileConfig,
"pyproject.toml",
f"tool.hatch.build.hooks.nativelib.pcfile[{i}]",
)
if pcfile.enable_if and not Marker(pcfile.enable_if).evaluate():
print(
f"{pcfile.pcfile} skipped because enable_if did not match current environment"
)
continue
pcfiles.append(pcfile)
return pcfiles
# TODO: this belongs in a separate script/api that can be used from multiple tools
def _write_libinit_py(
init_py: pathlib.Path,
libs: T.List[pathlib.Path],
requires: T.List[str],
):
"""
:param init_py: the _init module for the library(ies) that is written out
:param libs: for each library that is being initialized, this is the
path to that library
:param requires: other pkgconf packages that these libraries depend on.
Their init_py will be looked up and imported first.
"""
contents = [
"# This file is automatically generated, DO NOT EDIT",
"# fmt: off",
"",
]
for req in requires:
r = pkgconf.run_pkgconf(
req, f"--variable={INITPY_VARNAME}", capture_output=True
)
# TODO: should this be a fatal error
if r.returncode == 0:
module = r.stdout.decode("utf-8").strip() # type: ignore[arg-type, union-attr]
contents.append(f"import {module}")
else:
raise Exception("Could not find ", req)
if contents[-1] != "":
contents.append("")
if libs:
contents += [
"def __load_library():",
" from os.path import abspath, join, dirname, exists",
]
if is_macos:
contents += [" from ctypes import CDLL, RTLD_GLOBAL"]
else:
contents += [" from ctypes import cdll", ""]
if len(libs) > 1:
contents.append(" libs = []")
contents.append(" root = abspath(dirname(__file__))")
for lib in libs:
rel = lib.relative_to(init_py.parent)
components = ", ".join(map(repr, rel.parts))
contents += [
"",
f" lib_path = join(root, {components})",
"",
" try:",
]
if is_macos:
load = "CDLL(lib_path, mode=RTLD_GLOBAL)"
else:
load = "cdll.LoadLibrary(lib_path)"
if len(libs) > 1:
contents.append(f" libs.append({load})")
else:
contents.append(f" return {load}")
contents += [
" except FileNotFoundError:",
f" if not exists(lib_path):",
f' raise FileNotFoundError("{lib.name} was not found on your system. Is this package correctly installed?")',
]
if is_windows:
contents.append(
f' raise Exception("{lib.name} could not be loaded. Do you have Visual Studio C++ Redistributible installed?")'
)
else:
contents.append(
f' raise FileNotFoundError("{lib.name} could not be loaded. There is a missing dependency.")'
)
if len(libs) > 1:
contents += [" return libs"]
contents += ["", "__lib = __load_library()", ""]
content = ("\n".join(contents)) + "\n"
init_py.parent.mkdir(parents=True, exist_ok=True)
with open(init_py, "w") as fp:
fp.write(content)
def main():
pyproject_toml = sys.argv[1]
libinit_file = pathlib.Path(sys.argv[2])
pc_file = pathlib.Path(sys.argv[3])
pkgcfgs = [pathlib.Path(x) for x in sys.argv[4:]]
hack_pkgconfig(pkgcfgs)
with open(pyproject_toml, "rb") as fp:
raw_config = tomli.load(fp)
nativelib_cfg = raw_config["tool"]["hatch"]["build"]["hooks"]["nativelib"]
metadata = raw_config["project"]
generator = NativelibHook(pc_file, libinit_file, nativelib_cfg, metadata)
generator.initialize()
if __name__ == "__main__":
main()

43
shared/bazel/rules/robotpy/hatchlib_native_port/validate.py Normal file
View File

@@ -0,0 +1,43 @@
import typing
import validobj.validation
from validobj import errors
T = typing.TypeVar("T")
class ValidationError(Exception):
pass
def _convert_validation_error(
fname, ve: errors.ValidationError, prefix: str
) -> ValidationError:
locs = []
msgs = []
e: typing.Optional[BaseException] = ve
while e is not None:
if isinstance(e, errors.WrongFieldError):
locs.append(f".{e.wrong_field}")
elif isinstance(e, errors.WrongListItemError):
locs.append(f"[{e.wrong_index}]")
else:
msgs.append(str(e))
e = e.__cause__
loc = "".join(locs)
if loc.startswith("."):
loc = loc[1:]
msg = "\n ".join(msgs)
vmsg = f"{fname}: {prefix}{loc}:\n {msg}"
return ValidationError(vmsg)
def parse_input(value: typing.Any, spec: typing.Type[T], fname, prefix: str) -> T:
try:
return validobj.validation.parse_input(value, spec)
except errors.ValidationError as ve:
raise _convert_validation_error(fname, ve, prefix) from None

236
shared/bazel/rules/robotpy/pybind_build_file_template.jinja2 Normal file
View File

@@ -0,0 +1,236 @@
# THIS FILE IS AUTO GENERATED
{% if publish_casters_targets %}
load("@rules_cc//cc:cc_library.bzl", "cc_library")
{%- endif %}
load("//shared/bazel/rules/robotpy:pybind_rules.bzl", "create_pybind_library", "robotpy_library")
load("//shared/bazel/rules/robotpy:semiwrap_helpers.bzl", "gen_libinit", "gen_modinit_hpp", "gen_pkgconf", {% if publish_casters_targets %}"publish_casters", {% endif %}"resolve_casters", "run_header_gen")
load("//shared/bazel/rules/robotpy:semiwrap_tool_helpers.bzl", "scan_headers", "update_yaml_files")
{% for extension_module in extension_modules%}
def {{extension_module.name}}_extension(srcs = [], header_to_dat_deps = [], extra_hdrs = [], includes = [], extra_pyi_deps = []):
{{extension_module.name|upper}}_HEADER_GEN = [
{%- for header_cfg in extension_module.generation_data.values() %}
struct(
class_name = "{{header_cfg.class_name}}",
yml_file = "{{header_cfg.yml_file}}",
header_root = "{{header_cfg.include_root}}",
header_file = "{{header_cfg.include_file}}",
{%- if header_cfg.templates|length > 0 %}
tmpl_class_names = [
{%- for tmpl in header_cfg.templates %}
("{{ tmpl[0] }}", "{{ tmpl[1] }}"),
{%- endfor %}
],
{%- else %}
tmpl_class_names = [],
{%- endif %}
{%- if header_cfg.trampolines|length > 0 %}
trampolines = [
{%- for trampoline in header_cfg.trampolines %}
("{{ trampoline[0] }}", "{{ trampoline[1] }}"),
{%- endfor %}
],
{%- else %}
trampolines = [],
{%- endif %}
),
{%- endfor %}
]
resolve_casters(
name = "{{extension_module.name}}.resolve_casters",
caster_deps = {{ extension_module.resolve_casters.caster_deps | jsonify }},
casters_pkl_file = "{{ extension_module.resolve_casters.pkl_file }}",
dep_file = "{{ extension_module.resolve_casters.dep_file }}",
)
gen_libinit(
name = "{{extension_module.name}}.gen_lib_init",
output_file = "{{stripped_include_prefix}}/{{extension_module.gen_libinit.install_path}}/{{extension_module.gen_libinit.output_file}}",
modules = {{extension_module.gen_libinit.modules | jsonify}},
)
gen_pkgconf(
name = "{{extension_module.name}}.gen_pkgconf",
libinit_py = "{{ extension_module.gen_pkgconf.libinit_py }}",
module_pkg_name = "{{ extension_module.gen_pkgconf.module_pkg_name }}",
output_file = "{{ extension_module.gen_pkgconf.output_file }}",
pkg_name = "{{ extension_module.gen_pkgconf.pkg_name }}",
install_path = "{{stripped_include_prefix}}/{{ extension_module.gen_pkgconf.install_path }}",
project_file = "{{ stripped_include_prefix }}/{{ extension_module.gen_pkgconf.project_file }}",
package_root = "{{package_root_file}}",
)
gen_modinit_hpp(
name = "{{extension_module.name}}.gen_modinit_hpp",
input_dats = [x.class_name for x in {{extension_module.name|upper}}_HEADER_GEN],
libname = "{{ extension_module.gen_modinit.lib_name }}",
output_file = "{{ extension_module.gen_modinit.output_file }}",
)
run_header_gen(
name = "{{extension_module.name}}",
casters_pickle = "{{ extension_module.resolve_casters.pkl_file }}",
header_gen_config = {{extension_module.name|upper}}_HEADER_GEN,
trampoline_subpath = "{{stripped_include_prefix}}/{{ extension_module.install_path }}",
deps = header_to_dat_deps,
local_native_libraries = [
{%- for header_path in extension_module.native_wrapper_dependencies|sort %}
"{{header_path}}",
{%- endfor %}
],
{%- if extension_module.get_defines() %}
generation_defines = [{%-for da in extension_module.get_defines() %}"{{da.replace("=", " ")}}"{% endfor %}],
{%- endif %}
{%- if yml_prefix %}
yml_prefix = "{{yml_prefix}}",
{%- endif %}
)
create_pybind_library(
name = "{{extension_module.name}}",
install_path = "{{stripped_include_prefix}}/{{extension_module.install_path}}/",
extension_name = "{{ extension_module.gen_modinit.lib_name }}",
generated_srcs = [":{{extension_module.name}}.generated_srcs"],
semiwrap_header = [":{{extension_module.name}}.gen_modinit_hpp"],
deps = [
":{{extension_module.name}}.tmpl_hdrs",
":{{extension_module.name}}.trampoline_hdrs",
{%- for dep in extension_module.local_extension_dependencies %}
"{{dep}}",
{%- endfor %}
],
dynamic_deps = [
{%- for dep in extension_module.dynamic_dependencies %}
"{{dep}}",
{%- endfor %}
],
extra_hdrs = extra_hdrs,
extra_srcs = srcs,
includes = includes,
{%- if extension_module.get_defines() %}
local_defines = [{%-for da in extension_module.get_defines() %}"{{da.replace(' ', '=')}}"{% endfor %}],
{%- endif %}
)
native.filegroup(
name = "{{extension_module.name}}.generated_files",
srcs = [
"{{extension_module.name}}.gen_modinit_hpp.gen",
"{{extension_module.name}}.header_gen_files",
"{{extension_module.name}}.gen_pkgconf",
"{{extension_module.name}}.gen_lib_init",
],
tags = ["manual", "robotpy"],
)
{% endfor %}
{%- for caster_info in publish_casters_targets %}
def publish_library_casters():
publish_casters(
name = "publish_casters",
caster_name = "{{caster_info.casters_name}}",
output_json = "{{stripped_include_prefix}}/{{caster_info.install_path}}/{{caster_info.json_output}}",
output_pc = "{{stripped_include_prefix}}/{{caster_info.install_path}}/{{caster_info.pc_output}}",
project_config = "{{ stripped_include_prefix }}/{{caster_info.project_file}}",
package_root = "{{package_root_file}}",
typecasters_srcs = native.glob([{% for inc in caster_info.include_paths %}"{{ inc }}/**"{% if not loop.last%}, {% endif %}{% endfor %}]),
)
cc_library(
name = "{{caster_info.casters_name}}",
hdrs = native.glob([{% for inc in caster_info.include_paths %}"{{ inc }}/*.h"{% if not loop.last%}, {% endif %}{% endfor %}]),
includes = [{% for inc in caster_info.include_paths %}"{{ inc }}"{% if not loop.last%}, {% endif %}{% endfor %}],
visibility = ["//visibility:public"],
tags = ["robotpy"],
)
{% endfor %}
def define_pybind_library(name, pkgcfgs = []):
# Helper used to generate all files with one target.
native.filegroup(
name = "{}.generated_files".format(name),
srcs = [
{%- for em in extension_modules %}
"{{em.name}}.generated_files",
{%- endfor %}
],
tags = ["manual", "robotpy"],
visibility = ["//visibility:public"],
)
# Files that will be included in the wheel as data deps
native.filegroup(
name = "{}.generated_pkgcfg_files".format(name),
srcs = [
{%- for em in extension_modules %}
"{{stripped_include_prefix}}/{{em.gen_pkgconf.install_path}}/{{ em.gen_pkgconf.output_file }}",
{%- endfor %}
{%- for caster_info in publish_casters_targets %}
"{{stripped_include_prefix}}/{{caster_info.install_path}}/{{caster_info.pc_output}}",
"{{stripped_include_prefix}}/{{caster_info.install_path}}/{{caster_info.json_output}}",
{%- endfor %}
],
tags = ["manual", "robotpy"],
visibility = ["//visibility:public"],
)
# Contains all of the non-python files that need to be included in the wheel
native.filegroup(
name = "{}.extra_files".format(name),
srcs = native.glob(["{{stripped_include_prefix}}/{{top_level_name}}/**"], exclude = ["{{stripped_include_prefix}}/{{top_level_name}}/**/*.py"], allow_empty = True),
tags = ["manual", "robotpy"],
)
robotpy_library(
name = name,
srcs = native.glob(["{{stripped_include_prefix}}/{{top_level_name}}/**/*.py"]) + [
{%- for em in extension_modules %}
"{{stripped_include_prefix}}/{{ em.gen_pkgconf.libinit_py.replace(".", "/") }}.py",
{%- endfor %}
],
data = [
"{}.generated_pkgcfg_files".format(name),
"{}.extra_files".format(name),
{%- for em in extension_modules %}
":{{stripped_include_prefix}}/{{em.install_path}}/{{em.gen_modinit.lib_name}}",
{%- endfor %}
{%- for em in extension_modules %}
":{{em.name}}.trampoline_hdr_files",
{%- endfor %}
],
imports = ["{{stripped_include_prefix}}"],
deps = [
{%- for d in python_deps %}
"{{d}}",
{%- endfor %}
],
visibility = ["//visibility:public"],
)
update_yaml_files(
name = "{}-update-yaml".format(name),
yaml_output_directory = "{{ stripped_include_prefix }}/semiwrap",
extra_hdrs = native.glob(["{{stripped_include_prefix}}/**/*.h"], allow_empty = True) + [
{%- if python_deps %}
{% for d in python_deps %}
{%- if "native" in d %}"{{d}}.copy_headers",{%- endif %}
{%- endfor %}
{%- endif %}
],
package_root_file = "{{package_root_file}}",
pkgcfgs = pkgcfgs,
pyproject_toml = "{{ stripped_include_prefix }}/pyproject.toml",
yaml_files = native.glob(["{{stripped_include_prefix}}/semiwrap/**"]),
)
scan_headers(
name = "{}-scan-headers".format(name),
extra_hdrs = native.glob(["{{stripped_include_prefix}}/**/*.h"], allow_empty = True) + [
{%- if python_deps %}
{% for d in python_deps %}
{%- if "native" in d %}"{{d}}.copy_headers",{%- endif %}
{%- endfor %}
{%- endif %}
],
package_root_file = "{{package_root_file}}",
pkgcfgs = pkgcfgs,
pyproject_toml = "{{ stripped_include_prefix }}/pyproject.toml",
)

197
shared/bazel/rules/robotpy/pybind_rules.bzl Normal file
View File

@@ -0,0 +1,197 @@
load("@aspect_bazel_lib//lib:copy_file.bzl", "copy_file")
load("@pybind11_bazel//:build_defs.bzl", "pybind_extension", "pybind_library")
load("@rules_python//python:defs.bzl", "py_library")
load("//shared/bazel/rules/robotpy:compatibility_select.bzl", "robotpy_compatibility_select")
def create_pybind_library(
name,
extension_name,
install_path,
generated_srcs = [],
extra_hdrs = [],
extra_srcs = [],
deps = [],
dynamic_deps = [],
semiwrap_header = [],
includes = [],
local_defines = []):
"""
Function to create a pybind C++ extension library that has been defined by a semiwrap config
Outputs:
<name>_pybind_library - A pybind_library that functions like a header-only cc_library. It will include all
of the extra_hdrs, resolve the include paths, and add a dependency on the semiwrap headrs
<install_path + extension_name> - A pybind_extension that wraps the pybind_library and compiles all the source files.
Params:
extension_name - The name of the pybind extension. Should be sourced from pyproject
install_path - The subpath where the library will be installed. Should be source from pyproject
generated_srcs - List of auto-generated sources to be compiled into the extension.
extra_hdrs - Any non-autogenerated headers
extra_srcs - Any non-autogenerated sources
deps - cc_library deps used to create the pybind_library
dynamic_deps - cc_shared_library deps used to filter objects from the pybind_extension
semiwrap_header - Auto-generated file used to initialize the extension
includes - see cc_library#includes. Used during the creating of the pybind_library
local_defines - see cc_library#local_defines. Used during the compilation of the extension
"""
pybind_library(
name = "{}_pybind_library".format(name),
hdrs = extra_hdrs,
target_compatible_with = robotpy_compatibility_select(),
deps = deps + [
"//shared/bazel/rules/robotpy:semiwrap_headers",
],
includes = includes,
visibility = ["//visibility:public"],
tags = ["robotpy"],
)
extension_name = extension_name or "_{}".format(name)
pybind_extension(
name = install_path + extension_name,
srcs = extra_srcs + generated_srcs,
deps = [":{}_pybind_library".format(name)] + semiwrap_header,
dynamic_deps = dynamic_deps,
copts = select({
"@bazel_tools//src/conditions:darwin": [
"-Wno-deprecated-declarations",
"-Wno-overloaded-virtual",
"-Wno-pessimizing-move",
"-Wno-unused-value",
],
"@bazel_tools//src/conditions:linux_x86_64": [
"-Wno-attributes",
"-Wno-unused-value",
"-Wno-deprecated",
"-Wno-deprecated-declarations",
"-Wno-unused-parameter",
"-Wno-redundant-move",
"-Wno-unused-but-set-variable",
"-Wno-unused-variable",
"-Wno-pessimizing-move",
"-Wno-overloaded-virtual",
],
"@bazel_tools//src/conditions:windows": [
],
}),
target_compatible_with = robotpy_compatibility_select(),
local_defines = local_defines,
tags = ["robotpy"],
)
def robotpy_library(
name,
data = [],
**kwargs):
"""
Defines a python library that is wrapping a series of pybind extensions.
Outputs:
<name> - The python library
"""
py_library(
name = name,
data = data,
tags = ["robotpy"],
**kwargs
)
def copy_native_file(name, library, base_path):
"""
Copies a compiled shared library into a naming format that can be used by robotpy rules. The libraries are named
differently on OSX / Linux / Windows and this creates a handy alias to for easier use downstream
"""
copy_file(
name = name + ".win_copy_lib",
src = library,
out = "{}lib/{}.dll".format(base_path, name),
tags = ["manual"],
visibility = ["//visibility:public"],
)
copy_file(
name = name + ".osx_copy_lib",
src = library,
out = "{}lib/lib{}.dylib".format(base_path, name),
tags = ["manual"],
visibility = ["//visibility:public"],
)
copy_file(
name = name + ".linux_copy_lib",
src = library,
out = "{}lib/lib{}.so".format(base_path, name),
tags = ["manual"],
visibility = ["//visibility:public"],
)
native.alias(
name = "{}.copy_lib".format(name),
actual = select({
"@bazel_tools//src/conditions:darwin": name + ".osx_copy_lib",
"@bazel_tools//src/conditions:windows": name + ".win_copy_lib",
"//conditions:default": name + ".linux_copy_lib",
}),
visibility = ["//visibility:public"],
tags = ["robotpy"],
)
def _folder_prefix(name):
if "/" in name:
last_slash = name.rfind("/")
return (name[0:last_slash], name[last_slash + 1:])
else:
return ("", name)
def native_wrappery_library(
name,
pyproject_toml,
libinit_file,
pc_file,
pc_deps,
native_shared_library,
install_path,
headers,
deps = []):
"""
This function provides a sugar wrapper for defining a python library that wraps an allwpilib native library
"""
cmd = "$(locations //shared/bazel/rules/robotpy/hatchlib_native_port:generate_native_lib_files) "
cmd += " $(location " + pyproject_toml + ")"
cmd += " $(OUTS) "
for pc_dep in pc_deps:
cmd += " $(location " + pc_dep + ")"
native.genrule(
name = name + ".gen",
srcs = [pyproject_toml],
outs = [libinit_file, pc_file],
cmd = cmd,
tools = ["//shared/bazel/rules/robotpy/hatchlib_native_port:generate_native_lib_files"] + pc_deps,
visibility = ["//visibility:public"],
tags = ["robotpy"],
)
prefix, libname = _folder_prefix(native_shared_library)
copy_native_file(
name = libname,
library = native_shared_library,
base_path = install_path,
)
native.filegroup(
name = name + ".pc_wrapper",
srcs = [pc_file],
)
py_library(
name = name,
srcs = [libinit_file],
data = [pc_file, ":{}.copy_lib".format(libname), headers],
deps = deps,
imports = ["."],
visibility = ["//visibility:public"],
tags = ["robotpy"],
)

17
shared/bazel/rules/robotpy/pytest_util.bzl Normal file
View File

@@ -0,0 +1,17 @@
load("@rules_python_pytest//python_pytest:defs.bzl", "py_pytest_test")
load("//shared/bazel/rules/robotpy:compatibility_select.bzl", "robotpy_compatibility_select")
def robotpy_py_test(name, srcs, **kwargs):
py_pytest_test(
name = name,
size = "small",
srcs = srcs,
target_compatible_with = robotpy_compatibility_select(),
tags = [
"no-asan",
"no-tsan",
"robotpy",
],
legacy_create_init = 0,
**kwargs
)

357
shared/bazel/rules/robotpy/semiwrap_helpers.bzl Normal file
View File

@@ -0,0 +1,357 @@
load("@rules_cc//cc:defs.bzl", "cc_library")
load("//shared/bazel/rules/robotpy:compatibility_select.bzl", "robotpy_compatibility_select")
RESOLVE_CASTERS_DIR = "generated/resolve_casters/"
HEADER_DAT_DIR = "generated/header_to_dat/"
DAT_TO_CC_DIR = "generated/dat_to_cc/"
DAT_TO_TMPL_CC_DIR = "generated/dat_to_tmpl_cc/"
DAT_TO_TMPL_HDR_DIR = "generated/dat_to_tmpl_hdr/"
GEN_MODINIT_HDR_DIR = "generated/gen_modinit_hdr/"
def _location_helper(filename):
return " $(locations " + filename + ")"
def _wrapper():
return "$(locations //shared/bazel/rules/robotpy:wrapper) "
def _wrapper_dep():
return ["//shared/bazel/rules/robotpy:wrapper"]
def _semiwrap_caster():
return "//shared/bazel/rules/robotpy:semiwrap_casters_files"
def publish_casters(
name,
project_config,
caster_name,
output_json,
output_pc,
typecasters_srcs,
package_root):
"""
Sugar wrapper for the semiwrap.cmd.publish_casters tool
"""
cmd = _wrapper() + " semiwrap.cmd.publish_casters"
cmd += " $(SRCS) " + caster_name + " $(OUTS)"
native.genrule(
name = name,
srcs = [project_config],
outs = [output_json, output_pc],
cmd = cmd,
tools = _wrapper_dep() + typecasters_srcs + [package_root],
target_compatible_with = robotpy_compatibility_select(),
visibility = ["//visibility:public"],
tags = ["robotpy"],
)
def resolve_casters(
name,
casters_pkl_file,
dep_file,
caster_files = [],
caster_deps = []):
"""
Sugar wrapper for the semiwrap.cmd.resolve_casters tool
"""
cmd = _wrapper() + " semiwrap.cmd.resolve_casters "
cmd += " $(OUTS)"
cmd += _location_helper(_semiwrap_caster()) + "/semiwrap/semiwrap.pybind11.json"
resolved_caster_files = []
deps = []
for dep in caster_deps:
deps.append(dep)
cmd += _location_helper(dep)
for cfd in caster_files:
if cfd.startswith(":"):
resolved_caster_files.append(cfd)
cmd += _location_helper(cfd)
else:
cmd += " " + cfd
native.genrule(
name = name,
srcs = resolved_caster_files + deps,
outs = [RESOLVE_CASTERS_DIR + casters_pkl_file, RESOLVE_CASTERS_DIR + dep_file],
cmd = cmd,
tools = _wrapper_dep() + [_semiwrap_caster()],
target_compatible_with = robotpy_compatibility_select(),
tags = ["robotpy"],
)
def gen_libinit(
name,
output_file,
modules):
"""
Sugar wrapper for the semiwrap.cmd.gen_libinit tool
"""
cmd = _wrapper() + " semiwrap.cmd.gen_libinit "
cmd += " $(OUTS) "
cmd += " ".join(modules)
native.genrule(
name = name,
outs = [output_file],
cmd = cmd,
tools = _wrapper_dep(),
target_compatible_with = robotpy_compatibility_select(),
tags = ["robotpy"],
)
def gen_pkgconf(
name,
project_file,
module_pkg_name,
pkg_name,
output_file,
libinit_py,
install_path,
package_root):
"""
Sugar wrapper for the semiwrap.cmd.gen_pkgconf tool
"""
cmd = _wrapper() + " semiwrap.cmd.gen_pkgconf "
cmd += " " + module_pkg_name + " " + pkg_name
cmd += _location_helper(project_file)
cmd += " $(OUTS)"
if libinit_py:
cmd += " --libinit-py " + libinit_py
OUT_FILE = install_path + "/" + output_file
native.genrule(
name = name,
srcs = [package_root],
outs = [OUT_FILE],
cmd = cmd,
tools = _wrapper_dep() + [project_file],
target_compatible_with = robotpy_compatibility_select(),
visibility = ["//visibility:public"],
tags = ["robotpy"],
)
def header_to_dat(
name,
casters_pickle,
include_root,
class_name,
yml_file,
header_location,
generation_includes = [],
header_to_dat_deps = [],
extra_defines = [],
deps = []):
"""
Sugar wrapper for the semiwrap.cmd.header2dat tool
"""
cmd = _wrapper() + " semiwrap.cmd.header2dat "
cmd += "--cpp 202002L " # TODO(pj) - This is the option when I ran on linux. Does its value really matter?
cmd += class_name
cmd += _location_helper(yml_file)
cmd += " -I " + include_root
for inc in generation_includes:
cmd += " -I " + inc
for d in extra_defines:
cmd += " -D '" + d + "'"
cmd += " " + header_location
cmd += " " + include_root
cmd += _location_helper(RESOLVE_CASTERS_DIR + casters_pickle)
cmd += " $(OUTS)"
cmd += " bogus c++20 ccache c++ -- -std=c++20" # TODO(pj) Does it matter what these values are?
native.genrule(
name = name + "." + class_name,
srcs = [RESOLVE_CASTERS_DIR + casters_pickle] + deps + header_to_dat_deps,
outs = [HEADER_DAT_DIR + class_name + ".dat", HEADER_DAT_DIR + class_name + ".d"],
cmd = cmd,
tools = _wrapper_dep() + [yml_file],
target_compatible_with = robotpy_compatibility_select(),
tags = ["robotpy"],
)
def dat_to_cc(
name,
class_name):
dat_file = HEADER_DAT_DIR + class_name + ".dat"
cmd = _wrapper() + " semiwrap.cmd.dat2cpp "
cmd += _location_helper(dat_file)
cmd += " $(OUTS)"
native.genrule(
name = name + "." + class_name,
outs = [DAT_TO_CC_DIR + class_name + ".cpp"],
cmd = cmd,
tools = _wrapper_dep() + [dat_file],
target_compatible_with = robotpy_compatibility_select(),
tags = ["robotpy"],
)
def dat_to_tmpl_cpp(name, base_class_name, specialization, tmp_class_name):
cmd = _wrapper() + " semiwrap.cmd.dat2tmplcpp "
cmd += _location_helper(HEADER_DAT_DIR + base_class_name + ".dat")
cmd += " " + specialization
cmd += " $(OUTS)"
native.genrule(
name = name + "." + tmp_class_name,
outs = [DAT_TO_TMPL_CC_DIR + tmp_class_name + ".cpp"],
cmd = cmd,
tools = _wrapper_dep() + [HEADER_DAT_DIR + base_class_name + ".dat"],
target_compatible_with = robotpy_compatibility_select(),
tags = ["robotpy"],
)
def dat_to_tmpl_hpp(name, class_name):
dat_file = HEADER_DAT_DIR + class_name + ".dat"
cmd = _wrapper() + " semiwrap.cmd.dat2tmplhpp "
cmd += _location_helper(dat_file)
cmd += " $(OUTS)"
native.genrule(
name = name + "." + class_name,
outs = [DAT_TO_TMPL_HDR_DIR + class_name + "_tmpl.hpp"],
cmd = cmd,
tools = _wrapper_dep() + [dat_file],
target_compatible_with = robotpy_compatibility_select(),
tags = ["robotpy"],
)
def dat_to_trampoline(name, dat_file, class_name, output_file):
cmd = _wrapper() + " semiwrap.cmd.dat2trampoline "
cmd += _location_helper(HEADER_DAT_DIR + dat_file)
cmd += " " + class_name
cmd += " $(OUTS)"
native.genrule(
name = name + "." + output_file,
outs = [output_file],
cmd = cmd,
tools = _wrapper_dep() + [HEADER_DAT_DIR + dat_file],
target_compatible_with = robotpy_compatibility_select(),
tags = ["robotpy"],
)
def gen_modinit_hpp(
name,
libname,
input_dats,
output_file):
input_dats = [HEADER_DAT_DIR + x + ".dat" for x in input_dats]
cmd = _wrapper() + " semiwrap.cmd.gen_modinit_hpp "
cmd += " " + libname
cmd += " $(OUTS)"
for input_dat in input_dats:
cmd += _location_helper(input_dat)
native.genrule(
name = name + ".gen",
outs = [GEN_MODINIT_HDR_DIR + output_file],
cmd = cmd,
tools = _wrapper_dep() + input_dats,
target_compatible_with = robotpy_compatibility_select(),
tags = ["robotpy"],
)
cc_library(
name = name,
hdrs = [GEN_MODINIT_HDR_DIR + output_file],
strip_include_prefix = GEN_MODINIT_HDR_DIR,
tags = ["robotpy"],
target_compatible_with = robotpy_compatibility_select(),
)
def run_header_gen(name, casters_pickle, trampoline_subpath, header_gen_config, deps = [], generation_defines = [], local_native_libraries = [], header_to_dat_deps = [], yml_prefix = "src/main/python/"):
generation_includes = []
header_to_dat_deps = list(header_to_dat_deps)
for dep in local_native_libraries:
header_to_dat_deps.append(dep)
generation_includes.append("$(execpath " + dep + ")")
for header_gen in header_gen_config:
header_to_dat(
name = name + ".header_to_dat",
casters_pickle = casters_pickle,
include_root = header_gen.header_root,
class_name = header_gen.class_name,
yml_file = yml_prefix + header_gen.yml_file,
header_location = header_gen.header_file,
deps = deps,
generation_includes = generation_includes,
extra_defines = generation_defines,
header_to_dat_deps = header_to_dat_deps,
)
generated_cc_files = []
for header_gen in header_gen_config:
dat_to_cc(
name = name + ".dat_to_cc",
class_name = header_gen.class_name,
)
generated_cc_files.append(DAT_TO_CC_DIR + header_gen.class_name + ".cpp")
tmpl_hdrs = []
for header_gen in header_gen_config:
if header_gen.tmpl_class_names:
dat_to_tmpl_hpp(
name = name + ".dat_to_tmpl_hpp",
class_name = header_gen.class_name,
)
tmpl_hdrs.append(DAT_TO_TMPL_HDR_DIR + header_gen.class_name + "_tmpl.hpp")
for tmpl_class_name, specialization in header_gen.tmpl_class_names:
dat_to_tmpl_cpp(
name = name + ".dat_to_tmpl_cpp",
base_class_name = header_gen.class_name,
specialization = specialization,
tmp_class_name = tmpl_class_name,
)
generated_cc_files.append(DAT_TO_TMPL_CC_DIR + tmpl_class_name + ".cpp")
trampoline_hdrs = []
for header_gen in header_gen_config:
for trampoline_symbol, trampoline_header in header_gen.trampolines:
output_path = trampoline_subpath + "/trampolines/" + trampoline_header
dat_to_trampoline(
name = name + ".dat2trampoline",
dat_file = header_gen.class_name + ".dat",
class_name = trampoline_symbol,
output_file = output_path,
)
trampoline_hdrs.append(output_path)
cc_library(
name = name + ".tmpl_hdrs",
hdrs = tmpl_hdrs,
strip_include_prefix = DAT_TO_TMPL_HDR_DIR,
tags = ["robotpy"],
)
cc_library(
name = name + ".trampoline_hdrs",
hdrs = trampoline_hdrs,
strip_include_prefix = trampoline_subpath,
tags = ["robotpy"],
)
native.filegroup(
name = name + ".generated_srcs",
srcs = generated_cc_files,
tags = ["manual", "robotpy"],
)
native.filegroup(
name = name + ".trampoline_hdr_files",
srcs = trampoline_hdrs,
tags = ["manual", "robotpy"],
)
native.filegroup(
name = name + ".header_gen_files",
srcs = tmpl_hdrs + trampoline_hdrs + generated_cc_files,
tags = ["manual", "robotpy"],
)

92
shared/bazel/rules/robotpy/semiwrap_tool_helpers.bzl Normal file
View File

@@ -0,0 +1,92 @@
load("@allwpilib_pip_deps//:requirements.bzl", "requirement")
load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files")
load("@rules_python//python:defs.bzl", "py_test")
load("//shared/bazel/rules/robotpy:compatibility_select.bzl", "robotpy_compatibility_select")
def __update_yaml_files_impl(ctx):
output_dir = ctx.actions.declare_directory(ctx.attr.gen_dir)
args = ctx.actions.args()
args.add("semiwrap.tool")
args.add("update-yaml")
args.add("--write")
args.add("-v")
args.add("--project_file=" + ctx.files.pyproject_toml[0].path)
args.add("--override_output_directory=" + output_dir.path)
if ctx.files.pkgcfgs:
args.add("--pkgcfgs")
for f in ctx.files.pkgcfgs:
args.add(str(f.path))
ctx.actions.run(
inputs = ctx.files.package_root_file + ctx.files.pyproject_toml + ctx.files.pkgcfgs + ctx.files.extra_hdrs + ctx.files.yaml_files,
outputs = [output_dir],
executable = ctx.executable._tool,
arguments = [args],
)
return [DefaultInfo(files = depset([output_dir]))]
__update_yaml_files = rule(
implementation = __update_yaml_files_impl,
attrs = {
"extra_hdrs": attr.label_list(allow_files = True),
"gen_dir": attr.string(mandatory = True),
"package_root_file": attr.label(mandatory = True, allow_files = True),
"pkgcfgs": attr.label_list(allow_files = True),
"pyproject_toml": attr.label(mandatory = True, allow_files = True),
"yaml_files": attr.label_list(mandatory = True, allow_files = True),
"_tool": attr.label(
default = Label("//shared/bazel/rules/robotpy:wrapper"),
cfg = "exec",
executable = True,
),
},
)
def update_yaml_files(name, yaml_output_directory = "src/main/python/semiwrap", **kwargs):
__update_yaml_files(
name = name,
gen_dir = "{}_gen_update_yaml".format(name),
target_compatible_with = robotpy_compatibility_select(),
**kwargs
)
write_source_files(
name = "write_{}".format(name),
files = {
yaml_output_directory: ":" + name,
},
suggested_update_target = "//:write_all",
target_compatible_with = robotpy_compatibility_select(),
visibility = ["//visibility:public"],
)
def scan_headers(name, pyproject_toml, package_root_file, extra_hdrs, pkgcfgs):
if pkgcfgs:
pkgcfg_args = ["--pkgcfgs"]
for pkgcfg in pkgcfgs:
pkgcfg_args.append(" $(locations " + pkgcfg + ")")
else:
pkgcfg_args = []
py_test(
name = name,
srcs = [
"//shared/bazel/rules/robotpy:wrapper.py",
],
deps = [
"//shared/bazel/rules/robotpy:hack_pkgcfgs",
requirement("semiwrap"),
],
args = [
"semiwrap.tool",
"scan-headers",
"--pyproject=$(location " + pyproject_toml + ")",
] + pkgcfg_args,
data = extra_hdrs + pkgcfgs + [pyproject_toml, package_root_file],
main = "shared/bazel/rules/robotpy/wrapper.py",
size = "small",
target_compatible_with = robotpy_compatibility_select(),
)

55
shared/bazel/rules/robotpy/wrapper.py Normal file
View File

@@ -0,0 +1,55 @@
import importlib
import os
import pathlib
import sys
from shared.bazel.rules.robotpy.hack_pkgcfgs import hack_pkgconfig
"""
This file will wrap various semiwrap.tools executables. In the event that it fails
it will provide more helpful debug information for bazel users.
It can also "hack" the PKG_CONFIG_PATH environment variable. This allows us to use
generated package config files without having to install the libraries which decreases
build dependencies and increases the amount of parallelization that can happen during
the build.
"""
def main():
tool = sys.argv[1]
if "--pkgcfgs" in sys.argv[2:]:
pkgcfg_index = sys.argv.index("--pkgcfgs")
args = sys.argv[2:pkgcfg_index]
pkgcfgs = [pathlib.Path(x) for x in sys.argv[pkgcfg_index + 1 :]]
else:
args = sys.argv[2:]
pkgcfgs = []
hack_pkgconfig(pkgcfgs)
module = importlib.import_module(tool)
tool_main = getattr(module, "main")
sys.argv = [""] + args
try:
tool_main()
except SystemExit as e:
if e.code != 0:
raise Exception(
"sys.exit() explicitly called with a non-zero error code", e
)
except:
print("-------------------------------------")
print("Failed to run wrapped tool.")
print(f"CWD: {os.getcwd()}")
print(f"Tool: {tool}, Args:")
for a in args:
print(" ", a)
print("-------------------------------------")
raise
if __name__ == "__main__":
main()