[python] Improve robotpy generation (#8867)

The initial build file generation for robotpy projects was relatively
naive and purpose built to get `allwpilib` compiling, without supporting
all the available features.

This modifies the generation scripts to be able to support multiple
embedded libraries, which will be necessary for #8858, since `mrclib.so`
will need to be bundled along with the hal libraries. In addition some
cleanup was done to get the wheels looking more like what is in pypi.

shared/bazel/rules/robotpy/BUILD.bazel

@@ -31,6 +31,7 @@ py_binary(
     deps = [
         ":generation_utils",
         ":hack_pkgcfgs",
+        "//shared/bazel/rules/robotpy/hatchlib_native_port:generate_native_lib_files",
         requirement("semiwrap"),
         requirement("jinja2"),
     ],

shared/bazel/rules/robotpy/generate_native_build_file.py

@@ -3,12 +3,14 @@ import json
 
 import tomli
 from jinja2 import BaseLoader, Environment
+from packaging.markers import Marker
 
 from shared.bazel.rules.robotpy.generation_utils import (
     fixup_python_dep_name,
     fixup_root_package_name,
-    fixup_shared_lib_name,
 )
+from shared.bazel.rules.robotpy.hatchlib_native_port.config import PcFileConfig
+from shared.bazel.rules.robotpy.hatchlib_native_port.validate import parse_input
 
 
 def main():
@@ -40,20 +42,48 @@ def main():
     env.filters["double_quotes"] = double_quotes
     env.filters["get_pc_dep"] = get_pc_dep
     env.filters["get_python_dep"] = get_python_dep
+    env.filters["strip_src_prefix"] = lambda x: str(x).removeprefix("src/")
     template = env.from_string(BUILD_FILE_TEMPLATE)
 
     nativelib_config = raw_config["tool"]["hatch"]["build"]["hooks"]["nativelib"]
-    project_name = nativelib_config["pcfile"][0]["name"]
+    project_name = raw_config["project"]["name"].replace("robotpy-native-", "")
     root_package = fixup_root_package_name(project_name)
-    shared_library_name = fixup_shared_lib_name(project_name)
+    pc_files = []
+
+    local_pc_names = set()
+    for i, raw_pc in enumerate(nativelib_config.get("pcfile", [])):
+        pcfile = parse_input(
+            raw_pc,
+            PcFileConfig,
+            "pyproject.toml",
+            f"tool.hatch.build.hooks.nativelib.pcfile[{i}]",
+        )
+        if pcfile.enable_if and not Marker(pcfile.enable_if).evaluate():
+            continue
+        pc_files.append(pcfile)
+        local_pc_names.add(pcfile.get_pc_path().name[:-3])
+
+    requires = set()
+    for pcfile in pc_files:
+        if pcfile.requires:
+            for dep in pcfile.requires:
+                if dep not in local_pc_names:
+                    requires.add(dep)
+
+    maven_downloads = raw_config["tool"]["hatch"]["build"]["hooks"]["robotpy"][
+        "maven_lib_download"
+    ]
     with open(args.output_file, "w") as f:
         f.write(
             template.render(
                 raw_project_config=raw_config["project"],
                 nativelib_config=nativelib_config,
                 root_package=root_package,
-                shared_library_name=shared_library_name,
+                maven_downloads=maven_downloads,
                 third_party_dirs=args.third_party_dirs or [],
+                pc_files=pc_files,
+                requires=requires,
+                project_name=project_name,
             )
         )
 
@@ -61,9 +91,11 @@ def main():
 BUILD_FILE_TEMPLATE = """# THIS FILE IS AUTO GENERATED
 
 load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
-load("//shared/bazel/rules/robotpy:pybind_rules.bzl", "native_wrappery_library")
+load("//shared/bazel/rules/robotpy:robotpy_rules.bzl", "copy_native_file", "generate_native_files", "robotpy_library")
 
 def define_native_wrapper(name, pyproject_toml = None):
+    pyproject_toml = pyproject_toml or "src/main/python/native-pyproject.toml"
+
     copy_to_directory(
         name = "{}.copy_headers".format(name),
         srcs = native.glob(["src/main/native/include/**"]) + native.glob(["src/generated/main/native/include/**"], allow_empty = True){% if third_party_dirs %} + native.glob([
@@ -71,7 +103,7 @@ def define_native_wrapper(name, pyproject_toml = None):
             "src/main/native/thirdparty/{{dir}}/include/**",
         {%- endfor %}
         ]){%- endif %},
-        out = "native/{{nativelib_config.pcfile[0].name}}/include",
+        out = "native/{{project_name}}/include",
         root_paths = ["src/main/native/include/"],
         replace_prefixes = {
             "{{root_package}}/src/generated/main/native/include": "",
@@ -84,30 +116,57 @@ def define_native_wrapper(name, pyproject_toml = None):
         visibility = ["//visibility:public"],
     )
 
-    native_wrappery_library(
+    libinit_files = [{% for pcfile in pc_files %}"{{pcfile.get_init_module_path() | strip_src_prefix}}"{% if not loop.last %}, {% endif %}{%- endfor %}]
+
+    generate_native_files(
         name = name,
-        pyproject_toml = pyproject_toml or "src/main/python/native-pyproject.toml",
-        libinit_file = "native/{{nativelib_config.pcfile[0].name}}/_init_{{raw_project_config.name.replace("-", "_")}}.py",
-        pc_file = "native/{{nativelib_config.pcfile[0].name}}/{{raw_project_config.name}}.pc",
+        pyproject_toml = pyproject_toml,
         pc_deps = [
-        {%- for dep in nativelib_config.pcfile[0].requires | sort %}
+        {%- for dep in requires | sort %}
             "{{dep | get_pc_dep}}",
         {%- endfor %}
         ],
+        libinit_files = libinit_files,
+        pc_files = [{% for pcfile in pc_files %}"{{pcfile.pcfile | strip_src_prefix}}"{% if not loop.last %}, {% endif %}{%- endfor %}],
+    )
+    {%- for maven_info in maven_downloads %}
+    {%- for lib in maven_info["libs"] %}
+
+    copy_native_file(
+        name = "{{lib}}",
+        library = "shared/{{lib}}",
+        base_path = "native/{{project_name}}/",
+    )
+    {%- endfor %}
+    {%- endfor %}
+
+    robotpy_library(
+        name = name,
+        distribution = "{{raw_project_config.name}}",
+        srcs = libinit_files,
+        data = [
+            name + ".pc_wrapper",
+    {%- for maven_info in maven_downloads %}
+    {%- for lib in maven_info["libs"] %}
+            ":{{lib}}.copy_lib",
+    {%- endfor %}
+    {%- endfor %}
+            "{}.copy_headers".format(name),
+        ],
         deps = [
-        {%- for dep in nativelib_config.pcfile[0].requires | sort %}
+        {%- for dep in requires | sort %}
             "{{dep | get_python_dep}}",
         {%- endfor %}
         ],
-        headers = "{}.copy_headers".format(name),
-        native_shared_library = "shared/{{shared_library_name}}",
-        install_path = "native/{{nativelib_config.pcfile[0].name}}/",
-        strip_path_prefixes = ["{{root_package}}"],
-        requires = {{raw_project_config.dependencies | double_quotes}},
         summary = "{{raw_project_config.description}}",
+        requires = {{raw_project_config.dependencies | double_quotes}},
+        python_requires = "{{raw_project_config["requires-python"]}}",
+        strip_path_prefixes = ["{{root_package}}"],
         entry_points = {
             "pkg_config": [
-                "{{nativelib_config.pcfile[0].name}} = native.{{nativelib_config.pcfile[0].name}}",
+            {%- for pcfile in pc_files %}
+                "{{pcfile.name}} = native.{{pcfile.name}}",
+            {%- endfor %}
             ],
         },
     )

shared/bazel/rules/robotpy/generate_pybind_build_file.py

@@ -209,6 +209,8 @@ class BazelExtensionModule:
                 transitive_deps = set()
                 self._get_transitive_native_dependencies(dep_name, transitive_deps)
                 for d in transitive_deps:
+                    if d == "robotpy-native-mrclib":
+                        continue
                     base_library = fixup_root_package_name(
                         d.replace("robotpy-native-", "")
                     )

shared/bazel/rules/robotpy/hatchlib_native_port/generate_native_lib_files.py

@@ -1,4 +1,5 @@
 import functools
+import os
 import pathlib
 import platform
 import sys
@@ -22,9 +23,7 @@ is_macos = platform_sys == "Darwin"
 
 
 class NativelibHook:
-    def __init__(self, output_pcfile, output_libinit, config, metadata):
-        self.output_pcfile = output_pcfile
-        self.output_libinit = output_libinit
+    def __init__(self, output_pcfile, config, metadata):
 
         self.config = config
         self.root_pth = output_pcfile.parent.parent.parent
@@ -32,7 +31,19 @@ class NativelibHook:
 
     def initialize(self):
         for pcfg in self._pcfiles:
-            self._generate_pcfile(pcfg, {})
+            pcfile = self._generate_pcfile(pcfg, {})
+            self._add_pkg_config_path(str(pcfile.parent))
+
+    def _add_pkg_config_path(self, *paths: str) -> None:
+        current = os.environ.get("PKG_CONFIG_PATH")
+        entries = current.split(os.pathsep) if current else []
+
+        for path in paths:
+            if path not in entries:
+                entries.append(path)
+
+        if entries:
+            os.environ["PKG_CONFIG_PATH"] = os.pathsep.join(entries)
 
     def _get_pkg_from_path(self, path: pathlib.Path) -> str:
         rel = path.relative_to(self.root_pth)
@@ -43,7 +54,7 @@ class NativelibHook:
     ) -> pathlib.Path:
 
         pcfile_rel = pcfg.get_pc_path()
-        pcfile = self.output_pcfile
+        pcfile = self.root_pth / str(pcfile_rel).removeprefix("src/")
         prefix_rel = pcfile_rel.parent
         prefix_path = pcfile.parent
 
@@ -147,7 +158,7 @@ class NativelibHook:
         build_data: T.Dict[str, T.Any],
     ):
         libinit_py_rel = pcfg.get_init_module_path()
-        self.root_pth / libinit_py_rel
+        libinit_py = self.root_pth / str(libinit_py_rel).removeprefix("src/")
 
         libdir = prefix_path
         if pcfg.libdir:
@@ -165,7 +176,7 @@ class NativelibHook:
         else:
             requires = []
 
-        _write_libinit_py(self.output_libinit, lib_paths, requires)
+        _write_libinit_py(libinit_py, lib_paths, requires)
 
     def _make_shared_lib_fname(self, lib: str):
         if is_windows:
@@ -298,9 +309,8 @@ def _write_libinit_py(
 
 def main():
     pyproject_toml = sys.argv[1]
-    libinit_file = pathlib.Path(sys.argv[2])
-    pc_file = pathlib.Path(sys.argv[3])
-    pkgcfgs = [pathlib.Path(x) for x in sys.argv[4:]]
+    pc_file = pathlib.Path(sys.argv[2])
+    pkgcfgs = [pathlib.Path(x) for x in sys.argv[3:]]
 
     hack_pkgconfig(pkgcfgs)
 
@@ -310,7 +320,7 @@ def main():
     nativelib_cfg = raw_config["tool"]["hatch"]["build"]["hooks"]["nativelib"]
     metadata = raw_config["project"]
 
-    generator = NativelibHook(pc_file, libinit_file, nativelib_cfg, metadata)
+    generator = NativelibHook(pc_file, nativelib_cfg, metadata)
     generator.initialize()
 
 

shared/bazel/rules/robotpy/pybind_build_file_template.jinja2

@@ -8,7 +8,7 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library")
 {%- if version_file %}
 load("//shared/bazel/rules/gen:gen-version-file.bzl", "generate_version_file")
 {%- endif %}
-load("//shared/bazel/rules/robotpy:pybind_rules.bzl", "create_pybind_library", "robotpy_library")
+load("//shared/bazel/rules/robotpy:robotpy_rules.bzl", "create_pybind_library", "robotpy_library")
 load("//shared/bazel/rules/robotpy:semiwrap_helpers.bzl", "gen_libinit", "gen_modinit_hpp", "gen_pkgconf", {% if publish_casters_targets %}"publish_casters", {% endif %}"resolve_casters", "run_header_gen")
 load("//shared/bazel/rules/robotpy:semiwrap_tool_helpers.bzl", "scan_headers", "update_yaml_files")
 {% for extension_module in extension_modules%}
@@ -193,6 +193,7 @@ def define_pybind_library(name, pkgcfgs = []):
 {% endif %}
     robotpy_library(
         name = name,
+        distribution = "{{raw_project_config.name}}",
         srcs = native.glob(["{{stripped_include_prefix}}/{{top_level_name}}/**/*.py"]) + [
         {%- for em in extension_modules %}
             "{{stripped_include_prefix}}/{{ em.gen_pkgconf.libinit_py.replace(".", "/") }}.py",
@@ -222,6 +223,7 @@ def define_pybind_library(name, pkgcfgs = []):
         project_urls = {{raw_project_config.urls | jsonify}},
         author_email = "RobotPy Development Team <robotpy@googlegroups.com>",
         requires = {{raw_project_config.dependencies | jsonify}},
+        python_requires = {{raw_project_config["requires-python"] | jsonify}},
         entry_points = {
             {%- for group, entries in entry_points.items() %}
             "{{ group }}": {{entries | jsonify}},

shared/bazel/rules/robotpy/robotpy_rules.bzl

@@ -87,11 +87,14 @@ def robotpy_library(
         deps = [],
         data = [],
         strip_path_prefixes = None,
+        distribution = None,
         summary = None,
         project_urls = None,
         author_email = None,
         entry_points = None,
         requires = None,
+        description_file = None,
+        python_requires = None,
         **kwargs):
     """
     Defines a python library that is wrapping a series of pybind extensions.
@@ -110,7 +113,7 @@ def robotpy_library(
 
     py_wheel(
         name = "{}-wheel".format(name),
-        distribution = name,
+        distribution = distribution,
         stamp = 1,
         version = "$(ROBOTPY_VERSION)",
         summary = summary,
@@ -120,6 +123,8 @@ def robotpy_library(
         deps = data + [":{}-lib".format(name)],
         strip_path_prefixes = strip_path_prefixes,
         entry_points = entry_points,
+        description_file = description_file,
+        python_requires = python_requires,
         license = "BSD-3-Clause",
         tags = ["robotpy"],
     )
@@ -172,90 +177,25 @@ def copy_native_file(name, library, base_path):
         tags = ["robotpy"],
     )
 
-def _folder_prefix(name):
-    if "/" in name:
-        last_slash = name.rfind("/")
-        return (name[0:last_slash], name[last_slash + 1:])
-    else:
-        return ("", name)
-
-def native_wrappery_library(
-        name,
-        pyproject_toml,
-        libinit_file,
-        pc_file,
-        pc_deps,
-        native_shared_library,
-        install_path,
-        headers,
-        strip_path_prefixes = [],
-        summary = None,
-        project_urls = None,
-        author_email = None,
-        entry_points = None,
-        requires = None,
-        deps = []):
-    """
-    This function provides a sugar wrapper for defining a python library that wraps an allwpilib native library
-    """
+def generate_native_files(name, pyproject_toml, pc_deps, libinit_files, pc_files):
     cmd = "$(locations //shared/bazel/rules/robotpy/hatchlib_native_port:generate_native_lib_files) "
     cmd += "  $(location " + pyproject_toml + ")"
-    cmd += " $(OUTS) "
+    cmd += " $(location " + pc_files[0] + ") "
     for pc_dep in pc_deps:
         cmd += " $(location " + pc_dep + ")"
 
     native.genrule(
         name = name + ".gen",
         srcs = [pyproject_toml],
-        outs = [libinit_file, pc_file],
+        outs = libinit_files + pc_files,
         cmd = cmd,
         tools = ["//shared/bazel/rules/robotpy/hatchlib_native_port:generate_native_lib_files"] + pc_deps,
         visibility = ["//visibility:public"],
         tags = ["robotpy"],
-    )
-
-    prefix, libname = _folder_prefix(native_shared_library)
-
-    copy_native_file(
-        name = libname,
-        library = native_shared_library,
-        base_path = install_path,
+        target_compatible_with = robotpy_compatibility_select(),
     )
 
     native.filegroup(
         name = name + ".pc_wrapper",
-        srcs = [pc_file],
-    )
-
-    py_library(
-        name = name + "-lib",
-        srcs = [libinit_file],
-        data = [pc_file, ":{}.copy_lib".format(libname), headers],
-        deps = deps,
-        imports = ["."],
-        tags = ["robotpy"],
-    )
-
-    py_wheel(
-        name = "{}-wheel".format(name),
-        distribution = name,
-        stamp = 1,
-        version = "$(ROBOTPY_VERSION)",
-        summary = summary,
-        requires = requires,
-        project_urls = project_urls,
-        author_email = author_email,
-        deps = [name + "-lib", ":{}.copy_lib".format(libname), headers, name + ".pc_wrapper"],
-        strip_path_prefixes = strip_path_prefixes,
-        entry_points = entry_points,
-        tags = ["robotpy"],
-        license = "BSD-3-Clause",
-    )
-
-    pycross_wheel_library(
-        name = "{}".format(name),
-        wheel = "{}-wheel".format(name),
-        visibility = ["//visibility:public"],
-        tags = ["manual"],
-        deps = deps,
+        srcs = pc_files,
     )