Rewrite repo_missing for bpo + systemd split repo (MR 2410)

The "pmbootstrap repo_missing" action is used exclusively by bpo. It
calls it only with these arguments:

  pmbootstrap repo_missing --built --arch "$ARCH"

A blocker for merging systemd into pmaports master is, that the
current repo_missing code cannot display packages that are both in
extra-repos/systemd and in another path. I have considered just not
supporting this and discussed doing that with Caleb and Clayton, but we
figured it would be a major obstacle in the future to not be able to
easily override packages with systemd specific versions (currently we
need this for 3 packages).

Integrating this into the existing repo_missing code would be hacks upon
hacks. Also the scope of the current repo_missing code has many extra
features that are not used and would be extra effort to carry along:
* Allow specifying a pkgname
* Running without --built
* --overview

So I decided to replace the repo_missing code with a much simpler, more
modern implementation, that does exactly what is needed:
* Duplicate packages in systemd and non-systemd dirs are displayed
* The output always include all packages, no matter if they are already
  built or not (same behavior as with --built)
* Removed --overview and selecting specific packages too
* The code for filling "repo" (either "systemd" or None) is more
  resilient now, as it can use proper relative paths to the root of
  pmaports. Unlike the previous implementation, it will not fail if
  subdirs are added to the systemd dir.

I have made sure that the output is exactly the same as before on
current pmaports master.

Related: bpo issue 144
Related: bpo issue 140

pmb/helpers/frontend.py

@@ -285,9 +285,13 @@ def config(args: PmbArgs) -> None:
 
 
 def repo_missing(args: PmbArgs) -> None:
-    if args.arch is None or isinstance(args.package, list):
+    if args.arch is None:
         raise AssertionError
-    missing = pmb.helpers.repo_missing.generate(args.arch, args.overview, args.package, args.built)
+    if args.built:
+        logging.warning(
+            "WARNING: --built is deprecated (bpo#148: this warning is expected on build.postmarketos.org for now)"
+        )
+    missing = pmb.helpers.repo_missing.generate(args.arch)
     print(json.dumps(missing, indent=4))
 
 

pmb/helpers/repo_missing.py

@@ -1,171 +1,57 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2025 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-
-from typing import overload, Literal
-
 from pmb.core.arch import Arch
-from pmb.helpers import logging
+from pmb.core.context import get_context
-from pmb.types import Apkbuild
+from pathlib import Path
 
 import pmb.build
 import pmb.helpers.package
 import pmb.helpers.pmaports
+import glob
+import os
 
 
-def filter_missing_packages(arch: Arch, pkgnames: list[str]) -> list[str]:
+def generate(arch: Arch) -> list[dict[str, list[str] | str | None]]:
-    """Create a subset of pkgnames with missing or outdated binary packages.
+    """Get packages that need to be built, with all their dependencies. Include
+       packages from extra-repos, no matter if systemd is enabled or not. This
+       is used by bpo to fill its package database.
 
     :param arch: architecture (e.g. "armhf")
-    :param pkgnames: list of package names (e.g. ["hello-world", "test12"])
-    :returns: subset of pkgnames (e.g. ["hello-world"])
-    """
-    ret = []
-    for pkgname in pkgnames:
-        binary = pmb.parse.apkindex.package(pkgname, arch, False)
-        must_exist = False if binary else True
-        pmaport = pmb.helpers.pmaports.get(pkgname, must_exist)
-        if pmaport and pmb.build.get_status(arch, pmaport).necessary():
-            ret.append(pkgname)
-    return ret
-
-
-def filter_aport_packages(pkgnames: list[str]) -> list[str]:
-    """Create a subset of pkgnames where each one has an aport.
-
-    :param pkgnames: list of package names (e.g. ["hello-world", "test12"])
-    :returns: subset of pkgnames (e.g. ["hello-world"])
-    """
-    ret = []
-    for pkgname in pkgnames:
-        if pmb.helpers.pmaports.find_optional(pkgname):
-            ret += [pkgname]
-    return ret
-
-
-def filter_arch_packages(arch: Arch, pkgnames: list[str]) -> list[str]:
-    """Create a subset of pkgnames with packages removed that can not be built for a certain arch.
-
-    :param arch: architecture (e.g. "armhf")
-    :param pkgnames: list of package names (e.g. ["hello-world", "test12"])
-    :returns: subset of pkgnames (e.g. ["hello-world"])
-    """
-    ret = []
-    for pkgname in pkgnames:
-        if pmb.helpers.package.check_arch(pkgname, arch, False):
-            ret += [pkgname]
-    return ret
-
-
-def get_relevant_packages(arch: Arch, pkgname: str | None = None, built: bool = False) -> list[str]:
-    """Get all packages that can be built for the architecture in question.
-
-    :param arch: architecture (e.g. "armhf")
-    :param pkgname: only look at a specific package (and its dependencies)
-    :param built: include packages that have already been built
-    :returns: an alphabetically sorted list of pkgnames, e.g.:
-        ["devicepkg-dev", "hello-world", "osk-sdl"]
-    """
-    if pkgname:
-        if not pmb.helpers.package.check_arch(pkgname, arch, False):
-            raise RuntimeError(f"{pkgname} can't be built for {arch}.")
-        ret = pmb.helpers.package.depends_recurse(pkgname, arch)
-    else:
-        ret = pmb.helpers.pmaports.get_list()
-        ret = filter_arch_packages(arch, ret)
-    if built:
-        ret = filter_aport_packages(ret)
-        if not len(ret):
-            logging.info(
-                "NOTE: no aport found for any package in the"
-                " dependency tree, it seems they are all provided by"
-                " upstream (Alpine)."
-            )
-    else:
-        ret = filter_missing_packages(arch, ret)
-        if not len(ret):
-            logging.info(
-                "NOTE: all relevant packages are up to date, use"
-                " --built to include the ones that have already been"
-                " built."
-            )
-
-    # Sort alphabetically (to get a deterministic build order)
-    ret.sort()
-    return ret
-
-
-def generate_output_format(arch: Arch, pkgnames: list[str]) -> list[Apkbuild]:
-    """Generate the detailed output format.
-
-    :param arch: architecture
-    :param pkgnames: list of package names that should be in the output,
-        e.g.: ["hello-world", "pkg-depending-on-hello-world"]
-        :returns: a list like the following:
-        [{"pkgname": "hello-world",
-        "repo": "main",
-        "version": "1-r4",
-        "depends": []},
-        {"pkgname": "pkg-depending-on-hello-world",
-        "version": "0.5-r0",
-        "repo": "main",
-        "depends": ["hello-world"]}]
-    """
-    ret = []
-    for pkgname in pkgnames:
-        entry = pmb.helpers.package.get(pkgname, arch, True, try_other_arches=False)
-
-        if entry is None:
-            raise RuntimeError(f"Couldn't get package {pkgname} for arch {arch}")
-
-        ret += [
-            {
-                "pkgname": entry.pkgname,
-                "repo": pmb.helpers.pmaports.get_repo(pkgname),
-                "version": entry.version,
-                "depends": entry.depends,
-            }
-        ]
-    return ret
-
-
-@overload
-def generate(
-    arch: Arch, overview: Literal[False], pkgname: str | None = ..., built: bool = ...
-) -> list[Apkbuild]: ...
-
-
-@overload
-def generate(
-    arch: Arch, overview: Literal[True], pkgname: str | None = ..., built: bool = ...
-) -> list[str]: ...
-
-
-@overload
-def generate(
-    arch: Arch, overview: bool, pkgname: str | None = ..., built: bool = ...
-) -> list[Apkbuild] | list[str]: ...
-
-
-def generate(
-    arch: Arch, overview: bool, pkgname: str | None = None, built: bool = False
-) -> list[Apkbuild] | list[str]:
-    """Get packages that need to be built, with all their dependencies.
-
-    :param arch: architecture (e.g. "armhf")
-    :param pkgname: only look at a specific package
-    :param built: include packages that have already been built
     :returns: a list like the following:
-        [{"pkgname": "hello-world", "repo": "main", "version": "1-r4"},
+        [{"pkgname": "hello-world", "repo": None, "version": "1-r4"},
-        {"pkgname": "package-depending-on-hello-world", "version": "0.5-r0", "repo": "main"}]
+        {"pkgname": "package-depending-on-hello-world", "version": "0.5-r0", "repo": None}]
     """
-    # Log message
+    ret = []
-    packages_str = pkgname if pkgname else "all packages"
+    pmaports_dirs = list(map(lambda x: Path(x), get_context().config.aports))
-    logging.info(f"Calculate packages that need to be built ({packages_str}, {arch})")
 
-    # Order relevant packages
+    for pmaports_dir in pmaports_dirs:
-    ret = get_relevant_packages(arch, pkgname, built)
+        pattern = os.path.join(pmaports_dir, "**/*/APKBUILD")
 
-    # Output format
+        for apkbuild_path_str in glob.glob(pattern, recursive=True):
-    if overview:
+            apkbuild_path = Path(apkbuild_path_str)
-        return ret
+            pkgname = apkbuild_path.parent.name
-    return generate_output_format(arch, ret)
+
+            if not pmb.helpers.package.check_arch(pkgname, arch, False):
+                continue
+
+            relpath = apkbuild_path.relative_to(pmaports_dir)
+            repo = relpath.parts[1] if relpath.parts[0] == "extra-repos" else None
+
+            entry = pmb.helpers.package.get(pkgname, arch, True, try_other_arches=False)
+
+            if entry is None:
+                raise RuntimeError(f"Couldn't get package {pkgname} for arch {arch}")
+
+            ret += [
+                {
+                    "pkgname": entry.pkgname,
+                    "repo": repo,
+                    "version": entry.version,
+                    "depends": entry.depends,
+                }
+            ]
+
+    # "or -1" is needed for mypy
+    # https://github.com/python/mypy/issues/9765#issuecomment-1238263745
+    ret = sorted(ret, key=lambda d: d.get("pkgname") or -1)
+    return ret

pmb/parse/arguments.py

@@ -737,20 +737,21 @@ def arguments_repo_bootstrap(subparser: argparse._SubParsersAction) -> argparse.
 
 
 def arguments_repo_missing(subparser: argparse._SubParsersAction) -> argparse.ArgumentParser:
-    ret = subparser.add_parser("repo_missing")
+    ret = subparser.add_parser(
-    package = ret.add_argument(
+        "repo_missing",
-        "package", nargs="?", help="only look at a specific package and its dependencies"
+        help="list all packages + depends from pmaports for building the repository (used by bpo)",
     )
-    if "argcomplete" in sys.modules:
-        package.completer = package_completer
     ret.add_argument(
         "--arch", choices=Arch.supported(), default=Arch.native(), type=lambda x: Arch.from_str(x)
     )
+    # Deprecated argument that is currently kept so pmbootstrap can be called
+    # the same way for repo_missing by bpo with pmbv2 and pmbv3. Once we drop
+    # support for pmbv2 in bpo (can do that after v24.06 is EOL), we can adjust
+    # bpo to not use --built and remove this parameter from pmbootstrap.
     ret.add_argument(
-        "--built", action="store_true", help="include packages which exist in the binary repos"
+        "--built",
-    )
+        action="store_true",
-    ret.add_argument(
+        help=argparse.SUPPRESS,
-        "--overview", action="store_true", help="only print the pkgnames without any details"
     )
     return ret