diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index d1a6e4ec..bea8e846 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -285,9 +285,13 @@ def config(args: PmbArgs) -> None: def repo_missing(args: PmbArgs) -> None: - if args.arch is None or isinstance(args.package, list): + if args.arch is None: raise AssertionError - missing = pmb.helpers.repo_missing.generate(args.arch, args.overview, args.package, args.built) + if args.built: + logging.warning( + "WARNING: --built is deprecated (bpo#148: this warning is expected on build.postmarketos.org for now)" + ) + missing = pmb.helpers.repo_missing.generate(args.arch) print(json.dumps(missing, indent=4)) diff --git a/pmb/helpers/repo_missing.py b/pmb/helpers/repo_missing.py index fc127c2e..e0873db0 100644 --- a/pmb/helpers/repo_missing.py +++ b/pmb/helpers/repo_missing.py @@ -1,171 +1,57 @@ -# Copyright 2023 Oliver Smith +# Copyright 2025 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later - -from typing import overload, Literal - from pmb.core.arch import Arch -from pmb.helpers import logging -from pmb.types import Apkbuild +from pmb.core.context import get_context +from pathlib import Path import pmb.build import pmb.helpers.package import pmb.helpers.pmaports +import glob +import os -def filter_missing_packages(arch: Arch, pkgnames: list[str]) -> list[str]: - """Create a subset of pkgnames with missing or outdated binary packages. +def generate(arch: Arch) -> list[dict[str, list[str] | str | None]]: + """Get packages that need to be built, with all their dependencies. Include + packages from extra-repos, no matter if systemd is enabled or not. This + is used by bpo to fill its package database. :param arch: architecture (e.g. "armhf") - :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) - :returns: subset of pkgnames (e.g. ["hello-world"]) - """ - ret = [] - for pkgname in pkgnames: - binary = pmb.parse.apkindex.package(pkgname, arch, False) - must_exist = False if binary else True - pmaport = pmb.helpers.pmaports.get(pkgname, must_exist) - if pmaport and pmb.build.get_status(arch, pmaport).necessary(): - ret.append(pkgname) - return ret - - -def filter_aport_packages(pkgnames: list[str]) -> list[str]: - """Create a subset of pkgnames where each one has an aport. - - :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) - :returns: subset of pkgnames (e.g. ["hello-world"]) - """ - ret = [] - for pkgname in pkgnames: - if pmb.helpers.pmaports.find_optional(pkgname): - ret += [pkgname] - return ret - - -def filter_arch_packages(arch: Arch, pkgnames: list[str]) -> list[str]: - """Create a subset of pkgnames with packages removed that can not be built for a certain arch. - - :param arch: architecture (e.g. "armhf") - :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) - :returns: subset of pkgnames (e.g. ["hello-world"]) - """ - ret = [] - for pkgname in pkgnames: - if pmb.helpers.package.check_arch(pkgname, arch, False): - ret += [pkgname] - return ret - - -def get_relevant_packages(arch: Arch, pkgname: str | None = None, built: bool = False) -> list[str]: - """Get all packages that can be built for the architecture in question. - - :param arch: architecture (e.g. "armhf") - :param pkgname: only look at a specific package (and its dependencies) - :param built: include packages that have already been built - :returns: an alphabetically sorted list of pkgnames, e.g.: - ["devicepkg-dev", "hello-world", "osk-sdl"] - """ - if pkgname: - if not pmb.helpers.package.check_arch(pkgname, arch, False): - raise RuntimeError(f"{pkgname} can't be built for {arch}.") - ret = pmb.helpers.package.depends_recurse(pkgname, arch) - else: - ret = pmb.helpers.pmaports.get_list() - ret = filter_arch_packages(arch, ret) - if built: - ret = filter_aport_packages(ret) - if not len(ret): - logging.info( - "NOTE: no aport found for any package in the" - " dependency tree, it seems they are all provided by" - " upstream (Alpine)." - ) - else: - ret = filter_missing_packages(arch, ret) - if not len(ret): - logging.info( - "NOTE: all relevant packages are up to date, use" - " --built to include the ones that have already been" - " built." - ) - - # Sort alphabetically (to get a deterministic build order) - ret.sort() - return ret - - -def generate_output_format(arch: Arch, pkgnames: list[str]) -> list[Apkbuild]: - """Generate the detailed output format. - - :param arch: architecture - :param pkgnames: list of package names that should be in the output, - e.g.: ["hello-world", "pkg-depending-on-hello-world"] - :returns: a list like the following: - [{"pkgname": "hello-world", - "repo": "main", - "version": "1-r4", - "depends": []}, - {"pkgname": "pkg-depending-on-hello-world", - "version": "0.5-r0", - "repo": "main", - "depends": ["hello-world"]}] - """ - ret = [] - for pkgname in pkgnames: - entry = pmb.helpers.package.get(pkgname, arch, True, try_other_arches=False) - - if entry is None: - raise RuntimeError(f"Couldn't get package {pkgname} for arch {arch}") - - ret += [ - { - "pkgname": entry.pkgname, - "repo": pmb.helpers.pmaports.get_repo(pkgname), - "version": entry.version, - "depends": entry.depends, - } - ] - return ret - - -@overload -def generate( - arch: Arch, overview: Literal[False], pkgname: str | None = ..., built: bool = ... -) -> list[Apkbuild]: ... - - -@overload -def generate( - arch: Arch, overview: Literal[True], pkgname: str | None = ..., built: bool = ... -) -> list[str]: ... - - -@overload -def generate( - arch: Arch, overview: bool, pkgname: str | None = ..., built: bool = ... -) -> list[Apkbuild] | list[str]: ... - - -def generate( - arch: Arch, overview: bool, pkgname: str | None = None, built: bool = False -) -> list[Apkbuild] | list[str]: - """Get packages that need to be built, with all their dependencies. - - :param arch: architecture (e.g. "armhf") - :param pkgname: only look at a specific package - :param built: include packages that have already been built :returns: a list like the following: - [{"pkgname": "hello-world", "repo": "main", "version": "1-r4"}, - {"pkgname": "package-depending-on-hello-world", "version": "0.5-r0", "repo": "main"}] + [{"pkgname": "hello-world", "repo": None, "version": "1-r4"}, + {"pkgname": "package-depending-on-hello-world", "version": "0.5-r0", "repo": None}] """ - # Log message - packages_str = pkgname if pkgname else "all packages" - logging.info(f"Calculate packages that need to be built ({packages_str}, {arch})") + ret = [] + pmaports_dirs = list(map(lambda x: Path(x), get_context().config.aports)) - # Order relevant packages - ret = get_relevant_packages(arch, pkgname, built) + for pmaports_dir in pmaports_dirs: + pattern = os.path.join(pmaports_dir, "**/*/APKBUILD") - # Output format - if overview: - return ret - return generate_output_format(arch, ret) + for apkbuild_path_str in glob.glob(pattern, recursive=True): + apkbuild_path = Path(apkbuild_path_str) + pkgname = apkbuild_path.parent.name + + if not pmb.helpers.package.check_arch(pkgname, arch, False): + continue + + relpath = apkbuild_path.relative_to(pmaports_dir) + repo = relpath.parts[1] if relpath.parts[0] == "extra-repos" else None + + entry = pmb.helpers.package.get(pkgname, arch, True, try_other_arches=False) + + if entry is None: + raise RuntimeError(f"Couldn't get package {pkgname} for arch {arch}") + + ret += [ + { + "pkgname": entry.pkgname, + "repo": repo, + "version": entry.version, + "depends": entry.depends, + } + ] + + # "or -1" is needed for mypy + # https://github.com/python/mypy/issues/9765#issuecomment-1238263745 + ret = sorted(ret, key=lambda d: d.get("pkgname") or -1) + return ret diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 4abbe311..9cbdb39f 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -737,20 +737,21 @@ def arguments_repo_bootstrap(subparser: argparse._SubParsersAction) -> argparse. def arguments_repo_missing(subparser: argparse._SubParsersAction) -> argparse.ArgumentParser: - ret = subparser.add_parser("repo_missing") - package = ret.add_argument( - "package", nargs="?", help="only look at a specific package and its dependencies" + ret = subparser.add_parser( + "repo_missing", + help="list all packages + depends from pmaports for building the repository (used by bpo)", ) - if "argcomplete" in sys.modules: - package.completer = package_completer ret.add_argument( "--arch", choices=Arch.supported(), default=Arch.native(), type=lambda x: Arch.from_str(x) ) + # Deprecated argument that is currently kept so pmbootstrap can be called + # the same way for repo_missing by bpo with pmbv2 and pmbv3. Once we drop + # support for pmbv2 in bpo (can do that after v24.06 is EOL), we can adjust + # bpo to not use --built and remove this parameter from pmbootstrap. ret.add_argument( - "--built", action="store_true", help="include packages which exist in the binary repos" - ) - ret.add_argument( - "--overview", action="store_true", help="only print the pkgnames without any details" + "--built", + action="store_true", + help=argparse.SUPPRESS, ) return ret