chroot: apk: use apk.static in all cases (MR 2463)

In 0b4fb9119f (chroot: always run apk static v2 (MR 2423)) we adjusted
install_run_apk() to run apk static on the host and pass in the local
binary repo with "--repository". This function can call apk in two ways,
either with the progress bar handling or without, the second case was
never updated and still ran apk inside the chroot incorrectly and with
an incorrect --repository flag.

Let's finish the job by refactoring helpers/apk.py to support all our
usecases and pointing everything to it, removing the last few situations
where we call "pmb.chroot.root(["apk", ...]).

The apk_with_progress() function is replaced by a generic "run()"
function which takes a boolean to indicate if we should render apk
progress.

Additionally, a new cache_clean() function is added so that "pmbootstrap
zap --pkgs-online-mismatch" can FINALLY be refactored to not rely on a
chroot existing. This requires some hacks but nothing serious, see the
comments in the function for details.

The chroot.init() code is now simplified since handling the --root,
--arch, --cache-dir, and --repository flags is now all done by
apk._prepare_cmd() as and when appropriate.

Lastly, this fixes a (previously unnoticed) bug where apk.static was
actually using /var/cache/apk on your host machine for its cache... This
is definitely not good behaviour....

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>

pmb/chroot/apk.py

@@ -111,9 +111,7 @@ def packages_get_locally_built_apks(package_list: list[str], arch: Arch) -> list
         for channel in channels:
             apk_path = get_context().config.work / "packages" / channel / arch / apk_file
             if apk_path.exists():
-                # FIXME: use /mnt/pmb… until MR 2351 is reverted (pmb#2388)
+                local.append(apk_path)
-                # local.append(apk_path)
-                local.append(Path("/mnt/pmbootstrap/packages/") / channel / arch / apk_file)
                 break
 
         # Record all the packages we have visited so far
@@ -181,24 +179,17 @@ def install_run_apk(
         user_repo += ["--repository", context.config.work / "packages" / channel]
 
     for i, command in enumerate(commands):
-        # --no-interactive is a parameter to `add`, so it must be appended or apk
-        # gets confused
-        command += ["--no-interactive"]
         command = user_repo + command
 
         # Ignore missing repos before initial build (bpo#137)
         if os.getenv("PMB_APK_FORCE_MISSING_REPOSITORIES") == "1":
             command = ["--force-missing-repositories"] + command
 
-        if context.offline:
+        # Virtual package related commands don't actually install or remove
-            command = ["--no-network"] + command
+        # packages, but only mark the right ones as explicitly installed.
-        if i == 0:
+        # So only display a progress bar for the "apk add" command which is
-            pmb.helpers.apk.apk_with_progress(command, chroot)
+        # always the first one we process (i == 0).
-        else:
+        pmb.helpers.apk.run(command, chroot, with_progress=(i == 0))
-            # Virtual package related commands don't actually install or remove
-            # packages, but only mark the right ones as explicitly installed.
-            # They finish up almost instantly, so don't display a progress bar.
-            pmb.chroot.root(["apk", "--no-progress"] + command, chroot)
 
 
 def install(packages: list[str], chroot: Chroot, build: bool = True, quiet: bool = False) -> None:

pmb/chroot/apk_static.py

@@ -179,11 +179,12 @@ def init() -> None:
     extract(version, apk_static)
 
 
-def run(parameters):
+def run(parameters, with_progress=True):
     # --no-interactive is a parameter to `add`, so it must be appended or apk
     # gets confused
-    parameters += ["--no-interactive"]
+    if "add" in parameters:
+        parameters += ["--no-interactive"]
 
     if get_context().offline:
         parameters = ["--no-network"] + parameters
-    pmb.helpers.apk.apk_with_progress(parameters)
+    pmb.helpers.apk.run(parameters, with_progress=with_progress)

pmb/chroot/init.py

@@ -12,11 +12,13 @@ import pmb.chroot.apk
 import pmb.chroot.apk_static
 import pmb.config
 import pmb.config.workdir
+import pmb.helpers.apk
 import pmb.helpers.repo
 import pmb.helpers.run
 import pmb.helpers.other
 from pmb.core import Chroot, ChrootType
 from pmb.core.context import get_context
+from pmb.types import PathString
 
 
 class UsrMerge(enum.Enum):
@@ -145,10 +147,6 @@ def init(chroot: Chroot, usr_merge: UsrMerge = UsrMerge.AUTO) -> None:
 
     logging.info(f"({chroot}) Creating chroot")
 
-    # Initialize cache
-    apk_cache = config.work / f"cache_apk_{arch}"
-    pmb.helpers.run.root(["ln", "-s", "-f", "/var/cache/apk", chroot / "etc/apk/cache"])
-
     # Initialize /etc/apk/keys/, resolv.conf, repositories
     init_keys()
     copy_resolv_conf(chroot)
@@ -159,10 +157,8 @@ def init(chroot: Chroot, usr_merge: UsrMerge = UsrMerge.AUTO) -> None:
     # Install alpine-base
     pmb.helpers.repo.update(arch)
     pkgs = ["alpine-base"]
-    cmd = ["--root", chroot.path, "--cache-dir", apk_cache, "--initdb", "--arch", arch]
+    cmd: list[PathString] = ["--initdb"]
-    for channel in pmb.config.pmaports.all_channels():
+    pmb.helpers.apk.run(cmd + ["add", *pkgs], chroot)
-        cmd += ["--repository", config.work / "packages" / channel]
-    pmb.chroot.apk_static.run(cmd + ["add", *pkgs])
 
     # Merge /usr
     if usr_merge is UsrMerge.AUTO and pmb.config.is_systemd_selected(config):

pmb/chroot/initfs_hooks.py

@@ -51,7 +51,7 @@ def delete(hook: str, suffix: Chroot) -> None:
     if hook not in list_chroot(suffix):
         raise RuntimeError("There is no such hook installed!")
     prefix = pmb.config.initfs_hook_prefix
-    pmb.chroot.root(["apk", "del", f"{prefix}{hook}"], suffix)
+    pmb.helpers.apk.run(["del", f"{prefix}{hook}"], suffix)
 
 
 def update(suffix: Chroot) -> None:

pmb/chroot/zap.py

@@ -184,18 +184,6 @@ def zap_pkgs_online_mismatch(confirm=True, dry=False):
     # Iterate over existing apk caches
     for path in paths:
         arch = Arch.from_str(path.name.split("_", 2)[2])
-        if arch.is_native():
-            chroot = Chroot.native()
-        else:
-            chroot = Chroot.buildroot(arch)
 
-        # Skip if chroot does not exist
-        # FIXME: should we init the buildroot to do it anyway?
-        # what if we run apk.static with --arch instead?
-        if not chroot.exists():
-            continue
-
-        # Clean the cache with apk
-        logging.info(f"({chroot}) apk -v cache clean")
         if not dry:
-            pmb.chroot.root(["apk", "-v", "cache", "clean"], chroot)
+            pmb.helpers.apk.cache_clean(arch)

pmb/helpers/apk.py

@@ -4,6 +4,7 @@ import os
 import shlex
 from collections.abc import Sequence
 from pathlib import Path
+from typing import Literal
 
 import pmb.chroot
 import pmb.config.pmaports
@@ -23,8 +24,8 @@ from pmb.meta import Cache
 @Cache("root", "user_repository", mirrors_exclude=[])
 def update_repository_list(
     root: Path,
-    user_repository: bool = False,
+    user_repository: bool | Path = False,
-    mirrors_exclude: list[str] = [],
+    mirrors_exclude: list[str] | Literal[True] = [],
     check: bool = False,
 ) -> None:
     """
@@ -50,9 +51,15 @@ def update_repository_list(
     else:
         pmb.helpers.run.root(["mkdir", "-p", path.parent])
 
+    user_repo_dir: Path | None
+    if isinstance(user_repository, Path):
+        user_repo_dir = user_repository
+    else:
+        user_repo_dir = Path("/mnt/pmbootstrap/packages") if user_repository else None
+
     # Up to date: Save cache, return
     lines_new = pmb.helpers.repo.urls(
-        user_repository=user_repository, mirrors_exclude=mirrors_exclude
+        user_repository=user_repo_dir, mirrors_exclude=mirrors_exclude
     )
     if lines_old == lines_new:
         return
@@ -68,7 +75,10 @@ def update_repository_list(
     for line in lines_new:
         pmb.helpers.run.root(["sh", "-c", "echo " f"{shlex.quote(line)} >> {path}"])
     update_repository_list(
-        root, user_repository=user_repository, mirrors_exclude=mirrors_exclude, check=True
+        root,
+        user_repository=user_repository,
+        mirrors_exclude=mirrors_exclude,
+        check=True,
     )
 
 
@@ -99,7 +109,7 @@ def _create_command_with_progress(command, fifo):
     :param fifo: path of the fifo
     :returns: full command in list form
     """
-    flags = ["--no-progress", "--progress-fd", "3"]
+    flags = ["--progress-fd", "3"]
     command_full = [command[0]] + flags + command[1:]
     command_flat = pmb.helpers.run_core.flat_cmd([command_full])
     command_flat = f"exec 3>{fifo}; {command_flat}"
@@ -122,23 +132,15 @@ def _compute_progress(line):
     return cur / tot if tot > 0 else 0
 
 
-def apk_with_progress(command: Sequence[PathString], chroot: Chroot | None = None) -> None:
+def _apk_with_progress(command: Sequence[str]) -> None:
     """Run an apk subcommand while printing a progress bar to STDOUT.
 
     :param command: apk subcommand in list form
     :raises RuntimeError: when the apk command fails
     """
     fifo = _prepare_fifo()
-    _command: list[str] = [str(get_context().config.work / "apk.static")]
+    command_with_progress = _create_command_with_progress(command, fifo)
-    if chroot:
+    log_msg = " ".join(command)
-        _command.extend(["--root", str(chroot.path), "--arch", str(chroot.arch)])
-    for c in command:
-        if isinstance(c, Arch):
-            _command.append(str(c))
-        else:
-            _command.append(os.fspath(c))
-    command_with_progress = _create_command_with_progress(_command, fifo)
-    log_msg = " ".join(_command)
     with pmb.helpers.run.root(["cat", fifo], output="pipe") as p_cat:
         with pmb.helpers.run.root(command_with_progress, output="background") as p_apk:
             while p_apk.poll() is None:
@@ -152,6 +154,116 @@ def apk_with_progress(command: Sequence[PathString], chroot: Chroot | None = Non
             pmb.helpers.run_core.check_return_code(p_apk.returncode, log_msg)
 
 
+def _prepare_cmd(command: Sequence[PathString], chroot: Chroot | None) -> list[str]:
+    """Prepare the apk command.
+
+    Returns a tuple of the first part of the command with generic apk flags, and the second part
+    with the subcommand and its arguments.
+    """
+    config = get_context().config
+    # Our _apk_with_progress() wrapper also need --no-progress, since all that does is
+    # prevent apk itself from rendering progress bars. We instead want it to tell us
+    # the progress so we can render it. So we always set --no-progress.
+    _command: list[str] = [str(config.work / "apk.static"), "--no-progress"]
+    if chroot:
+        cache_dir = config.work / f"cache_apk_{chroot.arch}"
+        _command.extend(
+            [
+                "--root",
+                str(chroot.path),
+                "--arch",
+                str(chroot.arch),
+                "--cache-dir",
+                str(cache_dir),
+            ]
+        )
+        local_repos = pmb.helpers.repo.urls(
+            user_repository=config.work / "packages", mirrors_exclude=True
+        )
+        for repo in local_repos:
+            _command.extend(["--repository", repo])
+    if get_context().offline:
+        _command.append("--no-network")
+
+    for c in command:
+        _command.append(os.fspath(c))
+
+        # Always be non-interactive
+        if c == "add":
+            _command.append("--no-interactive")
+
+    return _command
+
+
+def run(command: Sequence[PathString], chroot: Chroot, with_progress: bool = True) -> None:
+    """Run an apk subcommand.
+
+    :param command: apk subcommand in list form
+    :param with_progress: whether to print a progress bar
+    :param chroot: chroot to run the command in
+    :raises RuntimeError: when the apk command fails
+    """
+    _command = _prepare_cmd(command, chroot)
+
+    if with_progress:
+        _apk_with_progress(_command)
+    else:
+        pmb.helpers.run.root(_command)
+
+
+def cache_clean(arch: Arch) -> None:
+    """Clean the APK cache for a specific architecture."""
+    work = get_context().config.work
+    cache_dir = work / f"cache_apk_{arch}"
+    if not cache_dir.exists():
+        return
+
+    # The problem here is that apk's "cache clean" command really
+    # expects to be run against an apk-managed rootfs and will return
+    # errors if it can't access certain paths (even though it will
+    # actually clean the cache like we want).
+    # We could just ignore the return value, but then we wouldn't know
+    # if something actually went wrong with apk...
+    # So we do this dance of creating a rootfs with only the files that
+    # APK needs to be happy
+    tmproot = work / "tmp" / "apk_root"
+    if not (tmproot / "etc/apk/repositories").exists():
+        tmproot.mkdir(exist_ok=True)
+        (tmproot / "var/cache").mkdir(exist_ok=True, parents=True)
+        (tmproot / "etc/apk").mkdir(exist_ok=True, parents=True)
+        (tmproot / "lib/apk/db").mkdir(exist_ok=True, parents=True)
+
+        (tmproot / "etc/apk/world").touch(exist_ok=True)
+        (tmproot / "lib/apk/db/installed").touch(exist_ok=True)
+        (tmproot / "lib/apk/db/triggers").touch(exist_ok=True)
+
+        (tmproot / "etc/apk/keys").symlink_to(work / "config_apk_keys")
+
+        # Our fake rootfs needs a valid repositories file for apk
+        # to have something to compare the cache against
+        update_repository_list(tmproot, user_repository=work / "packages")
+
+    # Point our tmproot cache dir to the real cache dir
+    # this is much simpler than passing --cache-dir to apk
+    # since even with that flag apk will also check it's
+    # "static cache".
+    (tmproot / "var/cache/apk").unlink(missing_ok=True)
+    (tmproot / "var/cache/apk").symlink_to(cache_dir)
+
+    command: list[PathString] = [
+        "-v",
+        "--root",
+        tmproot,
+        "--arch",
+        str(arch),
+    ]
+
+    command += ["cache", "clean"]
+    _command = _prepare_cmd(command, None)
+
+    pmb.helpers.run.root(_command)
+
+
 def check_outdated(version_installed, action_msg):
     """Check if the provided alpine version is outdated.
 

pmb/helpers/repo.py

@@ -16,6 +16,7 @@ from pmb.core.arch import Arch
 from pmb.core.pkgrepo import pkgrepo_names
 from pmb.helpers import logging
 from pathlib import Path
+from typing import Literal
 
 import pmb.config.pmaports
 from pmb.meta import Cache
@@ -54,11 +55,14 @@ def apkindex_hash(url: str, length: int = 8) -> Path:
 # FIXME: make config.mirrors a normal dict
 # mypy: disable-error-code="literal-required"
 @Cache("user_repository", "mirrors_exclude")
-def urls(user_repository: bool = False, mirrors_exclude: list[str] = []) -> list[str]:
+def urls(
+    user_repository: Path | None = None, mirrors_exclude: list[str] | Literal[True] = []
+) -> list[str]:
     """Get a list of repository URLs, as they are in /etc/apk/repositories.
 
     :param user_repository: add /mnt/pmbootstrap/packages
-    :param mirrors_exclude: mirrors to exclude (see pmb.core.config.Mirrors)
+    :param mirrors_exclude: mirrors to exclude (see pmb.core.config.Mirrors) or true to exclude
+                            all mirrors and only return the local repos
     :returns: list of mirror strings, like ["/mnt/pmbootstrap/packages",
                                             "http://...", ...]
     """
@@ -74,7 +78,10 @@ def urls(user_repository: bool = False, mirrors_exclude: list[str] = []) -> list
     # Local user repository (for packages compiled with pmbootstrap)
     if user_repository:
         for channel in pmb.config.pmaports.all_channels():
-            ret.append(f"/mnt/pmbootstrap/packages/{channel}")
+            ret.append(str(user_repository / channel))
+
+    if mirrors_exclude is True:
+        return ret
 
     # Don't add the systemd mirror if systemd is disabled
     if not pmb.config.is_systemd_selected(config):