chroot: always run apk static (MR 2252)

Testing by building postmarketos-initramfs (which installs >100 packages
but is very fast to build, so a worst-case scenario) this results in a
~15-20% speedup (which everything cached and doing multiple back to back
runs). From 32 seconds down to 25.

Doing a full install with --no-image, this takes us from 70 seconds on
my laptop down to 40s!

This also lets us drastically simplify pmb/helpers/apk.py!

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>

pmb/chroot/apk.py

@@ -1,5 +1,9 @@
 # Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
+import os
+from pathlib import Path
+import pmb.chroot.apk_static
+from pmb.core.chroot import ChrootType
 from pmb.helpers import logging
 import shlex
 from typing import List
@@ -11,6 +15,7 @@ import pmb.helpers.apk
 import pmb.helpers.other
 import pmb.helpers.pmaports
 import pmb.helpers.repo
+import pmb.helpers.run
 import pmb.parse.apkindex
 import pmb.parse.arch
 import pmb.parse.depends
@@ -140,7 +145,7 @@ def packages_split_to_add_del(packages):
     return (to_add, to_del)
 
 
-def packages_get_locally_built_apks(args: PmbArgs, packages, arch: str):
+def packages_get_locally_built_apks(args: PmbArgs, packages, arch: str) -> List[Path]:
     """
     Iterate over packages and if existing, get paths to locally built packages.
     This is used to force apk to upgrade packages to newer local versions, even
@@ -152,7 +157,7 @@ def packages_get_locally_built_apks(args: PmbArgs, packages, arch: str):
               ["/mnt/pmbootstrap/packages/x86_64/hello-world-1-r6.apk", ...]
     """
     channel: str = pmb.config.pmaports.read_config(args)["channel"]
-    ret = []
+    ret: List[Path] = []
 
     for package in packages:
         data_repo = pmb.parse.apkindex.package(args, package, arch, False)
@@ -160,10 +165,11 @@ def packages_get_locally_built_apks(args: PmbArgs, packages, arch: str):
             continue
 
         apk_file = f"{package}-{data_repo['version']}.apk"
-        if not (pmb.config.work / "packages" / channel / arch / apk_file ).exists():
+        apk_path = pmb.config.work / "packages" / channel / arch / apk_file
+        if not apk_path.exists():
             continue
 
-        ret.append(f"/mnt/pmbootstrap/packages/{arch}/{apk_file}")
+        ret.append(apk_path)
 
     return ret
 
@@ -182,7 +188,7 @@ def install_run_apk(args: PmbArgs, to_add, to_add_local, to_del, chroot: Chroot)
     """
     # Sanitize packages: don't allow '--allow-untrusted' and other options
     # to be passed to apk!
-    for package in to_add + to_add_local + to_del:
+    for package in to_add + [os.fspath(p) for p in to_add_local] + to_del:
         if package.startswith("-"):
             raise ValueError(f"Invalid package name: {package}")
 
@@ -197,10 +203,19 @@ def install_run_apk(args: PmbArgs, to_add, to_add_local, to_del, chroot: Chroot)
     if to_del:
         commands += [["del"] + to_del]
 
+    # For systemd we use a fork of apk-tools, to easily handle this
+    # we expect apk.static to be installed in the native chroot (which
+    # will be the systemd version if building for systemd) and run
+    # it from there.
+    apk_static = Chroot.native() / "sbin/apk.static"
+    arch = pmb.parse.arch.from_chroot_suffix(args, chroot)
+    apk_cache = pmb.config.work / f"cache_apk_{arch}"
+
     for (i, command) in enumerate(commands):
         # --no-interactive is a parameter to `add`, so it must be appended or apk
         # gets confused
         command += ["--no-interactive"]
+        command = ["--root", chroot.path, "--arch", arch, "--cache-dir", apk_cache] + command
 
         # Ignore missing repos before initial build (bpo#137)
         if os.getenv("PMB_APK_FORCE_MISSING_REPOSITORIES") == "1":
@@ -209,14 +224,12 @@ def install_run_apk(args: PmbArgs, to_add, to_add_local, to_del, chroot: Chroot)
         if args.offline:
             command = ["--no-network"] + command
         if i == 0:
-            pmb.helpers.apk.apk_with_progress(args, ["apk"] + command,
-                                              run_in_chroot=True, chroot=chroot)
+            pmb.helpers.apk.apk_with_progress(args, [apk_static] + command)
         else:
             # Virtual package related commands don't actually install or remove
             # packages, but only mark the right ones as explicitly installed.
             # They finish up almost instantly, so don't display a progress bar.
-            pmb.chroot.root(args, ["apk", "--no-progress"] + command,
-                            chroot=chroot)
+            pmb.helpers.run.root(args, [apk_static, "--no-progress"] + command)
 
 
 def install(args: PmbArgs, packages, chroot: Chroot, build=True):

pmb/chroot/apk_static.py

@@ -173,4 +173,4 @@ def run(args: PmbArgs, parameters):
     if args.offline:
         parameters = ["--no-network"] + parameters
     pmb.helpers.apk.apk_with_progress(
-        args, [pmb.config.work / "apk.static"] + parameters, run_in_chroot=False)
+        args, [pmb.config.work / "apk.static"] + parameters)

pmb/chroot/init.py

@@ -160,10 +160,16 @@ def init(args: PmbArgs, chroot: Chroot=Chroot.native(), usr_merge=UsrMerge.AUTO,
 
     # Install alpine-base
     pmb.helpers.repo.update(args, arch)
+    pkgs = ["alpine-base"]
+    # install apk static in the native chroot so we can run it
+    # we have a forked apk for systemd and this is the easiest
+    # way to install/run it.
+    if chroot.type == ChrootType.NATIVE:
+        pkgs += ["apk-tools-static"]
     pmb.chroot.apk_static.run(args, ["--root", chroot.path,
                                      "--cache-dir", apk_cache,
                                      "--initdb", "--arch", arch,
-                                     "add", "alpine-base"])
+                                     "add"] + pkgs)
 
     # Merge /usr
     if usr_merge is UsrMerge.AUTO and pmb.config.is_systemd_selected(args):

pmb/helpers/apk.py

@@ -11,27 +11,9 @@ import pmb.helpers.cli
 import pmb.helpers.run
 import pmb.helpers.run_core
 import pmb.parse.version
-from pmb.core import Chroot
 
 
-def _run(args: PmbArgs, command, run_in_chroot=False, chroot: Chroot=Chroot.native(), output="log"):
-    """Run a command.
-
-    :param command: command in list form
-    :param chroot: whether to run the command inside the chroot or on the host
-    :param suffix: chroot suffix. Only applies if the "chroot" parameter is
-                   set to True.
-
-    See pmb.helpers.run_core.core() for a detailed description of all other
-    arguments and the return value.
-    """
-    if run_in_chroot:
-        return pmb.chroot.root(args, command, output=output, chroot=chroot,
-                               disable_timeout=True)
-    return pmb.helpers.run.root(args, command, output=output)
-
-
-def _prepare_fifo(args: PmbArgs, run_in_chroot=False, chroot: Chroot=Chroot.native()):
+def _prepare_fifo(args: PmbArgs):
     """Prepare the progress fifo for reading / writing.
 
     :param chroot: whether to run the command inside the chroot or on the host
@@ -42,15 +24,11 @@ def _prepare_fifo(args: PmbArgs, run_in_chroot=False, chroot: Chroot=Chroot.nati
               path of the fifo as needed by cat to read from it (always
               relative to the host)
     """
-    if run_in_chroot:
-        fifo = Path("/tmp/apk_progress_fifo")
-        fifo_outside = chroot / fifo
-    else:
-        _run(args, ["mkdir", "-p", pmb.config.work / "tmp"])
-        fifo = fifo_outside = pmb.config.work / "tmp/apk_progress_fifo"
+    pmb.helpers.run.root(args, ["mkdir", "-p", pmb.config.work / "tmp"])
+    fifo = fifo_outside = pmb.config.work / "tmp/apk_progress_fifo"
     if os.path.exists(fifo_outside):
-        _run(args, ["rm", "-f", fifo_outside])
-    _run(args, ["mkfifo", fifo_outside])
+        pmb.helpers.run.root(args, ["rm", "-f", fifo_outside])
+    pmb.helpers.run.root(args, ["mkfifo", fifo_outside])
     return (fifo, fifo_outside)
 
 
@@ -84,7 +62,7 @@ def _compute_progress(line):
     return cur / tot if tot > 0 else 0
 
 
-def apk_with_progress(args: PmbArgs, command: Sequence[PathString], run_in_chroot=False, chroot: Chroot=Chroot.native()):
+def apk_with_progress(args: PmbArgs, command: Sequence[PathString]):
     """Run an apk subcommand while printing a progress bar to STDOUT.
 
     :param command: apk subcommand in list form
@@ -93,13 +71,13 @@ def apk_with_progress(args: PmbArgs, command: Sequence[PathString], run_in_chroo
                    set to True.
     :raises RuntimeError: when the apk command fails
     """
-    fifo, fifo_outside = _prepare_fifo(args, run_in_chroot, chroot)
+    fifo, fifo_outside = _prepare_fifo(args)
     _command: List[str] = [os.fspath(c) for c in command]
     command_with_progress = _create_command_with_progress(_command, fifo)
     log_msg = " ".join(_command)
-    with _run(args, ['cat', fifo], run_in_chroot=run_in_chroot, chroot=chroot,
+    with pmb.helpers.run.root(args, ['cat', fifo],
               output="pipe") as p_cat:
-        with _run(args, command_with_progress, run_in_chroot=run_in_chroot, chroot=chroot,
+        with pmb.helpers.run.root(args, command_with_progress,
                   output="background") as p_apk:
             while p_apk.poll() is None:
                 line = p_cat.stdout.readline().decode('utf-8')

pmb/helpers/repo.py

@@ -66,7 +66,8 @@ def urls(args: PmbArgs, user_repository=True, postmarketos_mirror=True, alpine=T
 
     # Local user repository (for packages compiled with pmbootstrap)
     if user_repository:
-        ret.append("/mnt/pmbootstrap/packages")
+        channel = pmb.config.pmaports.read_config(args)["channel"]
+        ret.append(str(pmb.config.work / "packages" / channel))
 
     # Upstream postmarketOS binary repository
     if postmarketos_mirror: