build: multiple binary repos (MR 2252)

Use a different binary repo depending on the source repository for the
package. This makes it possible to split out systemd packages into their
own repository.

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>

pmb/build/_package.py

@@ -2,7 +2,9 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
 import enum
+from typing import Dict, List
 from pmb.core.context import Context
+from pmb.core.pkgrepo import pkgrepo_paths, pkgrepo_relative_path
 from pmb.helpers import logging
 from pathlib import Path
 
@@ -10,7 +12,7 @@ import pmb.build
 import pmb.build.autodetect
 import pmb.chroot
 import pmb.chroot.apk
-from pmb.types import PmbArgs
+import pmb.config.pmaports
 import pmb.helpers.pmaports
 import pmb.helpers.repo
 import pmb.helpers.mount
@@ -356,7 +358,8 @@ def mount_pmaports(chroot: Chroot=Chroot.native()) -> Dict[str, Path]:
         
     return dest_paths
 
-def link_to_git_dir(suffix):
+
+def link_to_git_dir(chroot: Chroot):
     """ Make ``/home/pmos/build/.git`` point to the .git dir from pmaports.git, with a
     symlink so abuild does not fail (#1841).
 
@@ -383,7 +386,11 @@ def link_to_git_dir(suffix):
                            "/home/pmos/build/.git"], chroot)
 
 
-def run_abuild(context: Context, apkbuild, arch, strict=False, force=False, cross=None,
+def output_path(arch: str, pkgname: str, pkgver: str, pkgrel: str) -> Path:
+    return Path(arch) / f"{pkgname}-{pkgver}-r{pkgrel}.apk"
+
+
+def run_abuild(context: Context, apkbuild, channel, arch, strict=False, force=False, cross=None,
                suffix: Chroot=Chroot.native(), src=None, bootstrap_stage=BootstrapStage.NONE):
     """
     Set up all environment variables and construct the abuild command (all
@@ -403,12 +410,21 @@ def run_abuild(context: Context, apkbuild, arch, strict=False, force=False, cros
         logging.info("WARNING: Option !tracedeps is not set, but we're"
                      " cross-compiling in the native chroot. This will"
                      " probably fail!")
+    pkgdir = context.config.work / "packages" / channel
+    if not pkgdir.exists():
+        pmb.helpers.run.root(["mkdir", "-p", pkgdir])
+        pmb.helpers.run.root(["chown", "-R", f"{pmb.config.chroot_uid_user}:{pmb.config.chroot_uid_user}",
+                              pkgdir.parent])
+
+    pmb.chroot.rootm([["mkdir", "-p", "/home/pmos/packages"],
+                      ["rm", "-f", "/home/pmos/packages/pmos"],
+                      ["ln", "-sf", f"/mnt/pmbootstrap/packages/{channel}",
+                     "/home/pmos/packages/pmos"]], suffix)
 
     # Pretty log message
     pkgver = get_pkgver(apkbuild["pkgver"], src is None)
-    output = (arch + "/" + apkbuild["pkgname"] + "-" + pkgver +
-              "-r" + apkbuild["pkgrel"] + ".apk")
-    message = f"({suffix}) build " + output
+    output = output_path(arch, apkbuild["pkgname"], pkgver, apkbuild["pkgrel"])
+    message = f"({suffix}) build {channel}/{output}"
     if src:
         message += " (source: " + src + ")"
     logging.info(message)
@@ -465,10 +481,9 @@ def run_abuild(context: Context, apkbuild, arch, strict=False, force=False, cros
     return (output, cmd, env)
 
 
-def finish(apkbuild, arch, output: str, chroot: Chroot, strict=False):
+def finish(apkbuild, channel, arch, output: str, chroot: Chroot, strict=False):
     """Various finishing tasks that need to be done after a build."""
     # Verify output file
-    channel: str = pmb.config.pmaports.read_config()["channel"]
     out_dir = (get_context().config.work / "packages" / channel)
     if not (out_dir / output).exists():
         raise RuntimeError(f"Package not found after build: {(out_dir / output)}")
@@ -523,10 +538,12 @@ def package(context: Context, pkgname, arch=None, force=False, strict=False,
         return
 
     # Only build when APKBUILD exists
-    apkbuild = get_apkbuild(pkgname, arch)
+    aports, apkbuild = get_apkbuild(pkgname, arch)
     if not apkbuild:
         return
 
+    channel: str = pmb.config.pmaports.read_config(aports)["channel"]
+
     # Detect the build environment (skip unnecessary builds)
     if not check_build_for_arch(pkgname, arch):
         return
@@ -538,9 +555,9 @@ def package(context: Context, pkgname, arch=None, force=False, strict=False,
 
     # Build and finish up
     try:
-        (output, cmd, env) = run_abuild(context, apkbuild, arch, strict, force, cross,
+        (output, cmd, env) = run_abuild(context, apkbuild, channel, arch, strict, force, cross,
                                         chroot, src, bootstrap_stage)
     except RuntimeError:
         raise BuildFailedError(f"Build for {arch}/{pkgname} failed!")
-    finish(apkbuild, arch, output, chroot, strict)
+    finish(apkbuild, channel, arch, output, chroot, strict)
     return output

pmb/chroot/mount.py

@@ -84,7 +84,7 @@ def mount(chroot: Chroot):
 
     # Get all mountpoints
     arch = chroot.arch
-    channel = pmb.config.pmaports.read_config()["channel"]
+    channel = pmb.config.pmaports.read_config(support_systemd=False)["channel"]
     mountpoints: Dict[Path, Path] = {}
     for src_template, target_template in pmb.config.chroot_mount_bind.items():
         src_template = src_template.replace("$WORK", os.fspath(get_context().config.work))

pmb/config/pmaports.py

@@ -81,7 +81,7 @@ def read_config_repos():
     return ret
 
 
-def read_config(aports: Optional[Path] = None):
+def read_config(aports: Optional[Path] = None, support_systemd=True):
     """Read and verify pmaports.cfg."""
     if not aports:
         aports = pkgrepo_default_path()
@@ -98,10 +98,6 @@ def read_config(aports: Optional[Path] = None):
     # so jump up the main aports dir
     if "extra-repos" in aports.parts:
         aports = pkgrepo_relative_path(aports)[0]
-    # Try cache first
-    cache_key = "pmb.config.pmaports.read_config"
-    if aports.name in pmb.helpers.other.cache[cache_key]:
-        return pmb.helpers.other.cache[cache_key][aports.name]
 
     # Migration message
     if not os.path.exists(aports):
@@ -137,17 +133,11 @@ def read_config(aports: Optional[Path] = None):
 
 def all_channels() -> List[str]:
     """Get a list of all channels for all pkgrepos."""
-    ret = []
+    ret = set()
     for repo in pkgrepo_paths():
-        if repo.name == "systemd":
-            channel = "systemd"
-        else:
-            channel = read_config(repo)["channel"]
+        ret.add(read_config(repo)["channel"])
 
-        if channel not in ret:
-            ret.append(channel)
-
-    return ret
+    return list(ret)
 
 
 def read_config_channel():
@@ -162,7 +152,7 @@ def read_config_channel():
 
     """
     aports = pkgrepo_default_path()
-    channel = read_config()["channel"]
+    channel = read_config(support_systemd=False)["channel"]
     channels_cfg = pmb.helpers.git.parse_channels_cfg(aports)
 
     if channel in channels_cfg["channels"]:

pmb/parse/arguments.py

@@ -665,7 +665,7 @@ def get_parser():
                              " 128)")
     parser.add_argument("-p", "--aports",
                         help="postmarketos aports (pmaports) path",
-                        type=lambda x: Path(x))
+                        type=lambda x: [Path(p.strip()) for p in x.split(",")])
     parser.add_argument("-t", "--timeout", help="seconds after which processes"
                         " get killed that stopped writing any output (default:"
                         " 900)", default=900, type=float)

pmb/parse/deviceinfo.py

@@ -71,13 +71,6 @@ def deviceinfo(device=None, kernel=None) -> "Deviceinfo":
     if device in pmb.helpers.other.cache["deviceinfo"]:
         return pmb.helpers.other.cache["deviceinfo"][device]
 
-    aports = context.config.aports
-    if not aports.exists():
-        logging.fatal(f"Aports directory is missing, expected: {aports}")
-        logging.fatal("Please provide a path to the aports directory using the"
-                      " -p flag")
-        raise RuntimeError("Aports directory missing")
-
     path = pmb.helpers.devices.find_path(device, 'deviceinfo')
     if not path:
         raise RuntimeError(