Add "pmbootstrap repo_bootstrap" (MR 2273)

Related: https://postmarketos.org/pmaports.cfg bootstrap_1 etc.

pmb/build/__init__.py

@@ -6,4 +6,4 @@ from pmb.build.kconfig import menuconfig
 from pmb.build.newapkbuild import newapkbuild
 from pmb.build.other import copy_to_buildpath, is_necessary, \
     index_repo
-from pmb.build._package import mount_pmaports, package
+from pmb.build._package import BootstrapStage, mount_pmaports, package

pmb/build/_package.py

@@ -1,6 +1,7 @@
 # Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
+import enum
 import logging
 import os
 
@@ -14,6 +15,15 @@ import pmb.parse
 import pmb.parse.arch
 
 
+class BootstrapStage(enum.IntEnum):
+    """
+    Pass a BOOTSTRAP= environment variable with the given value to abuild. See
+    bootstrap_1 etc. at https://postmarketos.org/pmaports.cfg for details.
+    """
+    NONE = 0
+    # We don't need explicit representations of the other numbers.
+
+
 def skip_already_built(pkgname, arch):
     """
     Check if the package was already built in this session, and add it
@@ -364,7 +374,7 @@ def link_to_git_dir(args, suffix):
 
 
 def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
-               suffix="native", src=None):
+               suffix="native", src=None, bootstrap_stage=BootstrapStage.NONE):
     """
     Set up all environment variables and construct the abuild command (all
     depending on the cross-compiler method and target architecture), copy
@@ -372,6 +382,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
 
     :param cross: None, "native", or "crossdirect"
     :param src: override source used to build the package with a local folder
+    :param bootstrap_stage: pass a BOOTSTRAP= env var with the value to abuild
     :returns: (output, cmd, env), output is the destination apk path relative
               to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are
               used by the test case, and they are the full abuild command and
@@ -421,6 +432,9 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
     if args.go_mod_cache:
         env["GOMODCACHE"] = "/home/pmos/go/pkg/mod"
 
+    if bootstrap_stage:
+        env["BOOTSTRAP"] = str(bootstrap_stage)
+
     # Build the abuild command
     cmd = ["abuild", "-D", "postmarketOS"]
     if strict or "pmb:strict" in apkbuild["options"]:
@@ -468,7 +482,8 @@ def finish(args, apkbuild, arch, output, strict=False, suffix="native"):
 
 
 def package(args, pkgname, arch=None, force=False, strict=False,
-            skip_init_buildenv=False, src=None):
+            skip_init_buildenv=False, src=None,
+            bootstrap_stage=BootstrapStage.NONE):
     """
     Build a package and its dependencies with Alpine Linux' abuild.
 
@@ -487,6 +502,7 @@ def package(args, pkgname, arch=None, force=False, strict=False,
                                something during initialization of the build
                                environment (e.g. qemu aarch64 bug workaround)
     :param src: override source used to build the package with a local folder
+    :param bootstrap_stage: pass a BOOTSTRAP= env var with the value to abuild
     :returns: None if the build was not necessary
               output path relative to the packages folder ("armhf/ab-1-r2.apk")
     """
@@ -515,6 +531,6 @@ def package(args, pkgname, arch=None, force=False, strict=False,
 
     # Build and finish up
     (output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross,
-                                    suffix, src)
+                                    suffix, src, bootstrap_stage)
     finish(args, apkbuild, arch, output, strict, suffix)
     return output

pmb/helpers/frontend.py

@@ -24,6 +24,7 @@ import pmb.helpers.logging
 import pmb.helpers.pkgrel_bump
 import pmb.helpers.pmaports
 import pmb.helpers.repo
+import pmb.helpers.repo_bootstrap
 import pmb.helpers.repo_missing
 import pmb.helpers.run
 import pmb.helpers.status
@@ -218,6 +219,10 @@ def config(args):
     pmb.helpers.logging.disable()
 
 
+def repo_bootstrap(args):
+    pmb.helpers.repo_bootstrap.main(args)
+
+
 def repo_missing(args):
     missing = pmb.helpers.repo_missing.generate(args, args.arch, args.overview,
                                                 args.package, args.built)

pmb/helpers/repo_bootstrap.py

@@ -0,0 +1,157 @@
+# Copyright 2024 Oliver Smith
+# SPDX-License-Identifier: GPL-3.0-or-later
+import logging
+import glob
+
+import pmb.config.pmaports
+
+
+progress_done = 0
+progress_total = 0
+progress_step = None
+
+
+def get_arch(args):
+    if args.arch:
+        return args.arch
+
+    if args.build_default_device_arch:
+        return args.deviceinfo["arch"]
+
+    return pmb.config.arch_native
+
+
+def check_repo_arg(args):
+    cfg = pmb.config.pmaports.read_config_repos(args)
+    repo = args.repository
+
+    if repo in cfg:
+        return
+
+    if not cfg:
+        raise ValueError("pmaports.cfg of current branch does not have any"
+                         " sections starting with 'repo:'")
+
+    logging.info(f"Valid repositories: {', '.join(cfg.keys())}")
+    raise ValueError(f"Couldn't find section 'repo:{repo}' in pmaports.cfg of"
+                     " current branch")
+
+
+def check_existing_pkgs(args, arch):
+    channel = pmb.config.pmaports.read_config(args)["channel"]
+    path = f"{args.work}/packages/{channel}/{arch}"
+
+    if glob.glob(f"{path}/*"):
+        logging.info(f"Packages path: {path}")
+
+        msg = f"Found previously built packages for {channel}/{arch}, run" \
+              " 'pmbootstrap zap -p' first"
+        if pmb.parse.arch.cpu_emulation_required(arch):
+            msg += " or remove the path manually (to keep cross compilers if" \
+                   " you just built them)"
+
+        raise RuntimeError(f"{msg}!")
+
+
+def get_steps(args):
+    cfg = pmb.config.pmaports.read_config_repos(args)
+    prev_step = 0
+    ret = {}
+
+    for key, packages in cfg[args.repository].items():
+        if not key.startswith("bootstrap_"):
+            continue
+
+        step = int(key.split("bootstrap_", 1)[1])
+        assert step == prev_step + 1, (f"{key}: wrong order of steps, expected"
+            f" bootstrap_{prev_step + 1} (previous: bootstrap_{prev_step})")
+        prev_step = step
+
+        ret[key] = packages
+
+    return ret
+
+
+def get_suffix(args, arch):
+    if pmb.parse.arch.cpu_emulation_required(arch):
+        return f"buildroot_{arch}"
+    return "native"
+
+
+def get_packages(bootstrap_line):
+    ret = []
+    for word in bootstrap_line.split(" "):
+        if word.startswith("["):
+            continue
+        ret += [word]
+    return ret
+
+
+def set_progress_total(args, steps, arch):
+    global progress_total
+
+    progress_total = 0
+
+    # Add one progress point per package
+    for step, bootstrap_line in steps.items():
+        progress_total += len(get_packages(bootstrap_line))
+
+    # Add progress points per bootstrap step
+    progress_total += len(steps) * 2
+
+    # Foreign arch: need to initialize one additional chroot each step
+    if pmb.parse.arch.cpu_emulation_required(arch):
+        progress_total += len(steps)
+
+
+def log_progress(msg):
+    global progress_done
+
+    percent = int(100 * progress_done / progress_total)
+    logging.info(f"*** {percent}% [{progress_step}] {msg} ***")
+
+    progress_done += 1
+
+
+def run_steps(args, steps, arch, suffix):
+    global progress_step
+
+    for step, bootstrap_line in steps.items():
+        progress_step = step.replace("bootstrap_", "BOOTSTRAP=")
+
+        log_progress("zapping")
+        pmb.chroot.zap(args, confirm=False)
+
+        usr_merge = pmb.chroot.UsrMerge.OFF
+        if "[usr_merge]" in bootstrap_line:
+            usr_merge = pmb.chroot.UsrMerge.ON
+
+        if suffix != "native":
+            log_progress(f"initializing native chroot (merge /usr: {usr_merge.name})")
+            # Native chroot needs pmOS binary package repo for cross compilers
+            pmb.chroot.init(args, "native", usr_merge)
+
+        log_progress(f"initializing {suffix} chroot (merge /usr: {usr_merge.name})")
+        # Initialize without pmOS binary package repo
+        pmb.chroot.init(args, suffix, usr_merge, postmarketos_mirror=False)
+
+        for package in get_packages(bootstrap_line):
+            log_progress(f"building {package}")
+            bootstrap_stage = int(step.split("bootstrap_", 1)[1])
+            pmb.build.package(args, package, arch, force=True,
+                              strict=True, bootstrap_stage=bootstrap_stage)
+
+    log_progress("bootstrap complete!")
+
+
+def main(args):
+    check_repo_arg(args)
+
+    arch = get_arch(args)
+    check_existing_pkgs(args, arch)
+
+    steps = get_steps(args)
+    suffix = get_suffix(args, arch)
+
+    set_progress_total(args, steps, arch)
+    run_steps(args, steps, arch, suffix)

pmb/parse/arguments.py

@@ -517,6 +517,17 @@ def arguments_kconfig(subparser):
     add_kernel_arg(migrate)
 
 
+def arguments_repo_bootstrap(subparser):
+    arch_native = pmb.config.arch_native
+    arch_choices = set(pmb.config.build_device_architectures + [arch_native])
+
+    ret = subparser.add_parser("repo_bootstrap")
+    ret.add_argument("repository",
+                     help="which repository to bootstrap (e.g. systemd)")
+    ret.add_argument("--arch", choices=arch_choices, dest="arch")
+    return ret
+
+
 def arguments_repo_missing(subparser):
     ret = subparser.add_parser("repo_missing")
     package = ret.add_argument("package", nargs="?", help="only look at a"
@@ -697,6 +708,7 @@ def arguments():
     sub.add_parser("work_migrate", help="run this before using pmbootstrap"
                                         " non-interactively to migrate the"
                                         " work folder version on demand")
+    arguments_repo_bootstrap(sub)
     arguments_repo_missing(sub)
     arguments_kconfig(sub)
     arguments_export(sub)