commands: port repo_bootstrap (MR 2252)

The repo_bootstrap command is totally standalone and has it's own state,
making it a good candidate for the pmb.commands submodule. Port it over
and move the require_bootstrap() helper function over to
pmb/helpers/pmaports.py

We also fix the call to pmb.build.package() which broke during rework.

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>

pmb/commands/__init__.py

@@ -11,6 +11,7 @@ from pmb.helpers import frontend
 from .base import Command
 from .log import Log
 from .index import Index
+from .repo_bootstrap import RepoBootstrap
 
 """New way to model pmbootstrap subcommands that can be invoked without PmbArgs."""
 
@@ -19,7 +20,6 @@ unmigrated_commands = [
     "init",
     "shutdown",
     "work_migrate",
-    "repo_bootstrap",
     "repo_missing",
     "kconfig",
     "export",
@@ -64,6 +64,8 @@ def run_command(args: PmbArgs):
     elif args.action == "index":
         # FIXME: should index support --arch?
         command = Index()
+    elif args.action == "repo_bootstrap":
+        command = RepoBootstrap(args.arch, args.repository)
     else:
         raise NotImplementedError(f"Command '{args.action}' is not implemented.")
 

pmb/commands/repo_bootstrap.py

@@ -0,0 +1,164 @@
+# Copyright 2024 Oliver Smith
+# SPDX-License-Identifier: GPL-3.0-or-later
+from typing import Optional
+from pmb.core.chroot import Chroot, ChrootType
+from pmb.core.context import Context
+from pmb.helpers import logging
+import glob
+
+import pmb.config.pmaports
+import pmb.helpers.repo
+import pmb.build
+from pmb.types import Config
+from pmb.core import get_context
+
+from pmb import commands
+
+class RepoBootstrap(commands.Command):
+    arch: str
+    repo: str
+    context: Context
+
+    progress_done: int = 0
+    progress_total: int = 0
+    progress_step: str
+
+    def check_repo_arg(self):
+        cfg = pmb.config.pmaports.read_config_repos()
+
+        if self.repo in cfg:
+            return
+
+        if not cfg:
+            raise ValueError("pmaports.cfg of current branch does not have any"
+                                " sections starting with 'repo:'")
+
+        logging.info(f"Valid repositories: {', '.join(cfg.keys())}")
+        raise ValueError(f"Couldn't find section 'repo:{self.repo}' in pmaports.cfg of"
+                            " current branch")
+
+
+    def __init__(self, arch: Optional[str], repository: str):
+        context = get_context()
+        if arch:
+            self.arch = arch
+        else:
+            if context.config.build_default_device_arch:
+                self.arch = pmb.parse.deviceinfo().arch
+            else:
+                self.arch = pmb.config.arch_native
+
+        self.repo = repository
+        self.context = context
+
+        self.check_repo_arg()
+
+
+    def get_packages(self, bootstrap_line):
+        ret = []
+        for word in bootstrap_line.split(" "):
+            if word.startswith("["):
+                continue
+            ret += [word]
+        return ret
+
+
+    def set_progress_total(self, steps):
+        self.progress_total = 0
+
+        # Add one progress point per package
+        for step, bootstrap_line in steps.items():
+            self.progress_total += len(self.get_packages(bootstrap_line))
+
+        # Add progress points per bootstrap step
+        self.progress_total += len(steps) * 2
+
+        # Foreign arch: need to initialize one additional chroot each step
+        if pmb.parse.arch.cpu_emulation_required(self.arch):
+            self.progress_total += len(steps)
+
+
+    def log_progress(self, msg):
+        percent = int(100 * self.progress_done / self.progress_total)
+        logging.info(f"*** {percent}% [{self.progress_step}] {msg} ***")
+
+        self.progress_done += 1
+
+
+    def run_steps(self, steps):
+        chroot: Chroot
+        if pmb.parse.arch.cpu_emulation_required(self.arch):
+            chroot = Chroot(ChrootType.BUILDROOT, self.arch)
+        else:
+            chroot = Chroot.native()
+
+        for step, bootstrap_line in steps.items():
+            self.progress_step = step.replace("bootstrap_", "BOOTSTRAP=")
+
+            self.log_progress("zapping")
+            pmb.chroot.zap(confirm=False)
+
+            usr_merge = pmb.chroot.UsrMerge.OFF
+            if "[usr_merge]" in bootstrap_line:
+                usr_merge = pmb.chroot.UsrMerge.ON
+
+            if chroot != Chroot.native():
+                self.log_progress(f"initializing native chroot (merge /usr: {usr_merge.name})")
+                # Native chroot needs pmOS binary package repo for cross compilers
+                pmb.chroot.init(Chroot.native(), usr_merge)
+
+            self.log_progress(f"initializing {chroot} chroot (merge /usr: {usr_merge.name})")
+            # Initialize without pmOS binary package repo
+            pmb.chroot.init(chroot, usr_merge, postmarketos_mirror=False)
+
+            for package in self.get_packages(bootstrap_line):
+                self.log_progress(f"building {package}")
+                bootstrap_stage = int(step.split("bootstrap_", 1)[1])
+                pmb.build.package(self.context, package, self.arch, force=True,
+                                strict=True, bootstrap_stage=bootstrap_stage)
+
+        self.log_progress("bootstrap complete!")
+
+
+    def check_existing_pkgs(self):
+        channel = pmb.config.pmaports.read_config()["channel"]
+        path = self.context.config.work / "packages" / channel / self.arch
+
+        if glob.glob(f"{path}/*"):
+            logging.info(f"Packages path: {path}")
+
+            msg = f"Found previously built packages for {channel}/{self.arch}, run" \
+                " 'pmbootstrap zap -p' first"
+            if pmb.parse.arch.cpu_emulation_required(self.arch):
+                msg += " or remove the path manually (to keep cross compilers if" \
+                    " you just built them)"
+
+            raise RuntimeError(f"{msg}!")
+
+
+    def get_steps(self):
+        cfg = pmb.config.pmaports.read_config_repos()
+        prev_step = 0
+        ret = {}
+
+        for key, packages in cfg[self.repo].items():
+            if not key.startswith("bootstrap_"):
+                continue
+
+            step = int(key.split("bootstrap_", 1)[1])
+            assert step == prev_step + 1, (f"{key}: wrong order of steps, expected"
+                f" bootstrap_{prev_step + 1} (previous: bootstrap_{prev_step})")
+            prev_step = step
+
+            ret[key] = packages
+
+        return ret
+
+
+    def run(self):  # noqa: F821
+        self.check_existing_pkgs()
+
+        steps = self.get_steps()
+
+        self.set_progress_total(steps)
+        self.run_steps(steps)

pmb/helpers/frontend.py

@@ -27,7 +27,6 @@ import pmb.helpers.logging
 import pmb.helpers.pkgrel_bump
 import pmb.helpers.pmaports
 import pmb.helpers.repo
-import pmb.helpers.repo_bootstrap
 import pmb.helpers.repo_missing
 import pmb.helpers.run
 import pmb.helpers.status
@@ -119,7 +118,7 @@ def build(args: PmbArgs):
     # Ensure repo_bootstrap is done for all arches we intend to build for
     for package in args.packages:
         arch_package = args.arch or pmb.build.autodetect.arch(package)
-        pmb.helpers.repo_bootstrap.require_bootstrap(arch_package,
+        pmb.helpers.pmaports.require_bootstrap(arch_package,
             f"build {package} for {arch_package}")
 
     context = get_context()
@@ -243,10 +242,6 @@ def config(args: PmbArgs):
     pmb.helpers.logging.disable()
 
 
-def repo_bootstrap(args: PmbArgs):
-    pmb.helpers.repo_bootstrap.main(args.arch, args.repository)
-
-
 def repo_missing(args: PmbArgs):
     missing = pmb.helpers.repo_missing.generate(args.arch, args.overview,
                                                 args.package, args.built)
@@ -274,7 +269,7 @@ def install(args: PmbArgs):
         raise ValueError("Installation using rsync"
                         " is not currently supported on btrfs filesystem.")
 
-    pmb.helpers.repo_bootstrap.require_bootstrap(deviceinfo.arch,
+    pmb.helpers.pmaports.require_bootstrap(deviceinfo.arch,
         f"do 'pmbootstrap install' for {deviceinfo.arch}"
         " (deviceinfo_arch)")
 

pmb/helpers/pmaports.py

@@ -314,3 +314,32 @@ def get_channel_new(channel: str) -> str:
         logging.verbose(f"Legacy channel '{channel}' translated to '{ret}'")
         return ret
     return channel
+
+def require_bootstrap_error(repo, arch, trigger_str):
+    """
+    Tell the user that they need to do repo_bootstrap, with some context.
+
+    :param repo: which repository
+    :param arch: for which architecture
+    :param trigger_str: message for the user to understand what caused this
+    """
+    logging.info(f"ERROR: Trying to {trigger_str} with {repo} enabled, but the"
+                 f" {repo} repo needs to be bootstrapped first.")
+    raise RuntimeError(f"Run 'pmbootstrap repo_bootstrap {repo} --arch={arch}'"
+                       " and then try again.")
+
+
+def require_bootstrap(arch, trigger_str):
+    """
+    Check if repo_bootstrap was done, if any is needed.
+
+    :param arch: for which architecture
+    :param trigger_str: message for the user to understand what caused this
+    """
+    if pmb.config.other.is_systemd_selected(get_context().config):
+        pmb.helpers.repo.update(arch)
+        pkg = pmb.parse.apkindex.package("postmarketos-base-systemd",
+                                         arch, False)
+        if not pkg:
+            require_bootstrap_error("systemd", arch, trigger_str)
+

pmb/helpers/repo_bootstrap.py

@@ -1,188 +0,0 @@
-# Copyright 2024 Oliver Smith
-# SPDX-License-Identifier: GPL-3.0-or-later
-from typing import Optional
-from pmb.core.chroot import Chroot
-from pmb.helpers import logging
-import glob
-
-import pmb.config.pmaports
-import pmb.helpers.repo
-from pmb.types import Config
-from pmb.core import get_context
-
-
-progress_done = 0
-progress_total = 0
-progress_step: str
-
-
-def get_arch(config: Config):
-    if config.build_default_device_arch:
-        return pmb.parse.deviceinfo().arch
-
-    return pmb.config.arch_native
-
-
-def check_repo_arg(repo: str):
-    cfg = pmb.config.pmaports.read_config_repos()
-
-    if repo in cfg:
-        return
-
-    if not cfg:
-        raise ValueError("pmaports.cfg of current branch does not have any"
-                         " sections starting with 'repo:'")
-
-    logging.info(f"Valid repositories: {', '.join(cfg.keys())}")
-    raise ValueError(f"Couldn't find section 'repo:{repo}' in pmaports.cfg of"
-                     " current branch")
-
-
-def check_existing_pkgs(config: Config, arch):
-    channel = pmb.config.pmaports.read_config()["channel"]
-    path = config.work / "packages" / channel / arch
-
-    if glob.glob(f"{path}/*"):
-        logging.info(f"Packages path: {path}")
-
-        msg = f"Found previously built packages for {channel}/{arch}, run" \
-              " 'pmbootstrap zap -p' first"
-        if pmb.parse.arch.cpu_emulation_required(arch):
-            msg += " or remove the path manually (to keep cross compilers if" \
-                   " you just built them)"
-
-        raise RuntimeError(f"{msg}!")
-
-
-def get_steps(repo: str):
-    cfg = pmb.config.pmaports.read_config_repos()
-    prev_step = 0
-    ret = {}
-
-    for key, packages in cfg[repo].items():
-        if not key.startswith("bootstrap_"):
-            continue
-
-        step = int(key.split("bootstrap_", 1)[1])
-        assert step == prev_step + 1, (f"{key}: wrong order of steps, expected"
-            f" bootstrap_{prev_step + 1} (previous: bootstrap_{prev_step})")
-        prev_step = step
-
-        ret[key] = packages
-
-    return ret
-
-
-def get_suffix(arch):
-    if pmb.parse.arch.cpu_emulation_required(arch):
-        return f"buildroot_{arch}"
-    return "native"
-
-
-def get_packages(bootstrap_line):
-    ret = []
-    for word in bootstrap_line.split(" "):
-        if word.startswith("["):
-            continue
-        ret += [word]
-    return ret
-
-
-def set_progress_total(steps, arch):
-    global progress_total
-
-    progress_total = 0
-
-    # Add one progress point per package
-    for step, bootstrap_line in steps.items():
-        progress_total += len(get_packages(bootstrap_line))
-
-    # Add progress points per bootstrap step
-    progress_total += len(steps) * 2
-
-    # Foreign arch: need to initialize one additional chroot each step
-    if pmb.parse.arch.cpu_emulation_required(arch):
-        progress_total += len(steps)
-
-
-def log_progress(msg):
-    global progress_done
-
-    percent = int(100 * progress_done / progress_total)
-    logging.info(f"*** {percent}% [{progress_step}] {msg} ***")
-
-    progress_done += 1
-
-
-def run_steps(steps, arch, chroot: Chroot):
-    global progress_step
-
-    for step, bootstrap_line in steps.items():
-        progress_step = step.replace("bootstrap_", "BOOTSTRAP=")
-
-        log_progress("zapping")
-        pmb.chroot.zap(confirm=False)
-
-        usr_merge = pmb.chroot.UsrMerge.OFF
-        if "[usr_merge]" in bootstrap_line:
-            usr_merge = pmb.chroot.UsrMerge.ON
-
-        if chroot != Chroot.native():
-            log_progress(f"initializing native chroot (merge /usr: {usr_merge.name})")
-            # Native chroot needs pmOS binary package repo for cross compilers
-            pmb.chroot.init(Chroot.native(), usr_merge)
-
-        log_progress(f"initializing {chroot} chroot (merge /usr: {usr_merge.name})")
-        # Initialize without pmOS binary package repo
-        pmb.chroot.init(chroot, usr_merge, postmarketos_mirror=False)
-
-        for package in get_packages(bootstrap_line):
-            log_progress(f"building {package}")
-            bootstrap_stage = int(step.split("bootstrap_", 1)[1])
-            pmb.build.package(package, arch, force=True,
-                              strict=True, bootstrap_stage=bootstrap_stage)
-
-    log_progress("bootstrap complete!")
-
-
-def main(arch: Optional[str], repository: str):  # noqa: F821
-    config = get_context().config
-    check_repo_arg(repository)
-
-    arch = arch or get_arch(config)
-    check_existing_pkgs(config, arch)
-
-    steps = get_steps(repository)
-    suffix = get_suffix(arch)
-
-    set_progress_total(steps, arch)
-    run_steps(steps, arch, suffix)
-
-
-def require_bootstrap_error(repo, arch, trigger_str):
-    """
-    Tell the user that they need to do repo_bootstrap, with some context.
-
-    :param repo: which repository
-    :param arch: for which architecture
-    :param trigger_str: message for the user to understand what caused this
-    """
-    logging.info(f"ERROR: Trying to {trigger_str} with {repo} enabled, but the"
-                 f" {repo} repo needs to be bootstrapped first.")
-    raise RuntimeError(f"Run 'pmbootstrap repo_bootstrap {repo} --arch={arch}'"
-                       " and then try again.")
-
-
-def require_bootstrap(arch, trigger_str):
-    """
-    Check if repo_bootstrap was done, if any is needed.
-
-    :param arch: for which architecture
-    :param trigger_str: message for the user to understand what caused this
-    """
-    if pmb.config.other.is_systemd_selected(get_context().config):
-        pmb.helpers.repo.update(arch)
-        pkg = pmb.parse.apkindex.package("postmarketos-base-systemd",
-                                         arch, False)
-        if not pkg:
-            require_bootstrap_error("systemd", arch, trigger_str)