diff --git a/pmb/build/__init__.py b/pmb/build/__init__.py index fb764e52..de84d62b 100644 --- a/pmb/build/__init__.py +++ b/pmb/build/__init__.py @@ -6,4 +6,4 @@ from pmb.build.kconfig import menuconfig from pmb.build.newapkbuild import newapkbuild from pmb.build.other import copy_to_buildpath, is_necessary, \ index_repo -from pmb.build._package import mount_pmaports, package +from pmb.build._package import BootstrapStage, mount_pmaports, package diff --git a/pmb/build/_package.py b/pmb/build/_package.py index 5c00c121..ef2b5d62 100644 --- a/pmb/build/_package.py +++ b/pmb/build/_package.py @@ -1,6 +1,7 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later import datetime +import enum import logging import os @@ -14,6 +15,15 @@ import pmb.parse import pmb.parse.arch +class BootstrapStage(enum.IntEnum): + """ + Pass a BOOTSTRAP= environment variable with the given value to abuild. See + bootstrap_1 etc. at https://postmarketos.org/pmaports.cfg for details. + """ + NONE = 0 + # We don't need explicit representations of the other numbers. + + def skip_already_built(pkgname, arch): """ Check if the package was already built in this session, and add it @@ -364,7 +374,7 @@ def link_to_git_dir(args, suffix): def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None, - suffix="native", src=None): + suffix="native", src=None, bootstrap_stage=BootstrapStage.NONE): """ Set up all environment variables and construct the abuild command (all depending on the cross-compiler method and target architecture), copy @@ -372,6 +382,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None, :param cross: None, "native", or "crossdirect" :param src: override source used to build the package with a local folder + :param bootstrap_stage: pass a BOOTSTRAP= env var with the value to abuild :returns: (output, cmd, env), output is the destination apk path relative to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are used by the test case, and they are the full abuild command and @@ -421,6 +432,9 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None, if args.go_mod_cache: env["GOMODCACHE"] = "/home/pmos/go/pkg/mod" + if bootstrap_stage: + env["BOOTSTRAP"] = str(bootstrap_stage) + # Build the abuild command cmd = ["abuild", "-D", "postmarketOS"] if strict or "pmb:strict" in apkbuild["options"]: @@ -468,7 +482,8 @@ def finish(args, apkbuild, arch, output, strict=False, suffix="native"): def package(args, pkgname, arch=None, force=False, strict=False, - skip_init_buildenv=False, src=None): + skip_init_buildenv=False, src=None, + bootstrap_stage=BootstrapStage.NONE): """ Build a package and its dependencies with Alpine Linux' abuild. @@ -487,6 +502,7 @@ def package(args, pkgname, arch=None, force=False, strict=False, something during initialization of the build environment (e.g. qemu aarch64 bug workaround) :param src: override source used to build the package with a local folder + :param bootstrap_stage: pass a BOOTSTRAP= env var with the value to abuild :returns: None if the build was not necessary output path relative to the packages folder ("armhf/ab-1-r2.apk") """ @@ -515,6 +531,6 @@ def package(args, pkgname, arch=None, force=False, strict=False, # Build and finish up (output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross, - suffix, src) + suffix, src, bootstrap_stage) finish(args, apkbuild, arch, output, strict, suffix) return output diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 18f47e44..7d89b621 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -24,6 +24,7 @@ import pmb.helpers.logging import pmb.helpers.pkgrel_bump import pmb.helpers.pmaports import pmb.helpers.repo +import pmb.helpers.repo_bootstrap import pmb.helpers.repo_missing import pmb.helpers.run import pmb.helpers.status @@ -218,6 +219,10 @@ def config(args): pmb.helpers.logging.disable() +def repo_bootstrap(args): + pmb.helpers.repo_bootstrap.main(args) + + def repo_missing(args): missing = pmb.helpers.repo_missing.generate(args, args.arch, args.overview, args.package, args.built) diff --git a/pmb/helpers/repo_bootstrap.py b/pmb/helpers/repo_bootstrap.py new file mode 100644 index 00000000..b46f53b6 --- /dev/null +++ b/pmb/helpers/repo_bootstrap.py @@ -0,0 +1,157 @@ +# Copyright 2024 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later +import logging +import glob + +import pmb.config.pmaports + + +progress_done = 0 +progress_total = 0 +progress_step = None + + +def get_arch(args): + if args.arch: + return args.arch + + if args.build_default_device_arch: + return args.deviceinfo["arch"] + + return pmb.config.arch_native + + +def check_repo_arg(args): + cfg = pmb.config.pmaports.read_config_repos(args) + repo = args.repository + + if repo in cfg: + return + + if not cfg: + raise ValueError("pmaports.cfg of current branch does not have any" + " sections starting with 'repo:'") + + logging.info(f"Valid repositories: {', '.join(cfg.keys())}") + raise ValueError(f"Couldn't find section 'repo:{repo}' in pmaports.cfg of" + " current branch") + + +def check_existing_pkgs(args, arch): + channel = pmb.config.pmaports.read_config(args)["channel"] + path = f"{args.work}/packages/{channel}/{arch}" + + if glob.glob(f"{path}/*"): + logging.info(f"Packages path: {path}") + + msg = f"Found previously built packages for {channel}/{arch}, run" \ + " 'pmbootstrap zap -p' first" + if pmb.parse.arch.cpu_emulation_required(arch): + msg += " or remove the path manually (to keep cross compilers if" \ + " you just built them)" + + raise RuntimeError(f"{msg}!") + + +def get_steps(args): + cfg = pmb.config.pmaports.read_config_repos(args) + prev_step = 0 + ret = {} + + for key, packages in cfg[args.repository].items(): + if not key.startswith("bootstrap_"): + continue + + step = int(key.split("bootstrap_", 1)[1]) + assert step == prev_step + 1, (f"{key}: wrong order of steps, expected" + f" bootstrap_{prev_step + 1} (previous: bootstrap_{prev_step})") + prev_step = step + + ret[key] = packages + + return ret + + +def get_suffix(args, arch): + if pmb.parse.arch.cpu_emulation_required(arch): + return f"buildroot_{arch}" + return "native" + + +def get_packages(bootstrap_line): + ret = [] + for word in bootstrap_line.split(" "): + if word.startswith("["): + continue + ret += [word] + return ret + + +def set_progress_total(args, steps, arch): + global progress_total + + progress_total = 0 + + # Add one progress point per package + for step, bootstrap_line in steps.items(): + progress_total += len(get_packages(bootstrap_line)) + + # Add progress points per bootstrap step + progress_total += len(steps) * 2 + + # Foreign arch: need to initialize one additional chroot each step + if pmb.parse.arch.cpu_emulation_required(arch): + progress_total += len(steps) + + +def log_progress(msg): + global progress_done + + percent = int(100 * progress_done / progress_total) + logging.info(f"*** {percent}% [{progress_step}] {msg} ***") + + progress_done += 1 + + +def run_steps(args, steps, arch, suffix): + global progress_step + + for step, bootstrap_line in steps.items(): + progress_step = step.replace("bootstrap_", "BOOTSTRAP=") + + log_progress("zapping") + pmb.chroot.zap(args, confirm=False) + + usr_merge = pmb.chroot.UsrMerge.OFF + if "[usr_merge]" in bootstrap_line: + usr_merge = pmb.chroot.UsrMerge.ON + + if suffix != "native": + log_progress(f"initializing native chroot (merge /usr: {usr_merge.name})") + # Native chroot needs pmOS binary package repo for cross compilers + pmb.chroot.init(args, "native", usr_merge) + + log_progress(f"initializing {suffix} chroot (merge /usr: {usr_merge.name})") + # Initialize without pmOS binary package repo + pmb.chroot.init(args, suffix, usr_merge, postmarketos_mirror=False) + + for package in get_packages(bootstrap_line): + log_progress(f"building {package}") + bootstrap_stage = int(step.split("bootstrap_", 1)[1]) + pmb.build.package(args, package, arch, force=True, + strict=True, bootstrap_stage=bootstrap_stage) + + log_progress("bootstrap complete!") + + +def main(args): + check_repo_arg(args) + + arch = get_arch(args) + check_existing_pkgs(args, arch) + + steps = get_steps(args) + suffix = get_suffix(args, arch) + + set_progress_total(args, steps, arch) + run_steps(args, steps, arch, suffix) diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 8075a7b8..7c739fe2 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -517,6 +517,17 @@ def arguments_kconfig(subparser): add_kernel_arg(migrate) +def arguments_repo_bootstrap(subparser): + arch_native = pmb.config.arch_native + arch_choices = set(pmb.config.build_device_architectures + [arch_native]) + + ret = subparser.add_parser("repo_bootstrap") + ret.add_argument("repository", + help="which repository to bootstrap (e.g. systemd)") + ret.add_argument("--arch", choices=arch_choices, dest="arch") + return ret + + def arguments_repo_missing(subparser): ret = subparser.add_parser("repo_missing") package = ret.add_argument("package", nargs="?", help="only look at a" @@ -697,6 +708,7 @@ def arguments(): sub.add_parser("work_migrate", help="run this before using pmbootstrap" " non-interactively to migrate the" " work folder version on demand") + arguments_repo_bootstrap(sub) arguments_repo_missing(sub) arguments_kconfig(sub) arguments_export(sub)