forked from Mirror/pmbootstrap
Remove "pmbootstrap repo_bootstrap"
The repo bootstrap logic has worked well enough in the initial systemd bringup. But we have decided to replace the approach of building packages multiple times during the repo_bootstrap with building each package only once and adding -stage0 packages where necessary (currently only a systemd-stage0 package). Advantages: * Replace the often breaking repo_bootstrap logic with something less fragile (repo_bootstrap is currently broken again). * Can get rid of the whole bootstrap logic in pmaports.cfg, pmbootstrap, build.postmarketos.org. This will make pmbootstrap and bpo easier to maintain. * Fix problems we have seen when upgrading two or more of (systemd, dbus, linux-pam-pmos) at once: it doesn't pass in CI and it doesn't pass in BPO * You don't need to do a whole bootstrap at once, it is again broken down by package. This means if building one package fails (locally or in bpo), we don't need to do the whole thing again, just start at the package that failed. This also means it is much easier to optimize e.g. the stage0 packages to build faster. * Fixes some specific bugs we currently have (abuild twice in pmaports, bpo tests disabled because of that, ...) Part-of: https://gitlab.postmarketos.org/postmarketOS/pmbootstrap/-/merge_requests/2588
This commit is contained in:
Oliver Smith
parent
f6b0ebd78c
commit
47be020c7a
6 changed files with 0 additions and 236 deletions
|
|
@ -76,14 +76,6 @@ pmb.commands.pull module
|
|||
:undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
pmb.commands.repo_bootstrap module
|
||||
----------------------------------
|
||||
|
||||
.. automodule:: pmb.commands.repo_bootstrap
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
pmb.commands.shutdown module
|
||||
----------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from .aportgen import Aportgen
|
|||
from .flasher import Flasher
|
||||
from .log import Log
|
||||
from .index import Index
|
||||
from .repo_bootstrap import RepoBootstrap
|
||||
from .shutdown import Shutdown
|
||||
from .test import Test
|
||||
from .pkgrel_bump import PkgrelBump
|
||||
|
|
@ -77,8 +76,6 @@ def run_command(args: PmbArgs) -> None:
|
|||
case "index":
|
||||
# FIXME: should index support --arch?
|
||||
command = Index()
|
||||
case "repo_bootstrap":
|
||||
command = RepoBootstrap(args.arch, args.repository)
|
||||
case "shutdown":
|
||||
command = Shutdown()
|
||||
case "test":
|
||||
|
|
|
|||
|
|
@ -1,177 +0,0 @@
|
|||
# Copyright 2024 Oliver Smith
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
from pmb.core.arch import Arch
|
||||
from pmb.core.chroot import Chroot, ChrootType
|
||||
from pmb.core.context import Context
|
||||
from pmb.helpers import logging
|
||||
import glob
|
||||
|
||||
import pmb.config.pmaports
|
||||
import pmb.helpers.repo
|
||||
import pmb.build
|
||||
import pmb.chroot
|
||||
import pmb.chroot.apk
|
||||
from pmb.build import BuildQueueItem
|
||||
from pmb.core.context import get_context
|
||||
|
||||
from pmb import commands
|
||||
|
||||
|
||||
class RepoBootstrap(commands.Command):
|
||||
arch: Arch
|
||||
repo: str
|
||||
context: Context
|
||||
|
||||
progress_done: int = 0
|
||||
progress_total: int = 0
|
||||
progress_step: str
|
||||
|
||||
def check_repo_arg(self) -> None:
|
||||
cfg = pmb.config.pmaports.read_config_repos()
|
||||
|
||||
if self.repo in cfg:
|
||||
return
|
||||
|
||||
if not cfg:
|
||||
raise ValueError(
|
||||
"pmaports.cfg of current branch does not have any sections starting with 'repo:'"
|
||||
)
|
||||
|
||||
logging.info(f"Valid repositories: {', '.join(cfg.keys())}")
|
||||
raise ValueError(
|
||||
f"Couldn't find section 'repo:{self.repo}' in pmaports.cfg of current branch"
|
||||
)
|
||||
|
||||
def __init__(self, arch: Arch | None, repository: str):
|
||||
context = get_context()
|
||||
if arch:
|
||||
self.arch = arch
|
||||
else:
|
||||
if context.config.build_default_device_arch:
|
||||
self.arch = pmb.parse.deviceinfo().arch
|
||||
else:
|
||||
self.arch = Arch.native()
|
||||
|
||||
self.repo = repository
|
||||
self.context = context
|
||||
|
||||
self.check_repo_arg()
|
||||
|
||||
def get_packages(self, bootstrap_line: str) -> list[str]:
|
||||
ret = []
|
||||
for word in bootstrap_line.split(" "):
|
||||
if word.startswith("["):
|
||||
continue
|
||||
ret += [word]
|
||||
return ret
|
||||
|
||||
def set_progress_total(self, steps: dict[str, str]) -> None:
|
||||
self.progress_total = 0
|
||||
|
||||
# Add one progress point per package
|
||||
for step, bootstrap_line in steps.items():
|
||||
self.progress_total += len(self.get_packages(bootstrap_line))
|
||||
|
||||
# Add progress points per bootstrap step
|
||||
self.progress_total += len(steps) * 2
|
||||
|
||||
# Foreign arch: need to initialize one additional chroot each step
|
||||
if self.arch.cpu_emulation_required():
|
||||
self.progress_total += len(steps)
|
||||
|
||||
def log_progress(self, msg: str) -> None:
|
||||
percent = int(100 * self.progress_done / self.progress_total)
|
||||
logging.info(f"*** {percent}% [{self.progress_step}] {msg} ***")
|
||||
|
||||
self.progress_done += 1
|
||||
|
||||
def run_steps(self, steps: dict[str, str]) -> None:
|
||||
chroot: Chroot
|
||||
if self.arch.cpu_emulation_required():
|
||||
chroot = Chroot(ChrootType.BUILDROOT, self.arch)
|
||||
else:
|
||||
chroot = Chroot.native()
|
||||
|
||||
for step, bootstrap_line in steps.items():
|
||||
self.progress_step = step.replace("bootstrap_", "BOOTSTRAP=")
|
||||
|
||||
self.log_progress("zapping")
|
||||
pmb.chroot.zap(confirm=False)
|
||||
|
||||
usr_merge = pmb.chroot.UsrMerge.OFF
|
||||
if "[usr_merge]" in bootstrap_line:
|
||||
usr_merge = pmb.chroot.UsrMerge.ON
|
||||
|
||||
if chroot != Chroot.native():
|
||||
self.log_progress(f"initializing native chroot (merge /usr: {usr_merge.name})")
|
||||
# Native chroot needs pmOS binary package repo for cross compilers
|
||||
pmb.chroot.init(Chroot.native(), usr_merge)
|
||||
|
||||
self.log_progress(f"initializing {chroot} chroot (merge /usr: {usr_merge.name})")
|
||||
# Initialize without pmOS binary package repo
|
||||
pmb.helpers.apk.update_repository_list(chroot.path, mirrors_exclude=[self.repo])
|
||||
pmb.chroot.init(chroot, usr_merge)
|
||||
|
||||
bootstrap_stage = int(step.split("bootstrap_", 1)[1])
|
||||
|
||||
def log_wrapper(pkg: BuildQueueItem) -> None:
|
||||
self.log_progress(f"building {pkg['name']}")
|
||||
|
||||
packages = self.get_packages(bootstrap_line)
|
||||
pmb.build.packages(
|
||||
self.context,
|
||||
packages,
|
||||
self.arch,
|
||||
force=True,
|
||||
strict=True,
|
||||
bootstrap_stage=bootstrap_stage,
|
||||
log_callback=log_wrapper,
|
||||
)
|
||||
|
||||
self.log_progress("bootstrap complete!")
|
||||
|
||||
def check_existing_pkgs(self) -> None:
|
||||
channel = pmb.config.pmaports.read_config()["channel"]
|
||||
path = self.context.config.work / "packages" / channel / self.arch
|
||||
|
||||
if glob.glob(f"{path}/*"):
|
||||
logging.info(f"Packages path: {path}")
|
||||
|
||||
msg = (
|
||||
f"Found previously built packages for {channel}/{self.arch}, run"
|
||||
" 'pmbootstrap zap -p' first"
|
||||
)
|
||||
if self.arch.cpu_emulation_required():
|
||||
msg += (
|
||||
" or remove the path manually (to keep cross compilers if you just built them)"
|
||||
)
|
||||
|
||||
raise RuntimeError(f"{msg}!")
|
||||
|
||||
def get_steps(self) -> dict[str, str]:
|
||||
cfg = pmb.config.pmaports.read_config_repos()
|
||||
prev_step = 0
|
||||
ret: dict[str, str] = {}
|
||||
|
||||
for key, packages in cfg[self.repo].items():
|
||||
if not key.startswith("bootstrap_"):
|
||||
continue
|
||||
|
||||
step = int(key.split("bootstrap_", 1)[1])
|
||||
assert step == prev_step + 1, (
|
||||
f"{key}: wrong order of steps, expected"
|
||||
f" bootstrap_{prev_step + 1} (previous: bootstrap_{prev_step})"
|
||||
)
|
||||
prev_step = step
|
||||
|
||||
ret[key] = packages
|
||||
|
||||
return ret
|
||||
|
||||
def run(self) -> None:
|
||||
self.check_existing_pkgs()
|
||||
|
||||
steps = self.get_steps()
|
||||
|
||||
self.set_progress_total(steps)
|
||||
self.run_steps(steps)
|
||||
|
|
@ -119,11 +119,6 @@ def build(args: PmbArgs) -> None:
|
|||
if src and not os.path.exists(src):
|
||||
raise RuntimeError("Invalid path specified for --src: " + src)
|
||||
|
||||
# Ensure repo_bootstrap is done for all arches we intend to build for
|
||||
for package in args.packages:
|
||||
arch_package = args.arch or pmb.build.autodetect.arch(package)
|
||||
pmb.helpers.pmaports.require_bootstrap(arch_package, f"build {package} for {arch_package}")
|
||||
|
||||
context = get_context()
|
||||
# Build all packages
|
||||
built = pmb.build.packages(
|
||||
|
|
@ -321,10 +316,6 @@ def install(args: PmbArgs) -> None:
|
|||
if args.rsync and args.filesystem == "btrfs":
|
||||
raise ValueError("Installation using rsync is not currently supported on btrfs filesystem.")
|
||||
|
||||
pmb.helpers.pmaports.require_bootstrap(
|
||||
deviceinfo.arch, f"do 'pmbootstrap install' for {deviceinfo.arch} (deviceinfo_arch)"
|
||||
)
|
||||
|
||||
# On-device installer checks
|
||||
# Note that this can't be in the mutually exclusive group that has most of
|
||||
# the conflicting options, because then it would not work with --disk.
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ See also:
|
|||
- pmb/helpers/package.py (work with both)
|
||||
"""
|
||||
|
||||
from pmb.core.context import get_context
|
||||
from pmb.core.arch import Arch
|
||||
from pmb.core.pkgrepo import pkgrepo_iter_package_dirs
|
||||
from pmb.helpers import logging
|
||||
|
|
@ -405,31 +404,3 @@ def get_channel_new(channel: str) -> str:
|
|||
logging.verbose(f"Legacy channel '{channel}' translated to '{ret}'")
|
||||
return ret
|
||||
return channel
|
||||
|
||||
|
||||
def require_bootstrap_error(repo: str, arch: Arch, trigger_str: str) -> None:
|
||||
"""
|
||||
Tell the user that they need to do repo_bootstrap, with some context.
|
||||
|
||||
:param repo: which repository
|
||||
:param arch: for which architecture
|
||||
:param trigger_str: message for the user to understand what caused this
|
||||
"""
|
||||
logging.info(
|
||||
f"ERROR: Trying to {trigger_str} with {repo} enabled, but the {repo} repo needs to be bootstrapped first."
|
||||
)
|
||||
raise RuntimeError(f"Run 'pmbootstrap repo_bootstrap {repo} --arch={arch}' and then try again.")
|
||||
|
||||
|
||||
def require_bootstrap(arch: Arch, trigger_str: str) -> None:
|
||||
"""
|
||||
Check if repo_bootstrap was done, if any is needed.
|
||||
|
||||
:param arch: for which architecture
|
||||
:param trigger_str: message for the user to understand what caused this
|
||||
"""
|
||||
if pmb.config.other.is_systemd_selected(get_context().config):
|
||||
pmb.helpers.repo.update(arch)
|
||||
pkg = pmb.parse.apkindex.package("postmarketos-base-systemd", arch, False)
|
||||
if not pkg:
|
||||
require_bootstrap_error("systemd", arch, trigger_str)
|
||||
|
|
|
|||
|
|
@ -729,15 +729,6 @@ def arguments_kconfig(subparser: argparse._SubParsersAction) -> None:
|
|||
add_kernel_arg(migrate, nargs=1)
|
||||
|
||||
|
||||
def arguments_repo_bootstrap(subparser: argparse._SubParsersAction) -> argparse.ArgumentParser:
|
||||
arch_choices = Arch.supported()
|
||||
|
||||
ret = subparser.add_parser("repo_bootstrap")
|
||||
ret.add_argument("repository", help="which repository to bootstrap (e.g. systemd)")
|
||||
ret.add_argument("--arch", choices=arch_choices, dest="arch", type=lambda x: Arch.from_str(x))
|
||||
return ret
|
||||
|
||||
|
||||
def arguments_repo_missing(subparser: argparse._SubParsersAction) -> argparse.ArgumentParser:
|
||||
ret = subparser.add_parser(
|
||||
"repo_missing",
|
||||
|
|
@ -985,7 +976,6 @@ def get_parser() -> argparse.ArgumentParser:
|
|||
" non-interactively to migrate the"
|
||||
" work folder version on demand",
|
||||
)
|
||||
arguments_repo_bootstrap(sub)
|
||||
arguments_repo_missing(sub)
|
||||
arguments_kconfig(sub)
|
||||
arguments_export(sub)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue