Administration/pmbootstrap-meow
2
0
Fork 0
forked from Mirror/pmbootstrap
Code Pull requests Activity Actions

commands: port repo_bootstrap (MR 2252)

Browse source 
The repo_bootstrap command is totally standalone and has it's own state,
making it a good candidate for the pmb.commands submodule. Port it over
and move the require_bootstrap() helper function over to
pmb/helpers/pmaports.py

We also fix the call to pmb.build.package() which broke during rework.

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>
This commit is contained in:
Caleb Connolly 2024-06-08 01:09:03 +02:00 • committed by Oliver Smith
parent 97bd8b96ec
commit f422b5c7ca
No known key found for this signature in database
GPG key ID: 5AE7F5513E0885CB
5 changed files with 198 additions and 196 deletions
Download patch file Download diff file Expand all files Collapse all files

4
pmb/commands/__init__.py
View file

@ -11,6 +11,7 @@ from pmb.helpers import frontend
from .base import Command
from .log import Log
from .index import Index
from .repo_bootstrap import RepoBootstrap
"""New way to model pmbootstrap subcommands that can be invoked without PmbArgs."""
@ -19,7 +20,6 @@ unmigrated_commands = [
"init",
"shutdown",
"work_migrate",
"repo_bootstrap",
"repo_missing",
"kconfig",
"export",
@ -64,6 +64,8 @@ def run_command(args: PmbArgs):
elif args.action == "index":
# FIXME: should index support --arch?
command = Index()
elif args.action == "repo_bootstrap":
command = RepoBootstrap(args.arch, args.repository)
else:
raise NotImplementedError(f"Command '{args.action}' is not implemented.")

164
pmb/commands/repo_bootstrap.py Normal file
View file

@ -0,0 +1,164 @@
# Copyright 2024 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from typing import Optional
from pmb.core.chroot import Chroot, ChrootType
from pmb.core.context import Context
from pmb.helpers import logging
import glob
import pmb.config.pmaports
import pmb.helpers.repo
import pmb.build
from pmb.types import Config
from pmb.core import get_context
from pmb import commands
class RepoBootstrap(commands.Command):
arch: str
repo: str
context: Context
progress_done: int = 0
progress_total: int = 0
progress_step: str
def check_repo_arg(self):
cfg = pmb.config.pmaports.read_config_repos()
if self.repo in cfg:
return
if not cfg:
raise ValueError("pmaports.cfg of current branch does not have any"
" sections starting with 'repo:'")
logging.info(f"Valid repositories: {', '.join(cfg.keys())}")
raise ValueError(f"Couldn't find section 'repo:{self.repo}' in pmaports.cfg of"
" current branch")
def __init__(self, arch: Optional[str], repository: str):
context = get_context()
if arch:
self.arch = arch
else:
if context.config.build_default_device_arch:
self.arch = pmb.parse.deviceinfo().arch
else:
self.arch = pmb.config.arch_native
self.repo = repository
self.context = context
self.check_repo_arg()
def get_packages(self, bootstrap_line):
ret = []
for word in bootstrap_line.split(" "):
if word.startswith("["):
continue
ret += [word]
return ret
def set_progress_total(self, steps):
self.progress_total = 0
# Add one progress point per package
for step, bootstrap_line in steps.items():
self.progress_total += len(self.get_packages(bootstrap_line))
# Add progress points per bootstrap step
self.progress_total += len(steps) * 2
# Foreign arch: need to initialize one additional chroot each step
if pmb.parse.arch.cpu_emulation_required(self.arch):
self.progress_total += len(steps)
def log_progress(self, msg):
percent = int(100 * self.progress_done / self.progress_total)
logging.info(f"*** {percent}% [{self.progress_step}] {msg} ***")
self.progress_done += 1
def run_steps(self, steps):
chroot: Chroot
if pmb.parse.arch.cpu_emulation_required(self.arch):
chroot = Chroot(ChrootType.BUILDROOT, self.arch)
else:
chroot = Chroot.native()
for step, bootstrap_line in steps.items():
self.progress_step = step.replace("bootstrap_", "BOOTSTRAP=")
self.log_progress("zapping")
pmb.chroot.zap(confirm=False)
usr_merge = pmb.chroot.UsrMerge.OFF
if "[usr_merge]" in bootstrap_line:
usr_merge = pmb.chroot.UsrMerge.ON
if chroot != Chroot.native():
self.log_progress(f"initializing native chroot (merge /usr: {usr_merge.name})")
# Native chroot needs pmOS binary package repo for cross compilers
pmb.chroot.init(Chroot.native(), usr_merge)
self.log_progress(f"initializing {chroot} chroot (merge /usr: {usr_merge.name})")
# Initialize without pmOS binary package repo
pmb.chroot.init(chroot, usr_merge, postmarketos_mirror=False)
for package in self.get_packages(bootstrap_line):
self.log_progress(f"building {package}")
bootstrap_stage = int(step.split("bootstrap_", 1)[1])
pmb.build.package(self.context, package, self.arch, force=True,
strict=True, bootstrap_stage=bootstrap_stage)
self.log_progress("bootstrap complete!")
def check_existing_pkgs(self):
channel = pmb.config.pmaports.read_config()["channel"]
path = self.context.config.work / "packages" / channel / self.arch
if glob.glob(f"{path}/*"):
logging.info(f"Packages path: {path}")
msg = f"Found previously built packages for {channel}/{self.arch}, run" \
" 'pmbootstrap zap -p' first"
if pmb.parse.arch.cpu_emulation_required(self.arch):
msg += " or remove the path manually (to keep cross compilers if" \
" you just built them)"
raise RuntimeError(f"{msg}!")
def get_steps(self):
cfg = pmb.config.pmaports.read_config_repos()
prev_step = 0
ret = {}
for key, packages in cfg[self.repo].items():
if not key.startswith("bootstrap_"):
continue
step = int(key.split("bootstrap_", 1)[1])
assert step == prev_step + 1, (f"{key}: wrong order of steps, expected"
f" bootstrap_{prev_step + 1} (previous: bootstrap_{prev_step})")
prev_step = step
ret[key] = packages
return ret
def run(self): # noqa: F821
self.check_existing_pkgs()
steps = self.get_steps()
self.set_progress_total(steps)
self.run_steps(steps)

9
pmb/helpers/frontend.py
View file

@ -27,7 +27,6 @@ import pmb.helpers.logging
import pmb.helpers.pkgrel_bump
import pmb.helpers.pmaports
import pmb.helpers.repo
import pmb.helpers.repo_bootstrap
import pmb.helpers.repo_missing
import pmb.helpers.run
import pmb.helpers.status
@ -119,7 +118,7 @@ def build(args: PmbArgs):
# Ensure repo_bootstrap is done for all arches we intend to build for
for package in args.packages:
arch_package = args.arch or pmb.build.autodetect.arch(package)
pmb.helpers.repo_bootstrap.require_bootstrap(arch_package,
pmb.helpers.pmaports.require_bootstrap(arch_package,
f"build {package} for {arch_package}")
context = get_context()
@ -243,10 +242,6 @@ def config(args: PmbArgs):
pmb.helpers.logging.disable()
def repo_bootstrap(args: PmbArgs):
pmb.helpers.repo_bootstrap.main(args.arch, args.repository)
def repo_missing(args: PmbArgs):
missing = pmb.helpers.repo_missing.generate(args.arch, args.overview,
args.package, args.built)
@ -274,7 +269,7 @@ def install(args: PmbArgs):
raise ValueError("Installation using rsync"
" is not currently supported on btrfs filesystem.")
pmb.helpers.repo_bootstrap.require_bootstrap(deviceinfo.arch,
pmb.helpers.pmaports.require_bootstrap(deviceinfo.arch,
f"do 'pmbootstrap install' for {deviceinfo.arch}"
" (deviceinfo_arch)")

29
pmb/helpers/pmaports.py
View file

@ -314,3 +314,32 @@ def get_channel_new(channel: str) -> str:
logging.verbose(f"Legacy channel '{channel}' translated to '{ret}'")
return ret
return channel
def require_bootstrap_error(repo, arch, trigger_str):
"""
Tell the user that they need to do repo_bootstrap, with some context.
:param repo: which repository
:param arch: for which architecture
:param trigger_str: message for the user to understand what caused this
"""
logging.info(f"ERROR: Trying to {trigger_str} with {repo} enabled, but the"
f" {repo} repo needs to be bootstrapped first.")
raise RuntimeError(f"Run 'pmbootstrap repo_bootstrap {repo} --arch={arch}'"
" and then try again.")
def require_bootstrap(arch, trigger_str):
"""
Check if repo_bootstrap was done, if any is needed.
:param arch: for which architecture
:param trigger_str: message for the user to understand what caused this
"""
if pmb.config.other.is_systemd_selected(get_context().config):
pmb.helpers.repo.update(arch)
pkg = pmb.parse.apkindex.package("postmarketos-base-systemd",
arch, False)
if not pkg:
require_bootstrap_error("systemd", arch, trigger_str)

188
pmb/helpers/repo_bootstrap.py
View file

@ -1,188 +0,0 @@
# Copyright 2024 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from typing import Optional
from pmb.core.chroot import Chroot
from pmb.helpers import logging
import glob
import pmb.config.pmaports
import pmb.helpers.repo
from pmb.types import Config
from pmb.core import get_context
progress_done = 0
progress_total = 0
progress_step: str
def get_arch(config: Config):
if config.build_default_device_arch:
return pmb.parse.deviceinfo().arch
return pmb.config.arch_native
def check_repo_arg(repo: str):
cfg = pmb.config.pmaports.read_config_repos()
if repo in cfg:
return
if not cfg:
raise ValueError("pmaports.cfg of current branch does not have any"
" sections starting with 'repo:'")
logging.info(f"Valid repositories: {', '.join(cfg.keys())}")
raise ValueError(f"Couldn't find section 'repo:{repo}' in pmaports.cfg of"
" current branch")
def check_existing_pkgs(config: Config, arch):
channel = pmb.config.pmaports.read_config()["channel"]
path = config.work / "packages" / channel / arch
if glob.glob(f"{path}/*"):
logging.info(f"Packages path: {path}")
msg = f"Found previously built packages for {channel}/{arch}, run" \
" 'pmbootstrap zap -p' first"
if pmb.parse.arch.cpu_emulation_required(arch):
msg += " or remove the path manually (to keep cross compilers if" \
" you just built them)"
raise RuntimeError(f"{msg}!")
def get_steps(repo: str):
cfg = pmb.config.pmaports.read_config_repos()
prev_step = 0
ret = {}
for key, packages in cfg[repo].items():
if not key.startswith("bootstrap_"):
continue
step = int(key.split("bootstrap_", 1)[1])
assert step == prev_step + 1, (f"{key}: wrong order of steps, expected"
f" bootstrap_{prev_step + 1} (previous: bootstrap_{prev_step})")
prev_step = step
ret[key] = packages
return ret
def get_suffix(arch):
if pmb.parse.arch.cpu_emulation_required(arch):
return f"buildroot_{arch}"
return "native"
def get_packages(bootstrap_line):
ret = []
for word in bootstrap_line.split(" "):
if word.startswith("["):
continue
ret += [word]
return ret
def set_progress_total(steps, arch):
global progress_total
progress_total = 0
# Add one progress point per package
for step, bootstrap_line in steps.items():
progress_total += len(get_packages(bootstrap_line))
# Add progress points per bootstrap step
progress_total += len(steps) * 2
# Foreign arch: need to initialize one additional chroot each step
if pmb.parse.arch.cpu_emulation_required(arch):
progress_total += len(steps)
def log_progress(msg):
global progress_done
percent = int(100 * progress_done / progress_total)
logging.info(f"*** {percent}% [{progress_step}] {msg} ***")
progress_done += 1
def run_steps(steps, arch, chroot: Chroot):
global progress_step
for step, bootstrap_line in steps.items():
progress_step = step.replace("bootstrap_", "BOOTSTRAP=")
log_progress("zapping")
pmb.chroot.zap(confirm=False)
usr_merge = pmb.chroot.UsrMerge.OFF
if "[usr_merge]" in bootstrap_line:
usr_merge = pmb.chroot.UsrMerge.ON
if chroot != Chroot.native():
log_progress(f"initializing native chroot (merge /usr: {usr_merge.name})")
# Native chroot needs pmOS binary package repo for cross compilers
pmb.chroot.init(Chroot.native(), usr_merge)
log_progress(f"initializing {chroot} chroot (merge /usr: {usr_merge.name})")
# Initialize without pmOS binary package repo
pmb.chroot.init(chroot, usr_merge, postmarketos_mirror=False)
for package in get_packages(bootstrap_line):
log_progress(f"building {package}")
bootstrap_stage = int(step.split("bootstrap_", 1)[1])
pmb.build.package(package, arch, force=True,
strict=True, bootstrap_stage=bootstrap_stage)
log_progress("bootstrap complete!")
def main(arch: Optional[str], repository: str): # noqa: F821
config = get_context().config
check_repo_arg(repository)
arch = arch or get_arch(config)
check_existing_pkgs(config, arch)
steps = get_steps(repository)
suffix = get_suffix(arch)
set_progress_total(steps, arch)
run_steps(steps, arch, suffix)
def require_bootstrap_error(repo, arch, trigger_str):
"""
Tell the user that they need to do repo_bootstrap, with some context.
:param repo: which repository
:param arch: for which architecture
:param trigger_str: message for the user to understand what caused this
"""
logging.info(f"ERROR: Trying to {trigger_str} with {repo} enabled, but the"
f" {repo} repo needs to be bootstrapped first.")
raise RuntimeError(f"Run 'pmbootstrap repo_bootstrap {repo} --arch={arch}'"
" and then try again.")
def require_bootstrap(arch, trigger_str):
"""
Check if repo_bootstrap was done, if any is needed.
:param arch: for which architecture
:param trigger_str: message for the user to understand what caused this
"""
if pmb.config.other.is_systemd_selected(get_context().config):
pmb.helpers.repo.update(arch)
pkg = pmb.parse.apkindex.package("postmarketos-base-systemd",
arch, False)
if not pkg:
require_bootstrap_error("systemd", arch, trigger_str)