forked from Mirror/pmbootstrap
chroot: apk: use apk.static in all cases (MR 2463)
In 0b4fb9119f (chroot: always run apk static v2 (MR 2423)) we adjusted
install_run_apk() to run apk static on the host and pass in the local
binary repo with "--repository". This function can call apk in two ways,
either with the progress bar handling or without, the second case was
never updated and still ran apk inside the chroot incorrectly and with
an incorrect --repository flag.
Let's finish the job by refactoring helpers/apk.py to support all our
usecases and pointing everything to it, removing the last few situations
where we call "pmb.chroot.root(["apk", ...]).
The apk_with_progress() function is replaced by a generic "run()"
function which takes a boolean to indicate if we should render apk
progress.
Additionally, a new cache_clean() function is added so that "pmbootstrap
zap --pkgs-online-mismatch" can FINALLY be refactored to not rely on a
chroot existing. This requires some hacks but nothing serious, see the
comments in the function for details.
The chroot.init() code is now simplified since handling the --root,
--arch, --cache-dir, and --repository flags is now all done by
apk._prepare_cmd() as and when appropriate.
Lastly, this fixes a (previously unnoticed) bug where apk.static was
actually using /var/cache/apk on your host machine for its cache... This
is definitely not good behaviour....
Signed-off-by: Caleb Connolly <caleb@postmarketos.org>
This commit is contained in:
Caleb Connolly
parent
12846f3b8e
commit
42158637a7
7 changed files with 154 additions and 59 deletions
|
|
@ -111,9 +111,7 @@ def packages_get_locally_built_apks(package_list: list[str], arch: Arch) -> list
|
|||
for channel in channels:
|
||||
apk_path = get_context().config.work / "packages" / channel / arch / apk_file
|
||||
if apk_path.exists():
|
||||
# FIXME: use /mnt/pmb… until MR 2351 is reverted (pmb#2388)
|
||||
# local.append(apk_path)
|
||||
local.append(Path("/mnt/pmbootstrap/packages/") / channel / arch / apk_file)
|
||||
local.append(apk_path)
|
||||
break
|
||||
|
||||
# Record all the packages we have visited so far
|
||||
|
|
@ -181,24 +179,17 @@ def install_run_apk(
|
|||
user_repo += ["--repository", context.config.work / "packages" / channel]
|
||||
|
||||
for i, command in enumerate(commands):
|
||||
# --no-interactive is a parameter to `add`, so it must be appended or apk
|
||||
# gets confused
|
||||
command += ["--no-interactive"]
|
||||
command = user_repo + command
|
||||
|
||||
# Ignore missing repos before initial build (bpo#137)
|
||||
if os.getenv("PMB_APK_FORCE_MISSING_REPOSITORIES") == "1":
|
||||
command = ["--force-missing-repositories"] + command
|
||||
|
||||
if context.offline:
|
||||
command = ["--no-network"] + command
|
||||
if i == 0:
|
||||
pmb.helpers.apk.apk_with_progress(command, chroot)
|
||||
else:
|
||||
# Virtual package related commands don't actually install or remove
|
||||
# packages, but only mark the right ones as explicitly installed.
|
||||
# They finish up almost instantly, so don't display a progress bar.
|
||||
pmb.chroot.root(["apk", "--no-progress"] + command, chroot)
|
||||
# So only display a progress bar for the "apk add" command which is
|
||||
# always the first one we process (i == 0).
|
||||
pmb.helpers.apk.run(command, chroot, with_progress=(i == 0))
|
||||
|
||||
|
||||
def install(packages: list[str], chroot: Chroot, build: bool = True, quiet: bool = False) -> None:
|
||||
|
|
|
|||
|
|
@ -179,11 +179,12 @@ def init() -> None:
|
|||
extract(version, apk_static)
|
||||
|
||||
|
||||
def run(parameters):
|
||||
def run(parameters, with_progress=True):
|
||||
# --no-interactive is a parameter to `add`, so it must be appended or apk
|
||||
# gets confused
|
||||
if "add" in parameters:
|
||||
parameters += ["--no-interactive"]
|
||||
|
||||
if get_context().offline:
|
||||
parameters = ["--no-network"] + parameters
|
||||
pmb.helpers.apk.apk_with_progress(parameters)
|
||||
pmb.helpers.apk.run(parameters, with_progress=with_progress)
|
||||
|
|
|
|||
|
|
@ -12,11 +12,13 @@ import pmb.chroot.apk
|
|||
import pmb.chroot.apk_static
|
||||
import pmb.config
|
||||
import pmb.config.workdir
|
||||
import pmb.helpers.apk
|
||||
import pmb.helpers.repo
|
||||
import pmb.helpers.run
|
||||
import pmb.helpers.other
|
||||
from pmb.core import Chroot, ChrootType
|
||||
from pmb.core.context import get_context
|
||||
from pmb.types import PathString
|
||||
|
||||
|
||||
class UsrMerge(enum.Enum):
|
||||
|
|
@ -145,10 +147,6 @@ def init(chroot: Chroot, usr_merge: UsrMerge = UsrMerge.AUTO) -> None:
|
|||
|
||||
logging.info(f"({chroot}) Creating chroot")
|
||||
|
||||
# Initialize cache
|
||||
apk_cache = config.work / f"cache_apk_{arch}"
|
||||
pmb.helpers.run.root(["ln", "-s", "-f", "/var/cache/apk", chroot / "etc/apk/cache"])
|
||||
|
||||
# Initialize /etc/apk/keys/, resolv.conf, repositories
|
||||
init_keys()
|
||||
copy_resolv_conf(chroot)
|
||||
|
|
@ -159,10 +157,8 @@ def init(chroot: Chroot, usr_merge: UsrMerge = UsrMerge.AUTO) -> None:
|
|||
# Install alpine-base
|
||||
pmb.helpers.repo.update(arch)
|
||||
pkgs = ["alpine-base"]
|
||||
cmd = ["--root", chroot.path, "--cache-dir", apk_cache, "--initdb", "--arch", arch]
|
||||
for channel in pmb.config.pmaports.all_channels():
|
||||
cmd += ["--repository", config.work / "packages" / channel]
|
||||
pmb.chroot.apk_static.run(cmd + ["add", *pkgs])
|
||||
cmd: list[PathString] = ["--initdb"]
|
||||
pmb.helpers.apk.run(cmd + ["add", *pkgs], chroot)
|
||||
|
||||
# Merge /usr
|
||||
if usr_merge is UsrMerge.AUTO and pmb.config.is_systemd_selected(config):
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ def delete(hook: str, suffix: Chroot) -> None:
|
|||
if hook not in list_chroot(suffix):
|
||||
raise RuntimeError("There is no such hook installed!")
|
||||
prefix = pmb.config.initfs_hook_prefix
|
||||
pmb.chroot.root(["apk", "del", f"{prefix}{hook}"], suffix)
|
||||
pmb.helpers.apk.run(["del", f"{prefix}{hook}"], suffix)
|
||||
|
||||
|
||||
def update(suffix: Chroot) -> None:
|
||||
|
|
|
|||
|
|
@ -184,18 +184,6 @@ def zap_pkgs_online_mismatch(confirm=True, dry=False):
|
|||
# Iterate over existing apk caches
|
||||
for path in paths:
|
||||
arch = Arch.from_str(path.name.split("_", 2)[2])
|
||||
if arch.is_native():
|
||||
chroot = Chroot.native()
|
||||
else:
|
||||
chroot = Chroot.buildroot(arch)
|
||||
|
||||
# Skip if chroot does not exist
|
||||
# FIXME: should we init the buildroot to do it anyway?
|
||||
# what if we run apk.static with --arch instead?
|
||||
if not chroot.exists():
|
||||
continue
|
||||
|
||||
# Clean the cache with apk
|
||||
logging.info(f"({chroot}) apk -v cache clean")
|
||||
if not dry:
|
||||
pmb.chroot.root(["apk", "-v", "cache", "clean"], chroot)
|
||||
pmb.helpers.apk.cache_clean(arch)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import os
|
|||
import shlex
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import pmb.chroot
|
||||
import pmb.config.pmaports
|
||||
|
|
@ -23,8 +24,8 @@ from pmb.meta import Cache
|
|||
@Cache("root", "user_repository", mirrors_exclude=[])
|
||||
def update_repository_list(
|
||||
root: Path,
|
||||
user_repository: bool = False,
|
||||
mirrors_exclude: list[str] = [],
|
||||
user_repository: bool | Path = False,
|
||||
mirrors_exclude: list[str] | Literal[True] = [],
|
||||
check: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
|
|
@ -50,9 +51,15 @@ def update_repository_list(
|
|||
else:
|
||||
pmb.helpers.run.root(["mkdir", "-p", path.parent])
|
||||
|
||||
user_repo_dir: Path | None
|
||||
if isinstance(user_repository, Path):
|
||||
user_repo_dir = user_repository
|
||||
else:
|
||||
user_repo_dir = Path("/mnt/pmbootstrap/packages") if user_repository else None
|
||||
|
||||
# Up to date: Save cache, return
|
||||
lines_new = pmb.helpers.repo.urls(
|
||||
user_repository=user_repository, mirrors_exclude=mirrors_exclude
|
||||
user_repository=user_repo_dir, mirrors_exclude=mirrors_exclude
|
||||
)
|
||||
if lines_old == lines_new:
|
||||
return
|
||||
|
|
@ -68,7 +75,10 @@ def update_repository_list(
|
|||
for line in lines_new:
|
||||
pmb.helpers.run.root(["sh", "-c", "echo " f"{shlex.quote(line)} >> {path}"])
|
||||
update_repository_list(
|
||||
root, user_repository=user_repository, mirrors_exclude=mirrors_exclude, check=True
|
||||
root,
|
||||
user_repository=user_repository,
|
||||
mirrors_exclude=mirrors_exclude,
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -99,7 +109,7 @@ def _create_command_with_progress(command, fifo):
|
|||
:param fifo: path of the fifo
|
||||
:returns: full command in list form
|
||||
"""
|
||||
flags = ["--no-progress", "--progress-fd", "3"]
|
||||
flags = ["--progress-fd", "3"]
|
||||
command_full = [command[0]] + flags + command[1:]
|
||||
command_flat = pmb.helpers.run_core.flat_cmd([command_full])
|
||||
command_flat = f"exec 3>{fifo}; {command_flat}"
|
||||
|
|
@ -122,23 +132,15 @@ def _compute_progress(line):
|
|||
return cur / tot if tot > 0 else 0
|
||||
|
||||
|
||||
def apk_with_progress(command: Sequence[PathString], chroot: Chroot | None = None) -> None:
|
||||
def _apk_with_progress(command: Sequence[str]) -> None:
|
||||
"""Run an apk subcommand while printing a progress bar to STDOUT.
|
||||
|
||||
:param command: apk subcommand in list form
|
||||
:raises RuntimeError: when the apk command fails
|
||||
"""
|
||||
fifo = _prepare_fifo()
|
||||
_command: list[str] = [str(get_context().config.work / "apk.static")]
|
||||
if chroot:
|
||||
_command.extend(["--root", str(chroot.path), "--arch", str(chroot.arch)])
|
||||
for c in command:
|
||||
if isinstance(c, Arch):
|
||||
_command.append(str(c))
|
||||
else:
|
||||
_command.append(os.fspath(c))
|
||||
command_with_progress = _create_command_with_progress(_command, fifo)
|
||||
log_msg = " ".join(_command)
|
||||
command_with_progress = _create_command_with_progress(command, fifo)
|
||||
log_msg = " ".join(command)
|
||||
with pmb.helpers.run.root(["cat", fifo], output="pipe") as p_cat:
|
||||
with pmb.helpers.run.root(command_with_progress, output="background") as p_apk:
|
||||
while p_apk.poll() is None:
|
||||
|
|
@ -152,6 +154,116 @@ def apk_with_progress(command: Sequence[PathString], chroot: Chroot | None = Non
|
|||
pmb.helpers.run_core.check_return_code(p_apk.returncode, log_msg)
|
||||
|
||||
|
||||
def _prepare_cmd(command: Sequence[PathString], chroot: Chroot | None) -> list[str]:
|
||||
"""Prepare the apk command.
|
||||
|
||||
Returns a tuple of the first part of the command with generic apk flags, and the second part
|
||||
with the subcommand and its arguments.
|
||||
"""
|
||||
config = get_context().config
|
||||
# Our _apk_with_progress() wrapper also need --no-progress, since all that does is
|
||||
# prevent apk itself from rendering progress bars. We instead want it to tell us
|
||||
# the progress so we can render it. So we always set --no-progress.
|
||||
_command: list[str] = [str(config.work / "apk.static"), "--no-progress"]
|
||||
if chroot:
|
||||
cache_dir = config.work / f"cache_apk_{chroot.arch}"
|
||||
_command.extend(
|
||||
[
|
||||
"--root",
|
||||
str(chroot.path),
|
||||
"--arch",
|
||||
str(chroot.arch),
|
||||
"--cache-dir",
|
||||
str(cache_dir),
|
||||
]
|
||||
)
|
||||
local_repos = pmb.helpers.repo.urls(
|
||||
user_repository=config.work / "packages", mirrors_exclude=True
|
||||
)
|
||||
for repo in local_repos:
|
||||
_command.extend(["--repository", repo])
|
||||
if get_context().offline:
|
||||
_command.append("--no-network")
|
||||
|
||||
for c in command:
|
||||
_command.append(os.fspath(c))
|
||||
|
||||
# Always be non-interactive
|
||||
if c == "add":
|
||||
_command.append("--no-interactive")
|
||||
|
||||
return _command
|
||||
|
||||
|
||||
def run(command: Sequence[PathString], chroot: Chroot, with_progress: bool = True) -> None:
|
||||
"""Run an apk subcommand.
|
||||
|
||||
:param command: apk subcommand in list form
|
||||
:param with_progress: whether to print a progress bar
|
||||
:param chroot: chroot to run the command in
|
||||
:raises RuntimeError: when the apk command fails
|
||||
"""
|
||||
_command = _prepare_cmd(command, chroot)
|
||||
|
||||
if with_progress:
|
||||
_apk_with_progress(_command)
|
||||
else:
|
||||
pmb.helpers.run.root(_command)
|
||||
|
||||
|
||||
def cache_clean(arch: Arch) -> None:
|
||||
"""Clean the APK cache for a specific architecture."""
|
||||
work = get_context().config.work
|
||||
cache_dir = work / f"cache_apk_{arch}"
|
||||
if not cache_dir.exists():
|
||||
return
|
||||
|
||||
# The problem here is that apk's "cache clean" command really
|
||||
# expects to be run against an apk-managed rootfs and will return
|
||||
# errors if it can't access certain paths (even though it will
|
||||
# actually clean the cache like we want).
|
||||
# We could just ignore the return value, but then we wouldn't know
|
||||
# if something actually went wrong with apk...
|
||||
# So we do this dance of creating a rootfs with only the files that
|
||||
# APK needs to be happy
|
||||
tmproot = work / "tmp" / "apk_root"
|
||||
if not (tmproot / "etc/apk/repositories").exists():
|
||||
tmproot.mkdir(exist_ok=True)
|
||||
(tmproot / "var/cache").mkdir(exist_ok=True, parents=True)
|
||||
(tmproot / "etc/apk").mkdir(exist_ok=True, parents=True)
|
||||
(tmproot / "lib/apk/db").mkdir(exist_ok=True, parents=True)
|
||||
|
||||
(tmproot / "etc/apk/world").touch(exist_ok=True)
|
||||
(tmproot / "lib/apk/db/installed").touch(exist_ok=True)
|
||||
(tmproot / "lib/apk/db/triggers").touch(exist_ok=True)
|
||||
|
||||
(tmproot / "etc/apk/keys").symlink_to(work / "config_apk_keys")
|
||||
|
||||
# Our fake rootfs needs a valid repositories file for apk
|
||||
# to have something to compare the cache against
|
||||
update_repository_list(tmproot, user_repository=work / "packages")
|
||||
|
||||
# Point our tmproot cache dir to the real cache dir
|
||||
# this is much simpler than passing --cache-dir to apk
|
||||
# since even with that flag apk will also check it's
|
||||
# "static cache".
|
||||
(tmproot / "var/cache/apk").unlink(missing_ok=True)
|
||||
(tmproot / "var/cache/apk").symlink_to(cache_dir)
|
||||
|
||||
command: list[PathString] = [
|
||||
"-v",
|
||||
"--root",
|
||||
tmproot,
|
||||
"--arch",
|
||||
str(arch),
|
||||
]
|
||||
|
||||
command += ["cache", "clean"]
|
||||
_command = _prepare_cmd(command, None)
|
||||
|
||||
pmb.helpers.run.root(_command)
|
||||
|
||||
|
||||
def check_outdated(version_installed, action_msg):
|
||||
"""Check if the provided alpine version is outdated.
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from pmb.core.arch import Arch
|
|||
from pmb.core.pkgrepo import pkgrepo_names
|
||||
from pmb.helpers import logging
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import pmb.config.pmaports
|
||||
from pmb.meta import Cache
|
||||
|
|
@ -54,11 +55,14 @@ def apkindex_hash(url: str, length: int = 8) -> Path:
|
|||
# FIXME: make config.mirrors a normal dict
|
||||
# mypy: disable-error-code="literal-required"
|
||||
@Cache("user_repository", "mirrors_exclude")
|
||||
def urls(user_repository: bool = False, mirrors_exclude: list[str] = []) -> list[str]:
|
||||
def urls(
|
||||
user_repository: Path | None = None, mirrors_exclude: list[str] | Literal[True] = []
|
||||
) -> list[str]:
|
||||
"""Get a list of repository URLs, as they are in /etc/apk/repositories.
|
||||
|
||||
:param user_repository: add /mnt/pmbootstrap/packages
|
||||
:param mirrors_exclude: mirrors to exclude (see pmb.core.config.Mirrors)
|
||||
:param mirrors_exclude: mirrors to exclude (see pmb.core.config.Mirrors) or true to exclude
|
||||
all mirrors and only return the local repos
|
||||
:returns: list of mirror strings, like ["/mnt/pmbootstrap/packages",
|
||||
"http://...", ...]
|
||||
"""
|
||||
|
|
@ -74,7 +78,10 @@ def urls(user_repository: bool = False, mirrors_exclude: list[str] = []) -> list
|
|||
# Local user repository (for packages compiled with pmbootstrap)
|
||||
if user_repository:
|
||||
for channel in pmb.config.pmaports.all_channels():
|
||||
ret.append(f"/mnt/pmbootstrap/packages/{channel}")
|
||||
ret.append(str(user_repository / channel))
|
||||
|
||||
if mirrors_exclude is True:
|
||||
return ret
|
||||
|
||||
# Don't add the systemd mirror if systemd is disabled
|
||||
if not pmb.config.is_systemd_selected(config):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue