Administration/pmbootstrap-meow
2
0
Fork 0
forked from Mirror/pmbootstrap
Code Pull requests Activity Actions

treewide: adopt pathlib.Path and type hinting (MR 2252)

Browse source 
With the new chroot type, we can now write fancy paths in the pythonic
way. Convert most of the codebase over, as well as adding various other
type hints.

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>
This commit is contained in:
Caleb Connolly 2024-04-04 06:14:14 +02:00 • committed by Oliver Smith
parent 00383bf354
commit 31cc898dd5
No known key found for this signature in database
GPG key ID: 5AE7F5513E0885CB
64 changed files with 513 additions and 385 deletions
Download patch file Download diff file Expand all files Collapse all files

7
pmb/__init__.py
View file

@ -2,10 +2,9 @@
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
# PYTHON_ARGCOMPLETE_OK # PYTHON_ARGCOMPLETE_OK
import sys import sys
import logging
import os import os
import traceback import traceback
from argparse import Namespace from typing import Any, Optional
from pmb.helpers.exceptions import BuildFailedError, NonBugError from pmb.helpers.exceptions import BuildFailedError, NonBugError
@ -42,7 +41,9 @@ def print_log_hint(args: Any) -> None:
def main() -> int: def main() -> int:
# Wrap everything to display nice error messages # Wrap everything to display nice error messages
args = None
# FIXME: can't use PmbArgs here because it creates a circular import
args: Any
try: try:
# Parse arguments, set up logging # Parse arguments, set up logging
args = parse.arguments() args = parse.arguments()

4
pmb/aportgen/core.py
View file

@ -76,8 +76,8 @@ def rewrite(args: PmbArgs, pkgname, path_original="", fields={}, replace_pkgname
if path_original: if path_original:
lines_new = [ lines_new = [
"# Automatically generated aport, do not edit!\n", "# Automatically generated aport, do not edit!\n",
"# Generator: pmbootstrap aportgen " + pkgname + "\n", f"# Generator: pmbootstrap aportgen {pkgname}\n",
"# Based on: " + path_original + "\n", f"# Based on: {path_original}\n",
"\n", "\n",
] ]
else: else:

17
pmb/build/autodetect.py
View file

@ -12,7 +12,8 @@ import pmb.parse.arch
from pmb.core import Chroot, ChrootType from pmb.core import Chroot, ChrootType
def arch_from_deviceinfo(args: PmbArgs, pkgname, aport): # FIXME (#2324): type hint Arch
def arch_from_deviceinfo(args: PmbArgs, pkgname, aport: Path) -> Optional[str]:
""" """
The device- packages are noarch packages. But it only makes sense to build The device- packages are noarch packages. But it only makes sense to build
them for the device's architecture, which is specified in the deviceinfo them for the device's architecture, which is specified in the deviceinfo
@ -23,10 +24,10 @@ def arch_from_deviceinfo(args: PmbArgs, pkgname, aport):
""" """
# Require a deviceinfo file in the aport # Require a deviceinfo file in the aport
if not pkgname.startswith("device-"): if not pkgname.startswith("device-"):
return return None
deviceinfo = aport + "/deviceinfo" deviceinfo = aport / "deviceinfo"
if not os.path.exists(deviceinfo): if not deviceinfo.exists():
return return None
# Return its arch # Return its arch
device = pkgname.split("-", 1)[1] device = pkgname.split("-", 1)[1]
@ -35,7 +36,7 @@ def arch_from_deviceinfo(args: PmbArgs, pkgname, aport):
return arch return arch
def arch(args: PmbArgs, pkgname): def arch(args: PmbArgs, pkgname: str):
""" """
Find a good default in case the user did not specify for which architecture Find a good default in case the user did not specify for which architecture
a package should be built. a package should be built.
@ -47,6 +48,8 @@ def arch(args: PmbArgs, pkgname):
* first arch in the APKBUILD * first arch in the APKBUILD
""" """
aport = pmb.helpers.pmaports.find(args, pkgname) aport = pmb.helpers.pmaports.find(args, pkgname)
if not aport:
raise FileNotFoundError(f"APKBUILD not found for {pkgname}")
ret = arch_from_deviceinfo(args, pkgname, aport) ret = arch_from_deviceinfo(args, pkgname, aport)
if ret: if ret:
return ret return ret
@ -80,7 +83,7 @@ def chroot(apkbuild: Dict[str, str], arch: str) -> Chroot:
if "pmb:cross-native" in apkbuild["options"]: if "pmb:cross-native" in apkbuild["options"]:
return Chroot.native() return Chroot.native()
return Chroot(ChrootType.BUILDROOT, arch) return Chroot.buildroot(arch)
def crosscompile(args: PmbArgs, apkbuild, arch, suffix: Chroot): def crosscompile(args: PmbArgs, apkbuild, arch, suffix: Chroot):

28
pmb/build/envkernel.py
View file

@ -1,5 +1,6 @@
# Copyright 2023 Robert Yang # Copyright 2023 Robert Yang
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
from typing import List
from pmb.helpers import logging from pmb.helpers import logging
import os import os
from pathlib import Path from pathlib import Path
@ -9,7 +10,7 @@ import pmb.aportgen
import pmb.build import pmb.build
import pmb.build.autodetect import pmb.build.autodetect
import pmb.chroot import pmb.chroot
from pmb.core.types import PmbArgs from pmb.core.types import PathString, PmbArgs
import pmb.helpers import pmb.helpers
import pmb.helpers.mount import pmb.helpers.mount
import pmb.helpers.pmaports import pmb.helpers.pmaports
@ -89,7 +90,7 @@ def find_kbuild_output_dir(function_body):
"can't resolve it, please open an issue.") "can't resolve it, please open an issue.")
def modify_apkbuild(args: PmbArgs, pkgname, aport): def modify_apkbuild(args: PmbArgs, pkgname: str, aport: Path):
"""Modify kernel APKBUILD to package build output from envkernel.sh.""" """Modify kernel APKBUILD to package build output from envkernel.sh."""
apkbuild_path = aport + "/APKBUILD" apkbuild_path = aport + "/APKBUILD"
apkbuild = pmb.parse.apkbuild(apkbuild_path) apkbuild = pmb.parse.apkbuild(apkbuild_path)
@ -110,7 +111,7 @@ def modify_apkbuild(args: PmbArgs, pkgname, aport):
pmb.aportgen.core.rewrite(args, pkgname, apkbuild_path, fields=fields) pmb.aportgen.core.rewrite(args, pkgname, apkbuild_path, fields=fields)
def run_abuild(args: PmbArgs, pkgname, arch, apkbuild_path, kbuild_out): def run_abuild(args: PmbArgs, pkgname: str, arch: str, apkbuild_path: Path, kbuild_out):
""" """
Prepare build environment and run abuild. Prepare build environment and run abuild.
@ -142,17 +143,16 @@ def run_abuild(args: PmbArgs, pkgname, arch, apkbuild_path, kbuild_out):
pmb.build.copy_to_buildpath(args, pkgname) pmb.build.copy_to_buildpath(args, pkgname)
# Create symlink from abuild working directory to envkernel build directory # Create symlink from abuild working directory to envkernel build directory
build_output = Path("" if kbuild_out == "" else "/" + kbuild_out) if kbuild_out != "":
if False or build_output != "": if os.path.islink(chroot / "mnt/linux" / kbuild_out) and \
if os.path.islink(chroot / "mnt/linux" / build_output) and \ os.path.lexists(chroot / "mnt/linux" / kbuild_out):
os.path.lexists(chroot / "mnt/linux" / build_output): pmb.chroot.root(args, ["rm", "/mnt/linux" / kbuild_out])
pmb.chroot.root(args, ["rm", "/mnt/linux" / build_output])
pmb.chroot.root(args, ["ln", "-s", "/mnt/linux", pmb.chroot.root(args, ["ln", "-s", "/mnt/linux",
build_path / "src"]) build_path / "src"])
pmb.chroot.root(args, ["ln", "-s", kbuild_out_source, pmb.chroot.root(args, ["ln", "-s", kbuild_out_source,
build_path / "src" / build_output]) build_path / "src" / kbuild_out])
cmd = ["cp", apkbuild_path, chroot / build_path / "APKBUILD"] cmd: List[PathString] = ["cp", apkbuild_path, chroot / build_path / "APKBUILD"]
pmb.helpers.run.root(args, cmd) pmb.helpers.run.root(args, cmd)
# Create the apk package # Create the apk package
@ -167,10 +167,10 @@ def run_abuild(args: PmbArgs, pkgname, arch, apkbuild_path, kbuild_out):
pmb.helpers.mount.umount_all(args, chroot / "mnt/linux") pmb.helpers.mount.umount_all(args, chroot / "mnt/linux")
# Clean up symlinks # Clean up symlinks
if build_output != "": if kbuild_out != "":
if os.path.islink(chroot / "mnt/linux" / build_output) and \ if os.path.islink(chroot / "mnt/linux" / kbuild_out) and \
os.path.lexists(chroot / "mnt/linux" / build_output): os.path.lexists(chroot / "mnt/linux" / kbuild_out):
pmb.chroot.root(args, ["rm", "/mnt/linux" / build_output]) pmb.chroot.root(args, ["rm", "/mnt/linux" / kbuild_out])
pmb.chroot.root(args, ["rm", build_path / "src"]) pmb.chroot.root(args, ["rm", build_path / "src"])

7
pmb/build/other.py
View file

@ -33,13 +33,14 @@ def copy_to_buildpath(args: PmbArgs, package, chroot: Chroot=Chroot.native()):
# Copy aport contents with resolved symlinks # Copy aport contents with resolved symlinks
pmb.helpers.run.root(args, ["mkdir", "-p", build]) pmb.helpers.run.root(args, ["mkdir", "-p", build])
for entry in aport.iterdir(): for entry in aport.iterdir():
file = entry.name
# Don't copy those dirs, as those have probably been generated by running `abuild` # Don't copy those dirs, as those have probably been generated by running `abuild`
# on the host system directly and not cleaning up after itself. # on the host system directly and not cleaning up after itself.
# Those dirs might contain broken symlinks and cp fails resolving them. # Those dirs might contain broken symlinks and cp fails resolving them.
if entry.name in ["src", "pkg"]: if file in ["src", "pkg"]:
logging.warn(f"WARNING: Not copying {entry}, looks like a leftover from abuild") logging.warning(f"WARNING: Not copying {file}, looks like a leftover from abuild")
continue continue
pmb.helpers.run.root(args, ["cp", "-rL", aport / entry, build / entry]) pmb.helpers.run.root(args, ["cp", "-rL", aport / file, build / file])
pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", pmb.chroot.root(args, ["chown", "-R", "pmos:pmos",
"/home/pmos/build"], chroot) "/home/pmos/build"], chroot)

22
pmb/chroot/apk.py
View file

@ -168,7 +168,7 @@ def packages_get_locally_built_apks(args: PmbArgs, packages, arch: str):
return ret return ret
def install_run_apk(args: PmbArgs, to_add, to_add_local, to_del, suffix): def install_run_apk(args: PmbArgs, to_add, to_add_local, to_del, chroot: Chroot):
""" """
Run apk to add packages, and ensure only the desired packages get Run apk to add packages, and ensure only the desired packages get
explicitly marked as installed. explicitly marked as installed.
@ -178,7 +178,7 @@ def install_run_apk(args: PmbArgs, to_add, to_add_local, to_del, suffix):
:param to_del: list of pkgnames to be deleted, this should be set to :param to_del: list of pkgnames to be deleted, this should be set to
conflicting dependencies in any of the packages to be conflicting dependencies in any of the packages to be
installed or their dependencies (e.g. ["unl0kr"]) installed or their dependencies (e.g. ["unl0kr"])
:param suffix: the chroot suffix, e.g. "native" or "rootfs_qemu-amd64" :param chroot: the chroot suffix, e.g. "native" or "rootfs_qemu-amd64"
""" """
# Sanitize packages: don't allow '--allow-untrusted' and other options # Sanitize packages: don't allow '--allow-untrusted' and other options
# to be passed to apk! # to be passed to apk!
@ -210,16 +210,16 @@ def install_run_apk(args: PmbArgs, to_add, to_add_local, to_del, suffix):
command = ["--no-network"] + command command = ["--no-network"] + command
if i == 0: if i == 0:
pmb.helpers.apk.apk_with_progress(args, ["apk"] + command, pmb.helpers.apk.apk_with_progress(args, ["apk"] + command,
chroot=True, suffix=suffix) run_in_chroot=True, chroot=chroot)
else: else:
# Virtual package related commands don't actually install or remove # Virtual package related commands don't actually install or remove
# packages, but only mark the right ones as explicitly installed. # packages, but only mark the right ones as explicitly installed.
# They finish up almost instantly, so don't display a progress bar. # They finish up almost instantly, so don't display a progress bar.
pmb.chroot.root(args, ["apk", "--no-progress"] + command, pmb.chroot.root(args, ["apk", "--no-progress"] + command,
chroot=suffix) chroot=chroot)
def install(args: PmbArgs, packages, suffix: Chroot=Chroot.native(), build=True): def install(args: PmbArgs, packages, chroot: Chroot=Chroot.native(), build=True):
""" """
Install packages from pmbootstrap's local package index or the pmOS/Alpine Install packages from pmbootstrap's local package index or the pmOS/Alpine
binary package mirrors. Iterate over all dependencies recursively, and binary package mirrors. Iterate over all dependencies recursively, and
@ -232,7 +232,7 @@ def install(args: PmbArgs, packages, suffix: Chroot=Chroot.native(), build=True)
special case that all packages are expected to be in Alpine's special case that all packages are expected to be in Alpine's
repositories, set this to False for performance optimization. repositories, set this to False for performance optimization.
""" """
arch = pmb.parse.arch.from_chroot_suffix(args, suffix) arch = pmb.parse.arch.from_chroot_suffix(args, chroot)
if not packages: if not packages:
logging.verbose("pmb.chroot.apk.install called with empty packages list," logging.verbose("pmb.chroot.apk.install called with empty packages list,"
@ -240,10 +240,10 @@ def install(args: PmbArgs, packages, suffix: Chroot=Chroot.native(), build=True)
return return
# Initialize chroot # Initialize chroot
check_min_version(args, suffix) check_min_version(args, chroot)
pmb.chroot.init(args, suffix) pmb.chroot.init(args, chroot)
packages_with_depends = pmb.parse.depends.recurse(args, packages, suffix) packages_with_depends = pmb.parse.depends.recurse(args, packages, chroot)
to_add, to_del = packages_split_to_add_del(packages_with_depends) to_add, to_del = packages_split_to_add_del(packages_with_depends)
if build: if build:
@ -253,8 +253,8 @@ def install(args: PmbArgs, packages, suffix: Chroot=Chroot.native(), build=True)
to_add_local = packages_get_locally_built_apks(args, to_add, arch) to_add_local = packages_get_locally_built_apks(args, to_add, arch)
to_add_no_deps, _ = packages_split_to_add_del(packages) to_add_no_deps, _ = packages_split_to_add_del(packages)
logging.info(f"({suffix}) install {' '.join(to_add_no_deps)}") logging.info(f"({chroot}) install {' '.join(to_add_no_deps)}")
install_run_apk(args, to_add_no_deps, to_add_local, to_del, suffix) install_run_apk(args, to_add_no_deps, to_add_local, to_del, chroot)
def installed(args: PmbArgs, suffix: Chroot=Chroot.native()): def installed(args: PmbArgs, suffix: Chroot=Chroot.native()):

9
pmb/chroot/apk_static.py
View file

@ -67,8 +67,8 @@ def extract_temp(tar, sigfilename):
for ftype in ret.keys(): for ftype in ret.keys():
member = tar.getmember(ret[ftype]["filename"]) member = tar.getmember(ret[ftype]["filename"])
handle, path = tempfile.mkstemp(ftype, "pmbootstrap") fd, path = tempfile.mkstemp(ftype, "pmbootstrap")
handle = open(handle, "wb") handle = open(fd, "wb")
ret[ftype]["temp_path"] = path ret[ftype]["temp_path"] = path
shutil.copyfileobj(tar.extractfile(member), handle) shutil.copyfileobj(tar.extractfile(member), handle)
@ -119,8 +119,7 @@ def extract(args: PmbArgs, version, apk_path):
logging.debug("Verify the version reported by the apk.static binary" logging.debug("Verify the version reported by the apk.static binary"
f" (must match the package version {version})") f" (must match the package version {version})")
os.chmod(temp_path, os.stat(temp_path).st_mode | stat.S_IEXEC) os.chmod(temp_path, os.stat(temp_path).st_mode | stat.S_IEXEC)
version_bin = pmb.helpers.run.user(args, [temp_path, "--version"], version_bin = pmb.helpers.run.user_output(args, [temp_path, "--version"])
output_return=True)
version_bin = version_bin.split(" ")[1].split(",")[0] version_bin = version_bin.split(" ")[1].split(",")[0]
if not version.startswith(f"{version_bin}-r"): if not version.startswith(f"{version_bin}-r"):
os.unlink(temp_path) os.unlink(temp_path)
@ -174,4 +173,4 @@ def run(args: PmbArgs, parameters):
if args.offline: if args.offline:
parameters = ["--no-network"] + parameters parameters = ["--no-network"] + parameters
pmb.helpers.apk.apk_with_progress( pmb.helpers.apk.apk_with_progress(
args, [pmb.config.work / "apk.static"] + parameters, chroot=False) args, [pmb.config.work / "apk.static"] + parameters, run_in_chroot=False)

7
pmb/chroot/init.py
View file

@ -5,7 +5,6 @@ import filecmp
from typing import List from typing import List
from pmb.helpers import logging from pmb.helpers import logging
import os import os
import filecmp
import pmb.chroot import pmb.chroot
import pmb.chroot.apk_static import pmb.chroot.apk_static
@ -17,7 +16,7 @@ import pmb.helpers.run
import pmb.parse.arch import pmb.parse.arch
from pmb.core import Chroot, ChrootType from pmb.core import Chroot, ChrootType
cache_chroot_is_outdated = [] cache_chroot_is_outdated: List[str] = []
class UsrMerge(enum.Enum): class UsrMerge(enum.Enum):
""" """
@ -98,7 +97,7 @@ def warn_if_chroot_is_outdated(args: PmbArgs, chroot: Chroot):
global cache_chroot_is_outdated global cache_chroot_is_outdated
# Only check / display the warning once per session # Only check / display the warning once per session
if chroot in cache_chroot_is_outdated: if str(chroot) in cache_chroot_is_outdated:
return return
if pmb.config.workdir.chroots_outdated(args, chroot): if pmb.config.workdir.chroots_outdated(args, chroot):
@ -107,7 +106,7 @@ def warn_if_chroot_is_outdated(args: PmbArgs, chroot: Chroot):
f" {days_warn} days. Consider running" f" {days_warn} days. Consider running"
" 'pmbootstrap zap'.") " 'pmbootstrap zap'.")
cache_chroot_is_outdated += [chroot] cache_chroot_is_outdated += [str(chroot)]
def init(args: PmbArgs, chroot: Chroot=Chroot.native(), usr_merge=UsrMerge.AUTO, def init(args: PmbArgs, chroot: Chroot=Chroot.native(), usr_merge=UsrMerge.AUTO,

3
pmb/chroot/mount.py
View file

@ -93,6 +93,7 @@ def mount(args: PmbArgs, chroot: Chroot=Chroot.native()):
# Mount if necessary # Mount if necessary
for source, target in mountpoints.items(): for source, target in mountpoints.items():
target_outer = chroot / target target_outer = chroot / target
#raise RuntimeError("test")
pmb.helpers.mount.bind(args, source, target_outer) pmb.helpers.mount.bind(args, source, target_outer)
@ -101,7 +102,7 @@ def mount_native_into_foreign(args: PmbArgs, chroot: Chroot):
target = chroot / "native" target = chroot / "native"
pmb.helpers.mount.bind(args, source, target) pmb.helpers.mount.bind(args, source, target)
musl = next(source.glob("/lib/ld-musl-*.so.1")).name musl = next(source.glob("lib/ld-musl-*.so.1")).name
musl_link = (chroot / "lib" / musl) musl_link = (chroot / "lib" / musl)
if not musl_link.is_symlink(): if not musl_link.is_symlink():
pmb.helpers.run.root(args, ["ln", "-s", "/native/lib/" + musl, pmb.helpers.run.root(args, ["ln", "-s", "/native/lib/" + musl,

13
pmb/chroot/root.py
View file

@ -1,7 +1,7 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import os import os
from pathlib import Path from pathlib import Path, PurePath
import shutil import shutil
from typing import Sequence from typing import Sequence
@ -11,7 +11,7 @@ import pmb.chroot.binfmt
import pmb.helpers.run import pmb.helpers.run
import pmb.helpers.run_core import pmb.helpers.run_core
from pmb.core import Chroot from pmb.core import Chroot
from pmb.core.types import PathString, PmbArgs from pmb.core.types import Env, PathString, PmbArgs
def executables_absolute_path(): def executables_absolute_path():
@ -29,7 +29,7 @@ def executables_absolute_path():
return ret return ret
def root(args: PmbArgs, cmd: Sequence[PathString], chroot: Chroot=Chroot.native(), working_dir: Path=Path("/"), output="log", def root(args: PmbArgs, cmd: Sequence[PathString], chroot: Chroot=Chroot.native(), working_dir: PurePath=PurePath("/"), output="log",
output_return=False, check=None, env={}, auto_init=True, output_return=False, check=None, env={}, auto_init=True,
disable_timeout=False, add_proxy_env_vars=True): disable_timeout=False, add_proxy_env_vars=True):
""" """
@ -38,6 +38,7 @@ def root(args: PmbArgs, cmd: Sequence[PathString], chroot: Chroot=Chroot.native(
:param env: dict of environment variables to be passed to the command, e.g. :param env: dict of environment variables to be passed to the command, e.g.
{"JOBS": "5"} {"JOBS": "5"}
:param auto_init: automatically initialize the chroot :param auto_init: automatically initialize the chroot
:param working_dir: chroot-relative working directory
:param add_proxy_env_vars: if True, preserve HTTP_PROXY etc. vars from host :param add_proxy_env_vars: if True, preserve HTTP_PROXY etc. vars from host
environment. pmb.chroot.user sets this to False environment. pmb.chroot.user sets this to False
when calling pmb.chroot.root, because it already when calling pmb.chroot.root, because it already
@ -59,12 +60,12 @@ def root(args: PmbArgs, cmd: Sequence[PathString], chroot: Chroot=Chroot.native(
msg = f"({chroot}) % " msg = f"({chroot}) % "
for key, value in env.items(): for key, value in env.items():
msg += f"{key}={value} " msg += f"{key}={value} "
if working_dir != Path("/"): if working_dir != PurePath("/"):
msg += f"cd {working_dir}; " msg += f"cd {working_dir}; "
msg += " ".join(cmd_str) msg += " ".join(cmd_str)
# Merge env with defaults into env_all # Merge env with defaults into env_all
env_all = {"CHARSET": "UTF-8", env_all: Env = {"CHARSET": "UTF-8",
"HISTFILE": "~/.ash_history", "HISTFILE": "~/.ash_history",
"HOME": "/root", "HOME": "/root",
"LANG": "UTF-8", "LANG": "UTF-8",
@ -83,7 +84,7 @@ def root(args: PmbArgs, cmd: Sequence[PathString], chroot: Chroot=Chroot.native(
# cmd_sudo: ["sudo", "env", "-i", "sh", "-c", "PATH=... /sbin/chroot ..."] # cmd_sudo: ["sudo", "env", "-i", "sh", "-c", "PATH=... /sbin/chroot ..."]
executables = executables_absolute_path() executables = executables_absolute_path()
cmd_chroot = [executables["chroot"], chroot.path, "/bin/sh", "-c", cmd_chroot = [executables["chroot"], chroot.path, "/bin/sh", "-c",
pmb.helpers.run_core.flat_cmd(cmd_str, working_dir)] pmb.helpers.run_core.flat_cmd(cmd_str, Path(working_dir))]
cmd_sudo = pmb.config.sudo([ cmd_sudo = pmb.config.sudo([
"env", "-i", executables["sh"], "-c", "env", "-i", executables["sh"], "-c",
pmb.helpers.run_core.flat_cmd(cmd_chroot, env=env_all)] pmb.helpers.run_core.flat_cmd(cmd_chroot, env=env_all)]

7
pmb/chroot/zap.py
View file

@ -11,7 +11,7 @@ from pmb.core.types import PmbArgs
import pmb.helpers.pmaports import pmb.helpers.pmaports
import pmb.helpers.run import pmb.helpers.run
import pmb.parse.apkindex import pmb.parse.apkindex
from pmb.core import Chroot, ChrootType from pmb.core import Chroot
def zap(args: PmbArgs, confirm=True, dry=False, pkgs_local=False, http=False, def zap(args: PmbArgs, confirm=True, dry=False, pkgs_local=False, http=False,
@ -65,6 +65,7 @@ def zap(args: PmbArgs, confirm=True, dry=False, pkgs_local=False, http=False,
# Delete everything matching the patterns # Delete everything matching the patterns
for pattern in patterns: for pattern in patterns:
logging.debug(f"Deleting {pattern}")
pattern = os.path.realpath(f"{pmb.config.work}/{pattern}") pattern = os.path.realpath(f"{pmb.config.work}/{pattern}")
matches = glob.glob(pattern) matches = glob.glob(pattern)
for match in matches: for match in matches:
@ -114,7 +115,7 @@ def zap_pkgs_local_mismatch(args: PmbArgs, confirm=True, dry=False):
continue continue
# Aport path # Aport path
aport_path = pmb.helpers.pmaports.find(args, origin, False) aport_path = pmb.helpers.pmaports.find_optional(args, origin)
if not aport_path: if not aport_path:
logging.info(f"% rm {apk_path_short}" logging.info(f"% rm {apk_path_short}"
f" ({origin} aport not found)") f" ({origin} aport not found)")
@ -154,7 +155,7 @@ def zap_pkgs_online_mismatch(args: PmbArgs, confirm=True, dry=False):
suffix = Chroot.native() suffix = Chroot.native()
else: else:
try: try:
suffix = Chroot(ChrootType.BUILDROOT, arch) suffix = Chroot.buildroot(arch)
except ValueError: except ValueError:
continue # Ignore invalid directory name continue # Ignore invalid directory name

24
pmb/config/__init__.py
View file

@ -3,14 +3,16 @@
import multiprocessing import multiprocessing
import os import os
from pathlib import Path from pathlib import Path
from pmb.core.types import PathString from pmb.core.types import AportGenEntry, PathString
import pmb.parse.arch import pmb.parse.arch
import sys import sys
from typing import Sequence from typing import Dict, List, Sequence, TypedDict
# #
# Exported functions # Exported functions
# #
# FIXME (#2324): this sucks, we should re-organise this and not rely on "lifting"
# this functions this way
from pmb.config.load import load, sanity_checks from pmb.config.load import load, sanity_checks
from pmb.config.save import save from pmb.config.save import save
from pmb.config.merge_with_args import merge_with_args from pmb.config.merge_with_args import merge_with_args
@ -24,7 +26,7 @@ from pmb.config.other import is_systemd_selected
pmb_src: Path = Path(Path(__file__) / "../../..").resolve() pmb_src: Path = Path(Path(__file__) / "../../..").resolve()
apk_keys_path: Path = (pmb_src / "pmb/data/keys") apk_keys_path: Path = (pmb_src / "pmb/data/keys")
arch_native = pmb.parse.arch.alpine_native() arch_native = pmb.parse.arch.alpine_native()
work: Path = Path("/unitialised/pmbootstrap/work/dir") work: Path
# apk-tools minimum version # apk-tools minimum version
# https://pkgs.alpinelinux.org/packages?name=apk-tools&branch=edge # https://pkgs.alpinelinux.org/packages?name=apk-tools&branch=edge
@ -67,7 +69,7 @@ required_programs = [
] ]
def sudo(cmd: Sequence[PathString]) -> Sequence[str]: def sudo(cmd: Sequence[PathString]) -> Sequence[PathString]:
"""Adapt a command to run as root.""" """Adapt a command to run as root."""
sudo = which_sudo() sudo = which_sudo()
if sudo: if sudo:
@ -81,7 +83,7 @@ def work_dir(_work: Path) -> None:
work directory before any other code is run. It is not meant to be used work directory before any other code is run. It is not meant to be used
anywhere else.""" anywhere else."""
global work global work
if work: if "work" in globals():
raise RuntimeError("work_dir() called multiple times!") raise RuntimeError("work_dir() called multiple times!")
work = _work work = _work
@ -947,10 +949,10 @@ flash_methods = [
# These folders will be mounted at the same location into the native # These folders will be mounted at the same location into the native
# chroot, before the flash programs get started. # chroot, before the flash programs get started.
flash_mount_bind = [ flash_mount_bind = [
"sys/bus/usb/devices/", Path("/sys/bus/usb/devices/"),
"sys/dev/", Path("/sys/dev/"),
"sys/devices/", Path("/sys/devices/"),
"dev/bus/usb/" Path("/dev/bus/usb/"),
] ]
""" """
@ -969,7 +971,7 @@ Fastboot specific: $KERNEL_CMDLINE
Heimdall specific: $PARTITION_INITFS Heimdall specific: $PARTITION_INITFS
uuu specific: $UUU_SCRIPT uuu specific: $UUU_SCRIPT
""" """
flashers = { flashers: Dict[str, Dict[str, bool | List[str] | Dict[str, List[List[str]]]]] = {
"fastboot": { "fastboot": {
"depends": [], # pmaports.cfg: supported_fastboot_depends "depends": [], # pmaports.cfg: supported_fastboot_depends
"actions": { "actions": {
@ -1129,7 +1131,7 @@ git_repos = {
# #
# APORTGEN # APORTGEN
# #
aportgen = { aportgen: Dict[str, AportGenEntry] = {
"cross": { "cross": {
"prefixes": ["busybox-static", "gcc", "musl", "grub-efi"], "prefixes": ["busybox-static", "gcc", "musl", "grub-efi"],
"confirm_overwrite": False, "confirm_overwrite": False,

7
pmb/config/init.py
View file

@ -423,8 +423,7 @@ def ask_for_device(args: PmbArgs):
device = f"{vendor}-{codename}" device = f"{vendor}-{codename}"
device_path = pmb.helpers.devices.find_path(args, device, 'deviceinfo') device_path = pmb.helpers.devices.find_path(args, device, 'deviceinfo')
device_exists = device_path is not None if device_path is None:
if not device_exists:
if device == args.device: if device == args.device:
raise RuntimeError( raise RuntimeError(
"This device does not exist anymore, check" "This device does not exist anymore, check"
@ -449,7 +448,7 @@ def ask_for_device(args: PmbArgs):
break break
kernel = ask_for_device_kernel(args, device) kernel = ask_for_device_kernel(args, device)
return (device, device_exists, kernel) return (device, device_path is not None, kernel)
def ask_for_additional_options(args: PmbArgs, cfg): def ask_for_additional_options(args: PmbArgs, cfg):
@ -739,7 +738,7 @@ def frontend(args: PmbArgs):
# Zap existing chroots # Zap existing chroots
if (work_exists and device_exists and if (work_exists and device_exists and
len(glob.glob(pmb.config.work / "chroot_*")) and len(list(Chroot.iter_patterns())) and
pmb.helpers.cli.confirm( pmb.helpers.cli.confirm(
args, "Zap existing chroots to apply configuration?", args, "Zap existing chroots to apply configuration?",
default=True)): default=True)):

19
pmb/config/pmaports.py
View file

@ -9,13 +9,14 @@ import pmb.config
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
import pmb.helpers.git import pmb.helpers.git
import pmb.helpers.pmaports import pmb.helpers.pmaports
import pmb.parse.version
def check_legacy_folder(): def check_legacy_folder():
# Existing pmbootstrap/aports must be a symlink # Existing pmbootstrap/aports must be a symlink
link = pmb.config.pmb_src + "/aports" link = pmb.config.pmb_src / "aports"
if os.path.exists(link) and not os.path.islink(link): if os.path.exists(link) and not os.path.islink(link):
raise RuntimeError("The path '" + link + "' should be a" raise RuntimeError(f"The path '{link}' should be a"
" symlink pointing to the new pmaports" " symlink pointing to the new pmaports"
" repository, which was split from the" " repository, which was split from the"
" pmbootstrap repository (#383). Consider" " pmbootstrap repository (#383). Consider"
@ -58,21 +59,21 @@ def check_version_pmaports(real):
raise RuntimeError("Run 'pmbootstrap pull' to update your pmaports.") raise RuntimeError("Run 'pmbootstrap pull' to update your pmaports.")
def check_version_pmbootstrap(min): def check_version_pmbootstrap(min_ver):
# Compare versions # Compare versions
real = pmb.__version__ real = pmb.__version__
if pmb.parse.version.compare(real, min) >= 0: if pmb.parse.version.compare(real, min_ver) >= 0:
return return
# Show versions # Show versions
logging.info("NOTE: you are using pmbootstrap version " + real + ", but" + logging.info(f"NOTE: you are using pmbootstrap version {real}, but"
" version " + min + " is required.") f" version {min_ver} is required.")
# Error for git clone # Error for git clone
pmb_src = pmb.config.pmb_src pmb_src = pmb.config.pmb_src
if os.path.exists(pmb_src + "/.git"): if os.path.exists(pmb_src / ".git"):
raise RuntimeError("Please update your local pmbootstrap repository." raise RuntimeError("Please update your local pmbootstrap repository."
" Usually with: 'git -C \"" + pmb_src + "\" pull'") f" Usually with: 'git -C \"{pmb_src}\" pull'")
# Error for package manager installation # Error for package manager installation
raise RuntimeError("Please update your pmbootstrap version (with your" raise RuntimeError("Please update your pmbootstrap version (with your"
@ -121,7 +122,7 @@ def read_config(args: PmbArgs):
path_cfg = args.aports / "pmaports.cfg" path_cfg = args.aports / "pmaports.cfg"
if not os.path.exists(path_cfg): if not os.path.exists(path_cfg):
raise RuntimeError("Invalid pmaports repository, could not find the" raise RuntimeError("Invalid pmaports repository, could not find the"
" config: " + path_cfg) f" config: {path_cfg}")
# Load the config # Load the config
cfg = configparser.ConfigParser() cfg = configparser.ConfigParser()

2
pmb/config/save.py
View file

@ -7,7 +7,7 @@ from pmb.core.types import PmbArgs
def save(args: PmbArgs, cfg): def save(args: PmbArgs, cfg):
logging.debug("Save config: " + args.config) logging.debug(f"Save config: {args.config}")
os.makedirs(os.path.dirname(args.config), 0o700, True) os.makedirs(os.path.dirname(args.config), 0o700, True)
with open(args.config, "w") as handle: with open(args.config, "w") as handle:
cfg.write(handle) cfg.write(handle)

13
pmb/core/chroot.py
View file

@ -4,7 +4,7 @@
from __future__ import annotations from __future__ import annotations
import enum import enum
from typing import Generator, Optional from typing import Generator, Optional
from pathlib import Path, PosixPath from pathlib import Path, PosixPath, PurePosixPath
import pmb.config import pmb.config
class ChrootType(enum.Enum): class ChrootType(enum.Enum):
@ -86,7 +86,7 @@ class Chroot:
def __truediv__(self, other: object) -> Path: def __truediv__(self, other: object) -> Path:
if isinstance(other, PosixPath): if isinstance(other, PosixPath) or isinstance(other, PurePosixPath):
# Convert the other path to a relative path # Convert the other path to a relative path
# FIXME: we should avoid creating absolute paths that we actually want # FIXME: we should avoid creating absolute paths that we actually want
# to make relative to the chroot... # to make relative to the chroot...
@ -99,8 +99,8 @@ class Chroot:
def __rtruediv__(self, other: object) -> Path: def __rtruediv__(self, other: object) -> Path:
if isinstance(other, PosixPath): if isinstance(other, PosixPath) or isinstance(other, PurePosixPath):
return other / self.path return Path(other) / self.path
if isinstance(other, str): if isinstance(other, str):
return other / self.path return other / self.path
@ -120,6 +120,11 @@ class Chroot:
return Chroot(ChrootType.NATIVE) return Chroot(ChrootType.NATIVE)
@staticmethod
def buildroot(arch: str) -> Chroot:
return Chroot(ChrootType.BUILDROOT, arch)
@staticmethod @staticmethod
def rootfs(device: str) -> Chroot: def rootfs(device: str) -> Chroot:
return Chroot(ChrootType.ROOTFS, device) return Chroot(ChrootType.ROOTFS, device)

45
pmb/core/types.py
View file

@ -3,9 +3,24 @@
from argparse import Namespace from argparse import Namespace
from pathlib import Path from pathlib import Path
from typing import Dict, Union from typing import Dict, List, Optional, Tuple, TypedDict, Union
PathString = Union[Path, str] PathString = Union[Path, str]
Env = Dict[str, PathString]
# These types are not definitive / API, they exist to describe the current
# state of things so that we can improve our type hinting coverage and make
# future refactoring efforts easier.
class PartitionLayout(TypedDict):
kernel: Optional[int]
boot: int
reserve: Optional[int]
root: int
class AportGenEntry(TypedDict):
prefixes: List[str]
confirm_overwrite: bool
# Property list generated with: # Property list generated with:
# $ rg --vimgrep "((^|\s)args\.\w+)" --only-matching | cut -d"." -f3 | sort | uniq # $ rg --vimgrep "((^|\s)args\.\w+)" --only-matching | cut -d"." -f3 | sort | uniq
@ -28,7 +43,7 @@ class PmbArgs(Namespace):
autoinstall: str autoinstall: str
boot_size: str boot_size: str
build_default_device_arch: str build_default_device_arch: str
build_pkgs_on_install: str build_pkgs_on_install: bool
buildroot: str buildroot: str
built: str built: str
ccache_size: str ccache_size: str
@ -38,17 +53,17 @@ class PmbArgs(Namespace):
command: str command: str
config: Path config: Path
config_channels: str config_channels: str
details: str details: bool
details_to_stdout: str details_to_stdout: str
device: str device: str
deviceinfo: Dict[str, str] deviceinfo: Dict[str, str]
deviceinfo_parse_kernel: str deviceinfo_parse_kernel: str
devices: str devices: str
disk: str disk: Path
dry: str dry: str
efi: str efi: str
envkernel: str envkernel: str
export_folder: str export_folder: Path
extra_packages: str extra_packages: str
extra_space: str extra_space: str
fast: str fast: str
@ -58,7 +73,8 @@ class PmbArgs(Namespace):
folder: str folder: str
force: str force: str
fork_alpine: str fork_alpine: str
from_argparse: str # This is a filthy lie
from_argparse: "PmbArgs"
full_disk_encryption: str full_disk_encryption: str
hook: str hook: str
host: str host: str
@ -68,7 +84,7 @@ class PmbArgs(Namespace):
install_base: str install_base: str
install_blockdev: str install_blockdev: str
install_cgpt: str install_cgpt: str
install_key: str install_key: bool
install_local_pkgs: str install_local_pkgs: str
install_recommends: str install_recommends: str
is_default_channel: str is_default_channel: str
@ -80,7 +96,7 @@ class PmbArgs(Namespace):
lines: str lines: str
log: Path log: Path
mirror_alpine: str mirror_alpine: str
mirrors_postmarketos: str mirrors_postmarketos: List[str]
name: str name: str
no_depends: str no_depends: str
no_fde: str no_fde: str
@ -91,15 +107,16 @@ class PmbArgs(Namespace):
no_sshd: str no_sshd: str
odin_flashable_tar: str odin_flashable_tar: str
offline: str offline: str
ondev_cp: str ondev_cp: List[Tuple[str, str]]
on_device_installer: str on_device_installer: str
ondev_no_rootfs: str ondev_no_rootfs: str
overview: str overview: str
package: str # FIXME (#2324): figure out the args.package vs args.packages situation
packages: str package: str | List[str]
packages: List[str]
partition: str partition: str
password: str password: str
path: str path: Path
pkgname: str pkgname: str
pkgname_pkgver_srcurl: str pkgname_pkgver_srcurl: str
port: str port: str
@ -122,7 +139,7 @@ class PmbArgs(Namespace):
rsync: str rsync: str
scripts: str scripts: str
second_storage: str second_storage: str
selected_providers: str selected_providers: Dict[str, str]
sparse: str sparse: str
split: str split: str
src: str src: str
@ -131,7 +148,7 @@ class PmbArgs(Namespace):
sudo_timer: str sudo_timer: str
suffix: str suffix: str
systemd: str systemd: str
timeout: str timeout: float
ui: str ui: str
ui_extras: str ui_extras: str
user: str user: str

2
pmb/export/frontend.py
View file

@ -28,7 +28,7 @@ def frontend(args: PmbArgs):
pmb.chroot.initfs.build(args, flavor, Chroot(ChrootType.ROOTFS, args.device)) pmb.chroot.initfs.build(args, flavor, Chroot(ChrootType.ROOTFS, args.device))
# Do the export, print all files # Do the export, print all files
logging.info("Export symlinks to: " + target) logging.info(f"Export symlinks to: {target}")
if args.odin_flashable_tar: if args.odin_flashable_tar:
pmb.export.odin(args, flavor, target) pmb.export.odin(args, flavor, target)
pmb.export.symlinks(args, flavor, target) pmb.export.symlinks(args, flavor, target)

2
pmb/export/symlinks.py
View file

@ -45,7 +45,7 @@ def symlinks(args: PmbArgs, flavor, folder: Path):
# Generate a list of patterns # Generate a list of patterns
chroot_native = Chroot.native() chroot_native = Chroot.native()
path_boot = Chroot(ChrootType.ROOTFS, args.device) / "boot" path_boot = Chroot(ChrootType.ROOTFS, args.device) / "boot"
chroot_buildroot = Chroot(ChrootType.BUILDROOT, args.deviceinfo['arch']) chroot_buildroot = Chroot.buildroot(args.deviceinfo['arch'])
files: List[Path] = [ files: List[Path] = [
path_boot / f"boot.img{suffix}", path_boot / f"boot.img{suffix}",
path_boot / f"uInitrd{suffix}", path_boot / f"uInitrd{suffix}",

2
pmb/flasher/frontend.py
View file

@ -93,7 +93,7 @@ def sideload(args: PmbArgs):
pmb.flasher.install_depends(args) pmb.flasher.install_depends(args)
# Mount the buildroot # Mount the buildroot
chroot = Chroot(ChrootType.BUILDROOT, args.deviceinfo["arch"]) chroot = Chroot.buildroot(args.deviceinfo["arch"])
mountpoint = "/mnt/" / chroot mountpoint = "/mnt/" / chroot
pmb.helpers.mount.bind(args, chroot.path, pmb.helpers.mount.bind(args, chroot.path,
Chroot.native().path / mountpoint) Chroot.native().path / mountpoint)

2
pmb/flasher/run.py
View file

@ -22,6 +22,8 @@ def run(args: PmbArgs, action, flavor=None):
# Verify action # Verify action
method = args.flash_method or args.deviceinfo["flash_method"] method = args.flash_method or args.deviceinfo["flash_method"]
cfg = pmb.config.flashers[method] cfg = pmb.config.flashers[method]
if not isinstance(cfg["actions"], dict):
raise TypeError(f"Flashers misconfigured! {method} key 'actions' should be a dictionary")
if action not in cfg["actions"]: if action not in cfg["actions"]:
raise RuntimeError("action " + action + " is not" raise RuntimeError("action " + action + " is not"
" configured for method " + method + "!" " configured for method " + method + "!"

4
pmb/flasher/variables.py
View file

@ -1,5 +1,6 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
from typing import Optional
import pmb.config.pmaports import pmb.config.pmaports
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
@ -15,6 +16,9 @@ def variables(args: PmbArgs, flavor, method):
# updated and minimum pmbootstrap version bumped. # updated and minimum pmbootstrap version bumped.
# See also https://gitlab.com/postmarketOS/pmbootstrap/-/issues/2243 # See also https://gitlab.com/postmarketOS/pmbootstrap/-/issues/2243
_partition_kernel: Optional[str]
_partition_rootfs: Optional[str]
if method.startswith("fastboot"): if method.startswith("fastboot"):
_partition_kernel = args.deviceinfo["flash_fastboot_partition_kernel"]\ _partition_kernel = args.deviceinfo["flash_fastboot_partition_kernel"]\
or "boot" or "boot"

25
pmb/helpers/apk.py
View file

@ -1,10 +1,12 @@
# Copyright 2023 Johannes Marbach, Oliver Smith # Copyright 2023 Johannes Marbach, Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import os import os
from pathlib import Path
from typing import List, Sequence
import pmb.chroot.root import pmb.chroot.root
import pmb.config.pmaports import pmb.config.pmaports
from pmb.core.types import PmbArgs from pmb.core.types import PathString, PmbArgs
import pmb.helpers.cli import pmb.helpers.cli
import pmb.helpers.run import pmb.helpers.run
import pmb.helpers.run_core import pmb.helpers.run_core
@ -12,7 +14,7 @@ import pmb.parse.version
from pmb.core import Chroot from pmb.core import Chroot
def _run(args: PmbArgs, command, chroot=False, suffix: Chroot=Chroot.native(), output="log"): def _run(args: PmbArgs, command, run_in_chroot=False, chroot: Chroot=Chroot.native(), output="log"):
"""Run a command. """Run a command.
:param command: command in list form :param command: command in list form
@ -23,8 +25,8 @@ def _run(args: PmbArgs, command, chroot=False, suffix: Chroot=Chroot.native(), o
See pmb.helpers.run_core.core() for a detailed description of all other See pmb.helpers.run_core.core() for a detailed description of all other
arguments and the return value. arguments and the return value.
""" """
if chroot: if run_in_chroot:
return pmb.chroot.root(args, command, output=output, suffix=suffix, return pmb.chroot.root(args, command, output=output, chroot=chroot,
disable_timeout=True) disable_timeout=True)
return pmb.helpers.run.root(args, command, output=output) return pmb.helpers.run.root(args, command, output=output)
@ -41,7 +43,7 @@ def _prepare_fifo(args: PmbArgs, run_in_chroot=False, chroot: Chroot=Chroot.nati
relative to the host) relative to the host)
""" """
if run_in_chroot: if run_in_chroot:
fifo = "/tmp/apk_progress_fifo" fifo = Path("/tmp/apk_progress_fifo")
fifo_outside = chroot / fifo fifo_outside = chroot / fifo
else: else:
_run(args, ["mkdir", "-p", pmb.config.work / "tmp"]) _run(args, ["mkdir", "-p", pmb.config.work / "tmp"])
@ -82,7 +84,7 @@ def _compute_progress(line):
return cur / tot if tot > 0 else 0 return cur / tot if tot > 0 else 0
def apk_with_progress(args: PmbArgs, command, chroot=False, suffix: Chroot=Chroot.native()): def apk_with_progress(args: PmbArgs, command: Sequence[PathString], run_in_chroot=False, chroot: Chroot=Chroot.native()):
"""Run an apk subcommand while printing a progress bar to STDOUT. """Run an apk subcommand while printing a progress bar to STDOUT.
:param command: apk subcommand in list form :param command: apk subcommand in list form
@ -91,12 +93,13 @@ def apk_with_progress(args: PmbArgs, command, chroot=False, suffix: Chroot=Chroo
set to True. set to True.
:raises RuntimeError: when the apk command fails :raises RuntimeError: when the apk command fails
""" """
fifo, fifo_outside = _prepare_fifo(args, chroot, suffix) fifo, fifo_outside = _prepare_fifo(args, run_in_chroot, chroot)
command_with_progress = _create_command_with_progress(command, fifo) _command: List[str] = [os.fspath(c) for c in command]
log_msg = " ".join(command) command_with_progress = _create_command_with_progress(_command, fifo)
with _run(args, ['cat', fifo], chroot=chroot, suffix=suffix, log_msg = " ".join(_command)
with _run(args, ['cat', fifo], run_in_chroot=run_in_chroot, chroot=chroot,
output="pipe") as p_cat: output="pipe") as p_cat:
with _run(args, command_with_progress, chroot=chroot, suffix=suffix, with _run(args, command_with_progress, run_in_chroot=run_in_chroot, chroot=chroot,
output="background") as p_apk: output="background") as p_apk:
while p_apk.poll() is None: while p_apk.poll() is None:
line = p_cat.stdout.readline().decode('utf-8') line = p_cat.stdout.readline().decode('utf-8')

7
pmb/helpers/aportupgrade.py
View file

@ -6,15 +6,15 @@ from pmb.helpers import logging
import os import os
import re import re
import urllib.parse import urllib.parse
from typing import Optional from typing import Dict, Optional
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
import pmb.helpers.file import pmb.helpers.file
import pmb.helpers.http import pmb.helpers.http
import pmb.helpers.pmaports import pmb.helpers.pmaports
req_headers = None req_headers: Dict[str, str]
req_headers_github = None req_headers_github: Dict[str, str]
ANITYA_API_BASE = "https://release-monitoring.org/api/v2" ANITYA_API_BASE = "https://release-monitoring.org/api/v2"
GITHUB_API_BASE = "https://api.github.com" GITHUB_API_BASE = "https://api.github.com"
@ -272,7 +272,6 @@ def upgrade(args: PmbArgs, pkgname, git=True, stable=True) -> None:
if stable: if stable:
upgrade_stable_package(args, pkgname, package) upgrade_stable_package(args, pkgname, package)
return False
def upgrade_all(args: PmbArgs) -> None: def upgrade_all(args: PmbArgs) -> None:
"""Upgrade all packages, based on args.all, args.all_git and args.all_stable.""" """Upgrade all packages, based on args.all, args.all_git and args.all_stable."""

6
pmb/helpers/args.py
View file

@ -1,11 +1,11 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import copy import copy
import os
from pathlib import Path from pathlib import Path
import pmb.config import pmb.config
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
import pmb.helpers.git import pmb.helpers.git
import pmb.helpers.args
"""This file constructs the args variable, which is passed to almost all """This file constructs the args variable, which is passed to almost all
functions in the pmbootstrap code base. Here's a listing of the kind of functions in the pmbootstrap code base. Here's a listing of the kind of
@ -75,7 +75,7 @@ def check_pmaports_path(args: PmbArgs):
""" """
if args.from_argparse.aports and not os.path.exists(args.aports): if args.from_argparse.aports and not os.path.exists(args.aports):
raise ValueError("pmaports path (specified with --aports) does" raise ValueError("pmaports path (specified with --aports) does"
" not exist: " + args.aports) f" not exist: {args.aports}")
def replace_placeholders(args: PmbArgs): def replace_placeholders(args: PmbArgs):
@ -108,7 +108,7 @@ def add_deviceinfo(args: PmbArgs):
" <https://postmarketos.org/newarch>") " <https://postmarketos.org/newarch>")
def init(args: PmbArgs): def init(args: PmbArgs) -> PmbArgs:
# Basic initialization # Basic initialization
fix_mirrors_postmarketos(args) fix_mirrors_postmarketos(args)
pmb.config.merge_with_args(args) pmb.config.merge_with_args(args)

6
pmb/helpers/devices.py
View file

@ -1,13 +1,13 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import os import os
import glob
from pathlib import Path from pathlib import Path
from typing import Optional
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
import pmb.parse import pmb.parse
def find_path(args: PmbArgs, codename: str, file='') -> Path: def find_path(args: PmbArgs, codename: str, file='') -> Optional[Path]:
"""Find path to device APKBUILD under `device/*/device-`. """Find path to device APKBUILD under `device/*/device-`.
:param codename: device codename :param codename: device codename
@ -58,7 +58,7 @@ def list_apkbuilds(args: PmbArgs):
""":returns: { "first-device": {"pkgname": ..., "pkgver": ...}, ... }""" """:returns: { "first-device": {"pkgname": ..., "pkgver": ...}, ... }"""
ret = {} ret = {}
for device in list_codenames(args): for device in list_codenames(args):
apkbuild_path = f"{args.aports}/device/*/device-{device}/APKBUILD" apkbuild_path = next(args.aports.glob(f"device/*/device-{device}/APKBUILD"))
ret[device] = pmb.parse.apkbuild(apkbuild_path) ret[device] = pmb.parse.apkbuild(apkbuild_path)
return ret return ret

11
pmb/helpers/file.py
View file

@ -7,16 +7,17 @@ import time
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
import pmb.helpers.run import pmb.helpers.run
import pmb.helpers.pmaports
def replace(path, old, new): def replace(path: Path, old: str, new: str):
text = "" text = ""
with open(path, "r", encoding="utf-8") as handle: with path.open("r", encoding="utf-8") as handle:
text = handle.read() text = handle.read()
text = text.replace(old, new) text = text.replace(old, new)
with open(path, "w", encoding="utf-8") as handle: with path.open("w", encoding="utf-8") as handle:
handle.write(text) handle.write(text)
@ -29,7 +30,7 @@ def replace_apkbuild(args: PmbArgs, pkgname, key, new, in_quotes=False):
:param in_quotes: expect the value to be in quotation marks ("") :param in_quotes: expect the value to be in quotation marks ("")
""" """
# Read old value # Read old value
path = pmb.helpers.pmaports.find(args, pkgname) + "/APKBUILD" path = pmb.helpers.pmaports.find(args, pkgname) / "APKBUILD"
apkbuild = pmb.parse.apkbuild(path) apkbuild = pmb.parse.apkbuild(path)
old = apkbuild[key] old = apkbuild[key]
@ -95,7 +96,7 @@ def symlink(args: PmbArgs, file: Path, link: Path):
if (os.path.islink(link) and if (os.path.islink(link) and
os.path.realpath(os.readlink(link)) == os.path.realpath(file)): os.path.realpath(os.readlink(link)) == os.path.realpath(file)):
return return
raise RuntimeError("File exists: " + link) raise RuntimeError(f"File exists: {link}")
elif link.is_symlink(): elif link.is_symlink():
link.unlink() link.unlink()

51
pmb/helpers/frontend.py
View file

@ -1,6 +1,7 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import json import json
from typing import List, Sequence
from pmb.helpers import logging from pmb.helpers import logging
import os import os
from pathlib import Path from pathlib import Path
@ -14,7 +15,7 @@ import pmb.chroot.initfs
import pmb.chroot.other import pmb.chroot.other
import pmb.ci import pmb.ci
import pmb.config import pmb.config
from pmb.core.types import PmbArgs from pmb.core.types import PathString, PmbArgs
import pmb.export import pmb.export
import pmb.flasher import pmb.flasher
import pmb.helpers.aportupgrade import pmb.helpers.aportupgrade
@ -35,7 +36,6 @@ import pmb.netboot
import pmb.parse import pmb.parse
import pmb.qemu import pmb.qemu
import pmb.sideload import pmb.sideload
from argparse import Namespace
from pmb.core import ChrootType, Chroot from pmb.core import ChrootType, Chroot
@ -48,13 +48,13 @@ def _parse_flavor(args: PmbArgs, autoinstall=True):
# identifier that is typically in the form # identifier that is typically in the form
# "postmarketos-<manufacturer>-<device/chip>", e.g. # "postmarketos-<manufacturer>-<device/chip>", e.g.
# "postmarketos-qcom-sdm845" # "postmarketos-qcom-sdm845"
suffix = Chroot(ChrootType.ROOTFS, args.device) chroot = Chroot(ChrootType.ROOTFS, args.device)
flavor = pmb.chroot.other.kernel_flavor_installed( flavor = pmb.chroot.other.kernel_flavor_installed(
args, suffix, autoinstall) args, chroot, autoinstall)
if not flavor: if not flavor:
raise RuntimeError( raise RuntimeError(
"No kernel flavors installed in chroot " + suffix + "! Please let" f"No kernel flavors installed in chroot '{chroot}'! Please let"
" your device package depend on a package starting with 'linux-'.") " your device package depend on a package starting with 'linux-'.")
return flavor return flavor
@ -64,9 +64,9 @@ def _parse_suffix(args: PmbArgs) -> Chroot:
return Chroot(ChrootType.ROOTFS, args.device) return Chroot(ChrootType.ROOTFS, args.device)
elif args.buildroot: elif args.buildroot:
if args.buildroot == "device": if args.buildroot == "device":
return Chroot(ChrootType.BUILDROOT, args.deviceinfo["arch"]) return Chroot.buildroot(args.deviceinfo["arch"])
else: else:
return Chroot(ChrootType.BUILDROOT, args.buildroot) return Chroot.buildroot(args.buildroot)
elif args.suffix: elif args.suffix:
(_t, s) = args.suffix.split("_") (_t, s) = args.suffix.split("_")
t: ChrootType = ChrootType(_t) t: ChrootType = ChrootType(_t)
@ -416,11 +416,16 @@ def kconfig(args: PmbArgs):
raise RuntimeError("kconfig check failed!") raise RuntimeError("kconfig check failed!")
# Default to all kernel packages # Default to all kernel packages
packages: List[str]
# FIXME (#2324): figure out the args.package vs args.packages situation
if isinstance(args.package, list):
packages = args.package packages = args.package
else:
packages = [args.package]
if not args.package: if not args.package:
for aport in pmb.helpers.pmaports.get_list(args): for pkg in pmb.helpers.pmaports.get_list(args):
if aport.startswith("linux-"): if pkg.startswith("linux-"):
packages.append(aport.split("linux-")[1]) packages.append(pkg.split("linux-")[1])
# Iterate over all kernels # Iterate over all kernels
error = False error = False
@ -431,7 +436,7 @@ def kconfig(args: PmbArgs):
pkgname = package if package.startswith("linux-") \ pkgname = package if package.startswith("linux-") \
else "linux-" + package else "linux-" + package
aport = pmb.helpers.pmaports.find(args, pkgname) aport = pmb.helpers.pmaports.find(args, pkgname)
apkbuild = pmb.parse.apkbuild(f"{aport}/APKBUILD") apkbuild = pmb.parse.apkbuild(aport)
if "!pmb:kconfigcheck" in apkbuild["options"]: if "!pmb:kconfigcheck" in apkbuild["options"]:
skipped += 1 skipped += 1
continue continue
@ -449,7 +454,7 @@ def kconfig(args: PmbArgs):
logging.info("kconfig check succeeded!") logging.info("kconfig check succeeded!")
elif args.action_kconfig in ["edit", "migrate"]: elif args.action_kconfig in ["edit", "migrate"]:
if args.package: if args.package:
pkgname = args.package pkgname = args.package if isinstance(args.package, str) else args.package[0]
else: else:
pkgname = args.deviceinfo["codename"] pkgname = args.deviceinfo["codename"]
use_oldconfig = args.action_kconfig == "migrate" use_oldconfig = args.action_kconfig == "migrate"
@ -472,7 +477,7 @@ def deviceinfo_parse(args: PmbArgs):
def apkbuild_parse(args: PmbArgs): def apkbuild_parse(args: PmbArgs):
# Default to all packages # Default to all packages
packages = args.packages packages: Sequence[str] = args.packages
if not packages: if not packages:
packages = pmb.helpers.pmaports.get_list(args) packages = pmb.helpers.pmaports.get_list(args)
@ -480,8 +485,7 @@ def apkbuild_parse(args: PmbArgs):
for package in packages: for package in packages:
print(package + ":") print(package + ":")
aport = pmb.helpers.pmaports.find(args, package) aport = pmb.helpers.pmaports.find(args, package)
path = aport + "/APKBUILD" print(json.dumps(pmb.parse.apkbuild(aport), indent=4,
print(json.dumps(pmb.parse.apkbuild(path), indent=4,
sort_keys=True)) sort_keys=True))
@ -489,8 +493,7 @@ def apkindex_parse(args: PmbArgs):
result = pmb.parse.apkindex.parse(args.apkindex_path) result = pmb.parse.apkindex.parse(args.apkindex_path)
if args.package: if args.package:
if args.package not in result: if args.package not in result:
raise RuntimeError("Package not found in the APKINDEX: " + raise RuntimeError(f"Package not found in the APKINDEX: {args.package}")
args.package)
result = result[args.package] result = result[args.package]
print(json.dumps(result, indent=4)) print(json.dumps(result, indent=4))
@ -536,14 +539,14 @@ def shutdown(args: PmbArgs):
def stats(args: PmbArgs): def stats(args: PmbArgs):
# Chroot suffix # Chroot suffix
suffix = "native" chroot = Chroot.native()
if args.arch != pmb.config.arch_native: if args.arch != pmb.config.arch_native:
suffix = "buildroot_" + args.arch chroot = Chroot.buildroot(args.arch)
# Install ccache and display stats # Install ccache and display stats
pmb.chroot.apk.install(args, ["ccache"], suffix) pmb.chroot.apk.install(args, ["ccache"], chroot)
logging.info(f"({suffix}) % ccache -s") logging.info(f"({chroot}) % ccache -s")
pmb.chroot.user(args, ["ccache", "-s"], suffix, output="stdout") pmb.chroot.user(args, ["ccache", "-s"], chroot, output="stdout")
def work_migrate(args: PmbArgs): def work_migrate(args: PmbArgs):
@ -558,7 +561,7 @@ def log(args: PmbArgs):
pmb.helpers.run.user(args, ["truncate", "-s", "0", args.log]) pmb.helpers.run.user(args, ["truncate", "-s", "0", args.log])
pmb.helpers.run.user(args, ["truncate", "-s", "0", log_testsuite]) pmb.helpers.run.user(args, ["truncate", "-s", "0", log_testsuite])
cmd = ["tail", "-n", args.lines, "-F"] cmd: List[PathString] = ["tail", "-n", args.lines, "-F"]
# Follow the testsuite's log file too if it exists. It will be created when # Follow the testsuite's log file too if it exists. It will be created when
# starting a test case that writes to it (git -C test grep log_testsuite). # starting a test case that writes to it (git -C test grep log_testsuite).
@ -620,7 +623,7 @@ def pull(args: PmbArgs):
def lint(args: PmbArgs): def lint(args: PmbArgs):
packages = args.packages packages: Sequence[str] = args.packages
if not packages: if not packages:
packages = pmb.helpers.pmaports.get_list(args) packages = pmb.helpers.pmaports.get_list(args)

28
pmb/helpers/git.py
View file

@ -1,6 +1,8 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import configparser import configparser
from pathlib import Path
from typing import Dict
from pmb.helpers import logging from pmb.helpers import logging
import os import os
from pathlib import Path from pathlib import Path
@ -22,7 +24,7 @@ def get_path(args: PmbArgs, name_repo):
""" """
if name_repo == "pmaports": if name_repo == "pmaports":
return args.aports return args.aports
return pmb.config.work / "cache_git/" + name_repo return pmb.config.work / "cache_git" / name_repo
def clone(args: PmbArgs, name_repo): def clone(args: PmbArgs, name_repo):
@ -66,7 +68,7 @@ def rev_parse(args: PmbArgs, path, revision="HEAD", extra_args: list = []):
or (with ``--abbrev-ref``): the branch name, e.g. "master" or (with ``--abbrev-ref``): the branch name, e.g. "master"
""" """
command = ["git", "rev-parse"] + extra_args + [revision] command = ["git", "rev-parse"] + extra_args + [revision]
rev = pmb.helpers.run.user(args, command, path, output_return=True) rev = pmb.helpers.run.user_output(args, command, path)
return rev.rstrip() return rev.rstrip()
@ -84,7 +86,7 @@ def can_fast_forward(args: PmbArgs, path, branch_upstream, branch="HEAD"):
def clean_worktree(args: PmbArgs, path): def clean_worktree(args: PmbArgs, path):
"""Check if there are not any modified files in the git dir.""" """Check if there are not any modified files in the git dir."""
command = ["git", "status", "--porcelain"] command = ["git", "status", "--porcelain"]
return pmb.helpers.run.user(args, command, path, output_return=True) == "" return pmb.helpers.run.user_output(args, command, path) == ""
def get_upstream_remote(args: PmbArgs, name_repo): def get_upstream_remote(args: PmbArgs, name_repo):
@ -95,7 +97,7 @@ def get_upstream_remote(args: PmbArgs, name_repo):
urls = pmb.config.git_repos[name_repo] urls = pmb.config.git_repos[name_repo]
path = get_path(args, name_repo) path = get_path(args, name_repo)
command = ["git", "remote", "-v"] command = ["git", "remote", "-v"]
output = pmb.helpers.run.user(args, command, path, output_return=True) output = pmb.helpers.run.user_output(args, command, path)
for line in output.split("\n"): for line in output.split("\n"):
if any(u in line for u in urls): if any(u in line for u in urls):
return line.split("\t", 1)[0] return line.split("\t", 1)[0]
@ -127,8 +129,8 @@ def parse_channels_cfg(args):
else: else:
remote = get_upstream_remote(args, "pmaports") remote = get_upstream_remote(args, "pmaports")
command = ["git", "show", f"{remote}/master:channels.cfg"] command = ["git", "show", f"{remote}/master:channels.cfg"]
stdout = pmb.helpers.run.user(args, command, args.aports, stdout = pmb.helpers.run.user_output(args, command, args.aports,
output_return=True, check=False) check=False)
try: try:
cfg.read_string(stdout) cfg.read_string(stdout)
except configparser.MissingSectionHeaderError: except configparser.MissingSectionHeaderError:
@ -139,7 +141,7 @@ def parse_channels_cfg(args):
" pmaports clone") " pmaports clone")
# Meta section # Meta section
ret = {"channels": {}} ret: Dict[str, Dict[str, str | Dict[str, str]]] = {"channels": {}}
ret["meta"] = {"recommended": cfg.get("channels.cfg", "recommended")} ret["meta"] = {"recommended": cfg.get("channels.cfg", "recommended")}
# Channels # Channels
@ -153,7 +155,8 @@ def parse_channels_cfg(args):
for key in ["description", "branch_pmaports", "branch_aports", for key in ["description", "branch_pmaports", "branch_aports",
"mirrordir_alpine"]: "mirrordir_alpine"]:
value = cfg.get(channel, key) value = cfg.get(channel, key)
ret["channels"][channel_new][key] = value # FIXME: how to type this properly??
ret["channels"][channel_new][key] = value # type: ignore[index]
pmb.helpers.other.cache[cache_key] = ret pmb.helpers.other.cache[cache_key] = ret
return ret return ret
@ -261,11 +264,10 @@ def get_files(args: PmbArgs, path):
:returns: all files in a git repository as list, relative to path :returns: all files in a git repository as list, relative to path
""" """
ret = [] ret = []
files = pmb.helpers.run.user(args, ["git", "ls-files"], path, files = pmb.helpers.run.user_output(args, ["git", "ls-files"], path).split("\n")
output_return=True).split("\n") files += pmb.helpers.run.user_output(args, ["git", "ls-files",
files += pmb.helpers.run.user(args, ["git", "ls-files", "--exclude-standard", "--other"],
"--exclude-standard", "--other"], path, path).split("\n")
output_return=True).split("\n")
for file in files: for file in files:
if os.path.exists(f"{path}/{file}"): if os.path.exists(f"{path}/{file}"):
ret += [file] ret += [file]

9
pmb/helpers/http.py
View file

@ -4,12 +4,17 @@ import hashlib
import json import json
from pmb.helpers import logging from pmb.helpers import logging
import os import os
from pathlib import Path
import shutil import shutil
import urllib.request import urllib.request
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
import pmb.helpers.run import pmb.helpers.run
def cache_file(prefix: str, url: str) -> Path:
prefix = prefix.replace("/", "_")
return Path(f"{prefix}_{hashlib.sha256(url.encode('utf-8')).hexdigest()}")
def download(args: PmbArgs, url, prefix, cache=True, loglevel=logging.INFO, def download(args: PmbArgs, url, prefix, cache=True, loglevel=logging.INFO,
allow_404=False): allow_404=False):
@ -33,9 +38,7 @@ def download(args: PmbArgs, url, prefix, cache=True, loglevel=logging.INFO,
pmb.helpers.run.user(args, ["mkdir", "-p", pmb.config.work / "cache_http"]) pmb.helpers.run.user(args, ["mkdir", "-p", pmb.config.work / "cache_http"])
# Check if file exists in cache # Check if file exists in cache
prefix = prefix.replace("/", "_") path = pmb.config.work / "cache_http" / cache_file(prefix, url)
path = (pmb.config.work / "cache_http/" + prefix + "_" +
hashlib.sha256(url.encode("utf-8")).hexdigest())
if os.path.exists(path): if os.path.exists(path):
if cache: if cache:
return path return path

8
pmb/helpers/lint.py
View file

@ -1,5 +1,6 @@
# Copyright 2023 Danct12 <danct12@disroot.org> # Copyright 2023 Danct12 <danct12@disroot.org>
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
from pathlib import Path
from pmb.helpers import logging from pmb.helpers import logging
import os import os
@ -20,7 +21,7 @@ def check(args: PmbArgs, pkgnames):
# Mount pmaports.git inside the chroot so that we don't have to copy the # Mount pmaports.git inside the chroot so that we don't have to copy the
# package folders # package folders
pmaports = "/mnt/pmaports" pmaports = Path("/mnt/pmaports")
pmb.build.mount_pmaports(args, pmaports) pmb.build.mount_pmaports(args, pmaports)
# Locate all APKBUILDs and make the paths be relative to the pmaports # Locate all APKBUILDs and make the paths be relative to the pmaports
@ -28,9 +29,8 @@ def check(args: PmbArgs, pkgnames):
apkbuilds = [] apkbuilds = []
for pkgname in pkgnames: for pkgname in pkgnames:
aport = pmb.helpers.pmaports.find(args, pkgname) aport = pmb.helpers.pmaports.find(args, pkgname)
if not os.path.exists(aport + "/APKBUILD"): if not (aport / "APKBUILD").exists():
raise ValueError("Path does not contain an APKBUILD file:" + raise ValueError(f"Path does not contain an APKBUILD file: {aport}")
aport)
relpath = os.path.relpath(aport, args.aports) relpath = os.path.relpath(aport, args.aports)
apkbuilds.append(f"{relpath}/APKBUILD") apkbuilds.append(f"{relpath}/APKBUILD")

35
pmb/helpers/logging.py
View file

@ -3,15 +3,29 @@
import logging import logging
import os import os
import sys import sys
from typing import TextIO
import pmb.config import pmb.config
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
logfd = None logfd: TextIO
CRITICAL = logging.CRITICAL
FATAL = logging.FATAL
ERROR = logging.ERROR
WARNING = logging.WARNING
WARN = logging.WARN
INFO = logging.INFO
DEBUG = logging.DEBUG
NOTSET = logging.NOTSET
VERBOSE = 5
class log_handler(logging.StreamHandler): class log_handler(logging.StreamHandler):
"""Write to stdout and to the already opened log file.""" """Write to stdout and to the already opened log file."""
_args = None _args: PmbArgs
def __init__(self, args: PmbArgs):
super().__init__()
self._args = args
def emit(self, record): def emit(self, record):
try: try:
@ -80,13 +94,13 @@ def add_verbose_log_level():
All stackoverflow user contributions are licensed as CC-BY-SA: All stackoverflow user contributions are licensed as CC-BY-SA:
https://creativecommons.org/licenses/by-sa/3.0/ https://creativecommons.org/licenses/by-sa/3.0/
""" """
logging.VERBOSE = 5 setattr(logging, "VERBOSE", VERBOSE)
logging.addLevelName(logging.VERBOSE, "VERBOSE") logging.addLevelName(VERBOSE, "VERBOSE")
logging.Logger.verbose = lambda inst, msg, * \ setattr(logging.Logger, "verbose", lambda inst, msg, * \
args, **kwargs: inst.log(logging.VERBOSE, msg, *args, **kwargs) args, **kwargs: inst.log(VERBOSE, msg, *args, **kwargs))
logging.verbose = lambda msg, *args, **kwargs: logging.log(logging.VERBOSE, setattr(logging, "verbose", lambda msg, *args, **kwargs: logging.log(VERBOSE,
msg, *args, msg, *args,
**kwargs) **kwargs))
def init(args: PmbArgs): def init(args: PmbArgs):
@ -118,11 +132,10 @@ def init(args: PmbArgs):
add_verbose_log_level() add_verbose_log_level()
root_logger.setLevel(logging.DEBUG) root_logger.setLevel(logging.DEBUG)
if args.verbose: if args.verbose:
root_logger.setLevel(logging.VERBOSE) root_logger.setLevel(VERBOSE)
# Add a custom log handler # Add a custom log handler
handler = log_handler() handler = log_handler(args)
log_handler._args = args
handler.setFormatter(formatter) handler.setFormatter(formatter)
root_logger.addHandler(handler) root_logger.addHandler(handler)

12
pmb/helpers/mount.py
View file

@ -37,6 +37,8 @@ def bind(args: PmbArgs, source: Path, destination: Path, create_folders=True, um
else: else:
return return
print(f"Mounting {source} -> {destination}")
# Check/create folders # Check/create folders
for path in [source, destination]: for path in [source, destination]:
if os.path.exists(path): if os.path.exists(path):
@ -44,8 +46,7 @@ def bind(args: PmbArgs, source: Path, destination: Path, create_folders=True, um
if create_folders: if create_folders:
pmb.helpers.run.root(args, ["mkdir", "-p", path]) pmb.helpers.run.root(args, ["mkdir", "-p", path])
else: else:
raise RuntimeError("Mount failed, folder does not exist: " + raise RuntimeError(f"Mount failed, folder does not exist: {path}")
path)
# Actually mount the folder # Actually mount the folder
pmb.helpers.run.root(args, ["mount", "--bind", source, destination]) pmb.helpers.run.root(args, ["mount", "--bind", source, destination])
@ -89,8 +90,7 @@ def umount_all_list(prefix: Path, source: Path=Path("/proc/mounts")) -> List[Pat
for line in handle: for line in handle:
words = line.split() words = line.split()
if len(words) < 2: if len(words) < 2:
raise RuntimeError("Failed to parse line in " + source + ": " + raise RuntimeError(f"Failed to parse line in {source}: {line}")
line)
mountpoint = Path(words[1].replace(r"\040(deleted)", "")) mountpoint = Path(words[1].replace(r"\040(deleted)", ""))
if mountpoint.is_relative_to(prefix): # is subpath if mountpoint.is_relative_to(prefix): # is subpath
ret.append(mountpoint) ret.append(mountpoint)
@ -103,7 +103,7 @@ def umount_all(args: PmbArgs, folder: Path):
for mountpoint in umount_all_list(folder): for mountpoint in umount_all_list(folder):
pmb.helpers.run.root(args, ["umount", mountpoint]) pmb.helpers.run.root(args, ["umount", mountpoint])
if ismount(mountpoint): if ismount(mountpoint):
raise RuntimeError("Failed to umount: " + mountpoint) raise RuntimeError(f"Failed to umount: {mountpoint}")
def mount_device_rootfs(args: PmbArgs, chroot_rootfs: Chroot) -> PurePath: def mount_device_rootfs(args: PmbArgs, chroot_rootfs: Chroot) -> PurePath:
@ -114,7 +114,7 @@ def mount_device_rootfs(args: PmbArgs, chroot_rootfs: Chroot) -> PurePath:
"rootfs_qemu-amd64") "rootfs_qemu-amd64")
:returns: the mountpoint (relative to the native chroot) :returns: the mountpoint (relative to the native chroot)
""" """
mountpoint = PurePath("/mnt", chroot_rootfs.dirname()) mountpoint = PurePath("/mnt", chroot_rootfs.dirname)
pmb.helpers.mount.bind(args, chroot_rootfs.path, pmb.helpers.mount.bind(args, chroot_rootfs.path,
Chroot.native() / mountpoint) Chroot.native() / mountpoint)
return mountpoint return mountpoint

2
pmb/helpers/other.py
View file

@ -12,6 +12,8 @@ import pmb.helpers.pmaports
import pmb.helpers.run import pmb.helpers.run
from typing import Dict, Any from typing import Dict, Any
from typing import Any, Dict
def folder_size(args: PmbArgs, path: Path): def folder_size(args: PmbArgs, path: Path):
"""Run `du` to calculate the size of a folder. """Run `du` to calculate the size of a folder.

3
pmb/helpers/package.py
View file

@ -9,6 +9,7 @@ See also:
- pmb/helpers/repo.py (work with binary package repos) - pmb/helpers/repo.py (work with binary package repos)
""" """
import copy import copy
from typing import Any, Dict
from pmb.helpers import logging from pmb.helpers import logging
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
@ -57,7 +58,7 @@ def get(args: PmbArgs, pkgname, arch, replace_subpkgnames=False, must_exist=True
] ]
# Find in pmaports # Find in pmaports
ret = None ret: Dict[str, Any] = {}
pmaport = pmb.helpers.pmaports.get(args, pkgname, False) pmaport = pmb.helpers.pmaports.get(args, pkgname, False)
if pmaport: if pmaport:
ret = {"arch": pmaport["arch"], ret = {"arch": pmaport["arch"],

10
pmb/helpers/pkgrel_bump.py
View file

@ -18,7 +18,7 @@ def package(args: PmbArgs, pkgname, reason="", dry=False):
:param dry: don't modify the APKBUILD, just print the message :param dry: don't modify the APKBUILD, just print the message
""" """
# Current and new pkgrel # Current and new pkgrel
path = pmb.helpers.pmaports.find(args, pkgname) + "/APKBUILD" path = pmb.helpers.pmaports.find(args, pkgname) / "APKBUILD"
apkbuild = pmb.parse.apkbuild(path) apkbuild = pmb.parse.apkbuild(path)
pkgrel = int(apkbuild["pkgrel"]) pkgrel = int(apkbuild["pkgrel"])
pkgrel_new = pkgrel + 1 pkgrel_new = pkgrel + 1
@ -91,7 +91,7 @@ def auto_apkindex_package(args: PmbArgs, arch, aport, apk, dry=False):
# (which means dynamic libraries that the package was linked # (which means dynamic libraries that the package was linked
# against) and packages for which no aport exists. # against) and packages for which no aport exists.
if (depend.startswith("so:") or if (depend.startswith("so:") or
not pmb.helpers.pmaports.find(args, depend, False)): not pmb.helpers.pmaports.find_optional(args, depend)):
missing.append(depend) missing.append(depend)
# Increase pkgrel # Increase pkgrel
@ -107,7 +107,7 @@ def auto(args: PmbArgs, dry=False):
for arch in pmb.config.build_device_architectures: for arch in pmb.config.build_device_architectures:
paths = pmb.helpers.repo.apkindex_files(args, arch, alpine=False) paths = pmb.helpers.repo.apkindex_files(args, arch, alpine=False)
for path in paths: for path in paths:
logging.info("scan " + path) logging.info(f"scan {path}")
index = pmb.parse.apkindex.parse(path, False) index = pmb.parse.apkindex.parse(path, False)
for pkgname, apk in index.items(): for pkgname, apk in index.items():
origin = apk["origin"] origin = apk["origin"]
@ -116,12 +116,12 @@ def auto(args: PmbArgs, dry=False):
logging.verbose( logging.verbose(
f"{pkgname}: origin '{origin}' found again") f"{pkgname}: origin '{origin}' found again")
continue continue
aport_path = pmb.helpers.pmaports.find(args, origin, False) aport_path = pmb.helpers.pmaports.find_optional(args, origin)
if not aport_path: if not aport_path:
logging.warning("{}: origin '{}' aport not found".format( logging.warning("{}: origin '{}' aport not found".format(
pkgname, origin)) pkgname, origin))
continue continue
aport = pmb.parse.apkbuild(f"{aport_path}/APKBUILD") aport = pmb.parse.apkbuild(aport_path)
if auto_apkindex_package(args, arch, aport, apk, dry): if auto_apkindex_package(args, arch, aport, apk, dry):
ret.append(pkgname) ret.append(pkgname)
return ret return ret

38
pmb/helpers/pmaports.py
View file

@ -8,7 +8,6 @@ See also:
""" """
import glob import glob
from pmb.helpers import logging from pmb.helpers import logging
import os
from pathlib import Path from pathlib import Path
from typing import Optional, Sequence, Dict from typing import Optional, Sequence, Dict
@ -44,7 +43,7 @@ def get_list(args: PmbArgs) -> Sequence[str]:
return list(_find_apkbuilds(args).keys()) return list(_find_apkbuilds(args).keys())
def guess_main_dev(args: PmbArgs, subpkgname): def guess_main_dev(args: PmbArgs, subpkgname) -> Optional[Path]:
"""Check if a package without "-dev" at the end exists in pmaports or not, and log the appropriate message. """Check if a package without "-dev" at the end exists in pmaports or not, and log the appropriate message.
Don't call this function directly, use guess_main() instead. Don't call this function directly, use guess_main() instead.
@ -57,7 +56,7 @@ def guess_main_dev(args: PmbArgs, subpkgname):
if path: if path:
logging.verbose(subpkgname + ": guessed to be a subpackage of " + logging.verbose(subpkgname + ": guessed to be a subpackage of " +
pkgname + " (just removed '-dev')") pkgname + " (just removed '-dev')")
return os.path.dirname(path) return path.parent
logging.verbose(subpkgname + ": guessed to be a subpackage of " + pkgname + logging.verbose(subpkgname + ": guessed to be a subpackage of " + pkgname +
", which we can't find in pmaports, so it's probably in" ", which we can't find in pmaports, so it's probably in"
@ -147,7 +146,7 @@ def find(args: PmbArgs, package: str, must_exist=True) -> Path:
""" """
# Try to get a cached result first (we assume that the aports don't change # Try to get a cached result first (we assume that the aports don't change
# in one pmbootstrap call) # in one pmbootstrap call)
ret = Path() ret: Optional[Path] = None
if package in pmb.helpers.other.cache["find_aport"]: if package in pmb.helpers.other.cache["find_aport"]:
ret = pmb.helpers.other.cache["find_aport"][package] ret = pmb.helpers.other.cache["find_aport"][package]
else: else:
@ -165,7 +164,7 @@ def find(args: PmbArgs, package: str, must_exist=True) -> Path:
guess = guess_main(args, package) guess = guess_main(args, package)
if guess: if guess:
# Parse the APKBUILD and verify if the guess was right # Parse the APKBUILD and verify if the guess was right
if _find_package_in_apkbuild(package, f'{guess}/APKBUILD'): if _find_package_in_apkbuild(package, guess / "APKBUILD"):
ret = guess ret = guess
else: else:
# Otherwise parse all APKBUILDs (takes time!), is the # Otherwise parse all APKBUILDs (takes time!), is the
@ -183,7 +182,7 @@ def find(args: PmbArgs, package: str, must_exist=True) -> Path:
ret = guess ret = guess
# Crash when necessary # Crash when necessary
if ret is None and must_exist: if ret is None:
raise RuntimeError("Could not find aport for package: " + raise RuntimeError("Could not find aport for package: " +
package) package)
@ -192,6 +191,13 @@ def find(args: PmbArgs, package: str, must_exist=True) -> Path:
return ret return ret
def find_optional(args: PmbArgs, package: str) -> Optional[Path]:
try:
return find(args, package)
except RuntimeError:
return None
def get(args: PmbArgs, pkgname, must_exist=True, subpackages=True): def get(args: PmbArgs, pkgname, must_exist=True, subpackages=True):
"""Find and parse an APKBUILD file. """Find and parse an APKBUILD file.
@ -213,9 +219,12 @@ def get(args: PmbArgs, pkgname, must_exist=True, subpackages=True):
""" """
pkgname = pmb.helpers.package.remove_operators(pkgname) pkgname = pmb.helpers.package.remove_operators(pkgname)
if subpackages: if subpackages:
aport = find(args, pkgname, must_exist) aport = find_optional(args, pkgname)
if aport: if aport:
return pmb.parse.apkbuild(f"{aport}/APKBUILD") return pmb.parse.apkbuild(aport / "APKBUILD")
elif must_exist:
raise RuntimeError("Could not find APKBUILD for package:"
f" {pkgname}")
else: else:
path = _find_apkbuilds(args).get(pkgname) path = _find_apkbuilds(args).get(pkgname)
if path: if path:
@ -251,7 +260,8 @@ def find_providers(args: PmbArgs, provide):
key=lambda p: p[1].get('provider_priority', 0)) key=lambda p: p[1].get('provider_priority', 0))
def get_repo(args: PmbArgs, pkgname, must_exist=True): # FIXME (#2324): split into an _optional variant or drop must_exist
def get_repo(args: PmbArgs, pkgname, must_exist=True) -> Optional[str]:
"""Get the repository folder of an aport. """Get the repository folder of an aport.
:pkgname: package name :pkgname: package name
@ -259,10 +269,14 @@ def get_repo(args: PmbArgs, pkgname, must_exist=True):
:returns: a string like "main", "device", "cross", ... :returns: a string like "main", "device", "cross", ...
or None when the aport could not be found or None when the aport could not be found
""" """
aport = find(args, pkgname, must_exist) aport: Optional[Path]
if must_exist:
aport = find(args, pkgname)
else:
aport = find_optional(args, pkgname)
if not aport: if not aport:
return None return None
return os.path.basename(os.path.dirname(aport)) return aport.parent.name
def check_arches(arches, arch): def check_arches(arches, arch):
@ -284,7 +298,7 @@ def check_arches(arches, arch):
return False return False
def get_channel_new(channel): def get_channel_new(channel: str) -> str:
"""Translate legacy channel names to the new ones. """Translate legacy channel names to the new ones.
Legacy names are still supported for compatibility with old branches (pmb#2015). Legacy names are still supported for compatibility with old branches (pmb#2015).

13
pmb/helpers/repo.py
View file

@ -17,9 +17,10 @@ import pmb.config.pmaports
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
import pmb.helpers.http import pmb.helpers.http
import pmb.helpers.run import pmb.helpers.run
import pmb.helpers.other
def hash(url, length=8): def apkindex_hash(url: str, length: int=8) -> Path:
r"""Generate the hash that APK adds to the APKINDEX and apk packages in its apk cache folder. r"""Generate the hash that APK adds to the APKINDEX and apk packages in its apk cache folder.
It is the "12345678" part in this example: It is the "12345678" part in this example:
@ -43,7 +44,7 @@ def hash(url, length=8):
ret += xd[(binary[i] >> 4) & 0xf] ret += xd[(binary[i] >> 4) & 0xf]
ret += xd[binary[i] & 0xf] ret += xd[binary[i] & 0xf]
return ret return Path(f"APKINDEX.{ret}.tar.gz")
def urls(args: PmbArgs, user_repository=True, postmarketos_mirror=True, alpine=True): def urls(args: PmbArgs, user_repository=True, postmarketos_mirror=True, alpine=True):
@ -112,7 +113,7 @@ def apkindex_files(args: PmbArgs, arch=None, user_repository=True, pmos=True,
# Resolve the APKINDEX.$HASH.tar.gz files # Resolve the APKINDEX.$HASH.tar.gz files
for url in urls(args, False, pmos, alpine): for url in urls(args, False, pmos, alpine):
ret.append(pmb.config.work / f"cache_apk_{arch}" / f"APKINDEX.{hash(url)}.tar.gz") ret.append(pmb.config.work / f"cache_apk_{arch}" / apkindex_hash(url))
return ret return ret
@ -151,7 +152,7 @@ def update(args: PmbArgs, arch=None, force=False, existing_only=False):
# APKINDEX file name from the URL # APKINDEX file name from the URL
url_full = url + "/" + arch + "/APKINDEX.tar.gz" url_full = url + "/" + arch + "/APKINDEX.tar.gz"
cache_apk_outside = pmb.config.work / f"cache_apk_{arch}" cache_apk_outside = pmb.config.work / f"cache_apk_{arch}"
apkindex = cache_apk_outside / f"APKINDEX.{hash(url)}.tar.gz" apkindex = cache_apk_outside / f"APKINDEX.{apkindex_hash(url)}.tar.gz"
# Find update reason, possibly skip non-existing or known 404 files # Find update reason, possibly skip non-existing or known 404 files
reason = None reason = None
@ -217,5 +218,5 @@ def alpine_apkindex_path(args: PmbArgs, repo="main", arch=None):
# Find it on disk # Find it on disk
channel_cfg = pmb.config.pmaports.read_config_channel(args) channel_cfg = pmb.config.pmaports.read_config_channel(args)
repo_link = f"{args.mirror_alpine}{channel_cfg['mirrordir_alpine']}/{repo}" repo_link = f"{args.mirror_alpine}{channel_cfg['mirrordir_alpine']}/{repo}"
cache_folder = pmb.config.work / "cache_apk_" + arch cache_folder = pmb.config.work / (f"cache_apk_{arch}")
return cache_folder + "/APKINDEX." + hash(repo_link) + ".tar.gz" return cache_folder / apkindex_hash(repo_link)

13
pmb/helpers/repo_bootstrap.py
View file

@ -1,5 +1,6 @@
# Copyright 2024 Oliver Smith # Copyright 2024 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
from pmb.core.chroot import Chroot
from pmb.helpers import logging from pmb.helpers import logging
import glob import glob
@ -10,7 +11,7 @@ from pmb.core.types import PmbArgs
progress_done = 0 progress_done = 0
progress_total = 0 progress_total = 0
progress_step = None progress_step: str
def get_arch(args: PmbArgs): def get_arch(args: PmbArgs):
@ -115,7 +116,7 @@ def log_progress(msg):
progress_done += 1 progress_done += 1
def run_steps(args: PmbArgs, steps, arch, suffix): def run_steps(args: PmbArgs, steps, arch, chroot: Chroot):
global progress_step global progress_step
for step, bootstrap_line in steps.items(): for step, bootstrap_line in steps.items():
@ -128,14 +129,14 @@ def run_steps(args: PmbArgs, steps, arch, suffix):
if "[usr_merge]" in bootstrap_line: if "[usr_merge]" in bootstrap_line:
usr_merge = pmb.chroot.UsrMerge.ON usr_merge = pmb.chroot.UsrMerge.ON
if suffix != "native": if chroot != Chroot.native():
log_progress(f"initializing native chroot (merge /usr: {usr_merge.name})") log_progress(f"initializing native chroot (merge /usr: {usr_merge.name})")
# Native chroot needs pmOS binary package repo for cross compilers # Native chroot needs pmOS binary package repo for cross compilers
pmb.chroot.init(args, "native", usr_merge) pmb.chroot.init(args, Chroot.native(), usr_merge)
log_progress(f"initializing {suffix} chroot (merge /usr: {usr_merge.name})") log_progress(f"initializing {chroot} chroot (merge /usr: {usr_merge.name})")
# Initialize without pmOS binary package repo # Initialize without pmOS binary package repo
pmb.chroot.init(args, suffix, usr_merge, postmarketos_mirror=False) pmb.chroot.init(args, chroot, usr_merge, postmarketos_mirror=False)
for package in get_packages(bootstrap_line): for package in get_packages(bootstrap_line):
log_progress(f"building {package}") log_progress(f"building {package}")

2
pmb/helpers/repo_missing.py
View file

@ -34,7 +34,7 @@ def filter_aport_packages(args: PmbArgs, arch, pkgnames):
""" """
ret = [] ret = []
for pkgname in pkgnames: for pkgname in pkgnames:
if pmb.helpers.pmaports.find(args, pkgname, False): if pmb.helpers.pmaports.find_optional(args, pkgname):
ret += [pkgname] ret += [pkgname]
return ret return ret

20
pmb/helpers/run.py
View file

@ -2,13 +2,14 @@
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import os import os
from pathlib import Path from pathlib import Path
import subprocess
import pmb.helpers.run_core import pmb.helpers.run_core
from typing import Any, Dict, List, Optional, Sequence from typing import Any, Dict, List, Optional, Sequence
from pmb.core.types import PathString, PmbArgs from pmb.core.types import Env, PathString, PmbArgs
def user(args: PmbArgs, cmd: Sequence[PathString], working_dir: Path=Path("/"), output: str="log", output_return: bool=False, def user(args: PmbArgs, cmd: Sequence[PathString], working_dir: Optional[Path] = None, output: str = "log", output_return: bool = False,
check: Optional[bool]=None, env: Dict[Any, Any]={}, sudo: bool=False) -> str: check: Optional[bool] = None, env: Env = {}, sudo: bool = False) -> str | int | subprocess.Popen:
""" """
Run a command on the host system as user. Run a command on the host system as user.
@ -22,8 +23,8 @@ def user(args: PmbArgs, cmd: Sequence[PathString], working_dir: Path=Path("/"),
# Readable log message (without all the escaping) # Readable log message (without all the escaping)
msg = "% " msg = "% "
for key, value in env.items(): for key, value in env.items():
msg += key + "=" + value + " " msg += f"{key}={value} "
if working_dir != Path("/"): if working_dir is not None:
msg += f"cd {os.fspath(working_dir)}; " msg += f"cd {os.fspath(working_dir)}; "
msg += " ".join(cmd_parts) msg += " ".join(cmd_parts)
@ -35,6 +36,15 @@ def user(args: PmbArgs, cmd: Sequence[PathString], working_dir: Path=Path("/"),
return pmb.helpers.run_core.core(args, msg, cmd_parts, working_dir, output, return pmb.helpers.run_core.core(args, msg, cmd_parts, working_dir, output,
output_return, check, sudo) output_return, check, sudo)
# FIXME: should probably use some kind of wrapper class / builder pattern for all these parameters...
def user_output(args: PmbArgs, cmd: Sequence[PathString], working_dir: Optional[Path] = None, output: str = "log",
check: Optional[bool] = None, env: Env = {}, sudo: bool = False) -> str:
ret = user(args, cmd, working_dir, output, output_return=True, check=check, env=env, sudo=sudo)
if not isinstance(ret, str):
raise TypeError("Expected str output, got " + str(ret))
return ret
def root(args: PmbArgs, cmd: Sequence[PathString], working_dir=None, output="log", output_return=False, def root(args: PmbArgs, cmd: Sequence[PathString], working_dir=None, output="log", output_return=False,
check=None, env={}): check=None, env={}):

30
pmb/helpers/run_core.py
View file

@ -1,6 +1,7 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import fcntl import fcntl
from pmb.core.types import PathString, Env
from pmb.helpers import logging from pmb.helpers import logging
import os import os
from pathlib import Path from pathlib import Path
@ -10,17 +11,17 @@ import subprocess
import sys import sys
import threading import threading
import time import time
from typing import Sequence from typing import Dict, Optional, Sequence
from pmb.core.chroot import Chroot
import pmb.helpers.run import pmb.helpers.run
from pmb.core.types import PathString, PmbArgs
from pmb.core.types import Env, PathString, PmbArgs
"""For a detailed description of all output modes, read the description of """For a detailed description of all output modes, read the description of
core() at the bottom. All other functions in this file get (indirectly) core() at the bottom. All other functions in this file get (indirectly)
called by core(). """ called by core(). """
def flat_cmd(cmd: Sequence[PathString], working_dir: Path=Path("/"), env={}): def flat_cmd(cmd: Sequence[PathString], working_dir: Optional[Path]=None, env: Env={}):
"""Convert a shell command passed as list into a flat shell string with proper escaping. """Convert a shell command passed as list into a flat shell string with proper escaping.
:param cmd: command as list, e.g. ["echo", "string with spaces"] :param cmd: command as list, e.g. ["echo", "string with spaces"]
@ -35,14 +36,14 @@ def flat_cmd(cmd: Sequence[PathString], working_dir: Path=Path("/"), env={}):
# Merge env and cmd into escaped list # Merge env and cmd into escaped list
escaped = [] escaped = []
for key, value in env.items(): for key, value in env.items():
escaped.append(key + "=" + shlex.quote(value)) escaped.append(key + "=" + shlex.quote(os.fspath(value)))
for i in range(len(cmd)): for i in range(len(cmd)):
escaped.append(shlex.quote(os.fspath(cmd[i]))) escaped.append(shlex.quote(os.fspath(cmd[i])))
# Prepend working dir # Prepend working dir
ret = " ".join(escaped) ret = " ".join(escaped)
if working_dir != Path("/"): if working_dir is not None:
ret = "cd " + shlex.quote(working_dir) + ";" + ret ret = "cd " + shlex.quote(str(working_dir)) + ";" + ret
return ret return ret
@ -84,6 +85,7 @@ def pipe(cmd, working_dir=None):
return ret return ret
# FIXME (#2324): These types make no sense at all
def pipe_read(process, output_to_stdout=False, output_return=False, def pipe_read(process, output_to_stdout=False, output_return=False,
output_return_buffer=False): output_return_buffer=False):
"""Read all output from a subprocess, copy it to the log and optionally stdout and a buffer variable. """Read all output from a subprocess, copy it to the log and optionally stdout and a buffer variable.
@ -179,17 +181,21 @@ def foreground_pipe(args: PmbArgs, cmd, working_dir=None, output_to_stdout=False
stdin=stdin) stdin=stdin)
# Make process.stdout non-blocking # Make process.stdout non-blocking
handle = process.stdout.fileno() stdout = process.stdout or None
if not stdout:
raise RuntimeError("Process has no stdout?!")
handle = stdout.fileno()
flags = fcntl.fcntl(handle, fcntl.F_GETFL) flags = fcntl.fcntl(handle, fcntl.F_GETFL)
fcntl.fcntl(handle, fcntl.F_SETFL, flags | os.O_NONBLOCK) fcntl.fcntl(handle, fcntl.F_SETFL, flags | os.O_NONBLOCK)
# While process exists wait for output (with timeout) # While process exists wait for output (with timeout)
output_buffer = [] output_buffer: list[bytes] = []
sel = selectors.DefaultSelector() sel = selectors.DefaultSelector()
sel.register(process.stdout, selectors.EVENT_READ) sel.register(stdout, selectors.EVENT_READ)
timeout = args.timeout if output_timeout else None timeout = args.timeout
while process.poll() is None: while process.poll() is None:
wait_start = time.perf_counter() if output_timeout else None wait_start = time.perf_counter()
sel.select(timeout) sel.select(timeout)
# On timeout raise error (we need to measure time on our own, because # On timeout raise error (we need to measure time on our own, because

4
pmb/helpers/ui.py
View file

@ -16,8 +16,8 @@ def list(args: PmbArgs, arch):
ret = [("none", "Bare minimum OS image for testing and manual" ret = [("none", "Bare minimum OS image for testing and manual"
" customization. The \"console\" UI should be selected if" " customization. The \"console\" UI should be selected if"
" a graphical UI is not desired.")] " a graphical UI is not desired.")]
for path in sorted(glob.glob(args.aports + "/main/postmarketos-ui-*")): for path in sorted(args.aports.glob("main/postmarketos-ui-*")):
apkbuild = pmb.parse.apkbuild(f"{path}/APKBUILD") apkbuild = pmb.parse.apkbuild(path)
ui = os.path.basename(path).split("-", 2)[2] ui = os.path.basename(path).split("-", 2)[2]
if pmb.helpers.package.check_arch(args, apkbuild["pkgname"], arch): if pmb.helpers.package.check_arch(args, apkbuild["pkgname"], arch):
ret.append((ui, apkbuild["pkgdesc"])) ret.append((ui, apkbuild["pkgdesc"]))

64
pmb/install/_install.py
View file

@ -6,8 +6,8 @@ import re
import glob import glob
import shlex import shlex
import sys import sys
from typing import Dict, List from typing import Dict, List, Optional, Sequence, TypedDict
from pathlib import Path, PurePath from pathlib import Path
import pmb.chroot import pmb.chroot
import pmb.chroot.apk import pmb.chroot.apk
@ -15,7 +15,7 @@ import pmb.chroot.other
import pmb.chroot.initfs import pmb.chroot.initfs
import pmb.config import pmb.config
import pmb.config.pmaports import pmb.config.pmaports
from pmb.core.types import PmbArgs from pmb.core.types import PartitionLayout, PmbArgs
import pmb.helpers.devices import pmb.helpers.devices
from pmb.helpers.mount import mount_device_rootfs from pmb.helpers.mount import mount_device_rootfs
import pmb.helpers.run import pmb.helpers.run
@ -60,8 +60,11 @@ def get_nonfree_packages(args: PmbArgs, device):
["device-nokia-n900-nonfree-firmware"] ["device-nokia-n900-nonfree-firmware"]
""" """
# Read subpackages # Read subpackages
apkbuild = pmb.parse.apkbuild(pmb.helpers.devices.find_path(args, device, device_path = pmb.helpers.devices.find_path(args, device, 'APKBUILD')
'APKBUILD')) if not device_path:
raise RuntimeError(f"Device package not found for {device}")
apkbuild = pmb.parse.apkbuild(device_path)
subpackages = apkbuild["subpackages"] subpackages = apkbuild["subpackages"]
# Check for firmware and userland # Check for firmware and userland
@ -123,7 +126,7 @@ def copy_files_from_chroot(args: PmbArgs, chroot: Chroot):
pmb.helpers.run.root(args, ["rm", fifo]) pmb.helpers.run.root(args, ["rm", fifo])
# Get all folders inside the device rootfs (except for home) # Get all folders inside the device rootfs (except for home)
folders: List[PurePath] = [] folders: List[str] = []
for path in mountpoint_outside.glob("*"): for path in mountpoint_outside.glob("*"):
if path.name == "home": if path.name == "home":
continue continue
@ -150,7 +153,7 @@ def create_home_from_skel(args: PmbArgs):
rootfs = (Chroot.native() / "mnt/install") rootfs = (Chroot.native() / "mnt/install")
# In btrfs, home subvol & home dir is created in format.py # In btrfs, home subvol & home dir is created in format.py
if args.filesystem != "btrfs": if args.filesystem != "btrfs":
pmb.helpers.run.root(args, ["mkdir", rootfs + "/home"]) pmb.helpers.run.root(args, ["mkdir", rootfs / "home"])
home = (rootfs / "home" / args.user) home = (rootfs / "home" / args.user)
if (rootfs / "etc/skel").exists(): if (rootfs / "etc/skel").exists():
@ -310,8 +313,7 @@ def copy_ssh_keys(args: PmbArgs):
target = Chroot.native() / "mnt/install/home/" / args.user / ".ssh" target = Chroot.native() / "mnt/install/home/" / args.user / ".ssh"
pmb.helpers.run.root(args, ["mkdir", target]) pmb.helpers.run.root(args, ["mkdir", target])
pmb.helpers.run.root(args, ["chmod", "700", target]) pmb.helpers.run.root(args, ["chmod", "700", target])
pmb.helpers.run.root(args, ["cp", authorized_keys, target + pmb.helpers.run.root(args, ["cp", authorized_keys, target / "authorized_keys"])
"/authorized_keys"])
pmb.helpers.run.root(args, ["rm", authorized_keys]) pmb.helpers.run.root(args, ["rm", authorized_keys])
pmb.helpers.run.root(args, ["chown", "-R", "10000:10000", target]) pmb.helpers.run.root(args, ["chown", "-R", "10000:10000", target])
@ -538,9 +540,9 @@ def generate_binary_list(args: PmbArgs, chroot: Chroot, step):
binaries = args.deviceinfo["sd_embed_firmware"].split(",") binaries = args.deviceinfo["sd_embed_firmware"].split(",")
for binary_offset in binaries: for binary_offset in binaries:
binary, offset = binary_offset.split(':') binary, _offset = binary_offset.split(':')
try: try:
offset = int(offset) offset = int(_offset)
except ValueError: except ValueError:
raise RuntimeError("Value for firmware binary offset is " raise RuntimeError("Value for firmware binary offset is "
f"not valid: {offset}") f"not valid: {offset}")
@ -627,9 +629,9 @@ def write_cgpt_kpart(args: PmbArgs, layout, suffix: Chroot):
def sanity_check_boot_size(args: PmbArgs): def sanity_check_boot_size(args: PmbArgs):
default = pmb.config.defaults["boot_size"] default = pmb.config.defaults["boot_size"]
if int(args.boot_size) >= int(default): if isinstance(default, str) and int(args.boot_size) >= int(default):
return return
logging.error("ERROR: your pmbootstrap has a small boot_size of" logging.error("ERROR: your pmbootstrap has a small/invalid boot_size of"
f" {args.boot_size} configured, probably because the config" f" {args.boot_size} configured, probably because the config"
" has been created with an old version.") " has been created with an old version.")
logging.error("This can lead to problems later on, we recommend setting it" logging.error("This can lead to problems later on, we recommend setting it"
@ -700,11 +702,12 @@ def get_partition_layout(reserve, kernel):
:returns: the partition layout, e.g. without reserve and kernel: :returns: the partition layout, e.g. without reserve and kernel:
{"kernel": None, "boot": 1, "reserve": None, "root": 2} {"kernel": None, "boot": 1, "reserve": None, "root": 2}
""" """
ret = {} ret: PartitionLayout = {
ret["kernel"] = None "kernel": None,
ret["boot"] = 1 "boot": 1,
ret["reserve"] = None "reserve": None,
ret["root"] = 2 "root": 2,
}
if kernel: if kernel:
ret["kernel"] = 1 ret["kernel"] = 1
@ -761,11 +764,11 @@ def create_fstab(args: PmbArgs, layout, chroot: Chroot):
# Do not install fstab into target rootfs when using on-device # Do not install fstab into target rootfs when using on-device
# installer. Provide fstab only to installer suffix # installer. Provide fstab only to installer suffix
if args.on_device_installer and "rootfs_" in chroot: if args.on_device_installer and chroot.type == ChrootType.ROOTFS:
return return
boot_dev = f"/dev/installp{layout['boot']}" boot_dev = Path(f"/dev/installp{layout['boot']}")
root_dev = f"/dev/installp{layout['root']}" root_dev = Path(f"/dev/installp{layout['root']}")
boot_mount_point = f"UUID={get_uuid(args, boot_dev)}" boot_mount_point = f"UUID={get_uuid(args, boot_dev)}"
root_mount_point = "/dev/mapper/root" if args.full_disk_encryption \ root_mount_point = "/dev/mapper/root" if args.full_disk_encryption \
@ -806,7 +809,7 @@ def create_fstab(args: PmbArgs, layout, chroot: Chroot):
def install_system_image(args: PmbArgs, size_reserve, chroot: Chroot, step, steps, def install_system_image(args: PmbArgs, size_reserve, chroot: Chroot, step, steps,
boot_label="pmOS_boot", root_label="pmOS_root", boot_label="pmOS_boot", root_label="pmOS_root",
split=False, disk=None): split=False, disk: Optional[Path]=None):
""" """
:param size_reserve: empty partition between root and boot in MiB (pma#463) :param size_reserve: empty partition between root and boot in MiB (pma#463)
:param suffix: the chroot suffix, where the rootfs that will be installed :param suffix: the chroot suffix, where the rootfs that will be installed
@ -912,6 +915,8 @@ def print_flash_info(args: PmbArgs):
method = args.deviceinfo["flash_method"] method = args.deviceinfo["flash_method"]
flasher = pmb.config.flashers.get(method, {}) flasher = pmb.config.flashers.get(method, {})
flasher_actions = flasher.get("actions", {}) flasher_actions = flasher.get("actions", {})
if not isinstance(flasher_actions, dict):
raise TypeError(f"flasher actions must be a dictionary, got: {flasher_actions}")
requires_split = flasher.get("split", False) requires_split = flasher.get("split", False)
if method == "none": if method == "none":
@ -929,11 +934,9 @@ def print_flash_info(args: PmbArgs):
logging.info("* pmbootstrap flasher flash_rootfs") logging.info("* pmbootstrap flasher flash_rootfs")
logging.info(" Flashes the generated rootfs image to your device:") logging.info(" Flashes the generated rootfs image to your device:")
if args.split: if args.split:
logging.info(f" {Chroot.native()}/home/pmos/rootfs/" logging.info(f" {Chroot.native() / 'home/pmos/rootfs' / args.device}-rootfs.img")
f"{args.device}-rootfs.img")
else: else:
logging.info(f" {Chroot.native()}/home/pmos/rootfs/" logging.info(f" {Chroot.native() / 'home/pmos/rootfs' / args.device}.img")
f"{args.device}.img")
logging.info(" (NOTE: This file has a partition table, which" logging.info(" (NOTE: This file has a partition table, which"
" contains /boot and / subpartitions. That way we" " contains /boot and / subpartitions. That way we"
" don't need to change the partition layout on your" " don't need to change the partition layout on your"
@ -975,8 +978,7 @@ def print_flash_info(args: PmbArgs):
" Use 'pmbootstrap flasher boot' to do that.)") " Use 'pmbootstrap flasher boot' to do that.)")
if "flash_lk2nd" in flasher_actions and \ if "flash_lk2nd" in flasher_actions and \
os.path.exists(f"{Chroot(ChrootType.ROOTFS, args.device)}" (Chroot(ChrootType.ROOTFS, args.device) / "/boot/lk2nd.img").exists():
"/boot/lk2nd.img"):
logging.info("* Your device supports and may even require" logging.info("* Your device supports and may even require"
" flashing lk2nd. You should flash it before" " flashing lk2nd. You should flash it before"
" flashing anything else. Use 'pmbootstrap flasher" " flashing anything else. Use 'pmbootstrap flasher"
@ -991,7 +993,7 @@ def print_flash_info(args: PmbArgs):
def install_recovery_zip(args: PmbArgs, steps): def install_recovery_zip(args: PmbArgs, steps):
logging.info(f"*** ({steps}/{steps}) CREATING RECOVERY-FLASHABLE ZIP ***") logging.info(f"*** ({steps}/{steps}) CREATING RECOVERY-FLASHABLE ZIP ***")
suffix = "buildroot_" + args.deviceinfo["arch"] suffix = "buildroot_" + args.deviceinfo["arch"]
mount_device_rootfs(args, Chroot(ChrootType.ROOTFS, args.device), suffix) mount_device_rootfs(args, Chroot.rootfs(args.device))
pmb.install.recovery.create_zip(args, suffix) pmb.install.recovery.create_zip(args, suffix)
# Flash information # Flash information
@ -1003,7 +1005,7 @@ def install_recovery_zip(args: PmbArgs, steps):
def install_on_device_installer(args: PmbArgs, step, steps): def install_on_device_installer(args: PmbArgs, step, steps):
# Generate the rootfs image # Generate the rootfs image
if not args.ondev_no_rootfs: if not args.ondev_no_rootfs:
suffix_rootfs = Chroot(ChrootType.ROOTFS, args.device) suffix_rootfs = Chroot.rootfs(args.device)
install_system_image(args, 0, suffix_rootfs, step=step, steps=steps, install_system_image(args, 0, suffix_rootfs, step=step, steps=steps,
split=True) split=True)
step += 2 step += 2
@ -1132,7 +1134,7 @@ def get_recommends(args: PmbArgs, packages) -> Sequence[str]:
""" """
global get_recommends_visited global get_recommends_visited
ret = [] ret: List[str] = []
if not args.install_recommends: if not args.install_recommends:
return ret return ret

9
pmb/install/blockdevice.py
View file

@ -1,5 +1,6 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
from typing import Optional
from pmb.helpers import logging from pmb.helpers import logging
import os import os
import glob import glob
@ -20,7 +21,7 @@ def previous_install(args: PmbArgs, path: Path):
:param path: path to disk block device (e.g. /dev/mmcblk0) :param path: path to disk block device (e.g. /dev/mmcblk0)
""" """
label = "" label = ""
for blockdevice_outside in [f"{path}1", f"{path}p1"]: for blockdevice_outside in [path.with_stem(f"{path.name}1"), path.with_stem(f"{path.name}p1")]:
if not os.path.exists(blockdevice_outside): if not os.path.exists(blockdevice_outside):
continue continue
blockdevice_inside = "/dev/diskp1" blockdevice_inside = "/dev/diskp1"
@ -39,14 +40,14 @@ def previous_install(args: PmbArgs, path: Path):
return "pmOS_boot" in label return "pmOS_boot" in label
def mount_disk(args: PmbArgs, path): def mount_disk(args: PmbArgs, path: Path):
""" """
:param path: path to disk block device (e.g. /dev/mmcblk0) :param path: path to disk block device (e.g. /dev/mmcblk0)
""" """
# Sanity checks # Sanity checks
if not os.path.exists(path): if not os.path.exists(path):
raise RuntimeError(f"The disk block device does not exist: {path}") raise RuntimeError(f"The disk block device does not exist: {path}")
for path_mount in glob.glob(f"{path}*"): for path_mount in path.parent.glob(f"{path.name}*"):
if pmb.helpers.mount.ismount(path_mount): if pmb.helpers.mount.ismount(path_mount):
raise RuntimeError(f"{path_mount} is mounted! Will not attempt to" raise RuntimeError(f"{path_mount} is mounted! Will not attempt to"
" format this!") " format this!")
@ -124,7 +125,7 @@ def create_and_mount_image(args: PmbArgs, size_boot, size_root, size_reserve,
pmb.helpers.mount.bind_file(args, device, Chroot.native() / mount_point) pmb.helpers.mount.bind_file(args, device, Chroot.native() / mount_point)
def create(args: PmbArgs, size_boot, size_root, size_reserve, split, disk): def create(args: PmbArgs, size_boot, size_root, size_reserve, split, disk: Optional[Path]):
""" """
Create /dev/install (the "install blockdevice"). Create /dev/install (the "install blockdevice").

41
pmb/install/losetup.py
View file

@ -2,11 +2,13 @@
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import glob import glob
import json import json
from pathlib import Path
from typing import List, Optional
from pmb.helpers import logging from pmb.helpers import logging
import os import os
import time import time
from pmb.core.types import PmbArgs from pmb.core.types import PathString, PmbArgs
import pmb.helpers.mount import pmb.helpers.mount
import pmb.helpers.run import pmb.helpers.run
import pmb.chroot import pmb.chroot
@ -14,19 +16,19 @@ from pmb.core import Chroot
def init(args: PmbArgs): def init(args: PmbArgs):
if not os.path.isdir("/sys/module/loop"): if not Path("/sys/module/loop").is_dir():
pmb.helpers.run.root(args, ["modprobe", "loop"]) pmb.helpers.run.root(args, ["modprobe", "loop"])
for loopdevice in glob.glob("/dev/loop*"): for loopdevice in Path("/dev/").glob("loop*"):
if os.path.isdir(loopdevice): if loopdevice.is_dir():
continue continue
pmb.helpers.mount.bind_file(args, loopdevice, Chroot.native() / loopdevice) pmb.helpers.mount.bind_file(args, loopdevice, Chroot.native() / loopdevice)
def mount(args: PmbArgs, img_path): def mount(args: PmbArgs, img_path: Path):
""" """
:param img_path: Path to the img file inside native chroot. :param img_path: Path to the img file inside native chroot.
""" """
logging.debug("(native) mount " + img_path + " (loop)") logging.debug(f"(native) mount {img_path} (loop)")
# Try to mount multiple times (let the kernel module initialize #1594) # Try to mount multiple times (let the kernel module initialize #1594)
for i in range(0, 5): for i in range(0, 5):
@ -39,20 +41,23 @@ def mount(args: PmbArgs, img_path):
# Mount and return on success # Mount and return on success
init(args) init(args)
losetup_cmd = ["losetup", "-f", img_path] losetup_cmd: List[PathString] = ["losetup", "-f", img_path]
sector_size = args.deviceinfo["rootfs_image_sector_size"] sector_size = args.deviceinfo["rootfs_image_sector_size"]
if sector_size: if sector_size:
losetup_cmd += ["-b", str(int(sector_size))] losetup_cmd += ["-b", str(int(sector_size))]
pmb.chroot.root(args, losetup_cmd, check=False) pmb.chroot.root(args, losetup_cmd, check=False)
if device_by_back_file(args, img_path): try:
device_by_back_file(args, img_path)
return return
except RuntimeError:
pass
# Failure: raise exception # Failure: raise exception
raise RuntimeError("Failed to mount loop device: " + img_path) raise RuntimeError(f"Failed to mount loop device: {img_path}")
def device_by_back_file(args: PmbArgs, back_file, auto_init=True): def device_by_back_file(args: PmbArgs, back_file: Path, auto_init=True) -> Path:
""" """
Get the /dev/loopX device that points to a specific image file. Get the /dev/loopX device that points to a specific image file.
""" """
@ -61,22 +66,24 @@ def device_by_back_file(args: PmbArgs, back_file, auto_init=True):
losetup_output = pmb.chroot.root(args, ["losetup", "--json", "--list"], losetup_output = pmb.chroot.root(args, ["losetup", "--json", "--list"],
output_return=True, auto_init=auto_init) output_return=True, auto_init=auto_init)
if not losetup_output: if not losetup_output:
return None raise RuntimeError("losetup failed")
# Find the back_file # Find the back_file
losetup = json.loads(losetup_output) losetup = json.loads(losetup_output)
for loopdevice in losetup["loopdevices"]: for loopdevice in losetup["loopdevices"]:
if loopdevice["back-file"] == back_file: if Path(loopdevice["back-file"]) == back_file:
return loopdevice["name"] return Path(loopdevice["name"])
return None raise RuntimeError(f"Failed to find loop device for {back_file}")
def umount(args: PmbArgs, img_path, auto_init=True): def umount(args: PmbArgs, img_path: Path, auto_init=True):
""" """
:param img_path: Path to the img file inside native chroot. :param img_path: Path to the img file inside native chroot.
""" """
device: Path
try:
device = device_by_back_file(args, img_path, auto_init) device = device_by_back_file(args, img_path, auto_init)
if not device: except RuntimeError:
return return
logging.debug("(native) umount " + device) logging.debug(f"(native) umount {device}")
pmb.chroot.root(args, ["losetup", "-d", device], auto_init=auto_init) pmb.chroot.root(args, ["losetup", "-d", device], auto_init=auto_init)

23
pmb/install/partition.py
View file

@ -1,5 +1,7 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
from pathlib import Path
from typing import Optional
from pmb.helpers import logging from pmb.helpers import logging
import os import os
import time import time
@ -10,25 +12,28 @@ import pmb.install.losetup
from pmb.core import Chroot from pmb.core import Chroot
def partitions_mount(args: PmbArgs, layout, disk): # FIXME (#2324): this function drops disk to a string because it's easier
# to manipulate, this is probably bad.
def partitions_mount(args: PmbArgs, layout, disk: Optional[Path]):
""" """
Mount blockdevices of partitions inside native chroot Mount blockdevices of partitions inside native chroot
:param layout: partition layout from get_partition_layout() :param layout: partition layout from get_partition_layout()
:param disk: path to disk block device (e.g. /dev/mmcblk0) or None :param disk: path to disk block device (e.g. /dev/mmcblk0) or None
""" """
prefix = disk
if not disk: if not disk:
img_path = "/home/pmos/rootfs/" + args.device + ".img" img_path = Path("/home/pmos/rootfs") / f"{args.device}.img"
prefix = pmb.install.losetup.device_by_back_file(args, img_path) disk = pmb.install.losetup.device_by_back_file(args, img_path)
logging.info(f"Mounting partitions of {disk} inside the chroot")
tries = 20 tries = 20
# Devices ending with a number have a "p" before the partition number, # Devices ending with a number have a "p" before the partition number,
# /dev/sda1 has no "p", but /dev/mmcblk0p1 has. See add_partition() in # /dev/sda1 has no "p", but /dev/mmcblk0p1 has. See add_partition() in
# block/partitions/core.c of linux.git. # block/partitions/core.c of linux.git.
partition_prefix = prefix partition_prefix = str(disk)
if str.isdigit(prefix[-1:]): if str.isdigit(disk.name[-1:]):
partition_prefix = f"{prefix}p" partition_prefix = f"{disk}p"
found = False found = False
for i in range(tries): for i in range(tries):
@ -40,7 +45,7 @@ def partitions_mount(args: PmbArgs, layout, disk):
time.sleep(0.1) time.sleep(0.1)
if not found: if not found:
raise RuntimeError(f"Unable to find the first partition of {prefix}, " raise RuntimeError(f"Unable to find the first partition of {disk}, "
f"expected it to be at {partition_prefix}1!") f"expected it to be at {partition_prefix}1!")
partitions = [layout["boot"], layout["root"]] partitions = [layout["boot"], layout["root"]]
@ -49,7 +54,7 @@ def partitions_mount(args: PmbArgs, layout, disk):
partitions += [layout["kernel"]] partitions += [layout["kernel"]]
for i in partitions: for i in partitions:
source = f"{partition_prefix}{i}" source = Path(f"{partition_prefix}{i}")
target = Chroot.native() / "dev" / f"installp{i}" target = Chroot.native() / "dev" / f"installp{i}"
pmb.helpers.mount.bind_file(args, source, target) pmb.helpers.mount.bind_file(args, source, target)

5
pmb/install/recovery.py
View file

@ -1,5 +1,6 @@
# Copyright 2023 Attila Szollosi # Copyright 2023 Attila Szollosi
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
from pathlib import Path
from pmb.helpers import logging from pmb.helpers import logging
import pmb.chroot import pmb.chroot
@ -13,7 +14,7 @@ def create_zip(args: PmbArgs, suffix):
""" """
Create android recovery compatible installer zip. Create android recovery compatible installer zip.
""" """
zip_root = "/var/lib/postmarketos-android-recovery-installer/" zip_root = Path("/var/lib/postmarketos-android-recovery-installer/")
rootfs = "/mnt/rootfs_" + args.device rootfs = "/mnt/rootfs_" + args.device
flavor = pmb.helpers.frontend._parse_flavor(args) flavor = pmb.helpers.frontend._parse_flavor(args)
method = args.deviceinfo["flash_method"] method = args.deviceinfo["flash_method"]
@ -70,4 +71,4 @@ def create_zip(args: PmbArgs, suffix):
["gzip", "-f1", "rootfs.tar"], ["gzip", "-f1", "rootfs.tar"],
["build-recovery-zip", args.device]] ["build-recovery-zip", args.device]]
for command in commands: for command in commands:
pmb.chroot.root(args, command, suffix, zip_root) pmb.chroot.root(args, command, suffix, working_dir=zip_root)

5
pmb/install/ui.py
View file

@ -1,18 +1,19 @@
# Copyright 2023 Dylan Van Assche # Copyright 2023 Dylan Van Assche
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
from typing import List
from pmb.helpers import logging from pmb.helpers import logging
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
import pmb.helpers.pmaports import pmb.helpers.pmaports
def get_groups(args: PmbArgs): def get_groups(args: PmbArgs) -> List[str]:
""" Get all groups to which the user additionally must be added. """ Get all groups to which the user additionally must be added.
The list of groups are listed in _pmb_groups of the UI and The list of groups are listed in _pmb_groups of the UI and
UI-extras package. UI-extras package.
:returns: list of groups, e.g. ["feedbackd", "udev"] """ :returns: list of groups, e.g. ["feedbackd", "udev"] """
ret = [] ret: List[str] = []
if args.ui == "none": if args.ui == "none":
return ret return ret

8
pmb/parse/_apkbuild.py
View file

@ -206,7 +206,7 @@ def _parse_attributes(path, lines, apkbuild_attributes, ret):
ret[attribute] = replace_variable(ret, value) ret[attribute] = replace_variable(ret, value)
if "subpackages" in apkbuild_attributes: if "subpackages" in apkbuild_attributes:
subpackages = OrderedDict() subpackages: OrderedDict[str, str] = OrderedDict()
for subpkg in ret["subpackages"].split(" "): for subpkg in ret["subpackages"].split(" "):
if subpkg: if subpkg:
_parse_subpackage(path, lines, ret, subpackages, subpkg) _parse_subpackage(path, lines, ret, subpackages, subpkg)
@ -326,8 +326,12 @@ def apkbuild(path: Path, check_pkgver=True, check_pkgname=True):
:returns: relevant variables from the APKBUILD. Arrays get returned as :returns: relevant variables from the APKBUILD. Arrays get returned as
arrays. arrays.
""" """
if path.is_dir(): if path.name != "APKBUILD":
path = path / "APKBUILD" path = path / "APKBUILD"
if not path.exists():
raise FileNotFoundError(f"{path.relative_to(pmb.config.work)} not found!")
# Try to get a cached result first (we assume that the aports don't change # Try to get a cached result first (we assume that the aports don't change
# in one pmbootstrap call) # in one pmbootstrap call)
if path in pmb.helpers.other.cache["apkbuild"]: if path in pmb.helpers.other.cache["apkbuild"]:

41
pmb/parse/apkindex.py
View file

@ -1,6 +1,7 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import collections import collections
from typing import Any, Dict, List
from pmb.helpers import logging from pmb.helpers import logging
from pathlib import Path from pathlib import Path
import tarfile import tarfile
@ -34,7 +35,7 @@ def parse_next_block(path: Path, lines, start):
:returns: None, when there are no more blocks :returns: None, when there are no more blocks
""" """
# Parse until we hit an empty line or end of file # Parse until we hit an empty line or end of file
ret = {} ret: Dict[str, Any] = {}
mapping = { mapping = {
"A": "arch", "A": "arch",
"D": "depends", "D": "depends",
@ -60,9 +61,8 @@ def parse_next_block(path: Path, lines, start):
for letter, key in mapping.items(): for letter, key in mapping.items():
if line.startswith(letter + ":"): if line.startswith(letter + ":"):
if key in ret: if key in ret:
raise RuntimeError( raise RuntimeError(f"Key {key} ({letter}:) specified twice"
"Key " + key + " (" + letter + ":) specified twice" f" in block: {ret}, file: {path}")
" in block: " + str(ret) + ", file: " + path)
ret[key] = line[2:-1] ret[key] = line[2:-1]
# Format and return the block # Format and return the block
@ -91,9 +91,9 @@ def parse_next_block(path: Path, lines, start):
# No more blocks # No more blocks
elif ret != {}: elif ret != {}:
raise RuntimeError("Last block in " + path + " does not end" raise RuntimeError(f"Last block in {path} does not end"
" with a new line! Delete the file and" " with a new line! Delete the file and"
" try again. Last block: " + str(ret)) f" try again. Last block: {ret}")
return None return None
@ -168,7 +168,7 @@ def parse(path: Path, multiple_providers=True):
# Require the file to exist # Require the file to exist
if not path.is_file(): if not path.is_file():
logging.verbose("NOTE: APKINDEX not found, assuming no binary packages" logging.verbose("NOTE: APKINDEX not found, assuming no binary packages"
" exist for that architecture: " + path) f" exist for that architecture: {path}")
return {} return {}
# Try to get a cached result first # Try to get a cached result first
@ -185,14 +185,14 @@ def parse(path: Path, multiple_providers=True):
# Read all lines # Read all lines
if tarfile.is_tarfile(path): if tarfile.is_tarfile(path):
with tarfile.open(path, "r:gz") as tar: with tarfile.open(path, "r:gz") as tar:
with tar.extractfile(tar.getmember("APKINDEX")) as handle: with tar.extractfile(tar.getmember("APKINDEX")) as handle: # type:ignore[union-attr]
lines = handle.readlines() lines = handle.readlines()
else: else:
with path.open("r", encoding="utf-8") as handle: with path.open("r", encoding="utf-8") as handle:
lines = handle.readlines() lines = handle.readlines()
# Parse the whole APKINDEX file # Parse the whole APKINDEX file
ret = collections.OrderedDict() ret: Dict[str, Any] = collections.OrderedDict()
start = [0] start = [0]
while True: while True:
block = parse_next_block(path, lines, start) block = parse_next_block(path, lines, start)
@ -201,8 +201,8 @@ def parse(path: Path, multiple_providers=True):
# Skip virtual packages # Skip virtual packages
if "timestamp" not in block: if "timestamp" not in block:
logging.verbose("Skipped virtual package " + str(block) + " in" logging.verbose(f"Skipped virtual package {block} in"
" file: " + path) f" file: {path}")
continue continue
# Add the next package and all aliases # Add the next package and all aliases
@ -232,11 +232,11 @@ def parse_blocks(path: Path):
""" """
# Parse all lines # Parse all lines
with tarfile.open(path, "r:gz") as tar: with tarfile.open(path, "r:gz") as tar:
with tar.extractfile(tar.getmember("APKINDEX")) as handle: with tar.extractfile(tar.getmember("APKINDEX")) as handle: # type:ignore[union-attr]
lines = handle.readlines() lines = handle.readlines()
# Parse lines into blocks # Parse lines into blocks
ret = [] ret: List[str] = []
start = [0] start = [0]
while True: while True:
block = pmb.parse.apkindex.parse_next_block(path, lines, start) block = pmb.parse.apkindex.parse_next_block(path, lines, start)
@ -251,7 +251,7 @@ def clear_cache(path: Path):
:returns: True on successful deletion, False otherwise :returns: True on successful deletion, False otherwise
""" """
logging.verbose("Clear APKINDEX cache for: " + path) logging.verbose(f"Clear APKINDEX cache for: {path}")
if path in pmb.helpers.other.cache["apkindex"]: if path in pmb.helpers.other.cache["apkindex"]:
del pmb.helpers.other.cache["apkindex"][path] del pmb.helpers.other.cache["apkindex"][path]
return True return True
@ -281,7 +281,7 @@ def providers(args: PmbArgs, package, arch=None, must_exist=True, indexes=None):
package = pmb.helpers.package.remove_operators(package) package = pmb.helpers.package.remove_operators(package)
ret = collections.OrderedDict() ret: Dict[str, Any] = collections.OrderedDict()
for path in indexes: for path in indexes:
# Skip indexes not providing the package # Skip indexes not providing the package
index_packages = parse(path) index_packages = parse(path)
@ -295,10 +295,8 @@ def providers(args: PmbArgs, package, arch=None, must_exist=True, indexes=None):
if provider_pkgname in ret: if provider_pkgname in ret:
version_last = ret[provider_pkgname]["version"] version_last = ret[provider_pkgname]["version"]
if pmb.parse.version.compare(version, version_last) == -1: if pmb.parse.version.compare(version, version_last) == -1:
logging.verbose(package + ": provided by: " + logging.verbose(f"{package}: provided by: {provider_pkgname}-{version}"
provider_pkgname + "-" + version + " in " + f"in {path} (but {version_last} is higher)")
path + " (but " + version_last + " is"
" higher)")
continue continue
# Add the provider to ret # Add the provider to ret
@ -306,7 +304,8 @@ def providers(args: PmbArgs, package, arch=None, must_exist=True, indexes=None):
ret[provider_pkgname] = provider ret[provider_pkgname] = provider
if ret == {} and must_exist: if ret == {} and must_exist:
logging.debug("Searched in APKINDEX files: " + ", ".join(indexes)) import os
logging.debug(f"Searched in APKINDEX files: {', '.join([os.fspath(x) for x in indexes])}")
raise RuntimeError("Could not find package '" + package + "'!") raise RuntimeError("Could not find package '" + package + "'!")
return ret return ret
@ -319,7 +318,7 @@ def provider_highest_priority(providers, pkgname):
:param pkgname: the package name we are interested in (for the log message) :param pkgname: the package name we are interested in (for the log message)
""" """
max_priority = 0 max_priority = 0
priority_providers = collections.OrderedDict() priority_providers: collections.OrderedDict[str, str] = collections.OrderedDict()
for provider_name, provider in providers.items(): for provider_name, provider in providers.items():
priority = int(provider.get("provider_priority", -1)) priority = int(provider.get("provider_priority", -1))
if priority > max_priority: if priority > max_priority:

19
pmb/parse/arguments.py
View file

@ -3,10 +3,13 @@
import argparse import argparse
import copy import copy
import os import os
from pathlib import Path
import sys import sys
from pmb.core.types import PmbArgs
try: try:
import argcomplete import argcomplete # type:ignore[import-untyped]
except ImportError: except ImportError:
pass pass
@ -109,7 +112,7 @@ def arguments_install(subparser):
help="do not create an image file, instead" help="do not create an image file, instead"
" write to the given block device (SD card, USB" " write to the given block device (SD card, USB"
" stick, etc.), for example: '/dev/mmcblk0'", " stick, etc.), for example: '/dev/mmcblk0'",
metavar="BLOCKDEV") metavar="BLOCKDEV", type=lambda x: Path(x))
group.add_argument("--android-recovery-zip", group.add_argument("--android-recovery-zip",
help="generate TWRP flashable zip (recommended read:" help="generate TWRP flashable zip (recommended read:"
" https://postmarketos.org/recoveryzip)", " https://postmarketos.org/recoveryzip)",
@ -205,7 +208,8 @@ def arguments_export(subparser):
ret.add_argument("export_folder", help="export folder, defaults to" ret.add_argument("export_folder", help="export folder, defaults to"
" /tmp/postmarketOS-export", " /tmp/postmarketOS-export",
default="/tmp/postmarketOS-export", nargs="?") default=Path("/tmp/postmarketOS-export"), nargs="?",
type=lambda x: Path(x))
ret.add_argument("--odin", help="odin flashable tar" ret.add_argument("--odin", help="odin flashable tar"
" (boot.img/kernel+initramfs only)", " (boot.img/kernel+initramfs only)",
action="store_true", dest="odin_flashable_tar") action="store_true", dest="odin_flashable_tar")
@ -651,8 +655,8 @@ def get_parser():
f" default: {mirrors_pmos_default}", f" default: {mirrors_pmos_default}",
metavar="URL", action="append", default=[]) metavar="URL", action="append", default=[])
parser.add_argument("-m", "--mirror-alpine", dest="mirror_alpine", parser.add_argument("-m", "--mirror-alpine", dest="mirror_alpine",
help="Alpine Linux mirror, default: " + help="Alpine Linux mirror, default: "
pmb.config.defaults["mirror_alpine"], f"{pmb.config.defaults['mirror_alpine']}",
metavar="URL") metavar="URL")
parser.add_argument("-j", "--jobs", help="parallel jobs when compiling") parser.add_argument("-j", "--jobs", help="parallel jobs when compiling")
parser.add_argument("-E", "--extra-space", parser.add_argument("-E", "--extra-space",
@ -923,7 +927,8 @@ def get_parser():
# Action: bootimg_analyze # Action: bootimg_analyze
bootimg_analyze = sub.add_parser("bootimg_analyze", help="Extract all the" bootimg_analyze = sub.add_parser("bootimg_analyze", help="Extract all the"
" information from an existing boot.img") " information from an existing boot.img")
bootimg_analyze.add_argument("path", help="path to the boot.img") bootimg_analyze.add_argument("path", help="path to the boot.img",
type=lambda x: Path(x))
bootimg_analyze.add_argument("--force", "-f", action="store_true", bootimg_analyze.add_argument("--force", "-f", action="store_true",
help="force even if the file seems to be" help="force even if the file seems to be"
" invalid") " invalid")
@ -940,7 +945,7 @@ def get_parser():
def arguments(): def arguments():
# Parse and extend arguments (also backup unmodified result from argparse) # Parse and extend arguments (also backup unmodified result from argparse)
args = get_parser().parse_args() args: PmbArgs = get_parser().parse_args() # type: ignore
setattr(args, "from_argparse", copy.deepcopy(args)) setattr(args, "from_argparse", copy.deepcopy(args))
setattr(args.from_argparse, "from_argparse", args.from_argparse) setattr(args.from_argparse, "from_argparse", args.from_argparse)

4
pmb/parse/binfmt_info.py
View file

@ -10,8 +10,8 @@ import pmb.config
def binfmt_info(arch_qemu): def binfmt_info(arch_qemu):
# Parse the info file # Parse the info file
full = {} full = {}
info = pmb.config.pmb_src + "/pmb/data/qemu-user-binfmt.txt" info = pmb.config.pmb_src / "pmb/data/qemu-user-binfmt.txt"
logging.verbose("parsing: " + info) logging.verbose(f"parsing: {info}")
with open(info, "r") as handle: with open(info, "r") as handle:
for line in handle: for line in handle:
if line.startswith('#') or "=" not in line: if line.startswith('#') or "=" not in line:

3
pmb/parse/cpuinfo.py
View file

@ -1,9 +1,10 @@
# Copyright 2023 Lary Gibaud # Copyright 2023 Lary Gibaud
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import re import re
from typing import Optional
def arm_big_little_first_group_ncpus(): def arm_big_little_first_group_ncpus() -> Optional[int]:
""" """
Infer from /proc/cpuinfo on aarch64 if this is a big/little architecture Infer from /proc/cpuinfo on aarch64 if this is a big/little architecture
(if there is different processor models) and the number of cores in the (if there is different processor models) and the number of cores in the

10
pmb/parse/depends.py
View file

@ -17,12 +17,12 @@ def package_from_aports(args: PmbArgs, pkgname_depend):
depends, version. The version is the combined pkgver and pkgrel. depends, version. The version is the combined pkgver and pkgrel.
""" """
# Get the aport # Get the aport
aport = pmb.helpers.pmaports.find(args, pkgname_depend, False) aport = pmb.helpers.pmaports.find_optional(args, pkgname_depend)
if not aport: if not aport:
return None return None
# Parse its version # Parse its version
apkbuild = pmb.parse.apkbuild(f"{aport}/APKBUILD") apkbuild = pmb.parse.apkbuild(aport / "APKBUILD")
pkgname = apkbuild["pkgname"] pkgname = apkbuild["pkgname"]
version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
@ -118,7 +118,7 @@ def package_from_index(args: PmbArgs, pkgname_depend, pkgnames_install, package_
return provider return provider
def recurse(args: PmbArgs, pkgnames, suffix: Chroot=Chroot.native()): def recurse(args: PmbArgs, pkgnames, suffix: Chroot=Chroot.native()) -> Sequence[str]:
""" """
Find all dependencies of the given pkgnames. Find all dependencies of the given pkgnames.
@ -134,8 +134,8 @@ def recurse(args: PmbArgs, pkgnames, suffix: Chroot=Chroot.native()):
# Iterate over todo-list until is is empty # Iterate over todo-list until is is empty
todo = list(pkgnames) todo = list(pkgnames)
required_by = {} required_by: Dict[str, Set[str]] = {}
ret = [] ret: List[str] = []
while len(todo): while len(todo):
# Skip already passed entries # Skip already passed entries
pkgname_depend = todo.pop(0) pkgname_depend = todo.pop(0)

3
pmb/parse/deviceinfo.py
View file

@ -113,7 +113,8 @@ def _parse_kernel_suffix(args: PmbArgs, info, device, kernel):
return ret return ret
def deviceinfo(args: PmbArgs, device=None, kernel=None): # FIXME (#2324): Make deviceinfo a type! (class!!!)
def deviceinfo(args: PmbArgs, device=None, kernel=None) -> Dict[str, str]:
""" """
:param device: defaults to args.device :param device: defaults to args.device
:param kernel: defaults to args.kernel :param kernel: defaults to args.kernel

14
pmb/parse/kconfig.py
View file

@ -253,21 +253,25 @@ def check(args: PmbArgs, pkgname, components_list=[], details=False, must_exist=
# Read all kernel configs in the aport # Read all kernel configs in the aport
ret = True ret = True
aport = pmb.helpers.pmaports.find(args, "linux-" + flavor, must_exist=must_exist) aport: Path
if aport is None: try:
aport = pmb.helpers.pmaports.find(args, "linux-" + flavor)
except RuntimeError as e:
if must_exist:
raise e
return None return None
apkbuild = pmb.parse.apkbuild(f"{aport}/APKBUILD") apkbuild = pmb.parse.apkbuild(aport / "APKBUILD")
pkgver = apkbuild["pkgver"] pkgver = apkbuild["pkgver"]
# We only enforce optional checks for community & main devices # We only enforce optional checks for community & main devices
enforce_check = aport.split("/")[-2] in ["community", "main"] enforce_check = aport.parts[-2] in ["community", "main"]
for name in get_all_component_names(): for name in get_all_component_names():
if f"pmb:kconfigcheck-{name}" in apkbuild["options"] and \ if f"pmb:kconfigcheck-{name}" in apkbuild["options"] and \
name not in components_list: name not in components_list:
components_list += [name] components_list += [name]
for config_path in glob.glob(aport + "/config-*"): for config_path in aport.glob("config-*"):
# The architecture of the config is in the name, so it just needs to be # The architecture of the config is in the name, so it just needs to be
# extracted # extracted
config_name = os.path.basename(config_path) config_name = os.path.basename(config_path)

8
pmb/parse/version.py
View file

@ -108,12 +108,12 @@ def parse_suffix(rest):
C equivalent: get_token(), case TOKEN_SUFFIX C equivalent: get_token(), case TOKEN_SUFFIX
""" """
suffixes = collections.OrderedDict([ name_suffixes = collections.OrderedDict([
("pre", ["alpha", "beta", "pre", "rc"]), ("pre", ["alpha", "beta", "pre", "rc"]),
("post", ["cvs", "svn", "git", "hg", "p"]), ("post", ["cvs", "svn", "git", "hg", "p"]),
]) ])
for name, suffixes in suffixes.items(): for name, suffixes in name_suffixes.items():
for i, suffix in enumerate(suffixes): for i, suffix in enumerate(suffixes):
if not rest.startswith(suffix): if not rest.startswith(suffix):
continue continue
@ -203,7 +203,7 @@ def validate(version):
return True return True
def compare(a_version, b_version, fuzzy=False): def compare(a_version: str, b_version: str, fuzzy=False):
""" """
Compare two versions A and B to find out which one is higher, or if Compare two versions A and B to find out which one is higher, or if
both are equal. both are equal.
@ -307,4 +307,4 @@ def check_string(a_version, rule):
# Compare # Compare
result = compare(a_version, b_version) result = compare(a_version, b_version)
return result in expected_results return not expected_results or result in expected_results

30
pmb/qemu/run.py
View file

@ -1,5 +1,6 @@
# Copyright 2023 Pablo Castellano, Oliver Smith # Copyright 2023 Pablo Castellano, Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import subprocess
from typing import Sequence from typing import Sequence
from pmb.helpers import logging from pmb.helpers import logging
import os import os
@ -16,7 +17,7 @@ import pmb.chroot.other
import pmb.chroot.initfs import pmb.chroot.initfs
import pmb.config import pmb.config
import pmb.config.pmaports import pmb.config.pmaports
from pmb.core.types import PmbArgs from pmb.core.types import PathString, PmbArgs
import pmb.helpers.run import pmb.helpers.run
import pmb.parse.arch import pmb.parse.arch
import pmb.parse.cpuinfo import pmb.parse.cpuinfo
@ -30,7 +31,7 @@ def system_image(args: PmbArgs):
""" """
path = Chroot.native() / "home/pmos/rootfs" / f"{args.device}.img" path = Chroot.native() / "home/pmos/rootfs" / f"{args.device}.img"
if not path.exists(): if not path.exists():
logging.debug("Could not find rootfs: " + path) logging.debug(f"Could not find rootfs: {path}")
raise RuntimeError("The rootfs has not been generated yet, please " raise RuntimeError("The rootfs has not been generated yet, please "
"run 'pmbootstrap install' first.") "run 'pmbootstrap install' first.")
return path return path
@ -76,10 +77,10 @@ def create_gdk_loader_cache(args: PmbArgs) -> Path:
cache_path = gdk_cache_dir / "loaders.cache" cache_path = gdk_cache_dir / "loaders.cache"
if not (chroot_native / cache_path).is_file(): if not (chroot_native / cache_path).is_file():
raise RuntimeError("gdk pixbuf cache file not found: " + cache_path) raise RuntimeError(f"gdk pixbuf cache file not found: {cache_path}")
pmb.chroot.root(args, ["cp", cache_path, custom_cache_path]) pmb.chroot.root(args, ["cp", cache_path, custom_cache_path])
cmd = ["sed", "-i", "-e", cmd: Sequence[PathString] = ["sed", "-i", "-e",
f"s@\"{gdk_cache_dir}@\"{chroot_native / gdk_cache_dir}@", f"s@\"{gdk_cache_dir}@\"{chroot_native / gdk_cache_dir}@",
custom_cache_path] custom_cache_path]
pmb.chroot.root(args, cmd) pmb.chroot.root(args, cmd)
@ -138,11 +139,12 @@ def command_qemu(args: PmbArgs, arch, img_path, img_path_2nd=None):
if "gtk" in args.qemu_display: if "gtk" in args.qemu_display:
gdk_cache = create_gdk_loader_cache(args) gdk_cache = create_gdk_loader_cache(args)
env.update({"GTK_THEME": "Default", # FIXME: why does mypy think the values here should all be paths??
"GDK_PIXBUF_MODULE_FILE": gdk_cache, env.update({"GTK_THEME": "Default", # type: ignore[dict-item]
"XDG_DATA_DIRS": ":".join([ "GDK_PIXBUF_MODULE_FILE": str(gdk_cache), # type: ignore[dict-item]
chroot_native / "usr/local/share", "XDG_DATA_DIRS": ":".join([ # type: ignore[dict-item]
chroot_native / "usr/share" str(chroot_native / "usr/local/share"),
str(chroot_native / "usr/share"),
])}) ])})
command = [] command = []
@ -162,9 +164,9 @@ def command_qemu(args: PmbArgs, arch, img_path, img_path_2nd=None):
command += [chroot_native / "lib" / f"ld-musl-{pmb.config.arch_native}.so.1"] command += [chroot_native / "lib" / f"ld-musl-{pmb.config.arch_native}.so.1"]
command += ["--library-path=" + ":".join([ command += ["--library-path=" + ":".join([
chroot_native / "lib", str(chroot_native / "lib"),
chroot_native / "usr/lib" + str(chroot_native / "usr/lib"),
chroot_native / "usr/lib/pulseaudio" str(chroot_native / "usr/lib/pulseaudio"),
])] ])]
command += [chroot_native / "usr/bin" / f"qemu-system-{arch}"] command += [chroot_native / "usr/bin" / f"qemu-system-{arch}"]
command += ["-L", chroot_native / "usr/share/qemu/"] command += ["-L", chroot_native / "usr/share/qemu/"]
@ -188,7 +190,7 @@ def command_qemu(args: PmbArgs, arch, img_path, img_path_2nd=None):
else: else:
command += ["stdio"] command += ["stdio"]
command += ["-drive", "file=" + img_path + ",format=raw,if=virtio"] command += ["-drive", f"file={img_path},format=raw,if=virtio"]
if img_path_2nd: if img_path_2nd:
command += ["-drive", "file=" + img_path_2nd + ",format=raw,if=virtio"] command += ["-drive", "file=" + img_path_2nd + ",format=raw,if=virtio"]
@ -387,5 +389,5 @@ def run(args: PmbArgs):
"send Ctrl+C to the VM, run:") "send Ctrl+C to the VM, run:")
logging.info("$ pmbootstrap config qemu_redir_stdio True") logging.info("$ pmbootstrap config qemu_redir_stdio True")
finally: finally:
if process: if isinstance(process, subprocess.Popen):
process.terminate() process.terminate()

13
pmb/sideload/__init__.py
View file

@ -4,9 +4,8 @@ import os
from typing import List from typing import List
from pmb.helpers import logging from pmb.helpers import logging
import shlex import shlex
from argparse import Namespace
from pmb.core.types import PmbArgs from pmb.core.types import PathString, PmbArgs
import pmb.helpers.run import pmb.helpers.run
import pmb.helpers.run_core import pmb.helpers.run_core
import pmb.parse.apkindex import pmb.parse.apkindex
@ -22,19 +21,19 @@ def scp_abuild_key(args: PmbArgs, user: str, host: str, port: str):
:param host: target device ssh hostname :param host: target device ssh hostname
:param port: target device ssh port """ :param port: target device ssh port """
keys = (pmb.config.work / "config_abuild").glob("*.pub") keys = list((pmb.config.work / "config_abuild").glob("*.pub"))
key = keys[0] key = keys[0]
key_name = os.path.basename(key) key_name = os.path.basename(key)
logging.info(f"Copying signing key ({key_name}) to {user}@{host}") logging.info(f"Copying signing key ({key_name}) to {user}@{host}")
command = ['scp', '-P', port, key, f'{user}@{host}:/tmp'] command: List[PathString] = ['scp', '-P', port, key, f'{user}@{host}:/tmp']
pmb.helpers.run.user(args, command, output="interactive") pmb.helpers.run.user(args, command, output="interactive")
logging.info(f"Installing signing key at {user}@{host}") logging.info(f"Installing signing key at {user}@{host}")
keyname = os.path.join("/tmp", os.path.basename(key)) keyname = os.path.join("/tmp", os.path.basename(key))
remote_cmd = ['sudo', '-p', pmb.config.sideload_sudo_prompt, remote_cmd_l: List[PathString] = ['sudo', '-p', pmb.config.sideload_sudo_prompt,
'-S', 'mv', '-n', keyname, "/etc/apk/keys/"] '-S', 'mv', '-n', keyname, "/etc/apk/keys/"]
remote_cmd = pmb.helpers.run_core.flat_cmd(remote_cmd) remote_cmd = pmb.helpers.run_core.flat_cmd(remote_cmd_l)
command = ['ssh', '-t', '-p', port, f'{user}@{host}', remote_cmd] command = ['ssh', '-t', '-p', port, f'{user}@{host}', remote_cmd]
pmb.helpers.run.user(args, command, output="tui") pmb.helpers.run.user(args, command, output="tui")
@ -43,7 +42,7 @@ def ssh_find_arch(args: PmbArgs, user: str, host: str, port: str) -> str:
"""Connect to a device via ssh and query the architecture.""" """Connect to a device via ssh and query the architecture."""
logging.info(f"Querying architecture of {user}@{host}") logging.info(f"Querying architecture of {user}@{host}")
command = ["ssh", "-p", port, f"{user}@{host}", "uname -m"] command = ["ssh", "-p", port, f"{user}@{host}", "uname -m"]
output = pmb.helpers.run.user(args, command, output_return=True) output = pmb.helpers.run.user_output(args, command)
# Split by newlines so we can pick out any irrelevant output, e.g. the "permanently # Split by newlines so we can pick out any irrelevant output, e.g. the "permanently
# added to list of known hosts" warnings. # added to list of known hosts" warnings.
output_lines = output.strip().splitlines() output_lines = output.strip().splitlines()

2
test/test_helpers_repo.py
View file

@ -25,7 +25,7 @@ def args(tmpdir, request):
def test_hash(): def test_hash():
url = "https://nl.alpinelinux.org/alpine/edge/testing" url = "https://nl.alpinelinux.org/alpine/edge/testing"
hash = "865a153c" hash = "865a153c"
assert pmb.helpers.repo.hash(url, 8) == hash assert pmb.helpers.repo.apkindex_hash(url, 8) == hash
def test_alpine_apkindex_path(args: PmbArgs): def test_alpine_apkindex_path(args: PmbArgs):

4
test/test_pkgrel_bump.py
View file

@ -73,7 +73,7 @@ def setup_work(args: PmbArgs, tmpdir):
pmb.helpers.run.user(args, ["./pmbootstrap.py", "shutdown"]) pmb.helpers.run.user(args, ["./pmbootstrap.py", "shutdown"])
# Link everything from work (except for "packages") to the tmpdir # Link everything from work (except for "packages") to the tmpdir
for path in glob.glob(pmb.config.work / "*"): for path in pmb.config.work.glob("*"):
if os.path.basename(path) != "packages": if os.path.basename(path) != "packages":
pmb.helpers.run.user(args, ["ln", "-s", path, tmpdir + "/"]) pmb.helpers.run.user(args, ["ln", "-s", path, tmpdir + "/"])
@ -91,7 +91,7 @@ def setup_work(args: PmbArgs, tmpdir):
f"{tmpdir}/_aports/main/{pkgname}"]) f"{tmpdir}/_aports/main/{pkgname}"])
# Copy pmaports.cfg # Copy pmaports.cfg
pmb.helpers.run.user(args, ["cp", args.aports + "/pmaports.cfg", tmpdir + pmb.helpers.run.user(args, ["cp", args.aports / "pmaports.cfg", tmpdir +
"/_aports"]) "/_aports"])
# Empty packages folder # Empty packages folder

2
test/test_qemu_running_processes.py
View file

@ -133,7 +133,7 @@ def is_running(args: PmbArgs, programs, timeout=300, sleep_before_retry=1):
ssh_works = False ssh_works = False
end = time.monotonic() + timeout end = time.monotonic() + timeout
last_try = 0 last_try = 0.0
while last_try < end: while last_try < end:
# Sleep only when last try exited immediately # Sleep only when last try exited immediately

3
test/test_run_core.py
View file

@ -1,6 +1,7 @@
# Copyright 2023 Oliver Smith # Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
""" Test pmb.helpers.run_core """ """ Test pmb.helpers.run_core """
from typing import Sequence
from pmb.core.types import PmbArgs from pmb.core.types import PmbArgs
import pytest import pytest
import re import re
@ -69,7 +70,7 @@ def test_pipe(args: PmbArgs):
def test_foreground_pipe(args: PmbArgs): def test_foreground_pipe(args: PmbArgs):
func = pmb.helpers.run_core.foreground_pipe func = pmb.helpers.run_core.foreground_pipe
cmd = ["echo", "test"] cmd: Sequence[str] = ["echo", "test"]
# Normal run # Normal run
assert func(args, cmd) == (0, "") assert func(args, cmd) == (0, "")