From 866e5bcfab5ee60fadb7a10b70cd3eb1e4c3a5d9 Mon Sep 17 00:00:00 2001 From: Caleb Connolly Date: Sat, 8 Jun 2024 21:27:27 +0200 Subject: [PATCH] core: add an Arch type (MR 2252) Move pmb/parse/arch.py over to core and refactor it as an Arch type, similar to how Chroot was done. Fix all the uses (that I can find) of arch in the codebase that need adjusting. The new Arch type is an Enum, making it clear what architectures can be represented and making it much easier to reason about. Since we support ~5 (kinda) different representations of an Architecture (Alpine, Kernel, target triple, platform, and QEMU), we now formalise that the Alpine format is what we represent internally, with methods to convert to any of the others as-needed. Signed-off-by: Caleb Connolly --- pmb/__init__.py | 2 +- pmb/aportgen/device.py | 2 +- pmb/aportgen/grub_efi.py | 3 +- pmb/aportgen/linux.py | 3 +- pmb/build/__init__.py | 2 +- pmb/build/_package.py | 19 ++-- pmb/build/autodetect.py | 15 ++- pmb/build/envkernel.py | 21 ++-- pmb/build/init.py | 14 +-- pmb/build/kconfig.py | 4 +- pmb/build/other.py | 2 - pmb/chroot/apk.py | 9 +- pmb/chroot/apk_static.py | 3 +- pmb/chroot/binfmt.py | 15 ++- pmb/chroot/init.py | 5 +- pmb/chroot/mount.py | 2 +- pmb/chroot/shutdown.py | 7 +- pmb/chroot/zap.py | 7 +- pmb/commands/repo_bootstrap.py | 13 +-- pmb/config/__init__.py | 14 +-- pmb/config/workdir.py | 5 +- pmb/core/arch.py | 191 +++++++++++++++++++++++++++++++++ pmb/core/chroot.py | 24 +++-- pmb/core/context.py | 5 +- pmb/helpers/apk.py | 8 +- pmb/helpers/args.py | 2 - pmb/helpers/frontend.py | 5 +- pmb/helpers/other.py | 3 +- pmb/helpers/package.py | 3 +- pmb/helpers/pkgrel_bump.py | 3 +- pmb/helpers/pmaports.py | 8 +- pmb/helpers/repo.py | 24 ++--- pmb/helpers/run_core.py | 3 + pmb/install/_install.py | 9 +- pmb/parse/__init__.py | 1 - pmb/parse/arch.py | 119 -------------------- pmb/parse/arguments.py | 47 ++++---- pmb/parse/depends.py | 2 - pmb/parse/deviceinfo.py | 16 +-- pmb/qemu/run.py | 40 +++---- pmb/sideload/__init__.py | 8 +- pmb/types.py | 4 +- 42 files changed, 389 insertions(+), 303 deletions(-) create mode 100644 pmb/core/arch.py delete mode 100644 pmb/parse/arch.py diff --git a/pmb/__init__.py b/pmb/__init__.py index de07e937..8e4a89f3 100644 --- a/pmb/__init__.py +++ b/pmb/__init__.py @@ -33,7 +33,7 @@ if version < (3, 9): def print_log_hint() -> None: context = get_context(allow_failure=True) - log = context.log if context else types.Config().work / "log.txt" + log = context.log if context else Config().work / "log.txt" # Hints about the log file (print to stdout only) log_hint = "Run 'pmbootstrap log' for details." if not os.path.exists(log): diff --git a/pmb/aportgen/device.py b/pmb/aportgen/device.py index afdea871..ea98ba2c 100644 --- a/pmb/aportgen/device.py +++ b/pmb/aportgen/device.py @@ -12,7 +12,7 @@ import pmb.parse def ask_for_architecture(): - architectures = pmb.config.build_device_architectures + architectures = Arch.supported() # Don't show armhf, new ports shouldn't use this architecture if "armhf" in architectures: architectures.remove("armhf") diff --git a/pmb/aportgen/grub_efi.py b/pmb/aportgen/grub_efi.py index 8a676093..6aff80f2 100644 --- a/pmb/aportgen/grub_efi.py +++ b/pmb/aportgen/grub_efi.py @@ -5,6 +5,7 @@ import pmb.aportgen.core import pmb.build import pmb.chroot.apk import pmb.chroot.apk_static +from pmb.core.arch import Arch from pmb.types import PmbArgs import pmb.helpers.run import pmb.parse.apkindex @@ -48,7 +49,7 @@ def generate(pkgname): pkgdesc="GRUB $_arch EFI files for every architecture" url="https://www.gnu.org/software/grub/" license="GPL-3.0-or-later" - arch="{pmb.config.arch_native}" + arch="{Arch.native()}" source="grub-efi-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk::$_mirror/{mirrordir}/main/$_arch/grub-efi-$pkgver-r$pkgrel.apk" package() {{ diff --git a/pmb/aportgen/linux.py b/pmb/aportgen/linux.py index decee098..7f1a319b 100644 --- a/pmb/aportgen/linux.py +++ b/pmb/aportgen/linux.py @@ -5,12 +5,11 @@ from pmb.parse.deviceinfo import Deviceinfo import pmb.helpers.run import pmb.aportgen.core import pmb.parse.apkindex -import pmb.parse.arch def generate_apkbuild(pkgname, deviceinfo: Deviceinfo, patches): device = "-".join(pkgname.split("-")[1:]) - carch = pmb.parse.arch.alpine_to_kernel(deviceinfo.arch) + carch = deviceinfo.arch.kernel() makedepends = ["bash", "bc", "bison", "devicepkg-dev", "findutils", "flex", "openssl-dev", "perl"] diff --git a/pmb/build/__init__.py b/pmb/build/__init__.py index de84d62b..03206137 100644 --- a/pmb/build/__init__.py +++ b/pmb/build/__init__.py @@ -6,4 +6,4 @@ from pmb.build.kconfig import menuconfig from pmb.build.newapkbuild import newapkbuild from pmb.build.other import copy_to_buildpath, is_necessary, \ index_repo -from pmb.build._package import BootstrapStage, mount_pmaports, package +from pmb.build._package import BootstrapStage, mount_pmaports, package, output_path diff --git a/pmb/build/_package.py b/pmb/build/_package.py index 8e18efc5..4cc7657b 100644 --- a/pmb/build/_package.py +++ b/pmb/build/_package.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: GPL-3.0-or-later import datetime import enum -from typing import Dict, List +from typing import Dict, List, Optional +from pmb.core.arch import Arch from pmb.core.context import Context from pmb.core.pkgrepo import pkgrepo_paths, pkgrepo_relative_path from pmb.helpers import logging @@ -17,7 +18,6 @@ import pmb.helpers.pmaports import pmb.helpers.repo import pmb.helpers.mount import pmb.parse -import pmb.parse.arch import pmb.parse.apkindex from pmb.helpers.exceptions import BuildFailedError @@ -221,7 +221,7 @@ def init_buildenv(context: Context, apkbuild, arch, strict=False, force=False, c depends_arch = arch if cross == "native": - depends_arch = pmb.config.arch_native + depends_arch = Arch.native() # Build dependencies depends, built = build_depends(context, apkbuild, depends_arch, strict) @@ -386,8 +386,9 @@ def link_to_git_dir(chroot: Chroot): "/home/pmos/build/.git"], chroot) -def output_path(arch: str, pkgname: str, pkgver: str, pkgrel: str) -> Path: - return Path(arch) / f"{pkgname}-{pkgver}-r{pkgrel}.apk" +def output_path(arch: Arch, pkgname: str, pkgver: str, pkgrel: str) -> Path: + # Yeahp, you can just treat an Arch like a path! + return arch / f"{pkgname}-{pkgver}-r{pkgrel}.apk" def run_abuild(context: Context, apkbuild, channel, arch, strict=False, force=False, cross=None, @@ -433,11 +434,11 @@ def run_abuild(context: Context, apkbuild, channel, arch, strict=False, force=Fa env = {"CARCH": arch, "SUDO_APK": "abuild-apk --no-progress"} if cross == "native": - hostspec = pmb.parse.arch.alpine_to_hostspec(arch) + hostspec = arch.alpine_triple() env["CROSS_COMPILE"] = hostspec + "-" env["CC"] = hostspec + "-gcc" if cross == "crossdirect": - env["PATH"] = ":".join(["/native/usr/lib/crossdirect/" + arch, + env["PATH"] = ":".join([f"/native/usr/lib/crossdirect/{arch}", pmb.config.chroot_path]) if not context.ccache: env["CCACHE_DISABLE"] = "1" @@ -503,7 +504,7 @@ def finish(apkbuild, channel, arch, output: str, chroot: Chroot, strict=False): pmb.chroot.init_keys() -def package(context: Context, pkgname, arch=None, force=False, strict=False, +def package(context: Context, pkgname, arch: Optional[Arch]=None, force=False, strict=False, skip_init_buildenv=False, src=None, bootstrap_stage=BootstrapStage.NONE): """ @@ -531,7 +532,7 @@ def package(context: Context, pkgname, arch=None, force=False, strict=False, logging.verbose(f"{pkgname}: running pmb.build._package.package") # Once per session is enough - arch = arch or pmb.config.arch_native + arch = arch or Arch.native() # the order of checks here is intentional, # skip_already_built() has side effects! if skip_already_built(pkgname, arch) and not force: diff --git a/pmb/build/autodetect.py b/pmb/build/autodetect.py index e5494b34..4466ed4d 100644 --- a/pmb/build/autodetect.py +++ b/pmb/build/autodetect.py @@ -1,14 +1,13 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later from pathlib import Path +from pmb.core.arch import Arch from pmb.helpers import logging from typing import Dict, Optional import pmb.config import pmb.chroot.apk -from pmb.types import PmbArgs import pmb.helpers.pmaports -import pmb.parse.arch from pmb.core import Chroot, ChrootType, get_context @@ -60,9 +59,9 @@ def arch(pkgname: str): if get_context().config.build_default_device_arch: preferred_arch = deviceinfo.arch - preferred_arch_2nd = pmb.config.arch_native + preferred_arch_2nd = Arch.native() else: - preferred_arch = pmb.config.arch_native + preferred_arch = Arch.native() preferred_arch_2nd = deviceinfo.arch if "noarch" in arches or "all" in arches or preferred_arch in arches: @@ -77,8 +76,8 @@ def arch(pkgname: str): return None -def chroot(apkbuild: Dict[str, str], arch: str) -> Chroot: - if arch == pmb.config.arch_native: +def chroot(apkbuild: Dict[str, str], arch: Arch) -> Chroot: + if arch == Arch.native(): return Chroot.native() if "pmb:cross-native" in apkbuild["options"]: @@ -87,13 +86,13 @@ def chroot(apkbuild: Dict[str, str], arch: str) -> Chroot: return Chroot.buildroot(arch) -def crosscompile(apkbuild, arch, suffix: Chroot): +def crosscompile(apkbuild, arch: Arch, suffix: Chroot): """ :returns: None, "native", "crossdirect" """ if not get_context().cross: return None - if not pmb.parse.arch.cpu_emulation_required(arch): + if not arch.cpu_emulation_required(): return None if suffix.type == ChrootType.NATIVE: return "native" diff --git a/pmb/build/envkernel.py b/pmb/build/envkernel.py index adfdf791..5155e0c0 100644 --- a/pmb/build/envkernel.py +++ b/pmb/build/envkernel.py @@ -1,6 +1,7 @@ # Copyright 2023 Robert Yang # SPDX-License-Identifier: GPL-3.0-or-later from typing import List +from pmb.core.arch import Arch from pmb.core.context import Context from pmb.helpers import logging import os @@ -114,7 +115,7 @@ def modify_apkbuild(pkgname: str, aport: Path): pmb.aportgen.core.rewrite(pkgname, apkbuild_path, fields=fields) -def run_abuild(context: Context, pkgname: str, arch: str, apkbuild_path: Path, kbuild_out): +def run_abuild(context: Context, pkgname: str, arch: Arch, apkbuild_path: Path, kbuild_out): """ Prepare build environment and run abuild. @@ -159,9 +160,9 @@ def run_abuild(context: Context, pkgname: str, arch: str, apkbuild_path: Path, k pmb.helpers.run.root(cmd) # Create the apk package - env = {"CARCH": arch, - "CHOST": arch, - "CBUILD": pmb.config.arch_native, + env = {"CARCH": str(arch), + "CHOST": str(arch), + "CBUILD": Arch.native(), "SUDO_APK": "abuild-apk --no-progress"} cmd = ["abuild", "rootpkg"] pmb.chroot.user(cmd, working_dir=build_path, env=env) @@ -201,15 +202,15 @@ def package_kernel(args: PmbArgs): # Install package dependencies depends, _ = pmb.build._package.build_depends( - context, apkbuild, pmb.config.arch_native, strict=False) + context, apkbuild, Arch.native(), strict=False) pmb.build.init(chroot) - if pmb.parse.arch.cpu_emulation_required(arch): - depends.append("binutils-" + arch) + if arch.cpu_emulation_required(): + depends.append(f"binutils-{arch}") pmb.chroot.apk.install(depends, chroot) - output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] + - "-r" + apkbuild["pkgrel"] + ".apk") - message = f"({chroot}) build " + output + output = pmb.build.output_path(arch, apkbuild["pkgname"], apkbuild["pkgver"], + apkbuild["pkgrel"]) + message = f"({chroot}) build {output}" logging.info(message) try: diff --git a/pmb/build/init.py b/pmb/build/init.py index e59d2b95..a5289b5b 100644 --- a/pmb/build/init.py +++ b/pmb/build/init.py @@ -1,5 +1,6 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later +from pmb.core.arch import Arch from pmb.core.context import Context from pmb.helpers import logging import os @@ -9,9 +10,7 @@ import pmb.build import pmb.config import pmb.chroot import pmb.chroot.apk -from pmb.types import PmbArgs import pmb.helpers.run -import pmb.parse.arch from pmb.core import Chroot, get_context @@ -66,7 +65,7 @@ def init(chroot: Chroot=Chroot.native()): apk_arch = chroot.arch # Add apk wrapper that runs native apk and lies about arch - if pmb.parse.arch.cpu_emulation_required(apk_arch) and \ + if apk_arch.cpu_emulation_required() and \ not (chroot / "usr/local/bin/abuild-apk").exists(): with (chroot / "tmp/apk_wrapper.sh").open("w") as handle: content = f""" @@ -112,14 +111,15 @@ def init(chroot: Chroot=Chroot.native()): pathlib.Path(marker).touch() -def init_compiler(context: Context, depends, cross, arch): +def init_compiler(context: Context, depends, cross, arch: Arch): + arch_str = str(arch) cross_pkgs = ["ccache-cross-symlinks", "abuild"] if "gcc4" in depends: - cross_pkgs += ["gcc4-" + arch] + cross_pkgs += ["gcc4-" + arch_str] elif "gcc6" in depends: - cross_pkgs += ["gcc6-" + arch] + cross_pkgs += ["gcc6-" + arch_str] else: - cross_pkgs += ["gcc-" + arch, "g++-" + arch] + cross_pkgs += ["gcc-" + arch_str, "g++-" + arch_str] if "clang" in depends or "clang-dev" in depends: cross_pkgs += ["clang"] if cross == "crossdirect": diff --git a/pmb/build/kconfig.py b/pmb/build/kconfig.py index 4ba1f1ce..824e713a 100644 --- a/pmb/build/kconfig.py +++ b/pmb/build/kconfig.py @@ -105,7 +105,7 @@ def menuconfig(args: PmbArgs, pkgname: str, use_oldconfig): arch = args.arch or get_arch(apkbuild) chroot = pmb.build.autodetect.chroot(apkbuild, arch) cross = pmb.build.autodetect.crosscompile(apkbuild, arch, chroot) - hostspec = pmb.parse.arch.alpine_to_hostspec(arch) + hostspec = arch.alpine_triple() # Set up build tools and makedepends pmb.build.init(chroot) @@ -143,7 +143,7 @@ def menuconfig(args: PmbArgs, pkgname: str, use_oldconfig): # Run make menuconfig outputdir = get_outputdir(args, pkgname, apkbuild) logging.info("(native) make " + kopt) - env = {"ARCH": pmb.parse.arch.alpine_to_kernel(arch), + env = {"ARCH": arch.kernel(), "DISPLAY": os.environ.get("DISPLAY"), "XAUTHORITY": "/home/pmos/.Xauthority"} if cross: diff --git a/pmb/build/other.py b/pmb/build/other.py index 413af331..0c371234 100644 --- a/pmb/build/other.py +++ b/pmb/build/other.py @@ -8,13 +8,11 @@ import datetime from typing import List import pmb.chroot -from pmb.types import PmbArgs import pmb.build import pmb.helpers.file import pmb.helpers.git import pmb.helpers.pmaports import pmb.helpers.run -import pmb.parse.arch import pmb.parse.apkindex import pmb.parse.version from pmb.core import Chroot, get_context diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index f1a5851d..a1fab083 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -3,7 +3,7 @@ import os from pathlib import Path import pmb.chroot.apk_static -from pmb.core.chroot import ChrootType +from pmb.core.arch import Arch from pmb.helpers import logging import shlex from typing import List @@ -11,14 +11,12 @@ from typing import List import pmb.build import pmb.chroot import pmb.config -from pmb.types import PmbArgs import pmb.helpers.apk import pmb.helpers.other import pmb.helpers.pmaports import pmb.helpers.repo import pmb.helpers.run import pmb.parse.apkindex -import pmb.parse.arch import pmb.parse.depends import pmb.parse.version from pmb.core import Chroot, get_context @@ -120,7 +118,7 @@ def packages_split_to_add_del(packages): return (to_add, to_del) -def packages_get_locally_built_apks(packages, arch: str) -> List[Path]: +def packages_get_locally_built_apks(packages, arch: Arch) -> List[Path]: """ Iterate over packages and if existing, get paths to locally built packages. This is used to force apk to upgrade packages to newer local versions, even @@ -183,6 +181,9 @@ def install_run_apk(to_add, to_add_local, to_del, chroot: Chroot): # we expect apk.static to be installed in the native chroot (which # will be the systemd version if building for systemd) and run # it from there. + # pmb.chroot.init(Chroot.native()) + # if chroot != Chroot.native(): + # pmb.chroot.init(chroot) apk_static = Chroot.native() / "sbin/apk.static" arch = chroot.arch apk_cache = get_context().config.work / f"cache_apk_{arch}" diff --git a/pmb/chroot/apk_static.py b/pmb/chroot/apk_static.py index 2d62a333..c62b47a9 100644 --- a/pmb/chroot/apk_static.py +++ b/pmb/chroot/apk_static.py @@ -1,6 +1,7 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later import os +from pmb.core.arch import Arch from pmb.helpers import logging import shutil import tarfile @@ -142,7 +143,7 @@ def download(file): """ channel_cfg = pmb.config.pmaports.read_config_channel() mirrordir = channel_cfg["mirrordir_alpine"] - base_url = f"{get_context().config.mirror_alpine}{mirrordir}/main/{pmb.config.arch_native}" + base_url = f"{get_context().config.mirror_alpine}{mirrordir}/main/{Arch.native()}" return pmb.helpers.http.download(f"{base_url}/{file}", file) diff --git a/pmb/chroot/binfmt.py b/pmb/chroot/binfmt.py index 643bffd5..209287b7 100644 --- a/pmb/chroot/binfmt.py +++ b/pmb/chroot/binfmt.py @@ -1,26 +1,25 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later import os +from pmb.core.arch import Arch from pmb.core.chroot import Chroot from pmb.helpers import logging -from pmb.types import PmbArgs import pmb.helpers.run import pmb.helpers.other import pmb.parse -import pmb.parse.arch import pmb.chroot.apk -def is_registered(arch_qemu): - return os.path.exists("/proc/sys/fs/binfmt_misc/qemu-" + arch_qemu) +def is_registered(arch_qemu: Arch): + return os.path.exists(f"/proc/sys/fs/binfmt_misc/qemu-{arch_qemu}") -def register(arch): +def register(arch: Arch): """ Get arch, magic, mask. """ - arch_qemu = pmb.parse.arch.alpine_to_qemu(arch) + arch_qemu = arch.qemu() chroot = Chroot.native() # always make sure the qemu- binary is installed, since registering @@ -58,8 +57,8 @@ def register(arch): pmb.helpers.run.root(["sh", "-c", 'echo "' + code + '" > ' + register]) -def unregister(arch): - arch_qemu = pmb.parse.arch.alpine_to_qemu(arch) +def unregister(arch: Arch): + arch_qemu = arch.qemu() binfmt_file = "/proc/sys/fs/binfmt_misc/qemu-" + arch_qemu if not os.path.exists(binfmt_file): return diff --git a/pmb/chroot/init.py b/pmb/chroot/init.py index 767c8e90..fee9f2dd 100644 --- a/pmb/chroot/init.py +++ b/pmb/chroot/init.py @@ -14,7 +14,6 @@ import pmb.config.workdir import pmb.helpers.repo import pmb.helpers.run import pmb.helpers.other -import pmb.parse.arch from pmb.core import Chroot, ChrootType, get_context cache_chroot_is_outdated: List[str] = [] @@ -58,10 +57,10 @@ def mark_in_chroot(chroot: Chroot=Chroot.native()): def setup_qemu_emulation(chroot: Chroot): arch = chroot.arch - if not pmb.parse.arch.cpu_emulation_required(arch): + if not arch.cpu_emulation_required(): return - arch_qemu = pmb.parse.arch.alpine_to_qemu(arch) + arch_qemu = arch.qemu() # mount --bind the qemu-user binary pmb.chroot.binfmt.register(arch) diff --git a/pmb/chroot/mount.py b/pmb/chroot/mount.py index 5ee81c25..49b9c7ff 100644 --- a/pmb/chroot/mount.py +++ b/pmb/chroot/mount.py @@ -88,7 +88,7 @@ def mount(chroot: Chroot): mountpoints: Dict[Path, Path] = {} for src_template, target_template in pmb.config.chroot_mount_bind.items(): src_template = src_template.replace("$WORK", os.fspath(get_context().config.work)) - src_template = src_template.replace("$ARCH", arch) + src_template = src_template.replace("$ARCH", str(arch)) src_template = src_template.replace("$CHANNEL", channel) mountpoints[Path(src_template)] = Path(target_template) diff --git a/pmb/chroot/shutdown.py b/pmb/chroot/shutdown.py index f8390cf1..640baa6f 100644 --- a/pmb/chroot/shutdown.py +++ b/pmb/chroot/shutdown.py @@ -1,14 +1,13 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later +from pmb.core.arch import Arch from pmb.helpers import logging import socket from contextlib import closing import pmb.chroot -from pmb.types import PmbArgs import pmb.helpers.mount import pmb.install.losetup -import pmb.parse.arch from pmb.core import Chroot, ChrootType, get_context @@ -100,7 +99,7 @@ def shutdown(only_install_related=False): pmb.helpers.mount.umount_all(path) # Clean up the rest - for arch in pmb.config.build_device_architectures: - if pmb.parse.arch.cpu_emulation_required(arch): + for arch in Arch.supported(): + if arch.cpu_emulation_required(): pmb.chroot.binfmt.unregister(arch) logging.debug("Shutdown complete") diff --git a/pmb/chroot/zap.py b/pmb/chroot/zap.py index 692e67f8..9da86785 100644 --- a/pmb/chroot/zap.py +++ b/pmb/chroot/zap.py @@ -1,6 +1,7 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later import glob +from pmb.core.arch import Arch from pmb.helpers import logging import os @@ -137,7 +138,7 @@ def zap_pkgs_local_mismatch(confirm=True, dry=False): def zap_pkgs_online_mismatch(confirm=True, dry=False): # Check whether we need to do anything - paths = glob.glob(f"{get_context().config.work}/cache_apk_*") + paths = list(get_context().config.work.glob("cache_apk_*")) if not len(paths): return if (confirm and not pmb.helpers.cli.confirm("Remove outdated" @@ -146,8 +147,8 @@ def zap_pkgs_online_mismatch(confirm=True, dry=False): # Iterate over existing apk caches for path in paths: - arch = os.path.basename(path).split("_", 2)[2] - if arch == pmb.config.arch_native: + arch = Arch.from_str(path.name.split("_", 2)[2]) + if arch == Arch.native(): suffix = Chroot.native() else: try: diff --git a/pmb/commands/repo_bootstrap.py b/pmb/commands/repo_bootstrap.py index c16186d8..b29a2a20 100644 --- a/pmb/commands/repo_bootstrap.py +++ b/pmb/commands/repo_bootstrap.py @@ -1,6 +1,7 @@ # Copyright 2024 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later from typing import Optional +from pmb.core.arch import Arch from pmb.core.chroot import Chroot, ChrootType from pmb.core.context import Context from pmb.helpers import logging @@ -15,7 +16,7 @@ from pmb.core import get_context from pmb import commands class RepoBootstrap(commands.Command): - arch: str + arch: Arch repo: str context: Context @@ -38,7 +39,7 @@ class RepoBootstrap(commands.Command): " current branch") - def __init__(self, arch: Optional[str], repository: str): + def __init__(self, arch: Optional[Arch], repository: str): context = get_context() if arch: self.arch = arch @@ -46,7 +47,7 @@ class RepoBootstrap(commands.Command): if context.config.build_default_device_arch: self.arch = pmb.parse.deviceinfo().arch else: - self.arch = pmb.config.arch_native + self.arch = Arch.native() self.repo = repository self.context = context @@ -74,7 +75,7 @@ class RepoBootstrap(commands.Command): self.progress_total += len(steps) * 2 # Foreign arch: need to initialize one additional chroot each step - if pmb.parse.arch.cpu_emulation_required(self.arch): + if self.arch.cpu_emulation_required(): self.progress_total += len(steps) @@ -87,7 +88,7 @@ class RepoBootstrap(commands.Command): def run_steps(self, steps): chroot: Chroot - if pmb.parse.arch.cpu_emulation_required(self.arch): + if self.arch.cpu_emulation_required(): chroot = Chroot(ChrootType.BUILDROOT, self.arch) else: chroot = Chroot.native() @@ -129,7 +130,7 @@ class RepoBootstrap(commands.Command): msg = f"Found previously built packages for {channel}/{self.arch}, run" \ " 'pmbootstrap zap -p' first" - if pmb.parse.arch.cpu_emulation_required(self.arch): + if self.arch.cpu_emulation_required(): msg += " or remove the path manually (to keep cross compilers if" \ " you just built them)" diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index 752d4e83..27e5f231 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -1,12 +1,10 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later -import multiprocessing import os from pathlib import Path from pmb.types import AportGenEntry, PathString -import pmb.parse.arch import sys -from typing import Dict, List, Sequence, TypedDict +from typing import Dict, List, Sequence # # Exported functions @@ -23,7 +21,6 @@ from pmb.config.other import is_systemd_selected # pmb_src: Path = Path(Path(__file__) / "../../..").resolve() apk_keys_path: Path = (pmb_src / "pmb/data/keys") -arch_native = pmb.parse.arch.alpine_native() # apk-tools minimum version # https://pkgs.alpinelinux.org/packages?name=apk-tools&branch=edge @@ -239,15 +236,6 @@ apkindex_retention_time = 4 # When chroot is considered outdated (in seconds) chroot_outdated = 3600 * 24 * 2 -# -# BUILD -# -# Officially supported host/target architectures for postmarketOS. Only -# specify architectures supported by Alpine here. For cross-compiling, -# we need to generate the "musl-$ARCH" and "gcc-$ARCH" packages (use -# "pmbootstrap aportgen musl-armhf" etc.). -build_device_architectures = ["armhf", "armv7", "aarch64", "x86_64", "x86", "riscv64"] - # Packages that will be installed in a chroot before it builds packages # for the first time build_packages = ["abuild", "build-base", "ccache", "git"] diff --git a/pmb/config/workdir.py b/pmb/config/workdir.py index fe7717c2..68ddf4ee 100644 --- a/pmb/config/workdir.py +++ b/pmb/config/workdir.py @@ -29,7 +29,7 @@ def chroot_save_init(suffix: Chroot): cfg[key] = {} # Update sections - channel = pmb.config.pmaports.read_config()["channel"] + channel = pmb.config.pmaports.read_config(support_systemd=False)["channel"] cfg["chroot-channels"][str(suffix)] = channel cfg["chroot-init-dates"][str(suffix)] = str(int(time.time())) @@ -82,7 +82,8 @@ def chroot_check_channel(chroot: Chroot): if key not in cfg or str(chroot) not in cfg[key]: raise RuntimeError(f"{msg_unknown} {msg_again}") - channel = pmb.config.pmaports.read_config()["channel"] + # Exclude systemd repo + channel = pmb.config.pmaports.read_config(support_systemd=False)["channel"] channel_cfg = cfg[key][str(chroot)] if channel != channel_cfg: raise RuntimeError(f"Chroot '{chroot}' was created for the" diff --git a/pmb/core/arch.py b/pmb/core/arch.py new file mode 100644 index 00000000..a90af631 --- /dev/null +++ b/pmb/core/arch.py @@ -0,0 +1,191 @@ +# Copyright 2023 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later +import enum +from pathlib import Path, PosixPath, PurePosixPath +import platform +from typing import Set + +# Initialised at the bottom +_cached_native_arch: "Arch" + + +class Arch(enum.Enum): + """Supported architectures according to the Alpine + APKBUILD format.""" + x86 = "x86" + x86_64 = "x86_64" + armhf = "armhf" + armv7 = "armv7" + aarch64 = "aarch64" + riscv64 = "riscv64" + s390x = "s390x" + ppc64le = "ppc64le" + # Arches Alpine can build for + armel = "armel" + loongarch32 = "loongarch32" + loongarchx32 = "loongarchx32" + loongarch64 = "loongarch64" + mips = "mips" + mips64 = "mips64" + mipsel = "mipsel" + mips64el = "mips64el" + ppc = "ppc" + ppc64 = "ppc64" + riscv32 = "riscv32" + + + def __str__(self) -> str: + return self.value + + + @staticmethod + def from_str(arch: str) -> "Arch": + try: + return Arch(arch) + except ValueError: + raise ValueError(f"Invalid architecture: {arch}") + + + @staticmethod + def from_machine_type(machine_type: str) -> "Arch": + mapping = { + "i686": Arch.x86, + "x86_64": Arch.x86_64, + "aarch64": Arch.aarch64, + "arm64": Arch.aarch64, + "armv6l": Arch.armhf, + "armv7l": Arch.armv7, + "armv8l": Arch.armv7, + } + return mapping[machine_type] + + + @staticmethod + def native() -> "Arch": + global _cached_native_arch + return _cached_native_arch + + + def is_native(self): + return self == Arch.native() + + + @staticmethod + def supported() -> Set["Arch"]: + """Officially supported host/target architectures for postmarketOS. Only + specify architectures supported by Alpine here. For cross-compiling, + we need to generate the "musl-$ARCH" and "gcc-$ARCH" packages (use + "pmbootstrap aportgen musl-armhf" etc.).""" + # FIXME: cache? + return set([ + Arch.armhf, + Arch.armv7, + Arch.aarch64, + Arch.x86_64, + Arch.x86, + Arch.riscv64, + Arch.native(), + ]) + + + def kernel(self): + mapping = { + Arch.x86: "x86", + Arch.x86_64: "x86_64", + Arch.armhf: "arm", + Arch.aarch64: "arm64", + Arch.riscv64: "riscv", + Arch.ppc64le: "powerpc", + Arch.s390x: "s390", + } + return mapping.get(self, self.value) + + def qemu(self): + mapping = { + Arch.x86: "i386", + Arch.armhf: "arm", + Arch.armv7: "arm", + Arch.ppc64le: "ppc64", + } + return mapping.get(self, self.value) + + + def alpine_triple(self): + mapping = { + Arch.aarch64: "aarch64-alpine-linux-musl", + Arch.armel: "armv5-alpine-linux-musleabi", + Arch.armhf: "armv6-alpine-linux-musleabihf", + Arch.armv7: "armv7-alpine-linux-musleabihf", + Arch.loongarch32: "loongarch32-alpine-linux-musl", + Arch.loongarchx32: "loongarchx32-alpine-linux-musl", + Arch.loongarch64: "loongarch64-alpine-linux-musl", + Arch.mips: "mips-alpine-linux-musl", + Arch.mips64: "mips64-alpine-linux-musl", + Arch.mipsel: "mipsel-alpine-linux-musl", + Arch.mips64el: "mips64el-alpine-linux-musl", + Arch.ppc: "powerpc-alpine-linux-musl", + Arch.ppc64: "powerpc64-alpine-linux-musl", + Arch.ppc64le: "powerpc64le-alpine-linux-musl", + Arch.riscv32: "riscv32-alpine-linux-musl", + Arch.riscv64: "riscv64-alpine-linux-musl", + Arch.s390x: "s390x-alpine-linux-musl", + Arch.x86: "i586-alpine-linux-musl", + Arch.x86_64: "x86_64-alpine-linux-musl", + } + + if self in mapping: + return mapping[self] + + raise ValueError(f"Can not map Alpine architecture '{self}'" + " to the right hostspec value") + + + def cpu_emulation_required(self): + # Obvious case: host arch is target arch + if self == Arch.native(): + return False + + # Other cases: host arch on the left, target archs on the right + not_required = { + Arch.x86_64: [Arch.x86], + Arch.armv7: [Arch.armel, Arch.armhf], + Arch.aarch64: [Arch.armv7], + } + if Arch.native() in not_required: + if self in not_required[Arch.native()]: + return False + + # No match: then it's required + return True + + + # Magic to let us use an arch as a Path element + def __truediv__(self, other: object) -> Path: + if isinstance(other, PosixPath) or isinstance(other, PurePosixPath): + # Convert the other path to a relative path + # FIXME: we should avoid creating absolute paths that we actually want + # to make relative to the chroot... + # if other.is_absolute(): + # logging.warning("FIXME: absolute path made relative to Arch??") + other = other.relative_to("/") if other.is_absolute() else other + return Path(str(self)).joinpath(other) + if isinstance(other, str): + # Let's us do Arch / "whatever.apk" and magically produce a path + # maybe this is a pattern we should avoid, but it seems somewhat + # sensible + return Path(str(self)).joinpath(other.strip("/")) + + return NotImplemented + + + def __rtruediv__(self, other: object) -> Path: + if isinstance(other, PosixPath) or isinstance(other, PurePosixPath): + # Important to produce a new Path object here, otherwise we + # end up with one object getting shared around and modified + # and lots of weird stuff happens. + return Path(other) / str(self) + # We don't support str / Arch since that is a weird pattern + + return NotImplemented + +_cached_native_arch = Arch.from_machine_type(platform.machine()) diff --git a/pmb/core/chroot.py b/pmb/core/chroot.py index 22065db8..d98d42ab 100644 --- a/pmb/core/chroot.py +++ b/pmb/core/chroot.py @@ -3,9 +3,10 @@ from __future__ import annotations import enum -from typing import Generator, Optional +from typing import Generator, Optional, Union from pathlib import Path, PosixPath, PurePosixPath import pmb.config +from pmb.core.arch import Arch from .context import get_context class ChrootType(enum.Enum): @@ -21,10 +22,10 @@ class Chroot: __type: ChrootType __name: str - def __init__(self, suffix_type: ChrootType, name: Optional[str] = ""): + def __init__(self, suffix_type: ChrootType, name: Optional[Union[str, Arch]] = ""): self.__type = suffix_type - self.__name = name or "" - + self.__name = str(name or "") + self.__validate() def __validate(self) -> None: @@ -74,16 +75,15 @@ class Chroot: @property - # FIXME: make an Arch type - def arch(self) -> str: + def arch(self) -> Arch: if self.type == ChrootType.NATIVE: - return pmb.config.arch_native + return Arch.native() if self.type == ChrootType.BUILDROOT: - return self.name() + return Arch.from_str(self.name()) # FIXME: this is quite delicate as it will only be valid # for certain pmbootstrap commands... It was like this # before but it should be fixed. - arch = pmb.core.get_context().device_arch + arch = pmb.parse.deviceinfo().arch if arch is not None: return arch @@ -118,8 +118,12 @@ class Chroot: def __rtruediv__(self, other: object) -> Path: if isinstance(other, PosixPath) or isinstance(other, PurePosixPath): + # Important to produce a new Path object here, otherwise we + # end up with one object getting shared around and modified + # and lots of weird stuff happens. return Path(other) / self.path if isinstance(other, str): + # This implicitly creates a new Path object return other / self.path return NotImplemented @@ -139,7 +143,7 @@ class Chroot: @staticmethod - def buildroot(arch: str) -> Chroot: + def buildroot(arch: Arch) -> Chroot: return Chroot(ChrootType.BUILDROOT, arch) diff --git a/pmb/core/context.py b/pmb/core/context.py index 44f687e1..e35739f0 100644 --- a/pmb/core/context.py +++ b/pmb/core/context.py @@ -5,7 +5,8 @@ from typing import List, Optional from pathlib import Path -from pmb.types import Config +from pmb.core.arch import Arch +from .config import Config class Context(): @@ -15,8 +16,6 @@ class Context(): sudo_timer: bool = False force: bool = False log: Path - # The architecture of the selected device - device_arch: Optional[str] = None # assume yes to prompts assume_yes: bool = False diff --git a/pmb/helpers/apk.py b/pmb/helpers/apk.py index 964e3cd1..c722c4d6 100644 --- a/pmb/helpers/apk.py +++ b/pmb/helpers/apk.py @@ -6,6 +6,7 @@ from typing import List, Sequence import pmb.chroot import pmb.config.pmaports +from pmb.core.arch import Arch from pmb.types import PathString, PmbArgs import pmb.helpers.cli import pmb.helpers.run @@ -72,7 +73,12 @@ def apk_with_progress(command: Sequence[PathString]): :raises RuntimeError: when the apk command fails """ fifo, fifo_outside = _prepare_fifo() - _command: List[str] = [os.fspath(c) for c in command] + _command: List[str] = [] + for c in command: + if isinstance(c, Arch): + _command.append(str(c)) + else: + _command.append(os.fspath(c)) command_with_progress = _create_command_with_progress(_command, fifo) log_msg = " ".join(_command) with pmb.helpers.run.root(['cat', fifo], diff --git a/pmb/helpers/args.py b/pmb/helpers/args.py index 17da7438..ddbd3d9e 100644 --- a/pmb/helpers/args.py +++ b/pmb/helpers/args.py @@ -99,8 +99,6 @@ def init(args: PmbArgs) -> PmbArgs: "pull", "shutdown", "zap"]: pmb.config.pmaports.read_config() pmb.helpers.git.parse_channels_cfg(pkgrepo_default_path()) - deviceinfo = pmb.parse.deviceinfo() - context.device_arch = deviceinfo.arch # Remove attributes from args so they don't get used by mistake delattr(args, "timeout") diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 2be3fbe8..690b3cc9 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import json from typing import List, Sequence, Tuple +from pmb.core.arch import Arch from pmb.helpers import logging import os from pathlib import Path @@ -67,7 +68,7 @@ def _parse_suffix(args: PmbArgs) -> Chroot: if args.buildroot == "device": return Chroot.buildroot(pmb.parse.deviceinfo().arch) else: - return Chroot.buildroot(args.buildroot) + return Chroot.buildroot(Arch.from_str(args.buildroot)) elif args.suffix: (_t, s) = args.suffix.split("_") t: ChrootType = ChrootType(_t) @@ -550,7 +551,7 @@ def shutdown(args: PmbArgs): def stats(args: PmbArgs): # Chroot suffix chroot = Chroot.native() - if args.arch != pmb.config.arch_native: + if args.arch != Arch.native(): chroot = Chroot.buildroot(args.arch) # Install ccache and display stats diff --git a/pmb/helpers/other.py b/pmb/helpers/other.py index 27227445..2eeed338 100644 --- a/pmb/helpers/other.py +++ b/pmb/helpers/other.py @@ -1,6 +1,7 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later from pmb.core import get_context +from pmb.core.arch import Arch from pmb.helpers import logging import os from pathlib import Path @@ -191,7 +192,7 @@ def migrate_work_folder(args: PmbArgs): # Move packages to edge subdir edge_path = context.config.work / "packages/edge" pmb.helpers.run.root(["mkdir", "-p", edge_path]) - for arch in pmb.config.build_device_architectures: + for arch in Arch.supported(): old_path = context.config.work / "packages" / arch new_path = edge_path / arch if old_path.exists(): diff --git a/pmb/helpers/package.py b/pmb/helpers/package.py index 1ad1637d..bc6c266b 100644 --- a/pmb/helpers/package.py +++ b/pmb/helpers/package.py @@ -10,6 +10,7 @@ See also: """ import copy from typing import Any, Dict +from pmb.core.arch import Arch from pmb.core.context import get_context from pmb.helpers import logging import pmb.build._package @@ -83,7 +84,7 @@ def get(pkgname, arch, replace_subpkgnames=False, must_exist=True): # Find in APKINDEX (other arches) if not ret: pmb.helpers.repo.update() - for arch_i in pmb.config.build_device_architectures: + for arch_i in Arch.supported(): if arch_i != arch: ret = pmb.parse.apkindex.package(pkgname, arch_i, False) if ret: diff --git a/pmb/helpers/pkgrel_bump.py b/pmb/helpers/pkgrel_bump.py index a9291118..519a3a8f 100644 --- a/pmb/helpers/pkgrel_bump.py +++ b/pmb/helpers/pkgrel_bump.py @@ -1,5 +1,6 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later +from pmb.core.arch import Arch from pmb.helpers import logging from pmb.types import PmbArgs @@ -104,7 +105,7 @@ def auto_apkindex_package(args: PmbArgs, arch, aport, apk, dry=False): def auto(args: PmbArgs, dry=False): """:returns: list of aport names, where the pkgrel needed to be changed""" ret = [] - for arch in pmb.config.build_device_architectures: + for arch in Arch.supported(): paths = pmb.helpers.repo.apkindex_files(args, arch, alpine=False) for path in paths: logging.info(f"scan {path}") diff --git a/pmb/helpers/pmaports.py b/pmb/helpers/pmaports.py index 08d354fc..3c612b9e 100644 --- a/pmb/helpers/pmaports.py +++ b/pmb/helpers/pmaports.py @@ -8,6 +8,7 @@ See also: """ import glob from pmb.core import get_context +from pmb.core.arch import Arch from pmb.core.pkgrepo import pkgrepo_iter_package_dirs from pmb.helpers import logging from pathlib import Path @@ -164,6 +165,7 @@ def find(package, must_exist=True, subpackages=True, skip_extra_repos=False): # Try to find an APKBUILD with the exact pkgname we are looking for path = _find_apkbuilds(skip_extra_repos).get(package) if path: + logging.verbose(f"{package}: found apkbuild: {path}") ret = path.parent elif subpackages: # No luck, take a guess what APKBUILD could have the package we are @@ -282,7 +284,7 @@ def get_repo(pkgname, must_exist=True) -> Optional[str]: return aport.parent.name -def check_arches(arches, arch): +def check_arches(arches, arch: Arch): """Check if building for a certain arch is allowed. :param arches: list of all supported arches, as it can be found in the @@ -293,9 +295,9 @@ def check_arches(arches, arch): :returns: True when building is allowed, False otherwise """ - if "!" + arch in arches: + if f"!{arch}" in arches: return False - for value in [arch, "all", "noarch"]: + for value in [str(arch), "all", "noarch"]: if value in arches: return True return False diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index f3b505fc..ccc310e3 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -10,10 +10,11 @@ See also: import os import hashlib from pmb.core import get_context +from pmb.core.arch import Arch from pmb.core.pkgrepo import pkgrepo_paths from pmb.helpers import logging from pathlib import Path -from typing import List +from typing import List, Optional import pmb.config.pmaports from pmb.types import PmbArgs @@ -69,7 +70,6 @@ def urls(user_repository=True, postmarketos_mirror=True, alpine=True): # Local user repository (for packages compiled with pmbootstrap) if user_repository: - channel = pmb.config.pmaports.read_config()["channel"] # FIXME: We shouldn't hardcod this here for channel in pmb.config.pmaports.all_channels(): ret.append(f"/mnt/pmbootstrap/packages/{channel}") @@ -98,7 +98,7 @@ def urls(user_repository=True, postmarketos_mirror=True, alpine=True): return ret -def apkindex_files(arch=None, user_repository=True, pmos=True, +def apkindex_files(arch: Optional[Arch]=None, user_repository=True, pmos=True, alpine=True) -> List[Path]: """Get a list of outside paths to all resolved APKINDEX.tar.gz files for a specific arch. @@ -109,7 +109,7 @@ def apkindex_files(arch=None, user_repository=True, pmos=True, :returns: list of absolute APKINDEX.tar.gz file paths """ if not arch: - arch = pmb.config.arch_native + arch = Arch.native() ret = [] # Local user repository (for packages compiled with pmbootstrap) @@ -144,7 +144,7 @@ def update(arch=None, force=False, existing_only=False): return False # Architectures and retention time - architectures = [arch] if arch else pmb.config.build_device_architectures + architectures = [arch] if arch else Arch.supported() retention_hours = pmb.config.apkindex_retention_time retention_seconds = retention_hours * 3600 @@ -152,13 +152,13 @@ def update(arch=None, force=False, existing_only=False): # outdated: {URL: apkindex_path, ... } # outdated_arches: ["armhf", "x86_64", ... ] outdated = {} - outdated_arches = [] + outdated_arches: List[Arch] = [] for url in urls(False): for arch in architectures: # APKINDEX file name from the URL - url_full = url + "/" + arch + "/APKINDEX.tar.gz" + url_full = f"{url}/{arch}/APKINDEX.tar.gz" cache_apk_outside = get_context().config.work / f"cache_apk_{arch}" - apkindex = cache_apk_outside / f"APKINDEX.{apkindex_hash(url)}.tar.gz" + apkindex = cache_apk_outside / f"{apkindex_hash(url)}.tar.gz" # Find update reason, possibly skip non-existing or known 404 files reason = None @@ -186,7 +186,7 @@ def update(arch=None, force=False, existing_only=False): # Bail out or show log message if not len(outdated): return False - logging.info("Update package index for " + ", ".join(outdated_arches) + + logging.info("Update package index for " + ", ".join([str(a) for a in outdated_arches]) + " (" + str(len(outdated)) + " file(s))") # Download and move to right location @@ -206,7 +206,7 @@ def update(arch=None, force=False, existing_only=False): return True -def alpine_apkindex_path(repo="main", arch=None): +def alpine_apkindex_path(repo="main", arch: Optional[Arch]=None): """Get the path to a specific Alpine APKINDEX file on disk and download it if necessary. :param repo: Alpine repository name (e.g. "main") @@ -215,10 +215,10 @@ def alpine_apkindex_path(repo="main", arch=None): """ # Repo sanity check if repo not in ["main", "community", "testing", "non-free"]: - raise RuntimeError("Invalid Alpine repository: " + repo) + raise RuntimeError(f"Invalid Alpine repository: {repo}") # Download the file - arch = arch or pmb.config.arch_native + arch = arch or Arch.native() update(arch) # Find it on disk diff --git a/pmb/helpers/run_core.py b/pmb/helpers/run_core.py index e71eb23d..614f2d00 100644 --- a/pmb/helpers/run_core.py +++ b/pmb/helpers/run_core.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import fcntl from pmb.core import get_context +from pmb.core.arch import Arch from pmb.types import PathString, Env from pmb.helpers import logging import os @@ -35,6 +36,8 @@ def flat_cmd(cmds: Sequence[Sequence[PathString]], working_dir: Optional[Path]=N # Merge env and cmd into escaped list escaped = [] for key, value in env.items(): + if isinstance(value, Arch): + value = str(value) escaped.append(key + "=" + shlex.quote(os.fspath(value))) for cmd in cmds: for i in range(len(cmd)): diff --git a/pmb/install/_install.py b/pmb/install/_install.py index 428b5f19..5b99655f 100644 --- a/pmb/install/_install.py +++ b/pmb/install/_install.py @@ -1,5 +1,6 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later +from pmb.core.arch import Arch from pmb.helpers import logging import os import re @@ -120,7 +121,7 @@ def copy_files_from_chroot(args: PmbArgs, chroot: Chroot): mountpoint_outside = Chroot.native() / mountpoint # Remove empty qemu-user binary stub (where the binary was bind-mounted) - arch_qemu = pmb.parse.arch.alpine_to_qemu(pmb.parse.deviceinfo().arch) + arch_qemu = pmb.parse.deviceinfo().arch.qemu() qemu_binary = mountpoint_outside / ("/usr/bin/qemu-" + arch_qemu + "-static") if os.path.exists(qemu_binary): pmb.helpers.run.root(["rm", qemu_binary]) @@ -367,7 +368,7 @@ def setup_keymap(config: Config): def setup_timezone(chroot: Chroot, timezone: str): # We don't care about the arch since it's built for all! - alpine_conf = pmb.helpers.package.get("alpine-conf", pmb.config.arch_native) + alpine_conf = pmb.helpers.package.get("alpine-conf", Arch.native()) version = alpine_conf["version"].split("-r")[0] setup_tz_cmd = ["setup-timezone"] @@ -476,7 +477,7 @@ def disable_firewall(chroot: Chroot): raise RuntimeError(f"Failed to disable firewall: {nftables_files}") -def print_firewall_info(disabled: bool, arch: str): +def print_firewall_info(disabled: bool, arch: Arch): pmaports_cfg = pmb.config.pmaports.read_config() pmaports_ok = pmaports_cfg.get("supported_firewall", None) == "nftables" @@ -981,7 +982,7 @@ def print_flash_info(device: str, deviceinfo: Deviceinfo, split: bool, have_disk " and flash outside of pmbootstrap.") -def install_recovery_zip(args: PmbArgs, device: str, arch: str, steps): +def install_recovery_zip(args: PmbArgs, device: str, arch: Arch, steps): logging.info(f"*** ({steps}/{steps}) CREATING RECOVERY-FLASHABLE ZIP ***") chroot = Chroot(ChrootType.BUILDROOT, arch) mount_device_rootfs(Chroot.rootfs(device)) diff --git a/pmb/parse/__init__.py b/pmb/parse/__init__.py index 3ea1df0c..9a9c77b4 100644 --- a/pmb/parse/__init__.py +++ b/pmb/parse/__init__.py @@ -8,4 +8,3 @@ from pmb.parse.deviceinfo import deviceinfo from pmb.parse.kconfig import check from pmb.parse.bootimg import bootimg from pmb.parse.cpuinfo import arm_big_little_first_group_ncpus -import pmb.parse.arch diff --git a/pmb/parse/arch.py b/pmb/parse/arch.py deleted file mode 100644 index 176d225f..00000000 --- a/pmb/parse/arch.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2023 Oliver Smith -# SPDX-License-Identifier: GPL-3.0-or-later -import fnmatch -import platform -import pmb.config -import pmb.parse.arch - -def alpine_native(): - machine = platform.machine() - - return machine_type_to_alpine(machine) - - -def alpine_to_qemu(arch): - """ - Convert the architecture to the string used in the QEMU packaging. - This corresponds to the package name of e.g. qemu-system-aarch64. - """ - - mapping = { - "x86": "i386", - "x86_64": "x86_64", - "armhf": "arm", - "armv7": "arm", - "aarch64": "aarch64", - "riscv64": "riscv64", - } - for pattern, arch_qemu in mapping.items(): - if fnmatch.fnmatch(arch, pattern): - return arch_qemu - raise ValueError("Can not map Alpine architecture '" + arch + "'" - " to the right Debian architecture.") - - -def alpine_to_kernel(arch): - """ - Convert the architecture to the string used inside the kernel sources. - You can read the mapping from the linux-vanilla APKBUILD for example. - """ - mapping = { - "aarch64*": "arm64", - "arm*": "arm", - "ppc*": "powerpc", - "s390*": "s390", - "riscv64*": "riscv", - } - for pattern, arch_kernel in mapping.items(): - if fnmatch.fnmatch(arch, pattern): - return arch_kernel - return arch - - -def alpine_to_hostspec(arch): - """ - See: abuild source code/functions.sh.in: arch_to_hostspec() - """ - mapping = { - "aarch64": "aarch64-alpine-linux-musl", - "armel": "armv5-alpine-linux-musleabi", - "armhf": "armv6-alpine-linux-musleabihf", - "armv7": "armv7-alpine-linux-musleabihf", - "loongarch32": "loongarch32-alpine-linux-musl", - "loongarchx32": "loongarchx32-alpine-linux-musl", - "loongarch64": "loongarch64-alpine-linux-musl", - "mips": "mips-alpine-linux-musl", - "mips64": "mips64-alpine-linux-musl", - "mipsel": "mipsel-alpine-linux-musl", - "mips64el": "mips64el-alpine-linux-musl", - "ppc": "powerpc-alpine-linux-musl", - "ppc64": "powerpc64-alpine-linux-musl", - "ppc64le": "powerpc64le-alpine-linux-musl", - "riscv32": "riscv32-alpine-linux-musl", - "riscv64": "riscv64-alpine-linux-musl", - "s390x": "s390x-alpine-linux-musl", - "x86": "i586-alpine-linux-musl", - "x86_64": "x86_64-alpine-linux-musl", - } - if arch in mapping: - return mapping[arch] - - raise ValueError("Can not map Alpine architecture '" + arch + "'" - " to the right hostspec value") - - -def cpu_emulation_required(arch): - # Obvious case: host arch is target arch - if pmb.config.arch_native == arch: - return False - - # Other cases: host arch on the left, target archs on the right - not_required = { - "x86_64": ["x86"], - "armv7": ["armel", "armhf"], - "aarch64": ["armv7"], - } - if pmb.config.arch_native in not_required: - if arch in not_required[pmb.config.arch_native]: - return False - - # No match: then it's required - return True - - -def machine_type_to_alpine(machine_type: str) -> str: - """Translate a machine type to an Alpine architecture. A machine type can come from - for example `$ uname -m` or platform.machine() from Python's standard library.""" - mapping = { - "i686": "x86", - "x86_64": "x86_64", - "aarch64": "aarch64", - "arm64": "aarch64", - "armv6l": "armhf", - "armv7l": "armv7", - "armv8l": "armv7", - } - if machine_type in mapping: - return mapping[machine_type] - raise ValueError(f"Can not map machine type '{machine_type}'" - " to the right Alpine Linux architecture") diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index f760c343..0ef334bc 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -1,7 +1,6 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later import argparse -import copy import os from pathlib import Path import sys @@ -16,7 +15,6 @@ except ImportError: pass import pmb.config -import pmb.parse.arch import pmb.helpers.args import pmb.helpers.pmaports @@ -234,7 +232,8 @@ def arguments_sideload(subparser): ret.add_argument("--user", help="use a different username than the" " one set in init") ret.add_argument("--arch", help="skip automatic architecture deduction and use the" - " given value") + " given value", + type=lambda x: Arch.from_str(x)) ret.add_argument("--install-key", help="install the apk key from this" " machine if needed", action="store_true", dest="install_key") @@ -483,8 +482,7 @@ def arguments_newapkbuild(subparser): def arguments_kconfig(subparser): # Allowed architectures - arch_native = pmb.config.arch_native - arch_choices = set(pmb.config.build_device_architectures + [arch_native]) + arch_choices = Arch.supported() # Kconfig subparser ret = subparser.add_parser("kconfig", help="change or edit kernel configs") @@ -496,7 +494,8 @@ def arguments_kconfig(subparser): check.add_argument("-f", "--force", action="store_true", help="check all" " kernels, even the ones that would be ignored by" " default") - check.add_argument("--arch", choices=arch_choices, dest="arch") + check.add_argument("--arch", choices=arch_choices, dest="arch", + type=lambda x: Arch.from_str(x)) check.add_argument("--file", help="check a file directly instead of a" " config in a package") check.add_argument("--no-details", action="store_false", @@ -511,7 +510,8 @@ def arguments_kconfig(subparser): # "pmbootstrap kconfig edit" edit = sub.add_parser("edit", help="edit kernel aport config") - edit.add_argument("--arch", choices=arch_choices, dest="arch") + edit.add_argument("--arch", choices=arch_choices, dest="arch", + type=lambda x: Arch.from_str(x)) edit.add_argument("-x", dest="xconfig", action="store_true", help="use xconfig rather than menuconfig for kernel" " configuration") @@ -526,18 +526,19 @@ def arguments_kconfig(subparser): "newer. Internally runs 'make oldconfig', " "which asks question for every new kernel " "config option.") - migrate.add_argument("--arch", choices=arch_choices, dest="arch") + migrate.add_argument("--arch", choices=arch_choices, dest="arch", + type=lambda x: Arch.from_str(x)) add_kernel_arg(migrate) def arguments_repo_bootstrap(subparser): - arch_native = pmb.config.arch_native - arch_choices = set(pmb.config.build_device_architectures + [arch_native]) + arch_choices = Arch.supported() ret = subparser.add_parser("repo_bootstrap") ret.add_argument("repository", help="which repository to bootstrap (e.g. systemd)") - ret.add_argument("--arch", choices=arch_choices, dest="arch") + ret.add_argument("--arch", choices=arch_choices, dest="arch", + type=lambda x: Arch.from_str(x)) return ret @@ -547,8 +548,9 @@ def arguments_repo_missing(subparser): " specific package and its dependencies") if "argcomplete" in sys.modules: package.completer = package_completer - ret.add_argument("--arch", choices=pmb.config.build_device_architectures, - default=pmb.config.arch_native) + ret.add_argument("--arch", choices=Arch.supported(), + default=Arch.native(), + type=lambda x: Arch.from_str(x)) ret.add_argument("--built", action="store_true", help="include packages which exist in the binary repos") ret.add_argument("--overview", action="store_true", @@ -635,8 +637,8 @@ def add_kernel_arg(subparser, name="package", nargs="?", *args, **kwargs): def get_parser(): parser = argparse.ArgumentParser(prog="pmbootstrap") - arch_native = pmb.config.arch_native - arch_choices = set(pmb.config.build_device_architectures + [arch_native]) + arch_native = Arch.native() + arch_choices = Arch.supported() default_config = Config() mirrors_pmos_default = ",".join(default_config.mirrors_postmarketos) @@ -774,13 +776,15 @@ def get_parser(): # Action: stats stats = sub.add_parser("stats", help="show ccache stats") - stats.add_argument("--arch", default=arch_native, choices=arch_choices) + stats.add_argument("--arch", default=arch_native, choices=arch_choices, + type=lambda x: Arch.from_str(x)) # Action: update update = sub.add_parser("update", help="update all existing APKINDEX" " files") update.add_argument("--arch", default=None, choices=arch_choices, - help="only update a specific architecture") + help="only update a specific architecture", + type=lambda x: Arch.from_str(x)) update.add_argument("--non-existing", action="store_true", help="do not" " only update the existing APKINDEX files, but all of" " them", dest="non_existing") @@ -817,7 +821,7 @@ def get_parser(): suffix.add_argument("-r", "--rootfs", action="store_true", help="Chroot for the device root file system") suffix.add_argument("-b", "--buildroot", nargs="?", const="device", - choices={"device"} | arch_choices, + choices={"device"} | {str(a) for a in arch_choices}, help="Chroot for building packages, defaults to" " device architecture") suffix.add_argument("-s", "--suffix", default=None, @@ -853,9 +857,10 @@ def get_parser(): build = sub.add_parser("build", help="create a package for a" " specific architecture") build.add_argument("--arch", choices=arch_choices, default=None, - help="CPU architecture to build for (default: " + - arch_native + " or first available architecture in" - " APKBUILD)") + help="CPU architecture to build for (default: " + f"{arch_native} or first available architecture in" + " APKBUILD)", + type=lambda x: Arch.from_str(x)) build.add_argument("--force", action="store_true", help="even build if not" " necessary") build.add_argument("--strict", action="store_true", help="(slower) zap and" diff --git a/pmb/parse/depends.py b/pmb/parse/depends.py index 770b502c..3566585a 100644 --- a/pmb/parse/depends.py +++ b/pmb/parse/depends.py @@ -4,10 +4,8 @@ from typing import Dict, List, Sequence, Set from pmb.helpers import logging import pmb.chroot import pmb.chroot.apk -from pmb.types import PmbArgs import pmb.helpers.pmaports import pmb.parse.apkindex -import pmb.parse.arch from pmb.core import Chroot, get_context diff --git a/pmb/parse/deviceinfo.py b/pmb/parse/deviceinfo.py index c65dfe05..490e68cd 100644 --- a/pmb/parse/deviceinfo.py +++ b/pmb/parse/deviceinfo.py @@ -4,6 +4,7 @@ import copy from pathlib import Path from typing import Dict, Optional from pmb.core import get_context +from pmb.core.arch import Arch from pmb.helpers import logging import os import pmb.config @@ -97,7 +98,7 @@ class Deviceinfo: codename: str year: str dtb: str - arch: str + arch: Arch # device chassis: str @@ -222,10 +223,10 @@ class Deviceinfo: if "arch" not in info or not info["arch"]: raise RuntimeError(f"Please add 'deviceinfo_arch' to: {path}") - arch = info["arch"] - if (arch != pmb.config.arch_native and - arch not in pmb.config.build_device_architectures): - raise ValueError("Arch '" + arch + "' is not available in" + arch = Arch.from_str(info["arch"]) + if (not arch.is_native() and + arch not in Arch.supported()): + raise ValueError(f"Arch '{arch}' is not available in" " postmarketOS. If you would like to add it, see:" " ") @@ -257,7 +258,10 @@ class Deviceinfo: # FIXME: something to turn on and fix in the future # if key not in Deviceinfo.__annotations__.keys(): # logging.warning(f"deviceinfo: {key} is not a known attribute") - setattr(self, key, value) + if key == "arch": + setattr(self, key, Arch(value)) + else: + setattr(self, key, value) if not self.flash_method: self.flash_method = "none" diff --git a/pmb/qemu/run.py b/pmb/qemu/run.py index 5ad91876..3e350239 100644 --- a/pmb/qemu/run.py +++ b/pmb/qemu/run.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import subprocess from typing import Sequence +from pmb.core.arch import Arch from pmb.core.context import get_context from pmb.helpers import logging import os @@ -20,7 +21,6 @@ import pmb.config import pmb.config.pmaports from pmb.types import PathString, PmbArgs import pmb.helpers.run -import pmb.parse.arch import pmb.parse.cpuinfo from pmb.core import Chroot, ChrootType @@ -48,15 +48,15 @@ def create_second_storage(args: PmbArgs, device: str): path = Chroot.native() / "home/pmos/rootfs" / f"{device}-2nd.img" pmb.helpers.run.root(["touch", path]) pmb.helpers.run.root(["chmod", "a+w", path]) - resize_image(args, args.second_storage, path) + resize_image(args.second_storage, path) return path -def which_qemu(arch): +def which_qemu(arch: Arch): """ Finds the qemu executable or raises an exception otherwise """ - executable = "qemu-system-" + arch + executable = "qemu-system-" + arch.qemu() if shutil.which(executable): return executable else: @@ -88,11 +88,11 @@ def create_gdk_loader_cache(args: PmbArgs) -> Path: return chroot_native / custom_cache_path -def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): +def command_qemu(args: PmbArgs, device: str, arch: Arch, img_path, img_path_2nd=None): """ Generate the full qemu command with arguments to run postmarketOS """ - cmdline = pmb.parse.deviceinfo().kernel_cmdline + cmdline = pmb.parse.deviceinfo().kernel_cmdline or "" if args.cmdline: cmdline = args.cmdline @@ -126,7 +126,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): # QEMU mach-virt's max CPU count is 8, limit it so it will work correctly # on systems with more than 8 CPUs - if arch != pmb.config.arch_native and ncpus > 8: + if not arch.is_native() and ncpus > 8: ncpus = 8 if args.host_qemu: @@ -149,7 +149,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): ])}) command = [] - if pmb.config.arch_native in ["aarch64", "armv7"]: + if Arch.native() in [Arch.aarch64, Arch.armv7]: # Workaround for QEMU failing on aarch64 asymmetric multiprocessor # arch (big/little architecture # https://en.wikipedia.org/wiki/ARM_big.LITTLE) see @@ -159,11 +159,11 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): ncpus = ncpus_bl logging.info("QEMU will run on big/little architecture on the" f" first {ncpus} cores (from /proc/cpuinfo)") - command += [chroot_native / "lib" / f"ld-musl-{pmb.config.arch_native}.so.1"] + command += [chroot_native / "lib" / f"ld-musl-{Arch.native()}.so.1"] command += [chroot_native / "usr/bin/taskset"] command += ["-c", "0-" + str(ncpus - 1)] - command += [chroot_native / "lib" / f"ld-musl-{pmb.config.arch_native}.so.1"] + command += [chroot_native / "lib" / f"ld-musl-{Arch.native()}.so.1"] command += ["--library-path=" + ":".join([ str(chroot_native / "lib"), str(chroot_native / "usr/lib"), @@ -203,13 +203,13 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): command += ["-netdev", f"user,id=net,hostfwd=tcp:127.0.0.1:{port_ssh}-:22"] command += ["-device", "virtio-net-pci,netdev=net"] - if arch == "x86_64": + if arch == Arch.x86_64: command += ["-device", "virtio-vga-gl"] - elif arch == "aarch64": + elif arch == Arch.aarch64: command += ["-M", "virt"] command += ["-cpu", "cortex-a57"] command += ["-device", "virtio-gpu-pci"] - elif arch == "riscv64": + elif arch == Arch.riscv64: command += ["-M", "virt"] command += ["-device", "virtio-gpu-pci"] else: @@ -221,7 +221,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): "if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF.fd"] # Kernel Virtual Machine (KVM) support - native = pmb.config.arch_native == pmb.parse.deviceinfo().arch + native = pmb.parse.deviceinfo().arch.is_native() if args.qemu_kvm and native and os.path.exists("/dev/kvm"): command += ["-enable-kvm"] command += ["-cpu", "host"] @@ -246,7 +246,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): return (command, env) -def resize_image(args: PmbArgs, img_size_new, img_path): +def resize_image(img_size_new, img_path): """ Truncates an image to a specific size. The value must be larger than the current image size, and it must be specified in MiB or GiB units (powers of @@ -292,7 +292,7 @@ def sigterm_handler(number, frame): " and killed the QEMU VM it was running.") -def install_depends(args: PmbArgs, arch): +def install_depends(args: PmbArgs, arch: Arch): """ Install any necessary qemu dependencies in native chroot """ @@ -309,7 +309,7 @@ def install_depends(args: PmbArgs, arch): "qemu-hw-display-virtio-gpu-pci", "qemu-hw-display-virtio-vga", "qemu-hw-display-virtio-vga-gl", - "qemu-system-" + arch, + "qemu-system-" + arch.qemu(), "qemu-ui-gtk", "qemu-ui-opengl", "qemu-ui-sdl", @@ -338,7 +338,7 @@ def run(args: PmbArgs): raise RuntimeError("'pmbootstrap qemu' can be only used with one of " "the QEMU device packages. Run 'pmbootstrap init' " "and select the 'qemu' vendor.") - arch = pmb.parse.arch.alpine_to_qemu(pmb.parse.deviceinfo().arch) + arch = pmb.parse.deviceinfo().arch img_path = system_image(device) img_path_2nd = None @@ -347,7 +347,7 @@ def run(args: PmbArgs): if not args.host_qemu: install_depends(args, arch) - logging.info("Running postmarketOS in QEMU VM (" + arch + ")") + logging.info("Running postmarketOS in QEMU VM (" + arch.qemu() + ")") qemu, env = command_qemu(args, device, arch, img_path, img_path_2nd) @@ -358,7 +358,7 @@ def run(args: PmbArgs): # Resize the rootfs (or show hint) if args.image_size: - resize_image(args, args.image_size, img_path) + resize_image(args.image_size, img_path) else: logging.info("NOTE: Run 'pmbootstrap qemu --image-size 2G' to set" " the rootfs size when you run out of space!") diff --git a/pmb/sideload/__init__.py b/pmb/sideload/__init__.py index 9173d828..487b3225 100644 --- a/pmb/sideload/__init__.py +++ b/pmb/sideload/__init__.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import os from typing import List +from pmb.core.arch import Arch from pmb.helpers import logging import shlex @@ -9,7 +10,6 @@ from pmb.types import PathString, PmbArgs import pmb.helpers.run import pmb.helpers.run_core import pmb.parse.apkindex -import pmb.parse.arch import pmb.config.pmaports import pmb.build from pmb.core import get_context @@ -39,7 +39,7 @@ def scp_abuild_key(args: PmbArgs, user: str, host: str, port: str): pmb.helpers.run.user(command, output="tui") -def ssh_find_arch(args: PmbArgs, user: str, host: str, port: str) -> str: +def ssh_find_arch(args: PmbArgs, user: str, host: str, port: str) -> Arch: """Connect to a device via ssh and query the architecture.""" logging.info(f"Querying architecture of {user}@{host}") command = ["ssh", "-p", port, f"{user}@{host}", "uname -m"] @@ -49,7 +49,7 @@ def ssh_find_arch(args: PmbArgs, user: str, host: str, port: str) -> str: output_lines = output.strip().splitlines() # Pick out last line which should contain the foreign device's architecture foreign_machine_type = output_lines[-1] - alpine_architecture = pmb.parse.arch.machine_type_to_alpine(foreign_machine_type) + alpine_architecture = Arch.from_machine_type(foreign_machine_type) return alpine_architecture @@ -81,7 +81,7 @@ def ssh_install_apks(args: PmbArgs, user, host, port, paths): pmb.helpers.run.user(command, output="tui") -def sideload(args: PmbArgs, user: str, host: str, port: str, arch: str, copy_key: bool, pkgnames): +def sideload(args: PmbArgs, user: str, host: str, port: str, arch: Arch, copy_key: bool, pkgnames): """ Build packages if necessary and install them via SSH. :param user: target device ssh username diff --git a/pmb/types.py b/pmb/types.py index 0cff7439..f1cc0e87 100644 --- a/pmb/types.py +++ b/pmb/types.py @@ -5,6 +5,8 @@ from argparse import Namespace from pathlib import Path from typing import Dict, List, Optional, Tuple, TypedDict, Union +from pmb.core.arch import Arch + PathString = Union[Path, str] Env = Dict[str, PathString] @@ -37,7 +39,7 @@ class PmbArgs(Namespace): android_recovery_zip: str aports: Optional[Path] _aports_real: str - arch: str + arch: Arch as_root: str assume_yes: str auto: str