diff --git a/pmb/__init__.py b/pmb/__init__.py index de07e937..8e4a89f3 100644 --- a/pmb/__init__.py +++ b/pmb/__init__.py @@ -33,7 +33,7 @@ if version < (3, 9): def print_log_hint() -> None: context = get_context(allow_failure=True) - log = context.log if context else types.Config().work / "log.txt" + log = context.log if context else Config().work / "log.txt" # Hints about the log file (print to stdout only) log_hint = "Run 'pmbootstrap log' for details." if not os.path.exists(log): diff --git a/pmb/aportgen/device.py b/pmb/aportgen/device.py index afdea871..ea98ba2c 100644 --- a/pmb/aportgen/device.py +++ b/pmb/aportgen/device.py @@ -12,7 +12,7 @@ import pmb.parse def ask_for_architecture(): - architectures = pmb.config.build_device_architectures + architectures = Arch.supported() # Don't show armhf, new ports shouldn't use this architecture if "armhf" in architectures: architectures.remove("armhf") diff --git a/pmb/aportgen/grub_efi.py b/pmb/aportgen/grub_efi.py index 8a676093..6aff80f2 100644 --- a/pmb/aportgen/grub_efi.py +++ b/pmb/aportgen/grub_efi.py @@ -5,6 +5,7 @@ import pmb.aportgen.core import pmb.build import pmb.chroot.apk import pmb.chroot.apk_static +from pmb.core.arch import Arch from pmb.types import PmbArgs import pmb.helpers.run import pmb.parse.apkindex @@ -48,7 +49,7 @@ def generate(pkgname): pkgdesc="GRUB $_arch EFI files for every architecture" url="https://www.gnu.org/software/grub/" license="GPL-3.0-or-later" - arch="{pmb.config.arch_native}" + arch="{Arch.native()}" source="grub-efi-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk::$_mirror/{mirrordir}/main/$_arch/grub-efi-$pkgver-r$pkgrel.apk" package() {{ diff --git a/pmb/aportgen/linux.py b/pmb/aportgen/linux.py index decee098..7f1a319b 100644 --- a/pmb/aportgen/linux.py +++ b/pmb/aportgen/linux.py @@ -5,12 +5,11 @@ from pmb.parse.deviceinfo import Deviceinfo import pmb.helpers.run import pmb.aportgen.core import pmb.parse.apkindex -import pmb.parse.arch def generate_apkbuild(pkgname, deviceinfo: Deviceinfo, patches): device = "-".join(pkgname.split("-")[1:]) - carch = pmb.parse.arch.alpine_to_kernel(deviceinfo.arch) + carch = deviceinfo.arch.kernel() makedepends = ["bash", "bc", "bison", "devicepkg-dev", "findutils", "flex", "openssl-dev", "perl"] diff --git a/pmb/build/__init__.py b/pmb/build/__init__.py index de84d62b..03206137 100644 --- a/pmb/build/__init__.py +++ b/pmb/build/__init__.py @@ -6,4 +6,4 @@ from pmb.build.kconfig import menuconfig from pmb.build.newapkbuild import newapkbuild from pmb.build.other import copy_to_buildpath, is_necessary, \ index_repo -from pmb.build._package import BootstrapStage, mount_pmaports, package +from pmb.build._package import BootstrapStage, mount_pmaports, package, output_path diff --git a/pmb/build/_package.py b/pmb/build/_package.py index 8e18efc5..4cc7657b 100644 --- a/pmb/build/_package.py +++ b/pmb/build/_package.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: GPL-3.0-or-later import datetime import enum -from typing import Dict, List +from typing import Dict, List, Optional +from pmb.core.arch import Arch from pmb.core.context import Context from pmb.core.pkgrepo import pkgrepo_paths, pkgrepo_relative_path from pmb.helpers import logging @@ -17,7 +18,6 @@ import pmb.helpers.pmaports import pmb.helpers.repo import pmb.helpers.mount import pmb.parse -import pmb.parse.arch import pmb.parse.apkindex from pmb.helpers.exceptions import BuildFailedError @@ -221,7 +221,7 @@ def init_buildenv(context: Context, apkbuild, arch, strict=False, force=False, c depends_arch = arch if cross == "native": - depends_arch = pmb.config.arch_native + depends_arch = Arch.native() # Build dependencies depends, built = build_depends(context, apkbuild, depends_arch, strict) @@ -386,8 +386,9 @@ def link_to_git_dir(chroot: Chroot): "/home/pmos/build/.git"], chroot) -def output_path(arch: str, pkgname: str, pkgver: str, pkgrel: str) -> Path: - return Path(arch) / f"{pkgname}-{pkgver}-r{pkgrel}.apk" +def output_path(arch: Arch, pkgname: str, pkgver: str, pkgrel: str) -> Path: + # Yeahp, you can just treat an Arch like a path! + return arch / f"{pkgname}-{pkgver}-r{pkgrel}.apk" def run_abuild(context: Context, apkbuild, channel, arch, strict=False, force=False, cross=None, @@ -433,11 +434,11 @@ def run_abuild(context: Context, apkbuild, channel, arch, strict=False, force=Fa env = {"CARCH": arch, "SUDO_APK": "abuild-apk --no-progress"} if cross == "native": - hostspec = pmb.parse.arch.alpine_to_hostspec(arch) + hostspec = arch.alpine_triple() env["CROSS_COMPILE"] = hostspec + "-" env["CC"] = hostspec + "-gcc" if cross == "crossdirect": - env["PATH"] = ":".join(["/native/usr/lib/crossdirect/" + arch, + env["PATH"] = ":".join([f"/native/usr/lib/crossdirect/{arch}", pmb.config.chroot_path]) if not context.ccache: env["CCACHE_DISABLE"] = "1" @@ -503,7 +504,7 @@ def finish(apkbuild, channel, arch, output: str, chroot: Chroot, strict=False): pmb.chroot.init_keys() -def package(context: Context, pkgname, arch=None, force=False, strict=False, +def package(context: Context, pkgname, arch: Optional[Arch]=None, force=False, strict=False, skip_init_buildenv=False, src=None, bootstrap_stage=BootstrapStage.NONE): """ @@ -531,7 +532,7 @@ def package(context: Context, pkgname, arch=None, force=False, strict=False, logging.verbose(f"{pkgname}: running pmb.build._package.package") # Once per session is enough - arch = arch or pmb.config.arch_native + arch = arch or Arch.native() # the order of checks here is intentional, # skip_already_built() has side effects! if skip_already_built(pkgname, arch) and not force: diff --git a/pmb/build/autodetect.py b/pmb/build/autodetect.py index e5494b34..4466ed4d 100644 --- a/pmb/build/autodetect.py +++ b/pmb/build/autodetect.py @@ -1,14 +1,13 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later from pathlib import Path +from pmb.core.arch import Arch from pmb.helpers import logging from typing import Dict, Optional import pmb.config import pmb.chroot.apk -from pmb.types import PmbArgs import pmb.helpers.pmaports -import pmb.parse.arch from pmb.core import Chroot, ChrootType, get_context @@ -60,9 +59,9 @@ def arch(pkgname: str): if get_context().config.build_default_device_arch: preferred_arch = deviceinfo.arch - preferred_arch_2nd = pmb.config.arch_native + preferred_arch_2nd = Arch.native() else: - preferred_arch = pmb.config.arch_native + preferred_arch = Arch.native() preferred_arch_2nd = deviceinfo.arch if "noarch" in arches or "all" in arches or preferred_arch in arches: @@ -77,8 +76,8 @@ def arch(pkgname: str): return None -def chroot(apkbuild: Dict[str, str], arch: str) -> Chroot: - if arch == pmb.config.arch_native: +def chroot(apkbuild: Dict[str, str], arch: Arch) -> Chroot: + if arch == Arch.native(): return Chroot.native() if "pmb:cross-native" in apkbuild["options"]: @@ -87,13 +86,13 @@ def chroot(apkbuild: Dict[str, str], arch: str) -> Chroot: return Chroot.buildroot(arch) -def crosscompile(apkbuild, arch, suffix: Chroot): +def crosscompile(apkbuild, arch: Arch, suffix: Chroot): """ :returns: None, "native", "crossdirect" """ if not get_context().cross: return None - if not pmb.parse.arch.cpu_emulation_required(arch): + if not arch.cpu_emulation_required(): return None if suffix.type == ChrootType.NATIVE: return "native" diff --git a/pmb/build/envkernel.py b/pmb/build/envkernel.py index adfdf791..5155e0c0 100644 --- a/pmb/build/envkernel.py +++ b/pmb/build/envkernel.py @@ -1,6 +1,7 @@ # Copyright 2023 Robert Yang # SPDX-License-Identifier: GPL-3.0-or-later from typing import List +from pmb.core.arch import Arch from pmb.core.context import Context from pmb.helpers import logging import os @@ -114,7 +115,7 @@ def modify_apkbuild(pkgname: str, aport: Path): pmb.aportgen.core.rewrite(pkgname, apkbuild_path, fields=fields) -def run_abuild(context: Context, pkgname: str, arch: str, apkbuild_path: Path, kbuild_out): +def run_abuild(context: Context, pkgname: str, arch: Arch, apkbuild_path: Path, kbuild_out): """ Prepare build environment and run abuild. @@ -159,9 +160,9 @@ def run_abuild(context: Context, pkgname: str, arch: str, apkbuild_path: Path, k pmb.helpers.run.root(cmd) # Create the apk package - env = {"CARCH": arch, - "CHOST": arch, - "CBUILD": pmb.config.arch_native, + env = {"CARCH": str(arch), + "CHOST": str(arch), + "CBUILD": Arch.native(), "SUDO_APK": "abuild-apk --no-progress"} cmd = ["abuild", "rootpkg"] pmb.chroot.user(cmd, working_dir=build_path, env=env) @@ -201,15 +202,15 @@ def package_kernel(args: PmbArgs): # Install package dependencies depends, _ = pmb.build._package.build_depends( - context, apkbuild, pmb.config.arch_native, strict=False) + context, apkbuild, Arch.native(), strict=False) pmb.build.init(chroot) - if pmb.parse.arch.cpu_emulation_required(arch): - depends.append("binutils-" + arch) + if arch.cpu_emulation_required(): + depends.append(f"binutils-{arch}") pmb.chroot.apk.install(depends, chroot) - output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] + - "-r" + apkbuild["pkgrel"] + ".apk") - message = f"({chroot}) build " + output + output = pmb.build.output_path(arch, apkbuild["pkgname"], apkbuild["pkgver"], + apkbuild["pkgrel"]) + message = f"({chroot}) build {output}" logging.info(message) try: diff --git a/pmb/build/init.py b/pmb/build/init.py index e59d2b95..a5289b5b 100644 --- a/pmb/build/init.py +++ b/pmb/build/init.py @@ -1,5 +1,6 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later +from pmb.core.arch import Arch from pmb.core.context import Context from pmb.helpers import logging import os @@ -9,9 +10,7 @@ import pmb.build import pmb.config import pmb.chroot import pmb.chroot.apk -from pmb.types import PmbArgs import pmb.helpers.run -import pmb.parse.arch from pmb.core import Chroot, get_context @@ -66,7 +65,7 @@ def init(chroot: Chroot=Chroot.native()): apk_arch = chroot.arch # Add apk wrapper that runs native apk and lies about arch - if pmb.parse.arch.cpu_emulation_required(apk_arch) and \ + if apk_arch.cpu_emulation_required() and \ not (chroot / "usr/local/bin/abuild-apk").exists(): with (chroot / "tmp/apk_wrapper.sh").open("w") as handle: content = f""" @@ -112,14 +111,15 @@ def init(chroot: Chroot=Chroot.native()): pathlib.Path(marker).touch() -def init_compiler(context: Context, depends, cross, arch): +def init_compiler(context: Context, depends, cross, arch: Arch): + arch_str = str(arch) cross_pkgs = ["ccache-cross-symlinks", "abuild"] if "gcc4" in depends: - cross_pkgs += ["gcc4-" + arch] + cross_pkgs += ["gcc4-" + arch_str] elif "gcc6" in depends: - cross_pkgs += ["gcc6-" + arch] + cross_pkgs += ["gcc6-" + arch_str] else: - cross_pkgs += ["gcc-" + arch, "g++-" + arch] + cross_pkgs += ["gcc-" + arch_str, "g++-" + arch_str] if "clang" in depends or "clang-dev" in depends: cross_pkgs += ["clang"] if cross == "crossdirect": diff --git a/pmb/build/kconfig.py b/pmb/build/kconfig.py index 4ba1f1ce..824e713a 100644 --- a/pmb/build/kconfig.py +++ b/pmb/build/kconfig.py @@ -105,7 +105,7 @@ def menuconfig(args: PmbArgs, pkgname: str, use_oldconfig): arch = args.arch or get_arch(apkbuild) chroot = pmb.build.autodetect.chroot(apkbuild, arch) cross = pmb.build.autodetect.crosscompile(apkbuild, arch, chroot) - hostspec = pmb.parse.arch.alpine_to_hostspec(arch) + hostspec = arch.alpine_triple() # Set up build tools and makedepends pmb.build.init(chroot) @@ -143,7 +143,7 @@ def menuconfig(args: PmbArgs, pkgname: str, use_oldconfig): # Run make menuconfig outputdir = get_outputdir(args, pkgname, apkbuild) logging.info("(native) make " + kopt) - env = {"ARCH": pmb.parse.arch.alpine_to_kernel(arch), + env = {"ARCH": arch.kernel(), "DISPLAY": os.environ.get("DISPLAY"), "XAUTHORITY": "/home/pmos/.Xauthority"} if cross: diff --git a/pmb/build/other.py b/pmb/build/other.py index 413af331..0c371234 100644 --- a/pmb/build/other.py +++ b/pmb/build/other.py @@ -8,13 +8,11 @@ import datetime from typing import List import pmb.chroot -from pmb.types import PmbArgs import pmb.build import pmb.helpers.file import pmb.helpers.git import pmb.helpers.pmaports import pmb.helpers.run -import pmb.parse.arch import pmb.parse.apkindex import pmb.parse.version from pmb.core import Chroot, get_context diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index f1a5851d..a1fab083 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -3,7 +3,7 @@ import os from pathlib import Path import pmb.chroot.apk_static -from pmb.core.chroot import ChrootType +from pmb.core.arch import Arch from pmb.helpers import logging import shlex from typing import List @@ -11,14 +11,12 @@ from typing import List import pmb.build import pmb.chroot import pmb.config -from pmb.types import PmbArgs import pmb.helpers.apk import pmb.helpers.other import pmb.helpers.pmaports import pmb.helpers.repo import pmb.helpers.run import pmb.parse.apkindex -import pmb.parse.arch import pmb.parse.depends import pmb.parse.version from pmb.core import Chroot, get_context @@ -120,7 +118,7 @@ def packages_split_to_add_del(packages): return (to_add, to_del) -def packages_get_locally_built_apks(packages, arch: str) -> List[Path]: +def packages_get_locally_built_apks(packages, arch: Arch) -> List[Path]: """ Iterate over packages and if existing, get paths to locally built packages. This is used to force apk to upgrade packages to newer local versions, even @@ -183,6 +181,9 @@ def install_run_apk(to_add, to_add_local, to_del, chroot: Chroot): # we expect apk.static to be installed in the native chroot (which # will be the systemd version if building for systemd) and run # it from there. + # pmb.chroot.init(Chroot.native()) + # if chroot != Chroot.native(): + # pmb.chroot.init(chroot) apk_static = Chroot.native() / "sbin/apk.static" arch = chroot.arch apk_cache = get_context().config.work / f"cache_apk_{arch}" diff --git a/pmb/chroot/apk_static.py b/pmb/chroot/apk_static.py index 2d62a333..c62b47a9 100644 --- a/pmb/chroot/apk_static.py +++ b/pmb/chroot/apk_static.py @@ -1,6 +1,7 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later import os +from pmb.core.arch import Arch from pmb.helpers import logging import shutil import tarfile @@ -142,7 +143,7 @@ def download(file): """ channel_cfg = pmb.config.pmaports.read_config_channel() mirrordir = channel_cfg["mirrordir_alpine"] - base_url = f"{get_context().config.mirror_alpine}{mirrordir}/main/{pmb.config.arch_native}" + base_url = f"{get_context().config.mirror_alpine}{mirrordir}/main/{Arch.native()}" return pmb.helpers.http.download(f"{base_url}/{file}", file) diff --git a/pmb/chroot/binfmt.py b/pmb/chroot/binfmt.py index 643bffd5..209287b7 100644 --- a/pmb/chroot/binfmt.py +++ b/pmb/chroot/binfmt.py @@ -1,26 +1,25 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later import os +from pmb.core.arch import Arch from pmb.core.chroot import Chroot from pmb.helpers import logging -from pmb.types import PmbArgs import pmb.helpers.run import pmb.helpers.other import pmb.parse -import pmb.parse.arch import pmb.chroot.apk -def is_registered(arch_qemu): - return os.path.exists("/proc/sys/fs/binfmt_misc/qemu-" + arch_qemu) +def is_registered(arch_qemu: Arch): + return os.path.exists(f"/proc/sys/fs/binfmt_misc/qemu-{arch_qemu}") -def register(arch): +def register(arch: Arch): """ Get arch, magic, mask. """ - arch_qemu = pmb.parse.arch.alpine_to_qemu(arch) + arch_qemu = arch.qemu() chroot = Chroot.native() # always make sure the qemu- binary is installed, since registering @@ -58,8 +57,8 @@ def register(arch): pmb.helpers.run.root(["sh", "-c", 'echo "' + code + '" > ' + register]) -def unregister(arch): - arch_qemu = pmb.parse.arch.alpine_to_qemu(arch) +def unregister(arch: Arch): + arch_qemu = arch.qemu() binfmt_file = "/proc/sys/fs/binfmt_misc/qemu-" + arch_qemu if not os.path.exists(binfmt_file): return diff --git a/pmb/chroot/init.py b/pmb/chroot/init.py index 767c8e90..fee9f2dd 100644 --- a/pmb/chroot/init.py +++ b/pmb/chroot/init.py @@ -14,7 +14,6 @@ import pmb.config.workdir import pmb.helpers.repo import pmb.helpers.run import pmb.helpers.other -import pmb.parse.arch from pmb.core import Chroot, ChrootType, get_context cache_chroot_is_outdated: List[str] = [] @@ -58,10 +57,10 @@ def mark_in_chroot(chroot: Chroot=Chroot.native()): def setup_qemu_emulation(chroot: Chroot): arch = chroot.arch - if not pmb.parse.arch.cpu_emulation_required(arch): + if not arch.cpu_emulation_required(): return - arch_qemu = pmb.parse.arch.alpine_to_qemu(arch) + arch_qemu = arch.qemu() # mount --bind the qemu-user binary pmb.chroot.binfmt.register(arch) diff --git a/pmb/chroot/mount.py b/pmb/chroot/mount.py index 5ee81c25..49b9c7ff 100644 --- a/pmb/chroot/mount.py +++ b/pmb/chroot/mount.py @@ -88,7 +88,7 @@ def mount(chroot: Chroot): mountpoints: Dict[Path, Path] = {} for src_template, target_template in pmb.config.chroot_mount_bind.items(): src_template = src_template.replace("$WORK", os.fspath(get_context().config.work)) - src_template = src_template.replace("$ARCH", arch) + src_template = src_template.replace("$ARCH", str(arch)) src_template = src_template.replace("$CHANNEL", channel) mountpoints[Path(src_template)] = Path(target_template) diff --git a/pmb/chroot/shutdown.py b/pmb/chroot/shutdown.py index f8390cf1..640baa6f 100644 --- a/pmb/chroot/shutdown.py +++ b/pmb/chroot/shutdown.py @@ -1,14 +1,13 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later +from pmb.core.arch import Arch from pmb.helpers import logging import socket from contextlib import closing import pmb.chroot -from pmb.types import PmbArgs import pmb.helpers.mount import pmb.install.losetup -import pmb.parse.arch from pmb.core import Chroot, ChrootType, get_context @@ -100,7 +99,7 @@ def shutdown(only_install_related=False): pmb.helpers.mount.umount_all(path) # Clean up the rest - for arch in pmb.config.build_device_architectures: - if pmb.parse.arch.cpu_emulation_required(arch): + for arch in Arch.supported(): + if arch.cpu_emulation_required(): pmb.chroot.binfmt.unregister(arch) logging.debug("Shutdown complete") diff --git a/pmb/chroot/zap.py b/pmb/chroot/zap.py index 692e67f8..9da86785 100644 --- a/pmb/chroot/zap.py +++ b/pmb/chroot/zap.py @@ -1,6 +1,7 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later import glob +from pmb.core.arch import Arch from pmb.helpers import logging import os @@ -137,7 +138,7 @@ def zap_pkgs_local_mismatch(confirm=True, dry=False): def zap_pkgs_online_mismatch(confirm=True, dry=False): # Check whether we need to do anything - paths = glob.glob(f"{get_context().config.work}/cache_apk_*") + paths = list(get_context().config.work.glob("cache_apk_*")) if not len(paths): return if (confirm and not pmb.helpers.cli.confirm("Remove outdated" @@ -146,8 +147,8 @@ def zap_pkgs_online_mismatch(confirm=True, dry=False): # Iterate over existing apk caches for path in paths: - arch = os.path.basename(path).split("_", 2)[2] - if arch == pmb.config.arch_native: + arch = Arch.from_str(path.name.split("_", 2)[2]) + if arch == Arch.native(): suffix = Chroot.native() else: try: diff --git a/pmb/commands/repo_bootstrap.py b/pmb/commands/repo_bootstrap.py index c16186d8..b29a2a20 100644 --- a/pmb/commands/repo_bootstrap.py +++ b/pmb/commands/repo_bootstrap.py @@ -1,6 +1,7 @@ # Copyright 2024 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later from typing import Optional +from pmb.core.arch import Arch from pmb.core.chroot import Chroot, ChrootType from pmb.core.context import Context from pmb.helpers import logging @@ -15,7 +16,7 @@ from pmb.core import get_context from pmb import commands class RepoBootstrap(commands.Command): - arch: str + arch: Arch repo: str context: Context @@ -38,7 +39,7 @@ class RepoBootstrap(commands.Command): " current branch") - def __init__(self, arch: Optional[str], repository: str): + def __init__(self, arch: Optional[Arch], repository: str): context = get_context() if arch: self.arch = arch @@ -46,7 +47,7 @@ class RepoBootstrap(commands.Command): if context.config.build_default_device_arch: self.arch = pmb.parse.deviceinfo().arch else: - self.arch = pmb.config.arch_native + self.arch = Arch.native() self.repo = repository self.context = context @@ -74,7 +75,7 @@ class RepoBootstrap(commands.Command): self.progress_total += len(steps) * 2 # Foreign arch: need to initialize one additional chroot each step - if pmb.parse.arch.cpu_emulation_required(self.arch): + if self.arch.cpu_emulation_required(): self.progress_total += len(steps) @@ -87,7 +88,7 @@ class RepoBootstrap(commands.Command): def run_steps(self, steps): chroot: Chroot - if pmb.parse.arch.cpu_emulation_required(self.arch): + if self.arch.cpu_emulation_required(): chroot = Chroot(ChrootType.BUILDROOT, self.arch) else: chroot = Chroot.native() @@ -129,7 +130,7 @@ class RepoBootstrap(commands.Command): msg = f"Found previously built packages for {channel}/{self.arch}, run" \ " 'pmbootstrap zap -p' first" - if pmb.parse.arch.cpu_emulation_required(self.arch): + if self.arch.cpu_emulation_required(): msg += " or remove the path manually (to keep cross compilers if" \ " you just built them)" diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index 752d4e83..27e5f231 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -1,12 +1,10 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later -import multiprocessing import os from pathlib import Path from pmb.types import AportGenEntry, PathString -import pmb.parse.arch import sys -from typing import Dict, List, Sequence, TypedDict +from typing import Dict, List, Sequence # # Exported functions @@ -23,7 +21,6 @@ from pmb.config.other import is_systemd_selected # pmb_src: Path = Path(Path(__file__) / "../../..").resolve() apk_keys_path: Path = (pmb_src / "pmb/data/keys") -arch_native = pmb.parse.arch.alpine_native() # apk-tools minimum version # https://pkgs.alpinelinux.org/packages?name=apk-tools&branch=edge @@ -239,15 +236,6 @@ apkindex_retention_time = 4 # When chroot is considered outdated (in seconds) chroot_outdated = 3600 * 24 * 2 -# -# BUILD -# -# Officially supported host/target architectures for postmarketOS. Only -# specify architectures supported by Alpine here. For cross-compiling, -# we need to generate the "musl-$ARCH" and "gcc-$ARCH" packages (use -# "pmbootstrap aportgen musl-armhf" etc.). -build_device_architectures = ["armhf", "armv7", "aarch64", "x86_64", "x86", "riscv64"] - # Packages that will be installed in a chroot before it builds packages # for the first time build_packages = ["abuild", "build-base", "ccache", "git"] diff --git a/pmb/config/workdir.py b/pmb/config/workdir.py index fe7717c2..68ddf4ee 100644 --- a/pmb/config/workdir.py +++ b/pmb/config/workdir.py @@ -29,7 +29,7 @@ def chroot_save_init(suffix: Chroot): cfg[key] = {} # Update sections - channel = pmb.config.pmaports.read_config()["channel"] + channel = pmb.config.pmaports.read_config(support_systemd=False)["channel"] cfg["chroot-channels"][str(suffix)] = channel cfg["chroot-init-dates"][str(suffix)] = str(int(time.time())) @@ -82,7 +82,8 @@ def chroot_check_channel(chroot: Chroot): if key not in cfg or str(chroot) not in cfg[key]: raise RuntimeError(f"{msg_unknown} {msg_again}") - channel = pmb.config.pmaports.read_config()["channel"] + # Exclude systemd repo + channel = pmb.config.pmaports.read_config(support_systemd=False)["channel"] channel_cfg = cfg[key][str(chroot)] if channel != channel_cfg: raise RuntimeError(f"Chroot '{chroot}' was created for the" diff --git a/pmb/core/arch.py b/pmb/core/arch.py new file mode 100644 index 00000000..a90af631 --- /dev/null +++ b/pmb/core/arch.py @@ -0,0 +1,191 @@ +# Copyright 2023 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later +import enum +from pathlib import Path, PosixPath, PurePosixPath +import platform +from typing import Set + +# Initialised at the bottom +_cached_native_arch: "Arch" + + +class Arch(enum.Enum): + """Supported architectures according to the Alpine + APKBUILD format.""" + x86 = "x86" + x86_64 = "x86_64" + armhf = "armhf" + armv7 = "armv7" + aarch64 = "aarch64" + riscv64 = "riscv64" + s390x = "s390x" + ppc64le = "ppc64le" + # Arches Alpine can build for + armel = "armel" + loongarch32 = "loongarch32" + loongarchx32 = "loongarchx32" + loongarch64 = "loongarch64" + mips = "mips" + mips64 = "mips64" + mipsel = "mipsel" + mips64el = "mips64el" + ppc = "ppc" + ppc64 = "ppc64" + riscv32 = "riscv32" + + + def __str__(self) -> str: + return self.value + + + @staticmethod + def from_str(arch: str) -> "Arch": + try: + return Arch(arch) + except ValueError: + raise ValueError(f"Invalid architecture: {arch}") + + + @staticmethod + def from_machine_type(machine_type: str) -> "Arch": + mapping = { + "i686": Arch.x86, + "x86_64": Arch.x86_64, + "aarch64": Arch.aarch64, + "arm64": Arch.aarch64, + "armv6l": Arch.armhf, + "armv7l": Arch.armv7, + "armv8l": Arch.armv7, + } + return mapping[machine_type] + + + @staticmethod + def native() -> "Arch": + global _cached_native_arch + return _cached_native_arch + + + def is_native(self): + return self == Arch.native() + + + @staticmethod + def supported() -> Set["Arch"]: + """Officially supported host/target architectures for postmarketOS. Only + specify architectures supported by Alpine here. For cross-compiling, + we need to generate the "musl-$ARCH" and "gcc-$ARCH" packages (use + "pmbootstrap aportgen musl-armhf" etc.).""" + # FIXME: cache? + return set([ + Arch.armhf, + Arch.armv7, + Arch.aarch64, + Arch.x86_64, + Arch.x86, + Arch.riscv64, + Arch.native(), + ]) + + + def kernel(self): + mapping = { + Arch.x86: "x86", + Arch.x86_64: "x86_64", + Arch.armhf: "arm", + Arch.aarch64: "arm64", + Arch.riscv64: "riscv", + Arch.ppc64le: "powerpc", + Arch.s390x: "s390", + } + return mapping.get(self, self.value) + + def qemu(self): + mapping = { + Arch.x86: "i386", + Arch.armhf: "arm", + Arch.armv7: "arm", + Arch.ppc64le: "ppc64", + } + return mapping.get(self, self.value) + + + def alpine_triple(self): + mapping = { + Arch.aarch64: "aarch64-alpine-linux-musl", + Arch.armel: "armv5-alpine-linux-musleabi", + Arch.armhf: "armv6-alpine-linux-musleabihf", + Arch.armv7: "armv7-alpine-linux-musleabihf", + Arch.loongarch32: "loongarch32-alpine-linux-musl", + Arch.loongarchx32: "loongarchx32-alpine-linux-musl", + Arch.loongarch64: "loongarch64-alpine-linux-musl", + Arch.mips: "mips-alpine-linux-musl", + Arch.mips64: "mips64-alpine-linux-musl", + Arch.mipsel: "mipsel-alpine-linux-musl", + Arch.mips64el: "mips64el-alpine-linux-musl", + Arch.ppc: "powerpc-alpine-linux-musl", + Arch.ppc64: "powerpc64-alpine-linux-musl", + Arch.ppc64le: "powerpc64le-alpine-linux-musl", + Arch.riscv32: "riscv32-alpine-linux-musl", + Arch.riscv64: "riscv64-alpine-linux-musl", + Arch.s390x: "s390x-alpine-linux-musl", + Arch.x86: "i586-alpine-linux-musl", + Arch.x86_64: "x86_64-alpine-linux-musl", + } + + if self in mapping: + return mapping[self] + + raise ValueError(f"Can not map Alpine architecture '{self}'" + " to the right hostspec value") + + + def cpu_emulation_required(self): + # Obvious case: host arch is target arch + if self == Arch.native(): + return False + + # Other cases: host arch on the left, target archs on the right + not_required = { + Arch.x86_64: [Arch.x86], + Arch.armv7: [Arch.armel, Arch.armhf], + Arch.aarch64: [Arch.armv7], + } + if Arch.native() in not_required: + if self in not_required[Arch.native()]: + return False + + # No match: then it's required + return True + + + # Magic to let us use an arch as a Path element + def __truediv__(self, other: object) -> Path: + if isinstance(other, PosixPath) or isinstance(other, PurePosixPath): + # Convert the other path to a relative path + # FIXME: we should avoid creating absolute paths that we actually want + # to make relative to the chroot... + # if other.is_absolute(): + # logging.warning("FIXME: absolute path made relative to Arch??") + other = other.relative_to("/") if other.is_absolute() else other + return Path(str(self)).joinpath(other) + if isinstance(other, str): + # Let's us do Arch / "whatever.apk" and magically produce a path + # maybe this is a pattern we should avoid, but it seems somewhat + # sensible + return Path(str(self)).joinpath(other.strip("/")) + + return NotImplemented + + + def __rtruediv__(self, other: object) -> Path: + if isinstance(other, PosixPath) or isinstance(other, PurePosixPath): + # Important to produce a new Path object here, otherwise we + # end up with one object getting shared around and modified + # and lots of weird stuff happens. + return Path(other) / str(self) + # We don't support str / Arch since that is a weird pattern + + return NotImplemented + +_cached_native_arch = Arch.from_machine_type(platform.machine()) diff --git a/pmb/core/chroot.py b/pmb/core/chroot.py index 22065db8..d98d42ab 100644 --- a/pmb/core/chroot.py +++ b/pmb/core/chroot.py @@ -3,9 +3,10 @@ from __future__ import annotations import enum -from typing import Generator, Optional +from typing import Generator, Optional, Union from pathlib import Path, PosixPath, PurePosixPath import pmb.config +from pmb.core.arch import Arch from .context import get_context class ChrootType(enum.Enum): @@ -21,10 +22,10 @@ class Chroot: __type: ChrootType __name: str - def __init__(self, suffix_type: ChrootType, name: Optional[str] = ""): + def __init__(self, suffix_type: ChrootType, name: Optional[Union[str, Arch]] = ""): self.__type = suffix_type - self.__name = name or "" - + self.__name = str(name or "") + self.__validate() def __validate(self) -> None: @@ -74,16 +75,15 @@ class Chroot: @property - # FIXME: make an Arch type - def arch(self) -> str: + def arch(self) -> Arch: if self.type == ChrootType.NATIVE: - return pmb.config.arch_native + return Arch.native() if self.type == ChrootType.BUILDROOT: - return self.name() + return Arch.from_str(self.name()) # FIXME: this is quite delicate as it will only be valid # for certain pmbootstrap commands... It was like this # before but it should be fixed. - arch = pmb.core.get_context().device_arch + arch = pmb.parse.deviceinfo().arch if arch is not None: return arch @@ -118,8 +118,12 @@ class Chroot: def __rtruediv__(self, other: object) -> Path: if isinstance(other, PosixPath) or isinstance(other, PurePosixPath): + # Important to produce a new Path object here, otherwise we + # end up with one object getting shared around and modified + # and lots of weird stuff happens. return Path(other) / self.path if isinstance(other, str): + # This implicitly creates a new Path object return other / self.path return NotImplemented @@ -139,7 +143,7 @@ class Chroot: @staticmethod - def buildroot(arch: str) -> Chroot: + def buildroot(arch: Arch) -> Chroot: return Chroot(ChrootType.BUILDROOT, arch) diff --git a/pmb/core/context.py b/pmb/core/context.py index 44f687e1..e35739f0 100644 --- a/pmb/core/context.py +++ b/pmb/core/context.py @@ -5,7 +5,8 @@ from typing import List, Optional from pathlib import Path -from pmb.types import Config +from pmb.core.arch import Arch +from .config import Config class Context(): @@ -15,8 +16,6 @@ class Context(): sudo_timer: bool = False force: bool = False log: Path - # The architecture of the selected device - device_arch: Optional[str] = None # assume yes to prompts assume_yes: bool = False diff --git a/pmb/helpers/apk.py b/pmb/helpers/apk.py index 964e3cd1..c722c4d6 100644 --- a/pmb/helpers/apk.py +++ b/pmb/helpers/apk.py @@ -6,6 +6,7 @@ from typing import List, Sequence import pmb.chroot import pmb.config.pmaports +from pmb.core.arch import Arch from pmb.types import PathString, PmbArgs import pmb.helpers.cli import pmb.helpers.run @@ -72,7 +73,12 @@ def apk_with_progress(command: Sequence[PathString]): :raises RuntimeError: when the apk command fails """ fifo, fifo_outside = _prepare_fifo() - _command: List[str] = [os.fspath(c) for c in command] + _command: List[str] = [] + for c in command: + if isinstance(c, Arch): + _command.append(str(c)) + else: + _command.append(os.fspath(c)) command_with_progress = _create_command_with_progress(_command, fifo) log_msg = " ".join(_command) with pmb.helpers.run.root(['cat', fifo], diff --git a/pmb/helpers/args.py b/pmb/helpers/args.py index 17da7438..ddbd3d9e 100644 --- a/pmb/helpers/args.py +++ b/pmb/helpers/args.py @@ -99,8 +99,6 @@ def init(args: PmbArgs) -> PmbArgs: "pull", "shutdown", "zap"]: pmb.config.pmaports.read_config() pmb.helpers.git.parse_channels_cfg(pkgrepo_default_path()) - deviceinfo = pmb.parse.deviceinfo() - context.device_arch = deviceinfo.arch # Remove attributes from args so they don't get used by mistake delattr(args, "timeout") diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 2be3fbe8..690b3cc9 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import json from typing import List, Sequence, Tuple +from pmb.core.arch import Arch from pmb.helpers import logging import os from pathlib import Path @@ -67,7 +68,7 @@ def _parse_suffix(args: PmbArgs) -> Chroot: if args.buildroot == "device": return Chroot.buildroot(pmb.parse.deviceinfo().arch) else: - return Chroot.buildroot(args.buildroot) + return Chroot.buildroot(Arch.from_str(args.buildroot)) elif args.suffix: (_t, s) = args.suffix.split("_") t: ChrootType = ChrootType(_t) @@ -550,7 +551,7 @@ def shutdown(args: PmbArgs): def stats(args: PmbArgs): # Chroot suffix chroot = Chroot.native() - if args.arch != pmb.config.arch_native: + if args.arch != Arch.native(): chroot = Chroot.buildroot(args.arch) # Install ccache and display stats diff --git a/pmb/helpers/other.py b/pmb/helpers/other.py index 27227445..2eeed338 100644 --- a/pmb/helpers/other.py +++ b/pmb/helpers/other.py @@ -1,6 +1,7 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later from pmb.core import get_context +from pmb.core.arch import Arch from pmb.helpers import logging import os from pathlib import Path @@ -191,7 +192,7 @@ def migrate_work_folder(args: PmbArgs): # Move packages to edge subdir edge_path = context.config.work / "packages/edge" pmb.helpers.run.root(["mkdir", "-p", edge_path]) - for arch in pmb.config.build_device_architectures: + for arch in Arch.supported(): old_path = context.config.work / "packages" / arch new_path = edge_path / arch if old_path.exists(): diff --git a/pmb/helpers/package.py b/pmb/helpers/package.py index 1ad1637d..bc6c266b 100644 --- a/pmb/helpers/package.py +++ b/pmb/helpers/package.py @@ -10,6 +10,7 @@ See also: """ import copy from typing import Any, Dict +from pmb.core.arch import Arch from pmb.core.context import get_context from pmb.helpers import logging import pmb.build._package @@ -83,7 +84,7 @@ def get(pkgname, arch, replace_subpkgnames=False, must_exist=True): # Find in APKINDEX (other arches) if not ret: pmb.helpers.repo.update() - for arch_i in pmb.config.build_device_architectures: + for arch_i in Arch.supported(): if arch_i != arch: ret = pmb.parse.apkindex.package(pkgname, arch_i, False) if ret: diff --git a/pmb/helpers/pkgrel_bump.py b/pmb/helpers/pkgrel_bump.py index a9291118..519a3a8f 100644 --- a/pmb/helpers/pkgrel_bump.py +++ b/pmb/helpers/pkgrel_bump.py @@ -1,5 +1,6 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later +from pmb.core.arch import Arch from pmb.helpers import logging from pmb.types import PmbArgs @@ -104,7 +105,7 @@ def auto_apkindex_package(args: PmbArgs, arch, aport, apk, dry=False): def auto(args: PmbArgs, dry=False): """:returns: list of aport names, where the pkgrel needed to be changed""" ret = [] - for arch in pmb.config.build_device_architectures: + for arch in Arch.supported(): paths = pmb.helpers.repo.apkindex_files(args, arch, alpine=False) for path in paths: logging.info(f"scan {path}") diff --git a/pmb/helpers/pmaports.py b/pmb/helpers/pmaports.py index 08d354fc..3c612b9e 100644 --- a/pmb/helpers/pmaports.py +++ b/pmb/helpers/pmaports.py @@ -8,6 +8,7 @@ See also: """ import glob from pmb.core import get_context +from pmb.core.arch import Arch from pmb.core.pkgrepo import pkgrepo_iter_package_dirs from pmb.helpers import logging from pathlib import Path @@ -164,6 +165,7 @@ def find(package, must_exist=True, subpackages=True, skip_extra_repos=False): # Try to find an APKBUILD with the exact pkgname we are looking for path = _find_apkbuilds(skip_extra_repos).get(package) if path: + logging.verbose(f"{package}: found apkbuild: {path}") ret = path.parent elif subpackages: # No luck, take a guess what APKBUILD could have the package we are @@ -282,7 +284,7 @@ def get_repo(pkgname, must_exist=True) -> Optional[str]: return aport.parent.name -def check_arches(arches, arch): +def check_arches(arches, arch: Arch): """Check if building for a certain arch is allowed. :param arches: list of all supported arches, as it can be found in the @@ -293,9 +295,9 @@ def check_arches(arches, arch): :returns: True when building is allowed, False otherwise """ - if "!" + arch in arches: + if f"!{arch}" in arches: return False - for value in [arch, "all", "noarch"]: + for value in [str(arch), "all", "noarch"]: if value in arches: return True return False diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index f3b505fc..ccc310e3 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -10,10 +10,11 @@ See also: import os import hashlib from pmb.core import get_context +from pmb.core.arch import Arch from pmb.core.pkgrepo import pkgrepo_paths from pmb.helpers import logging from pathlib import Path -from typing import List +from typing import List, Optional import pmb.config.pmaports from pmb.types import PmbArgs @@ -69,7 +70,6 @@ def urls(user_repository=True, postmarketos_mirror=True, alpine=True): # Local user repository (for packages compiled with pmbootstrap) if user_repository: - channel = pmb.config.pmaports.read_config()["channel"] # FIXME: We shouldn't hardcod this here for channel in pmb.config.pmaports.all_channels(): ret.append(f"/mnt/pmbootstrap/packages/{channel}") @@ -98,7 +98,7 @@ def urls(user_repository=True, postmarketos_mirror=True, alpine=True): return ret -def apkindex_files(arch=None, user_repository=True, pmos=True, +def apkindex_files(arch: Optional[Arch]=None, user_repository=True, pmos=True, alpine=True) -> List[Path]: """Get a list of outside paths to all resolved APKINDEX.tar.gz files for a specific arch. @@ -109,7 +109,7 @@ def apkindex_files(arch=None, user_repository=True, pmos=True, :returns: list of absolute APKINDEX.tar.gz file paths """ if not arch: - arch = pmb.config.arch_native + arch = Arch.native() ret = [] # Local user repository (for packages compiled with pmbootstrap) @@ -144,7 +144,7 @@ def update(arch=None, force=False, existing_only=False): return False # Architectures and retention time - architectures = [arch] if arch else pmb.config.build_device_architectures + architectures = [arch] if arch else Arch.supported() retention_hours = pmb.config.apkindex_retention_time retention_seconds = retention_hours * 3600 @@ -152,13 +152,13 @@ def update(arch=None, force=False, existing_only=False): # outdated: {URL: apkindex_path, ... } # outdated_arches: ["armhf", "x86_64", ... ] outdated = {} - outdated_arches = [] + outdated_arches: List[Arch] = [] for url in urls(False): for arch in architectures: # APKINDEX file name from the URL - url_full = url + "/" + arch + "/APKINDEX.tar.gz" + url_full = f"{url}/{arch}/APKINDEX.tar.gz" cache_apk_outside = get_context().config.work / f"cache_apk_{arch}" - apkindex = cache_apk_outside / f"APKINDEX.{apkindex_hash(url)}.tar.gz" + apkindex = cache_apk_outside / f"{apkindex_hash(url)}.tar.gz" # Find update reason, possibly skip non-existing or known 404 files reason = None @@ -186,7 +186,7 @@ def update(arch=None, force=False, existing_only=False): # Bail out or show log message if not len(outdated): return False - logging.info("Update package index for " + ", ".join(outdated_arches) + + logging.info("Update package index for " + ", ".join([str(a) for a in outdated_arches]) + " (" + str(len(outdated)) + " file(s))") # Download and move to right location @@ -206,7 +206,7 @@ def update(arch=None, force=False, existing_only=False): return True -def alpine_apkindex_path(repo="main", arch=None): +def alpine_apkindex_path(repo="main", arch: Optional[Arch]=None): """Get the path to a specific Alpine APKINDEX file on disk and download it if necessary. :param repo: Alpine repository name (e.g. "main") @@ -215,10 +215,10 @@ def alpine_apkindex_path(repo="main", arch=None): """ # Repo sanity check if repo not in ["main", "community", "testing", "non-free"]: - raise RuntimeError("Invalid Alpine repository: " + repo) + raise RuntimeError(f"Invalid Alpine repository: {repo}") # Download the file - arch = arch or pmb.config.arch_native + arch = arch or Arch.native() update(arch) # Find it on disk diff --git a/pmb/helpers/run_core.py b/pmb/helpers/run_core.py index e71eb23d..614f2d00 100644 --- a/pmb/helpers/run_core.py +++ b/pmb/helpers/run_core.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import fcntl from pmb.core import get_context +from pmb.core.arch import Arch from pmb.types import PathString, Env from pmb.helpers import logging import os @@ -35,6 +36,8 @@ def flat_cmd(cmds: Sequence[Sequence[PathString]], working_dir: Optional[Path]=N # Merge env and cmd into escaped list escaped = [] for key, value in env.items(): + if isinstance(value, Arch): + value = str(value) escaped.append(key + "=" + shlex.quote(os.fspath(value))) for cmd in cmds: for i in range(len(cmd)): diff --git a/pmb/install/_install.py b/pmb/install/_install.py index 428b5f19..5b99655f 100644 --- a/pmb/install/_install.py +++ b/pmb/install/_install.py @@ -1,5 +1,6 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later +from pmb.core.arch import Arch from pmb.helpers import logging import os import re @@ -120,7 +121,7 @@ def copy_files_from_chroot(args: PmbArgs, chroot: Chroot): mountpoint_outside = Chroot.native() / mountpoint # Remove empty qemu-user binary stub (where the binary was bind-mounted) - arch_qemu = pmb.parse.arch.alpine_to_qemu(pmb.parse.deviceinfo().arch) + arch_qemu = pmb.parse.deviceinfo().arch.qemu() qemu_binary = mountpoint_outside / ("/usr/bin/qemu-" + arch_qemu + "-static") if os.path.exists(qemu_binary): pmb.helpers.run.root(["rm", qemu_binary]) @@ -367,7 +368,7 @@ def setup_keymap(config: Config): def setup_timezone(chroot: Chroot, timezone: str): # We don't care about the arch since it's built for all! - alpine_conf = pmb.helpers.package.get("alpine-conf", pmb.config.arch_native) + alpine_conf = pmb.helpers.package.get("alpine-conf", Arch.native()) version = alpine_conf["version"].split("-r")[0] setup_tz_cmd = ["setup-timezone"] @@ -476,7 +477,7 @@ def disable_firewall(chroot: Chroot): raise RuntimeError(f"Failed to disable firewall: {nftables_files}") -def print_firewall_info(disabled: bool, arch: str): +def print_firewall_info(disabled: bool, arch: Arch): pmaports_cfg = pmb.config.pmaports.read_config() pmaports_ok = pmaports_cfg.get("supported_firewall", None) == "nftables" @@ -981,7 +982,7 @@ def print_flash_info(device: str, deviceinfo: Deviceinfo, split: bool, have_disk " and flash outside of pmbootstrap.") -def install_recovery_zip(args: PmbArgs, device: str, arch: str, steps): +def install_recovery_zip(args: PmbArgs, device: str, arch: Arch, steps): logging.info(f"*** ({steps}/{steps}) CREATING RECOVERY-FLASHABLE ZIP ***") chroot = Chroot(ChrootType.BUILDROOT, arch) mount_device_rootfs(Chroot.rootfs(device)) diff --git a/pmb/parse/__init__.py b/pmb/parse/__init__.py index 3ea1df0c..9a9c77b4 100644 --- a/pmb/parse/__init__.py +++ b/pmb/parse/__init__.py @@ -8,4 +8,3 @@ from pmb.parse.deviceinfo import deviceinfo from pmb.parse.kconfig import check from pmb.parse.bootimg import bootimg from pmb.parse.cpuinfo import arm_big_little_first_group_ncpus -import pmb.parse.arch diff --git a/pmb/parse/arch.py b/pmb/parse/arch.py deleted file mode 100644 index 176d225f..00000000 --- a/pmb/parse/arch.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2023 Oliver Smith -# SPDX-License-Identifier: GPL-3.0-or-later -import fnmatch -import platform -import pmb.config -import pmb.parse.arch - -def alpine_native(): - machine = platform.machine() - - return machine_type_to_alpine(machine) - - -def alpine_to_qemu(arch): - """ - Convert the architecture to the string used in the QEMU packaging. - This corresponds to the package name of e.g. qemu-system-aarch64. - """ - - mapping = { - "x86": "i386", - "x86_64": "x86_64", - "armhf": "arm", - "armv7": "arm", - "aarch64": "aarch64", - "riscv64": "riscv64", - } - for pattern, arch_qemu in mapping.items(): - if fnmatch.fnmatch(arch, pattern): - return arch_qemu - raise ValueError("Can not map Alpine architecture '" + arch + "'" - " to the right Debian architecture.") - - -def alpine_to_kernel(arch): - """ - Convert the architecture to the string used inside the kernel sources. - You can read the mapping from the linux-vanilla APKBUILD for example. - """ - mapping = { - "aarch64*": "arm64", - "arm*": "arm", - "ppc*": "powerpc", - "s390*": "s390", - "riscv64*": "riscv", - } - for pattern, arch_kernel in mapping.items(): - if fnmatch.fnmatch(arch, pattern): - return arch_kernel - return arch - - -def alpine_to_hostspec(arch): - """ - See: abuild source code/functions.sh.in: arch_to_hostspec() - """ - mapping = { - "aarch64": "aarch64-alpine-linux-musl", - "armel": "armv5-alpine-linux-musleabi", - "armhf": "armv6-alpine-linux-musleabihf", - "armv7": "armv7-alpine-linux-musleabihf", - "loongarch32": "loongarch32-alpine-linux-musl", - "loongarchx32": "loongarchx32-alpine-linux-musl", - "loongarch64": "loongarch64-alpine-linux-musl", - "mips": "mips-alpine-linux-musl", - "mips64": "mips64-alpine-linux-musl", - "mipsel": "mipsel-alpine-linux-musl", - "mips64el": "mips64el-alpine-linux-musl", - "ppc": "powerpc-alpine-linux-musl", - "ppc64": "powerpc64-alpine-linux-musl", - "ppc64le": "powerpc64le-alpine-linux-musl", - "riscv32": "riscv32-alpine-linux-musl", - "riscv64": "riscv64-alpine-linux-musl", - "s390x": "s390x-alpine-linux-musl", - "x86": "i586-alpine-linux-musl", - "x86_64": "x86_64-alpine-linux-musl", - } - if arch in mapping: - return mapping[arch] - - raise ValueError("Can not map Alpine architecture '" + arch + "'" - " to the right hostspec value") - - -def cpu_emulation_required(arch): - # Obvious case: host arch is target arch - if pmb.config.arch_native == arch: - return False - - # Other cases: host arch on the left, target archs on the right - not_required = { - "x86_64": ["x86"], - "armv7": ["armel", "armhf"], - "aarch64": ["armv7"], - } - if pmb.config.arch_native in not_required: - if arch in not_required[pmb.config.arch_native]: - return False - - # No match: then it's required - return True - - -def machine_type_to_alpine(machine_type: str) -> str: - """Translate a machine type to an Alpine architecture. A machine type can come from - for example `$ uname -m` or platform.machine() from Python's standard library.""" - mapping = { - "i686": "x86", - "x86_64": "x86_64", - "aarch64": "aarch64", - "arm64": "aarch64", - "armv6l": "armhf", - "armv7l": "armv7", - "armv8l": "armv7", - } - if machine_type in mapping: - return mapping[machine_type] - raise ValueError(f"Can not map machine type '{machine_type}'" - " to the right Alpine Linux architecture") diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index f760c343..0ef334bc 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -1,7 +1,6 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later import argparse -import copy import os from pathlib import Path import sys @@ -16,7 +15,6 @@ except ImportError: pass import pmb.config -import pmb.parse.arch import pmb.helpers.args import pmb.helpers.pmaports @@ -234,7 +232,8 @@ def arguments_sideload(subparser): ret.add_argument("--user", help="use a different username than the" " one set in init") ret.add_argument("--arch", help="skip automatic architecture deduction and use the" - " given value") + " given value", + type=lambda x: Arch.from_str(x)) ret.add_argument("--install-key", help="install the apk key from this" " machine if needed", action="store_true", dest="install_key") @@ -483,8 +482,7 @@ def arguments_newapkbuild(subparser): def arguments_kconfig(subparser): # Allowed architectures - arch_native = pmb.config.arch_native - arch_choices = set(pmb.config.build_device_architectures + [arch_native]) + arch_choices = Arch.supported() # Kconfig subparser ret = subparser.add_parser("kconfig", help="change or edit kernel configs") @@ -496,7 +494,8 @@ def arguments_kconfig(subparser): check.add_argument("-f", "--force", action="store_true", help="check all" " kernels, even the ones that would be ignored by" " default") - check.add_argument("--arch", choices=arch_choices, dest="arch") + check.add_argument("--arch", choices=arch_choices, dest="arch", + type=lambda x: Arch.from_str(x)) check.add_argument("--file", help="check a file directly instead of a" " config in a package") check.add_argument("--no-details", action="store_false", @@ -511,7 +510,8 @@ def arguments_kconfig(subparser): # "pmbootstrap kconfig edit" edit = sub.add_parser("edit", help="edit kernel aport config") - edit.add_argument("--arch", choices=arch_choices, dest="arch") + edit.add_argument("--arch", choices=arch_choices, dest="arch", + type=lambda x: Arch.from_str(x)) edit.add_argument("-x", dest="xconfig", action="store_true", help="use xconfig rather than menuconfig for kernel" " configuration") @@ -526,18 +526,19 @@ def arguments_kconfig(subparser): "newer. Internally runs 'make oldconfig', " "which asks question for every new kernel " "config option.") - migrate.add_argument("--arch", choices=arch_choices, dest="arch") + migrate.add_argument("--arch", choices=arch_choices, dest="arch", + type=lambda x: Arch.from_str(x)) add_kernel_arg(migrate) def arguments_repo_bootstrap(subparser): - arch_native = pmb.config.arch_native - arch_choices = set(pmb.config.build_device_architectures + [arch_native]) + arch_choices = Arch.supported() ret = subparser.add_parser("repo_bootstrap") ret.add_argument("repository", help="which repository to bootstrap (e.g. systemd)") - ret.add_argument("--arch", choices=arch_choices, dest="arch") + ret.add_argument("--arch", choices=arch_choices, dest="arch", + type=lambda x: Arch.from_str(x)) return ret @@ -547,8 +548,9 @@ def arguments_repo_missing(subparser): " specific package and its dependencies") if "argcomplete" in sys.modules: package.completer = package_completer - ret.add_argument("--arch", choices=pmb.config.build_device_architectures, - default=pmb.config.arch_native) + ret.add_argument("--arch", choices=Arch.supported(), + default=Arch.native(), + type=lambda x: Arch.from_str(x)) ret.add_argument("--built", action="store_true", help="include packages which exist in the binary repos") ret.add_argument("--overview", action="store_true", @@ -635,8 +637,8 @@ def add_kernel_arg(subparser, name="package", nargs="?", *args, **kwargs): def get_parser(): parser = argparse.ArgumentParser(prog="pmbootstrap") - arch_native = pmb.config.arch_native - arch_choices = set(pmb.config.build_device_architectures + [arch_native]) + arch_native = Arch.native() + arch_choices = Arch.supported() default_config = Config() mirrors_pmos_default = ",".join(default_config.mirrors_postmarketos) @@ -774,13 +776,15 @@ def get_parser(): # Action: stats stats = sub.add_parser("stats", help="show ccache stats") - stats.add_argument("--arch", default=arch_native, choices=arch_choices) + stats.add_argument("--arch", default=arch_native, choices=arch_choices, + type=lambda x: Arch.from_str(x)) # Action: update update = sub.add_parser("update", help="update all existing APKINDEX" " files") update.add_argument("--arch", default=None, choices=arch_choices, - help="only update a specific architecture") + help="only update a specific architecture", + type=lambda x: Arch.from_str(x)) update.add_argument("--non-existing", action="store_true", help="do not" " only update the existing APKINDEX files, but all of" " them", dest="non_existing") @@ -817,7 +821,7 @@ def get_parser(): suffix.add_argument("-r", "--rootfs", action="store_true", help="Chroot for the device root file system") suffix.add_argument("-b", "--buildroot", nargs="?", const="device", - choices={"device"} | arch_choices, + choices={"device"} | {str(a) for a in arch_choices}, help="Chroot for building packages, defaults to" " device architecture") suffix.add_argument("-s", "--suffix", default=None, @@ -853,9 +857,10 @@ def get_parser(): build = sub.add_parser("build", help="create a package for a" " specific architecture") build.add_argument("--arch", choices=arch_choices, default=None, - help="CPU architecture to build for (default: " + - arch_native + " or first available architecture in" - " APKBUILD)") + help="CPU architecture to build for (default: " + f"{arch_native} or first available architecture in" + " APKBUILD)", + type=lambda x: Arch.from_str(x)) build.add_argument("--force", action="store_true", help="even build if not" " necessary") build.add_argument("--strict", action="store_true", help="(slower) zap and" diff --git a/pmb/parse/depends.py b/pmb/parse/depends.py index 770b502c..3566585a 100644 --- a/pmb/parse/depends.py +++ b/pmb/parse/depends.py @@ -4,10 +4,8 @@ from typing import Dict, List, Sequence, Set from pmb.helpers import logging import pmb.chroot import pmb.chroot.apk -from pmb.types import PmbArgs import pmb.helpers.pmaports import pmb.parse.apkindex -import pmb.parse.arch from pmb.core import Chroot, get_context diff --git a/pmb/parse/deviceinfo.py b/pmb/parse/deviceinfo.py index c65dfe05..490e68cd 100644 --- a/pmb/parse/deviceinfo.py +++ b/pmb/parse/deviceinfo.py @@ -4,6 +4,7 @@ import copy from pathlib import Path from typing import Dict, Optional from pmb.core import get_context +from pmb.core.arch import Arch from pmb.helpers import logging import os import pmb.config @@ -97,7 +98,7 @@ class Deviceinfo: codename: str year: str dtb: str - arch: str + arch: Arch # device chassis: str @@ -222,10 +223,10 @@ class Deviceinfo: if "arch" not in info or not info["arch"]: raise RuntimeError(f"Please add 'deviceinfo_arch' to: {path}") - arch = info["arch"] - if (arch != pmb.config.arch_native and - arch not in pmb.config.build_device_architectures): - raise ValueError("Arch '" + arch + "' is not available in" + arch = Arch.from_str(info["arch"]) + if (not arch.is_native() and + arch not in Arch.supported()): + raise ValueError(f"Arch '{arch}' is not available in" " postmarketOS. If you would like to add it, see:" " ") @@ -257,7 +258,10 @@ class Deviceinfo: # FIXME: something to turn on and fix in the future # if key not in Deviceinfo.__annotations__.keys(): # logging.warning(f"deviceinfo: {key} is not a known attribute") - setattr(self, key, value) + if key == "arch": + setattr(self, key, Arch(value)) + else: + setattr(self, key, value) if not self.flash_method: self.flash_method = "none" diff --git a/pmb/qemu/run.py b/pmb/qemu/run.py index 5ad91876..3e350239 100644 --- a/pmb/qemu/run.py +++ b/pmb/qemu/run.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import subprocess from typing import Sequence +from pmb.core.arch import Arch from pmb.core.context import get_context from pmb.helpers import logging import os @@ -20,7 +21,6 @@ import pmb.config import pmb.config.pmaports from pmb.types import PathString, PmbArgs import pmb.helpers.run -import pmb.parse.arch import pmb.parse.cpuinfo from pmb.core import Chroot, ChrootType @@ -48,15 +48,15 @@ def create_second_storage(args: PmbArgs, device: str): path = Chroot.native() / "home/pmos/rootfs" / f"{device}-2nd.img" pmb.helpers.run.root(["touch", path]) pmb.helpers.run.root(["chmod", "a+w", path]) - resize_image(args, args.second_storage, path) + resize_image(args.second_storage, path) return path -def which_qemu(arch): +def which_qemu(arch: Arch): """ Finds the qemu executable or raises an exception otherwise """ - executable = "qemu-system-" + arch + executable = "qemu-system-" + arch.qemu() if shutil.which(executable): return executable else: @@ -88,11 +88,11 @@ def create_gdk_loader_cache(args: PmbArgs) -> Path: return chroot_native / custom_cache_path -def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): +def command_qemu(args: PmbArgs, device: str, arch: Arch, img_path, img_path_2nd=None): """ Generate the full qemu command with arguments to run postmarketOS """ - cmdline = pmb.parse.deviceinfo().kernel_cmdline + cmdline = pmb.parse.deviceinfo().kernel_cmdline or "" if args.cmdline: cmdline = args.cmdline @@ -126,7 +126,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): # QEMU mach-virt's max CPU count is 8, limit it so it will work correctly # on systems with more than 8 CPUs - if arch != pmb.config.arch_native and ncpus > 8: + if not arch.is_native() and ncpus > 8: ncpus = 8 if args.host_qemu: @@ -149,7 +149,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): ])}) command = [] - if pmb.config.arch_native in ["aarch64", "armv7"]: + if Arch.native() in [Arch.aarch64, Arch.armv7]: # Workaround for QEMU failing on aarch64 asymmetric multiprocessor # arch (big/little architecture # https://en.wikipedia.org/wiki/ARM_big.LITTLE) see @@ -159,11 +159,11 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): ncpus = ncpus_bl logging.info("QEMU will run on big/little architecture on the" f" first {ncpus} cores (from /proc/cpuinfo)") - command += [chroot_native / "lib" / f"ld-musl-{pmb.config.arch_native}.so.1"] + command += [chroot_native / "lib" / f"ld-musl-{Arch.native()}.so.1"] command += [chroot_native / "usr/bin/taskset"] command += ["-c", "0-" + str(ncpus - 1)] - command += [chroot_native / "lib" / f"ld-musl-{pmb.config.arch_native}.so.1"] + command += [chroot_native / "lib" / f"ld-musl-{Arch.native()}.so.1"] command += ["--library-path=" + ":".join([ str(chroot_native / "lib"), str(chroot_native / "usr/lib"), @@ -203,13 +203,13 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): command += ["-netdev", f"user,id=net,hostfwd=tcp:127.0.0.1:{port_ssh}-:22"] command += ["-device", "virtio-net-pci,netdev=net"] - if arch == "x86_64": + if arch == Arch.x86_64: command += ["-device", "virtio-vga-gl"] - elif arch == "aarch64": + elif arch == Arch.aarch64: command += ["-M", "virt"] command += ["-cpu", "cortex-a57"] command += ["-device", "virtio-gpu-pci"] - elif arch == "riscv64": + elif arch == Arch.riscv64: command += ["-M", "virt"] command += ["-device", "virtio-gpu-pci"] else: @@ -221,7 +221,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): "if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF.fd"] # Kernel Virtual Machine (KVM) support - native = pmb.config.arch_native == pmb.parse.deviceinfo().arch + native = pmb.parse.deviceinfo().arch.is_native() if args.qemu_kvm and native and os.path.exists("/dev/kvm"): command += ["-enable-kvm"] command += ["-cpu", "host"] @@ -246,7 +246,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None): return (command, env) -def resize_image(args: PmbArgs, img_size_new, img_path): +def resize_image(img_size_new, img_path): """ Truncates an image to a specific size. The value must be larger than the current image size, and it must be specified in MiB or GiB units (powers of @@ -292,7 +292,7 @@ def sigterm_handler(number, frame): " and killed the QEMU VM it was running.") -def install_depends(args: PmbArgs, arch): +def install_depends(args: PmbArgs, arch: Arch): """ Install any necessary qemu dependencies in native chroot """ @@ -309,7 +309,7 @@ def install_depends(args: PmbArgs, arch): "qemu-hw-display-virtio-gpu-pci", "qemu-hw-display-virtio-vga", "qemu-hw-display-virtio-vga-gl", - "qemu-system-" + arch, + "qemu-system-" + arch.qemu(), "qemu-ui-gtk", "qemu-ui-opengl", "qemu-ui-sdl", @@ -338,7 +338,7 @@ def run(args: PmbArgs): raise RuntimeError("'pmbootstrap qemu' can be only used with one of " "the QEMU device packages. Run 'pmbootstrap init' " "and select the 'qemu' vendor.") - arch = pmb.parse.arch.alpine_to_qemu(pmb.parse.deviceinfo().arch) + arch = pmb.parse.deviceinfo().arch img_path = system_image(device) img_path_2nd = None @@ -347,7 +347,7 @@ def run(args: PmbArgs): if not args.host_qemu: install_depends(args, arch) - logging.info("Running postmarketOS in QEMU VM (" + arch + ")") + logging.info("Running postmarketOS in QEMU VM (" + arch.qemu() + ")") qemu, env = command_qemu(args, device, arch, img_path, img_path_2nd) @@ -358,7 +358,7 @@ def run(args: PmbArgs): # Resize the rootfs (or show hint) if args.image_size: - resize_image(args, args.image_size, img_path) + resize_image(args.image_size, img_path) else: logging.info("NOTE: Run 'pmbootstrap qemu --image-size 2G' to set" " the rootfs size when you run out of space!") diff --git a/pmb/sideload/__init__.py b/pmb/sideload/__init__.py index 9173d828..487b3225 100644 --- a/pmb/sideload/__init__.py +++ b/pmb/sideload/__init__.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import os from typing import List +from pmb.core.arch import Arch from pmb.helpers import logging import shlex @@ -9,7 +10,6 @@ from pmb.types import PathString, PmbArgs import pmb.helpers.run import pmb.helpers.run_core import pmb.parse.apkindex -import pmb.parse.arch import pmb.config.pmaports import pmb.build from pmb.core import get_context @@ -39,7 +39,7 @@ def scp_abuild_key(args: PmbArgs, user: str, host: str, port: str): pmb.helpers.run.user(command, output="tui") -def ssh_find_arch(args: PmbArgs, user: str, host: str, port: str) -> str: +def ssh_find_arch(args: PmbArgs, user: str, host: str, port: str) -> Arch: """Connect to a device via ssh and query the architecture.""" logging.info(f"Querying architecture of {user}@{host}") command = ["ssh", "-p", port, f"{user}@{host}", "uname -m"] @@ -49,7 +49,7 @@ def ssh_find_arch(args: PmbArgs, user: str, host: str, port: str) -> str: output_lines = output.strip().splitlines() # Pick out last line which should contain the foreign device's architecture foreign_machine_type = output_lines[-1] - alpine_architecture = pmb.parse.arch.machine_type_to_alpine(foreign_machine_type) + alpine_architecture = Arch.from_machine_type(foreign_machine_type) return alpine_architecture @@ -81,7 +81,7 @@ def ssh_install_apks(args: PmbArgs, user, host, port, paths): pmb.helpers.run.user(command, output="tui") -def sideload(args: PmbArgs, user: str, host: str, port: str, arch: str, copy_key: bool, pkgnames): +def sideload(args: PmbArgs, user: str, host: str, port: str, arch: Arch, copy_key: bool, pkgnames): """ Build packages if necessary and install them via SSH. :param user: target device ssh username diff --git a/pmb/types.py b/pmb/types.py index 0cff7439..f1cc0e87 100644 --- a/pmb/types.py +++ b/pmb/types.py @@ -5,6 +5,8 @@ from argparse import Namespace from pathlib import Path from typing import Dict, List, Optional, Tuple, TypedDict, Union +from pmb.core.arch import Arch + PathString = Union[Path, str] Env = Dict[str, PathString] @@ -37,7 +39,7 @@ class PmbArgs(Namespace): android_recovery_zip: str aports: Optional[Path] _aports_real: str - arch: str + arch: Arch as_root: str assume_yes: str auto: str