Administration/pmbootstrap-meow
2
0
Fork 0
forked from Mirror/pmbootstrap
Code Pull requests Activity Actions

core: add an Arch type (MR 2252)

Browse source 
Move pmb/parse/arch.py over to core and refactor it as an Arch type,
similar to how Chroot was done. Fix all the uses (that I can find) of
arch in the codebase that need adjusting.

The new Arch type is an Enum, making it clear what architectures can be
represented and making it much easier to reason about. Since we support
~5 (kinda) different representations of an Architecture (Alpine, Kernel,
target triple, platform, and QEMU), we now formalise that the Alpine
format is what we represent internally, with methods to convert to any
of the others as-needed.

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>
This commit is contained in:
Caleb Connolly 2024-06-08 21:27:27 +02:00 • committed by Oliver Smith
parent 505165dc13
commit 866e5bcfab
No known key found for this signature in database
GPG key ID: 5AE7F5513E0885CB
42 changed files with 389 additions and 303 deletions
Download patch file Download diff file Expand all files Collapse all files

2
pmb/__init__.py
View file

@ -33,7 +33,7 @@ if version < (3, 9):
def print_log_hint() -> None:
context = get_context(allow_failure=True)
log = context.log if context else types.Config().work / "log.txt"
log = context.log if context else Config().work / "log.txt"
# Hints about the log file (print to stdout only)
log_hint = "Run 'pmbootstrap log' for details."
if not os.path.exists(log):

2
pmb/aportgen/device.py
View file

@ -12,7 +12,7 @@ import pmb.parse
def ask_for_architecture():
architectures = pmb.config.build_device_architectures
architectures = Arch.supported()
# Don't show armhf, new ports shouldn't use this architecture
if "armhf" in architectures:
architectures.remove("armhf")

3
pmb/aportgen/grub_efi.py
View file

@ -5,6 +5,7 @@ import pmb.aportgen.core
import pmb.build
import pmb.chroot.apk
import pmb.chroot.apk_static
from pmb.core.arch import Arch
from pmb.types import PmbArgs
import pmb.helpers.run
import pmb.parse.apkindex
@ -48,7 +49,7 @@ def generate(pkgname):
pkgdesc="GRUB $_arch EFI files for every architecture"
url="https://www.gnu.org/software/grub/"
license="GPL-3.0-or-later"
arch="{pmb.config.arch_native}"
arch="{Arch.native()}"
source="grub-efi-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk::$_mirror/{mirrordir}/main/$_arch/grub-efi-$pkgver-r$pkgrel.apk"
package() {{

3
pmb/aportgen/linux.py
View file

@ -5,12 +5,11 @@ from pmb.parse.deviceinfo import Deviceinfo
import pmb.helpers.run
import pmb.aportgen.core
import pmb.parse.apkindex
import pmb.parse.arch
def generate_apkbuild(pkgname, deviceinfo: Deviceinfo, patches):
device = "-".join(pkgname.split("-")[1:])
carch = pmb.parse.arch.alpine_to_kernel(deviceinfo.arch)
carch = deviceinfo.arch.kernel()
makedepends = ["bash", "bc", "bison", "devicepkg-dev", "findutils", "flex",
"openssl-dev", "perl"]

2
pmb/build/__init__.py
View file

@ -6,4 +6,4 @@ from pmb.build.kconfig import menuconfig
from pmb.build.newapkbuild import newapkbuild
from pmb.build.other import copy_to_buildpath, is_necessary, \
index_repo
from pmb.build._package import BootstrapStage, mount_pmaports, package
from pmb.build._package import BootstrapStage, mount_pmaports, package, output_path

19
pmb/build/_package.py
View file

@ -2,7 +2,8 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import datetime
import enum
from typing import Dict, List
from typing import Dict, List, Optional
from pmb.core.arch import Arch
from pmb.core.context import Context
from pmb.core.pkgrepo import pkgrepo_paths, pkgrepo_relative_path
from pmb.helpers import logging
@ -17,7 +18,6 @@ import pmb.helpers.pmaports
import pmb.helpers.repo
import pmb.helpers.mount
import pmb.parse
import pmb.parse.arch
import pmb.parse.apkindex
from pmb.helpers.exceptions import BuildFailedError
@ -221,7 +221,7 @@ def init_buildenv(context: Context, apkbuild, arch, strict=False, force=False, c
depends_arch = arch
if cross == "native":
depends_arch = pmb.config.arch_native
depends_arch = Arch.native()
# Build dependencies
depends, built = build_depends(context, apkbuild, depends_arch, strict)
@ -386,8 +386,9 @@ def link_to_git_dir(chroot: Chroot):
"/home/pmos/build/.git"], chroot)
def output_path(arch: str, pkgname: str, pkgver: str, pkgrel: str) -> Path:
return Path(arch) / f"{pkgname}-{pkgver}-r{pkgrel}.apk"
def output_path(arch: Arch, pkgname: str, pkgver: str, pkgrel: str) -> Path:
# Yeahp, you can just treat an Arch like a path!
return arch / f"{pkgname}-{pkgver}-r{pkgrel}.apk"
def run_abuild(context: Context, apkbuild, channel, arch, strict=False, force=False, cross=None,
@ -433,11 +434,11 @@ def run_abuild(context: Context, apkbuild, channel, arch, strict=False, force=Fa
env = {"CARCH": arch,
"SUDO_APK": "abuild-apk --no-progress"}
if cross == "native":
hostspec = pmb.parse.arch.alpine_to_hostspec(arch)
hostspec = arch.alpine_triple()
env["CROSS_COMPILE"] = hostspec + "-"
env["CC"] = hostspec + "-gcc"
if cross == "crossdirect":
env["PATH"] = ":".join(["/native/usr/lib/crossdirect/" + arch,
env["PATH"] = ":".join([f"/native/usr/lib/crossdirect/{arch}",
pmb.config.chroot_path])
if not context.ccache:
env["CCACHE_DISABLE"] = "1"
@ -503,7 +504,7 @@ def finish(apkbuild, channel, arch, output: str, chroot: Chroot, strict=False):
pmb.chroot.init_keys()
def package(context: Context, pkgname, arch=None, force=False, strict=False,
def package(context: Context, pkgname, arch: Optional[Arch]=None, force=False, strict=False,
skip_init_buildenv=False, src=None,
bootstrap_stage=BootstrapStage.NONE):
"""
@ -531,7 +532,7 @@ def package(context: Context, pkgname, arch=None, force=False, strict=False,
logging.verbose(f"{pkgname}: running pmb.build._package.package")
# Once per session is enough
arch = arch or pmb.config.arch_native
arch = arch or Arch.native()
# the order of checks here is intentional,
# skip_already_built() has side effects!
if skip_already_built(pkgname, arch) and not force:

15
pmb/build/autodetect.py
View file

@ -1,14 +1,13 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pathlib import Path
from pmb.core.arch import Arch
from pmb.helpers import logging
from typing import Dict, Optional
import pmb.config
import pmb.chroot.apk
from pmb.types import PmbArgs
import pmb.helpers.pmaports
import pmb.parse.arch
from pmb.core import Chroot, ChrootType, get_context
@ -60,9 +59,9 @@ def arch(pkgname: str):
if get_context().config.build_default_device_arch:
preferred_arch = deviceinfo.arch
preferred_arch_2nd = pmb.config.arch_native
preferred_arch_2nd = Arch.native()
else:
preferred_arch = pmb.config.arch_native
preferred_arch = Arch.native()
preferred_arch_2nd = deviceinfo.arch
if "noarch" in arches or "all" in arches or preferred_arch in arches:
@ -77,8 +76,8 @@ def arch(pkgname: str):
return None
def chroot(apkbuild: Dict[str, str], arch: str) -> Chroot:
if arch == pmb.config.arch_native:
def chroot(apkbuild: Dict[str, str], arch: Arch) -> Chroot:
if arch == Arch.native():
return Chroot.native()
if "pmb:cross-native" in apkbuild["options"]:
@ -87,13 +86,13 @@ def chroot(apkbuild: Dict[str, str], arch: str) -> Chroot:
return Chroot.buildroot(arch)
def crosscompile(apkbuild, arch, suffix: Chroot):
def crosscompile(apkbuild, arch: Arch, suffix: Chroot):
"""
:returns: None, "native", "crossdirect"
"""
if not get_context().cross:
return None
if not pmb.parse.arch.cpu_emulation_required(arch):
if not arch.cpu_emulation_required():
return None
if suffix.type == ChrootType.NATIVE:
return "native"

21
pmb/build/envkernel.py
View file

@ -1,6 +1,7 @@
# Copyright 2023 Robert Yang
# SPDX-License-Identifier: GPL-3.0-or-later
from typing import List
from pmb.core.arch import Arch
from pmb.core.context import Context
from pmb.helpers import logging
import os
@ -114,7 +115,7 @@ def modify_apkbuild(pkgname: str, aport: Path):
pmb.aportgen.core.rewrite(pkgname, apkbuild_path, fields=fields)
def run_abuild(context: Context, pkgname: str, arch: str, apkbuild_path: Path, kbuild_out):
def run_abuild(context: Context, pkgname: str, arch: Arch, apkbuild_path: Path, kbuild_out):
"""
Prepare build environment and run abuild.
@ -159,9 +160,9 @@ def run_abuild(context: Context, pkgname: str, arch: str, apkbuild_path: Path, k
pmb.helpers.run.root(cmd)
# Create the apk package
env = {"CARCH": arch,
"CHOST": arch,
"CBUILD": pmb.config.arch_native,
env = {"CARCH": str(arch),
"CHOST": str(arch),
"CBUILD": Arch.native(),
"SUDO_APK": "abuild-apk --no-progress"}
cmd = ["abuild", "rootpkg"]
pmb.chroot.user(cmd, working_dir=build_path, env=env)
@ -201,15 +202,15 @@ def package_kernel(args: PmbArgs):
# Install package dependencies
depends, _ = pmb.build._package.build_depends(
context, apkbuild, pmb.config.arch_native, strict=False)
context, apkbuild, Arch.native(), strict=False)
pmb.build.init(chroot)
if pmb.parse.arch.cpu_emulation_required(arch):
depends.append("binutils-" + arch)
if arch.cpu_emulation_required():
depends.append(f"binutils-{arch}")
pmb.chroot.apk.install(depends, chroot)
output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] +
"-r" + apkbuild["pkgrel"] + ".apk")
message = f"({chroot}) build " + output
output = pmb.build.output_path(arch, apkbuild["pkgname"], apkbuild["pkgver"],
apkbuild["pkgrel"])
message = f"({chroot}) build {output}"
logging.info(message)
try:

14
pmb/build/init.py
View file

@ -1,5 +1,6 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.core.arch import Arch
from pmb.core.context import Context
from pmb.helpers import logging
import os
@ -9,9 +10,7 @@ import pmb.build
import pmb.config
import pmb.chroot
import pmb.chroot.apk
from pmb.types import PmbArgs
import pmb.helpers.run
import pmb.parse.arch
from pmb.core import Chroot, get_context
@ -66,7 +65,7 @@ def init(chroot: Chroot=Chroot.native()):
apk_arch = chroot.arch
# Add apk wrapper that runs native apk and lies about arch
if pmb.parse.arch.cpu_emulation_required(apk_arch) and \
if apk_arch.cpu_emulation_required() and \
not (chroot / "usr/local/bin/abuild-apk").exists():
with (chroot / "tmp/apk_wrapper.sh").open("w") as handle:
content = f"""
@ -112,14 +111,15 @@ def init(chroot: Chroot=Chroot.native()):
pathlib.Path(marker).touch()
def init_compiler(context: Context, depends, cross, arch):
def init_compiler(context: Context, depends, cross, arch: Arch):
arch_str = str(arch)
cross_pkgs = ["ccache-cross-symlinks", "abuild"]
if "gcc4" in depends:
cross_pkgs += ["gcc4-" + arch]
cross_pkgs += ["gcc4-" + arch_str]
elif "gcc6" in depends:
cross_pkgs += ["gcc6-" + arch]
cross_pkgs += ["gcc6-" + arch_str]
else:
cross_pkgs += ["gcc-" + arch, "g++-" + arch]
cross_pkgs += ["gcc-" + arch_str, "g++-" + arch_str]
if "clang" in depends or "clang-dev" in depends:
cross_pkgs += ["clang"]
if cross == "crossdirect":

4
pmb/build/kconfig.py
View file

@ -105,7 +105,7 @@ def menuconfig(args: PmbArgs, pkgname: str, use_oldconfig):
arch = args.arch or get_arch(apkbuild)
chroot = pmb.build.autodetect.chroot(apkbuild, arch)
cross = pmb.build.autodetect.crosscompile(apkbuild, arch, chroot)
hostspec = pmb.parse.arch.alpine_to_hostspec(arch)
hostspec = arch.alpine_triple()
# Set up build tools and makedepends
pmb.build.init(chroot)
@ -143,7 +143,7 @@ def menuconfig(args: PmbArgs, pkgname: str, use_oldconfig):
# Run make menuconfig
outputdir = get_outputdir(args, pkgname, apkbuild)
logging.info("(native) make " + kopt)
env = {"ARCH": pmb.parse.arch.alpine_to_kernel(arch),
env = {"ARCH": arch.kernel(),
"DISPLAY": os.environ.get("DISPLAY"),
"XAUTHORITY": "/home/pmos/.Xauthority"}
if cross:

2
pmb/build/other.py
View file

@ -8,13 +8,11 @@ import datetime
from typing import List
import pmb.chroot
from pmb.types import PmbArgs
import pmb.build
import pmb.helpers.file
import pmb.helpers.git
import pmb.helpers.pmaports
import pmb.helpers.run
import pmb.parse.arch
import pmb.parse.apkindex
import pmb.parse.version
from pmb.core import Chroot, get_context

9
pmb/chroot/apk.py
View file

@ -3,7 +3,7 @@
import os
from pathlib import Path
import pmb.chroot.apk_static
from pmb.core.chroot import ChrootType
from pmb.core.arch import Arch
from pmb.helpers import logging
import shlex
from typing import List
@ -11,14 +11,12 @@ from typing import List
import pmb.build
import pmb.chroot
import pmb.config
from pmb.types import PmbArgs
import pmb.helpers.apk
import pmb.helpers.other
import pmb.helpers.pmaports
import pmb.helpers.repo
import pmb.helpers.run
import pmb.parse.apkindex
import pmb.parse.arch
import pmb.parse.depends
import pmb.parse.version
from pmb.core import Chroot, get_context
@ -120,7 +118,7 @@ def packages_split_to_add_del(packages):
return (to_add, to_del)
def packages_get_locally_built_apks(packages, arch: str) -> List[Path]:
def packages_get_locally_built_apks(packages, arch: Arch) -> List[Path]:
"""
Iterate over packages and if existing, get paths to locally built packages.
This is used to force apk to upgrade packages to newer local versions, even
@ -183,6 +181,9 @@ def install_run_apk(to_add, to_add_local, to_del, chroot: Chroot):
# we expect apk.static to be installed in the native chroot (which
# will be the systemd version if building for systemd) and run
# it from there.
# pmb.chroot.init(Chroot.native())
# if chroot != Chroot.native():
# pmb.chroot.init(chroot)
apk_static = Chroot.native() / "sbin/apk.static"
arch = chroot.arch
apk_cache = get_context().config.work / f"cache_apk_{arch}"

3
pmb/chroot/apk_static.py
View file

@ -1,6 +1,7 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
from pmb.core.arch import Arch
from pmb.helpers import logging
import shutil
import tarfile
@ -142,7 +143,7 @@ def download(file):
"""
channel_cfg = pmb.config.pmaports.read_config_channel()
mirrordir = channel_cfg["mirrordir_alpine"]
base_url = f"{get_context().config.mirror_alpine}{mirrordir}/main/{pmb.config.arch_native}"
base_url = f"{get_context().config.mirror_alpine}{mirrordir}/main/{Arch.native()}"
return pmb.helpers.http.download(f"{base_url}/{file}", file)

15
pmb/chroot/binfmt.py
View file

@ -1,26 +1,25 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
from pmb.core.arch import Arch
from pmb.core.chroot import Chroot
from pmb.helpers import logging
from pmb.types import PmbArgs
import pmb.helpers.run
import pmb.helpers.other
import pmb.parse
import pmb.parse.arch
import pmb.chroot.apk
def is_registered(arch_qemu):
return os.path.exists("/proc/sys/fs/binfmt_misc/qemu-" + arch_qemu)
def is_registered(arch_qemu: Arch):
return os.path.exists(f"/proc/sys/fs/binfmt_misc/qemu-{arch_qemu}")
def register(arch):
def register(arch: Arch):
"""
Get arch, magic, mask.
"""
arch_qemu = pmb.parse.arch.alpine_to_qemu(arch)
arch_qemu = arch.qemu()
chroot = Chroot.native()
# always make sure the qemu-<arch> binary is installed, since registering
@ -58,8 +57,8 @@ def register(arch):
pmb.helpers.run.root(["sh", "-c", 'echo "' + code + '" > ' + register])
def unregister(arch):
arch_qemu = pmb.parse.arch.alpine_to_qemu(arch)
def unregister(arch: Arch):
arch_qemu = arch.qemu()
binfmt_file = "/proc/sys/fs/binfmt_misc/qemu-" + arch_qemu
if not os.path.exists(binfmt_file):
return

5
pmb/chroot/init.py
View file

@ -14,7 +14,6 @@ import pmb.config.workdir
import pmb.helpers.repo
import pmb.helpers.run
import pmb.helpers.other
import pmb.parse.arch
from pmb.core import Chroot, ChrootType, get_context
cache_chroot_is_outdated: List[str] = []
@ -58,10 +57,10 @@ def mark_in_chroot(chroot: Chroot=Chroot.native()):
def setup_qemu_emulation(chroot: Chroot):
arch = chroot.arch
if not pmb.parse.arch.cpu_emulation_required(arch):
if not arch.cpu_emulation_required():
return
arch_qemu = pmb.parse.arch.alpine_to_qemu(arch)
arch_qemu = arch.qemu()
# mount --bind the qemu-user binary
pmb.chroot.binfmt.register(arch)

2
pmb/chroot/mount.py
View file

@ -88,7 +88,7 @@ def mount(chroot: Chroot):
mountpoints: Dict[Path, Path] = {}
for src_template, target_template in pmb.config.chroot_mount_bind.items():
src_template = src_template.replace("$WORK", os.fspath(get_context().config.work))
src_template = src_template.replace("$ARCH", arch)
src_template = src_template.replace("$ARCH", str(arch))
src_template = src_template.replace("$CHANNEL", channel)
mountpoints[Path(src_template)] = Path(target_template)

7
pmb/chroot/shutdown.py
View file

@ -1,14 +1,13 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.core.arch import Arch
from pmb.helpers import logging
import socket
from contextlib import closing
import pmb.chroot
from pmb.types import PmbArgs
import pmb.helpers.mount
import pmb.install.losetup
import pmb.parse.arch
from pmb.core import Chroot, ChrootType, get_context
@ -100,7 +99,7 @@ def shutdown(only_install_related=False):
pmb.helpers.mount.umount_all(path)
# Clean up the rest
for arch in pmb.config.build_device_architectures:
if pmb.parse.arch.cpu_emulation_required(arch):
for arch in Arch.supported():
if arch.cpu_emulation_required():
pmb.chroot.binfmt.unregister(arch)
logging.debug("Shutdown complete")

7
pmb/chroot/zap.py
View file

@ -1,6 +1,7 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import glob
from pmb.core.arch import Arch
from pmb.helpers import logging
import os
@ -137,7 +138,7 @@ def zap_pkgs_local_mismatch(confirm=True, dry=False):
def zap_pkgs_online_mismatch(confirm=True, dry=False):
# Check whether we need to do anything
paths = glob.glob(f"{get_context().config.work}/cache_apk_*")
paths = list(get_context().config.work.glob("cache_apk_*"))
if not len(paths):
return
if (confirm and not pmb.helpers.cli.confirm("Remove outdated"
@ -146,8 +147,8 @@ def zap_pkgs_online_mismatch(confirm=True, dry=False):
# Iterate over existing apk caches
for path in paths:
arch = os.path.basename(path).split("_", 2)[2]
if arch == pmb.config.arch_native:
arch = Arch.from_str(path.name.split("_", 2)[2])
if arch == Arch.native():
suffix = Chroot.native()
else:
try:

13
pmb/commands/repo_bootstrap.py
View file

@ -1,6 +1,7 @@
# Copyright 2024 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from typing import Optional
from pmb.core.arch import Arch
from pmb.core.chroot import Chroot, ChrootType
from pmb.core.context import Context
from pmb.helpers import logging
@ -15,7 +16,7 @@ from pmb.core import get_context
from pmb import commands
class RepoBootstrap(commands.Command):
arch: str
arch: Arch
repo: str
context: Context
@ -38,7 +39,7 @@ class RepoBootstrap(commands.Command):
" current branch")
def __init__(self, arch: Optional[str], repository: str):
def __init__(self, arch: Optional[Arch], repository: str):
context = get_context()
if arch:
self.arch = arch
@ -46,7 +47,7 @@ class RepoBootstrap(commands.Command):
if context.config.build_default_device_arch:
self.arch = pmb.parse.deviceinfo().arch
else:
self.arch = pmb.config.arch_native
self.arch = Arch.native()
self.repo = repository
self.context = context
@ -74,7 +75,7 @@ class RepoBootstrap(commands.Command):
self.progress_total += len(steps) * 2
# Foreign arch: need to initialize one additional chroot each step
if pmb.parse.arch.cpu_emulation_required(self.arch):
if self.arch.cpu_emulation_required():
self.progress_total += len(steps)
@ -87,7 +88,7 @@ class RepoBootstrap(commands.Command):
def run_steps(self, steps):
chroot: Chroot
if pmb.parse.arch.cpu_emulation_required(self.arch):
if self.arch.cpu_emulation_required():
chroot = Chroot(ChrootType.BUILDROOT, self.arch)
else:
chroot = Chroot.native()
@ -129,7 +130,7 @@ class RepoBootstrap(commands.Command):
msg = f"Found previously built packages for {channel}/{self.arch}, run" \
" 'pmbootstrap zap -p' first"
if pmb.parse.arch.cpu_emulation_required(self.arch):
if self.arch.cpu_emulation_required():
msg += " or remove the path manually (to keep cross compilers if" \
" you just built them)"

14
pmb/config/__init__.py
View file

@ -1,12 +1,10 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import multiprocessing
import os
from pathlib import Path
from pmb.types import AportGenEntry, PathString
import pmb.parse.arch
import sys
from typing import Dict, List, Sequence, TypedDict
from typing import Dict, List, Sequence
#
# Exported functions
@ -23,7 +21,6 @@ from pmb.config.other import is_systemd_selected
#
pmb_src: Path = Path(Path(__file__) / "../../..").resolve()
apk_keys_path: Path = (pmb_src / "pmb/data/keys")
arch_native = pmb.parse.arch.alpine_native()
# apk-tools minimum version
# https://pkgs.alpinelinux.org/packages?name=apk-tools&branch=edge
@ -239,15 +236,6 @@ apkindex_retention_time = 4
# When chroot is considered outdated (in seconds)
chroot_outdated = 3600 * 24 * 2
#
# BUILD
#
# Officially supported host/target architectures for postmarketOS. Only
# specify architectures supported by Alpine here. For cross-compiling,
# we need to generate the "musl-$ARCH" and "gcc-$ARCH" packages (use
# "pmbootstrap aportgen musl-armhf" etc.).
build_device_architectures = ["armhf", "armv7", "aarch64", "x86_64", "x86", "riscv64"]
# Packages that will be installed in a chroot before it builds packages
# for the first time
build_packages = ["abuild", "build-base", "ccache", "git"]

5
pmb/config/workdir.py
View file

@ -29,7 +29,7 @@ def chroot_save_init(suffix: Chroot):
cfg[key] = {}
# Update sections
channel = pmb.config.pmaports.read_config()["channel"]
channel = pmb.config.pmaports.read_config(support_systemd=False)["channel"]
cfg["chroot-channels"][str(suffix)] = channel
cfg["chroot-init-dates"][str(suffix)] = str(int(time.time()))
@ -82,7 +82,8 @@ def chroot_check_channel(chroot: Chroot):
if key not in cfg or str(chroot) not in cfg[key]:
raise RuntimeError(f"{msg_unknown} {msg_again}")
channel = pmb.config.pmaports.read_config()["channel"]
# Exclude systemd repo
channel = pmb.config.pmaports.read_config(support_systemd=False)["channel"]
channel_cfg = cfg[key][str(chroot)]
if channel != channel_cfg:
raise RuntimeError(f"Chroot '{chroot}' was created for the"

191
pmb/core/arch.py Normal file
View file

@ -0,0 +1,191 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import enum
from pathlib import Path, PosixPath, PurePosixPath
import platform
from typing import Set
# Initialised at the bottom
_cached_native_arch: "Arch"
class Arch(enum.Enum):
"""Supported architectures according to the Alpine
APKBUILD format."""
x86 = "x86"
x86_64 = "x86_64"
armhf = "armhf"
armv7 = "armv7"
aarch64 = "aarch64"
riscv64 = "riscv64"
s390x = "s390x"
ppc64le = "ppc64le"
# Arches Alpine can build for
armel = "armel"
loongarch32 = "loongarch32"
loongarchx32 = "loongarchx32"
loongarch64 = "loongarch64"
mips = "mips"
mips64 = "mips64"
mipsel = "mipsel"
mips64el = "mips64el"
ppc = "ppc"
ppc64 = "ppc64"
riscv32 = "riscv32"
def __str__(self) -> str:
return self.value
@staticmethod
def from_str(arch: str) -> "Arch":
try:
return Arch(arch)
except ValueError:
raise ValueError(f"Invalid architecture: {arch}")
@staticmethod
def from_machine_type(machine_type: str) -> "Arch":
mapping = {
"i686": Arch.x86,
"x86_64": Arch.x86_64,
"aarch64": Arch.aarch64,
"arm64": Arch.aarch64,
"armv6l": Arch.armhf,
"armv7l": Arch.armv7,
"armv8l": Arch.armv7,
}
return mapping[machine_type]
@staticmethod
def native() -> "Arch":
global _cached_native_arch
return _cached_native_arch
def is_native(self):
return self == Arch.native()
@staticmethod
def supported() -> Set["Arch"]:
"""Officially supported host/target architectures for postmarketOS. Only
specify architectures supported by Alpine here. For cross-compiling,
we need to generate the "musl-$ARCH" and "gcc-$ARCH" packages (use
"pmbootstrap aportgen musl-armhf" etc.)."""
# FIXME: cache?
return set([
Arch.armhf,
Arch.armv7,
Arch.aarch64,
Arch.x86_64,
Arch.x86,
Arch.riscv64,
Arch.native(),
])
def kernel(self):
mapping = {
Arch.x86: "x86",
Arch.x86_64: "x86_64",
Arch.armhf: "arm",
Arch.aarch64: "arm64",
Arch.riscv64: "riscv",
Arch.ppc64le: "powerpc",
Arch.s390x: "s390",
}
return mapping.get(self, self.value)
def qemu(self):
mapping = {
Arch.x86: "i386",
Arch.armhf: "arm",
Arch.armv7: "arm",
Arch.ppc64le: "ppc64",
}
return mapping.get(self, self.value)
def alpine_triple(self):
mapping = {
Arch.aarch64: "aarch64-alpine-linux-musl",
Arch.armel: "armv5-alpine-linux-musleabi",
Arch.armhf: "armv6-alpine-linux-musleabihf",
Arch.armv7: "armv7-alpine-linux-musleabihf",
Arch.loongarch32: "loongarch32-alpine-linux-musl",
Arch.loongarchx32: "loongarchx32-alpine-linux-musl",
Arch.loongarch64: "loongarch64-alpine-linux-musl",
Arch.mips: "mips-alpine-linux-musl",
Arch.mips64: "mips64-alpine-linux-musl",
Arch.mipsel: "mipsel-alpine-linux-musl",
Arch.mips64el: "mips64el-alpine-linux-musl",
Arch.ppc: "powerpc-alpine-linux-musl",
Arch.ppc64: "powerpc64-alpine-linux-musl",
Arch.ppc64le: "powerpc64le-alpine-linux-musl",
Arch.riscv32: "riscv32-alpine-linux-musl",
Arch.riscv64: "riscv64-alpine-linux-musl",
Arch.s390x: "s390x-alpine-linux-musl",
Arch.x86: "i586-alpine-linux-musl",
Arch.x86_64: "x86_64-alpine-linux-musl",
}
if self in mapping:
return mapping[self]
raise ValueError(f"Can not map Alpine architecture '{self}'"
" to the right hostspec value")
def cpu_emulation_required(self):
# Obvious case: host arch is target arch
if self == Arch.native():
return False
# Other cases: host arch on the left, target archs on the right
not_required = {
Arch.x86_64: [Arch.x86],
Arch.armv7: [Arch.armel, Arch.armhf],
Arch.aarch64: [Arch.armv7],
}
if Arch.native() in not_required:
if self in not_required[Arch.native()]:
return False
# No match: then it's required
return True
# Magic to let us use an arch as a Path element
def __truediv__(self, other: object) -> Path:
if isinstance(other, PosixPath) or isinstance(other, PurePosixPath):
# Convert the other path to a relative path
# FIXME: we should avoid creating absolute paths that we actually want
# to make relative to the chroot...
# if other.is_absolute():
# logging.warning("FIXME: absolute path made relative to Arch??")
other = other.relative_to("/") if other.is_absolute() else other
return Path(str(self)).joinpath(other)
if isinstance(other, str):
# Let's us do Arch / "whatever.apk" and magically produce a path
# maybe this is a pattern we should avoid, but it seems somewhat
# sensible
return Path(str(self)).joinpath(other.strip("/"))
return NotImplemented
def __rtruediv__(self, other: object) -> Path:
if isinstance(other, PosixPath) or isinstance(other, PurePosixPath):
# Important to produce a new Path object here, otherwise we
# end up with one object getting shared around and modified
# and lots of weird stuff happens.
return Path(other) / str(self)
# We don't support str / Arch since that is a weird pattern
return NotImplemented
_cached_native_arch = Arch.from_machine_type(platform.machine())

24
pmb/core/chroot.py
View file

@ -3,9 +3,10 @@
from __future__ import annotations
import enum
from typing import Generator, Optional
from typing import Generator, Optional, Union
from pathlib import Path, PosixPath, PurePosixPath
import pmb.config
from pmb.core.arch import Arch
from .context import get_context
class ChrootType(enum.Enum):
@ -21,10 +22,10 @@ class Chroot:
__type: ChrootType
__name: str
def __init__(self, suffix_type: ChrootType, name: Optional[str] = ""):
def __init__(self, suffix_type: ChrootType, name: Optional[Union[str, Arch]] = ""):
self.__type = suffix_type
self.__name = name or ""
self.__name = str(name or "")
self.__validate()
def __validate(self) -> None:
@ -74,16 +75,15 @@ class Chroot:
@property
# FIXME: make an Arch type
def arch(self) -> str:
def arch(self) -> Arch:
if self.type == ChrootType.NATIVE:
return pmb.config.arch_native
return Arch.native()
if self.type == ChrootType.BUILDROOT:
return self.name()
return Arch.from_str(self.name())
# FIXME: this is quite delicate as it will only be valid
# for certain pmbootstrap commands... It was like this
# before but it should be fixed.
arch = pmb.core.get_context().device_arch
arch = pmb.parse.deviceinfo().arch
if arch is not None:
return arch
@ -118,8 +118,12 @@ class Chroot:
def __rtruediv__(self, other: object) -> Path:
if isinstance(other, PosixPath) or isinstance(other, PurePosixPath):
# Important to produce a new Path object here, otherwise we
# end up with one object getting shared around and modified
# and lots of weird stuff happens.
return Path(other) / self.path
if isinstance(other, str):
# This implicitly creates a new Path object
return other / self.path
return NotImplemented
@ -139,7 +143,7 @@ class Chroot:
@staticmethod
def buildroot(arch: str) -> Chroot:
def buildroot(arch: Arch) -> Chroot:
return Chroot(ChrootType.BUILDROOT, arch)

5
pmb/core/context.py
View file

@ -5,7 +5,8 @@
from typing import List, Optional
from pathlib import Path
from pmb.types import Config
from pmb.core.arch import Arch
from .config import Config
class Context():
@ -15,8 +16,6 @@ class Context():
sudo_timer: bool = False
force: bool = False
log: Path
# The architecture of the selected device
device_arch: Optional[str] = None
# assume yes to prompts
assume_yes: bool = False

8
pmb/helpers/apk.py
View file

@ -6,6 +6,7 @@ from typing import List, Sequence
import pmb.chroot
import pmb.config.pmaports
from pmb.core.arch import Arch
from pmb.types import PathString, PmbArgs
import pmb.helpers.cli
import pmb.helpers.run
@ -72,7 +73,12 @@ def apk_with_progress(command: Sequence[PathString]):
:raises RuntimeError: when the apk command fails
"""
fifo, fifo_outside = _prepare_fifo()
_command: List[str] = [os.fspath(c) for c in command]
_command: List[str] = []
for c in command:
if isinstance(c, Arch):
_command.append(str(c))
else:
_command.append(os.fspath(c))
command_with_progress = _create_command_with_progress(_command, fifo)
log_msg = " ".join(_command)
with pmb.helpers.run.root(['cat', fifo],

2
pmb/helpers/args.py
View file

@ -99,8 +99,6 @@ def init(args: PmbArgs) -> PmbArgs:
"pull", "shutdown", "zap"]:
pmb.config.pmaports.read_config()
pmb.helpers.git.parse_channels_cfg(pkgrepo_default_path())
deviceinfo = pmb.parse.deviceinfo()
context.device_arch = deviceinfo.arch
# Remove attributes from args so they don't get used by mistake
delattr(args, "timeout")

5
pmb/helpers/frontend.py
View file

@ -2,6 +2,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import json
from typing import List, Sequence, Tuple
from pmb.core.arch import Arch
from pmb.helpers import logging
import os
from pathlib import Path
@ -67,7 +68,7 @@ def _parse_suffix(args: PmbArgs) -> Chroot:
if args.buildroot == "device":
return Chroot.buildroot(pmb.parse.deviceinfo().arch)
else:
return Chroot.buildroot(args.buildroot)
return Chroot.buildroot(Arch.from_str(args.buildroot))
elif args.suffix:
(_t, s) = args.suffix.split("_")
t: ChrootType = ChrootType(_t)
@ -550,7 +551,7 @@ def shutdown(args: PmbArgs):
def stats(args: PmbArgs):
# Chroot suffix
chroot = Chroot.native()
if args.arch != pmb.config.arch_native:
if args.arch != Arch.native():
chroot = Chroot.buildroot(args.arch)
# Install ccache and display stats

3
pmb/helpers/other.py
View file

@ -1,6 +1,7 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.core import get_context
from pmb.core.arch import Arch
from pmb.helpers import logging
import os
from pathlib import Path
@ -191,7 +192,7 @@ def migrate_work_folder(args: PmbArgs):
# Move packages to edge subdir
edge_path = context.config.work / "packages/edge"
pmb.helpers.run.root(["mkdir", "-p", edge_path])
for arch in pmb.config.build_device_architectures:
for arch in Arch.supported():
old_path = context.config.work / "packages" / arch
new_path = edge_path / arch
if old_path.exists():

3
pmb/helpers/package.py
View file

@ -10,6 +10,7 @@ See also:
"""
import copy
from typing import Any, Dict
from pmb.core.arch import Arch
from pmb.core.context import get_context
from pmb.helpers import logging
import pmb.build._package
@ -83,7 +84,7 @@ def get(pkgname, arch, replace_subpkgnames=False, must_exist=True):
# Find in APKINDEX (other arches)
if not ret:
pmb.helpers.repo.update()
for arch_i in pmb.config.build_device_architectures:
for arch_i in Arch.supported():
if arch_i != arch:
ret = pmb.parse.apkindex.package(pkgname, arch_i, False)
if ret:

3
pmb/helpers/pkgrel_bump.py
View file

@ -1,5 +1,6 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.core.arch import Arch
from pmb.helpers import logging
from pmb.types import PmbArgs
@ -104,7 +105,7 @@ def auto_apkindex_package(args: PmbArgs, arch, aport, apk, dry=False):
def auto(args: PmbArgs, dry=False):
""":returns: list of aport names, where the pkgrel needed to be changed"""
ret = []
for arch in pmb.config.build_device_architectures:
for arch in Arch.supported():
paths = pmb.helpers.repo.apkindex_files(args, arch, alpine=False)
for path in paths:
logging.info(f"scan {path}")

8
pmb/helpers/pmaports.py
View file

@ -8,6 +8,7 @@ See also:
"""
import glob
from pmb.core import get_context
from pmb.core.arch import Arch
from pmb.core.pkgrepo import pkgrepo_iter_package_dirs
from pmb.helpers import logging
from pathlib import Path
@ -164,6 +165,7 @@ def find(package, must_exist=True, subpackages=True, skip_extra_repos=False):
# Try to find an APKBUILD with the exact pkgname we are looking for
path = _find_apkbuilds(skip_extra_repos).get(package)
if path:
logging.verbose(f"{package}: found apkbuild: {path}")
ret = path.parent
elif subpackages:
# No luck, take a guess what APKBUILD could have the package we are
@ -282,7 +284,7 @@ def get_repo(pkgname, must_exist=True) -> Optional[str]:
return aport.parent.name
def check_arches(arches, arch):
def check_arches(arches, arch: Arch):
"""Check if building for a certain arch is allowed.
:param arches: list of all supported arches, as it can be found in the
@ -293,9 +295,9 @@ def check_arches(arches, arch):
:returns: True when building is allowed, False otherwise
"""
if "!" + arch in arches:
if f"!{arch}" in arches:
return False
for value in [arch, "all", "noarch"]:
for value in [str(arch), "all", "noarch"]:
if value in arches:
return True
return False

24
pmb/helpers/repo.py
View file

@ -10,10 +10,11 @@ See also:
import os
import hashlib
from pmb.core import get_context
from pmb.core.arch import Arch
from pmb.core.pkgrepo import pkgrepo_paths
from pmb.helpers import logging
from pathlib import Path
from typing import List
from typing import List, Optional
import pmb.config.pmaports
from pmb.types import PmbArgs
@ -69,7 +70,6 @@ def urls(user_repository=True, postmarketos_mirror=True, alpine=True):
# Local user repository (for packages compiled with pmbootstrap)
if user_repository:
channel = pmb.config.pmaports.read_config()["channel"]
# FIXME: We shouldn't hardcod this here
for channel in pmb.config.pmaports.all_channels():
ret.append(f"/mnt/pmbootstrap/packages/{channel}")
@ -98,7 +98,7 @@ def urls(user_repository=True, postmarketos_mirror=True, alpine=True):
return ret
def apkindex_files(arch=None, user_repository=True, pmos=True,
def apkindex_files(arch: Optional[Arch]=None, user_repository=True, pmos=True,
alpine=True) -> List[Path]:
"""Get a list of outside paths to all resolved APKINDEX.tar.gz files for a specific arch.
@ -109,7 +109,7 @@ def apkindex_files(arch=None, user_repository=True, pmos=True,
:returns: list of absolute APKINDEX.tar.gz file paths
"""
if not arch:
arch = pmb.config.arch_native
arch = Arch.native()
ret = []
# Local user repository (for packages compiled with pmbootstrap)
@ -144,7 +144,7 @@ def update(arch=None, force=False, existing_only=False):
return False
# Architectures and retention time
architectures = [arch] if arch else pmb.config.build_device_architectures
architectures = [arch] if arch else Arch.supported()
retention_hours = pmb.config.apkindex_retention_time
retention_seconds = retention_hours * 3600
@ -152,13 +152,13 @@ def update(arch=None, force=False, existing_only=False):
# outdated: {URL: apkindex_path, ... }
# outdated_arches: ["armhf", "x86_64", ... ]
outdated = {}
outdated_arches = []
outdated_arches: List[Arch] = []
for url in urls(False):
for arch in architectures:
# APKINDEX file name from the URL
url_full = url + "/" + arch + "/APKINDEX.tar.gz"
url_full = f"{url}/{arch}/APKINDEX.tar.gz"
cache_apk_outside = get_context().config.work / f"cache_apk_{arch}"
apkindex = cache_apk_outside / f"APKINDEX.{apkindex_hash(url)}.tar.gz"
apkindex = cache_apk_outside / f"{apkindex_hash(url)}.tar.gz"
# Find update reason, possibly skip non-existing or known 404 files
reason = None
@ -186,7 +186,7 @@ def update(arch=None, force=False, existing_only=False):
# Bail out or show log message
if not len(outdated):
return False
logging.info("Update package index for " + ", ".join(outdated_arches) +
logging.info("Update package index for " + ", ".join([str(a) for a in outdated_arches]) +
" (" + str(len(outdated)) + " file(s))")
# Download and move to right location
@ -206,7 +206,7 @@ def update(arch=None, force=False, existing_only=False):
return True
def alpine_apkindex_path(repo="main", arch=None):
def alpine_apkindex_path(repo="main", arch: Optional[Arch]=None):
"""Get the path to a specific Alpine APKINDEX file on disk and download it if necessary.
:param repo: Alpine repository name (e.g. "main")
@ -215,10 +215,10 @@ def alpine_apkindex_path(repo="main", arch=None):
"""
# Repo sanity check
if repo not in ["main", "community", "testing", "non-free"]:
raise RuntimeError("Invalid Alpine repository: " + repo)
raise RuntimeError(f"Invalid Alpine repository: {repo}")
# Download the file
arch = arch or pmb.config.arch_native
arch = arch or Arch.native()
update(arch)
# Find it on disk

3
pmb/helpers/run_core.py
View file

@ -2,6 +2,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import fcntl
from pmb.core import get_context
from pmb.core.arch import Arch
from pmb.types import PathString, Env
from pmb.helpers import logging
import os
@ -35,6 +36,8 @@ def flat_cmd(cmds: Sequence[Sequence[PathString]], working_dir: Optional[Path]=N
# Merge env and cmd into escaped list
escaped = []
for key, value in env.items():
if isinstance(value, Arch):
value = str(value)
escaped.append(key + "=" + shlex.quote(os.fspath(value)))
for cmd in cmds:
for i in range(len(cmd)):

9
pmb/install/_install.py
View file

@ -1,5 +1,6 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.core.arch import Arch
from pmb.helpers import logging
import os
import re
@ -120,7 +121,7 @@ def copy_files_from_chroot(args: PmbArgs, chroot: Chroot):
mountpoint_outside = Chroot.native() / mountpoint
# Remove empty qemu-user binary stub (where the binary was bind-mounted)
arch_qemu = pmb.parse.arch.alpine_to_qemu(pmb.parse.deviceinfo().arch)
arch_qemu = pmb.parse.deviceinfo().arch.qemu()
qemu_binary = mountpoint_outside / ("/usr/bin/qemu-" + arch_qemu + "-static")
if os.path.exists(qemu_binary):
pmb.helpers.run.root(["rm", qemu_binary])
@ -367,7 +368,7 @@ def setup_keymap(config: Config):
def setup_timezone(chroot: Chroot, timezone: str):
# We don't care about the arch since it's built for all!
alpine_conf = pmb.helpers.package.get("alpine-conf", pmb.config.arch_native)
alpine_conf = pmb.helpers.package.get("alpine-conf", Arch.native())
version = alpine_conf["version"].split("-r")[0]
setup_tz_cmd = ["setup-timezone"]
@ -476,7 +477,7 @@ def disable_firewall(chroot: Chroot):
raise RuntimeError(f"Failed to disable firewall: {nftables_files}")
def print_firewall_info(disabled: bool, arch: str):
def print_firewall_info(disabled: bool, arch: Arch):
pmaports_cfg = pmb.config.pmaports.read_config()
pmaports_ok = pmaports_cfg.get("supported_firewall", None) == "nftables"
@ -981,7 +982,7 @@ def print_flash_info(device: str, deviceinfo: Deviceinfo, split: bool, have_disk
" and flash outside of pmbootstrap.")
def install_recovery_zip(args: PmbArgs, device: str, arch: str, steps):
def install_recovery_zip(args: PmbArgs, device: str, arch: Arch, steps):
logging.info(f"*** ({steps}/{steps}) CREATING RECOVERY-FLASHABLE ZIP ***")
chroot = Chroot(ChrootType.BUILDROOT, arch)
mount_device_rootfs(Chroot.rootfs(device))

1
pmb/parse/__init__.py
View file

@ -8,4 +8,3 @@ from pmb.parse.deviceinfo import deviceinfo
from pmb.parse.kconfig import check
from pmb.parse.bootimg import bootimg
from pmb.parse.cpuinfo import arm_big_little_first_group_ncpus
import pmb.parse.arch

119
pmb/parse/arch.py
View file

@ -1,119 +0,0 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import fnmatch
import platform
import pmb.config
import pmb.parse.arch
def alpine_native():
machine = platform.machine()
return machine_type_to_alpine(machine)
def alpine_to_qemu(arch):
"""
Convert the architecture to the string used in the QEMU packaging.
This corresponds to the package name of e.g. qemu-system-aarch64.
"""
mapping = {
"x86": "i386",
"x86_64": "x86_64",
"armhf": "arm",
"armv7": "arm",
"aarch64": "aarch64",
"riscv64": "riscv64",
}
for pattern, arch_qemu in mapping.items():
if fnmatch.fnmatch(arch, pattern):
return arch_qemu
raise ValueError("Can not map Alpine architecture '" + arch + "'"
" to the right Debian architecture.")
def alpine_to_kernel(arch):
"""
Convert the architecture to the string used inside the kernel sources.
You can read the mapping from the linux-vanilla APKBUILD for example.
"""
mapping = {
"aarch64*": "arm64",
"arm*": "arm",
"ppc*": "powerpc",
"s390*": "s390",
"riscv64*": "riscv",
}
for pattern, arch_kernel in mapping.items():
if fnmatch.fnmatch(arch, pattern):
return arch_kernel
return arch
def alpine_to_hostspec(arch):
"""
See: abuild source code/functions.sh.in: arch_to_hostspec()
"""
mapping = {
"aarch64": "aarch64-alpine-linux-musl",
"armel": "armv5-alpine-linux-musleabi",
"armhf": "armv6-alpine-linux-musleabihf",
"armv7": "armv7-alpine-linux-musleabihf",
"loongarch32": "loongarch32-alpine-linux-musl",
"loongarchx32": "loongarchx32-alpine-linux-musl",
"loongarch64": "loongarch64-alpine-linux-musl",
"mips": "mips-alpine-linux-musl",
"mips64": "mips64-alpine-linux-musl",
"mipsel": "mipsel-alpine-linux-musl",
"mips64el": "mips64el-alpine-linux-musl",
"ppc": "powerpc-alpine-linux-musl",
"ppc64": "powerpc64-alpine-linux-musl",
"ppc64le": "powerpc64le-alpine-linux-musl",
"riscv32": "riscv32-alpine-linux-musl",
"riscv64": "riscv64-alpine-linux-musl",
"s390x": "s390x-alpine-linux-musl",
"x86": "i586-alpine-linux-musl",
"x86_64": "x86_64-alpine-linux-musl",
}
if arch in mapping:
return mapping[arch]
raise ValueError("Can not map Alpine architecture '" + arch + "'"
" to the right hostspec value")
def cpu_emulation_required(arch):
# Obvious case: host arch is target arch
if pmb.config.arch_native == arch:
return False
# Other cases: host arch on the left, target archs on the right
not_required = {
"x86_64": ["x86"],
"armv7": ["armel", "armhf"],
"aarch64": ["armv7"],
}
if pmb.config.arch_native in not_required:
if arch in not_required[pmb.config.arch_native]:
return False
# No match: then it's required
return True
def machine_type_to_alpine(machine_type: str) -> str:
"""Translate a machine type to an Alpine architecture. A machine type can come from
for example `$ uname -m` or platform.machine() from Python's standard library."""
mapping = {
"i686": "x86",
"x86_64": "x86_64",
"aarch64": "aarch64",
"arm64": "aarch64",
"armv6l": "armhf",
"armv7l": "armv7",
"armv8l": "armv7",
}
if machine_type in mapping:
return mapping[machine_type]
raise ValueError(f"Can not map machine type '{machine_type}'"
" to the right Alpine Linux architecture")

47
pmb/parse/arguments.py
View file

@ -1,7 +1,6 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import argparse
import copy
import os
from pathlib import Path
import sys
@ -16,7 +15,6 @@ except ImportError:
pass
import pmb.config
import pmb.parse.arch
import pmb.helpers.args
import pmb.helpers.pmaports
@ -234,7 +232,8 @@ def arguments_sideload(subparser):
ret.add_argument("--user", help="use a different username than the"
" one set in init")
ret.add_argument("--arch", help="skip automatic architecture deduction and use the"
" given value")
" given value",
type=lambda x: Arch.from_str(x))
ret.add_argument("--install-key", help="install the apk key from this"
" machine if needed",
action="store_true", dest="install_key")
@ -483,8 +482,7 @@ def arguments_newapkbuild(subparser):
def arguments_kconfig(subparser):
# Allowed architectures
arch_native = pmb.config.arch_native
arch_choices = set(pmb.config.build_device_architectures + [arch_native])
arch_choices = Arch.supported()
# Kconfig subparser
ret = subparser.add_parser("kconfig", help="change or edit kernel configs")
@ -496,7 +494,8 @@ def arguments_kconfig(subparser):
check.add_argument("-f", "--force", action="store_true", help="check all"
" kernels, even the ones that would be ignored by"
" default")
check.add_argument("--arch", choices=arch_choices, dest="arch")
check.add_argument("--arch", choices=arch_choices, dest="arch",
type=lambda x: Arch.from_str(x))
check.add_argument("--file", help="check a file directly instead of a"
" config in a package")
check.add_argument("--no-details", action="store_false",
@ -511,7 +510,8 @@ def arguments_kconfig(subparser):
# "pmbootstrap kconfig edit"
edit = sub.add_parser("edit", help="edit kernel aport config")
edit.add_argument("--arch", choices=arch_choices, dest="arch")
edit.add_argument("--arch", choices=arch_choices, dest="arch",
type=lambda x: Arch.from_str(x))
edit.add_argument("-x", dest="xconfig", action="store_true",
help="use xconfig rather than menuconfig for kernel"
" configuration")
@ -526,18 +526,19 @@ def arguments_kconfig(subparser):
"newer. Internally runs 'make oldconfig', "
"which asks question for every new kernel "
"config option.")
migrate.add_argument("--arch", choices=arch_choices, dest="arch")
migrate.add_argument("--arch", choices=arch_choices, dest="arch",
type=lambda x: Arch.from_str(x))
add_kernel_arg(migrate)
def arguments_repo_bootstrap(subparser):
arch_native = pmb.config.arch_native
arch_choices = set(pmb.config.build_device_architectures + [arch_native])
arch_choices = Arch.supported()
ret = subparser.add_parser("repo_bootstrap")
ret.add_argument("repository",
help="which repository to bootstrap (e.g. systemd)")
ret.add_argument("--arch", choices=arch_choices, dest="arch")
ret.add_argument("--arch", choices=arch_choices, dest="arch",
type=lambda x: Arch.from_str(x))
return ret
@ -547,8 +548,9 @@ def arguments_repo_missing(subparser):
" specific package and its dependencies")
if "argcomplete" in sys.modules:
package.completer = package_completer
ret.add_argument("--arch", choices=pmb.config.build_device_architectures,
default=pmb.config.arch_native)
ret.add_argument("--arch", choices=Arch.supported(),
default=Arch.native(),
type=lambda x: Arch.from_str(x))
ret.add_argument("--built", action="store_true",
help="include packages which exist in the binary repos")
ret.add_argument("--overview", action="store_true",
@ -635,8 +637,8 @@ def add_kernel_arg(subparser, name="package", nargs="?", *args, **kwargs):
def get_parser():
parser = argparse.ArgumentParser(prog="pmbootstrap")
arch_native = pmb.config.arch_native
arch_choices = set(pmb.config.build_device_architectures + [arch_native])
arch_native = Arch.native()
arch_choices = Arch.supported()
default_config = Config()
mirrors_pmos_default = ",".join(default_config.mirrors_postmarketos)
@ -774,13 +776,15 @@ def get_parser():
# Action: stats
stats = sub.add_parser("stats", help="show ccache stats")
stats.add_argument("--arch", default=arch_native, choices=arch_choices)
stats.add_argument("--arch", default=arch_native, choices=arch_choices,
type=lambda x: Arch.from_str(x))
# Action: update
update = sub.add_parser("update", help="update all existing APKINDEX"
" files")
update.add_argument("--arch", default=None, choices=arch_choices,
help="only update a specific architecture")
help="only update a specific architecture",
type=lambda x: Arch.from_str(x))
update.add_argument("--non-existing", action="store_true", help="do not"
" only update the existing APKINDEX files, but all of"
" them", dest="non_existing")
@ -817,7 +821,7 @@ def get_parser():
suffix.add_argument("-r", "--rootfs", action="store_true",
help="Chroot for the device root file system")
suffix.add_argument("-b", "--buildroot", nargs="?", const="device",
choices={"device"} | arch_choices,
choices={"device"} | {str(a) for a in arch_choices},
help="Chroot for building packages, defaults to"
" device architecture")
suffix.add_argument("-s", "--suffix", default=None,
@ -853,9 +857,10 @@ def get_parser():
build = sub.add_parser("build", help="create a package for a"
" specific architecture")
build.add_argument("--arch", choices=arch_choices, default=None,
help="CPU architecture to build for (default: " +
arch_native + " or first available architecture in"
" APKBUILD)")
help="CPU architecture to build for (default: "
f"{arch_native} or first available architecture in"
" APKBUILD)",
type=lambda x: Arch.from_str(x))
build.add_argument("--force", action="store_true", help="even build if not"
" necessary")
build.add_argument("--strict", action="store_true", help="(slower) zap and"

2
pmb/parse/depends.py
View file

@ -4,10 +4,8 @@ from typing import Dict, List, Sequence, Set
from pmb.helpers import logging
import pmb.chroot
import pmb.chroot.apk
from pmb.types import PmbArgs
import pmb.helpers.pmaports
import pmb.parse.apkindex
import pmb.parse.arch
from pmb.core import Chroot, get_context

16
pmb/parse/deviceinfo.py
View file

@ -4,6 +4,7 @@ import copy
from pathlib import Path
from typing import Dict, Optional
from pmb.core import get_context
from pmb.core.arch import Arch
from pmb.helpers import logging
import os
import pmb.config
@ -97,7 +98,7 @@ class Deviceinfo:
codename: str
year: str
dtb: str
arch: str
arch: Arch
# device
chassis: str
@ -222,10 +223,10 @@ class Deviceinfo:
if "arch" not in info or not info["arch"]:
raise RuntimeError(f"Please add 'deviceinfo_arch' to: {path}")
arch = info["arch"]
if (arch != pmb.config.arch_native and
arch not in pmb.config.build_device_architectures):
raise ValueError("Arch '" + arch + "' is not available in"
arch = Arch.from_str(info["arch"])
if (not arch.is_native() and
arch not in Arch.supported()):
raise ValueError(f"Arch '{arch}' is not available in"
" postmarketOS. If you would like to add it, see:"
" <https://postmarketos.org/newarch>")
@ -257,7 +258,10 @@ class Deviceinfo:
# FIXME: something to turn on and fix in the future
# if key not in Deviceinfo.__annotations__.keys():
# logging.warning(f"deviceinfo: {key} is not a known attribute")
setattr(self, key, value)
if key == "arch":
setattr(self, key, Arch(value))
else:
setattr(self, key, value)
if not self.flash_method:
self.flash_method = "none"

40
pmb/qemu/run.py
View file

@ -2,6 +2,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import subprocess
from typing import Sequence
from pmb.core.arch import Arch
from pmb.core.context import get_context
from pmb.helpers import logging
import os
@ -20,7 +21,6 @@ import pmb.config
import pmb.config.pmaports
from pmb.types import PathString, PmbArgs
import pmb.helpers.run
import pmb.parse.arch
import pmb.parse.cpuinfo
from pmb.core import Chroot, ChrootType
@ -48,15 +48,15 @@ def create_second_storage(args: PmbArgs, device: str):
path = Chroot.native() / "home/pmos/rootfs" / f"{device}-2nd.img"
pmb.helpers.run.root(["touch", path])
pmb.helpers.run.root(["chmod", "a+w", path])
resize_image(args, args.second_storage, path)
resize_image(args.second_storage, path)
return path
def which_qemu(arch):
def which_qemu(arch: Arch):
"""
Finds the qemu executable or raises an exception otherwise
"""
executable = "qemu-system-" + arch
executable = "qemu-system-" + arch.qemu()
if shutil.which(executable):
return executable
else:
@ -88,11 +88,11 @@ def create_gdk_loader_cache(args: PmbArgs) -> Path:
return chroot_native / custom_cache_path
def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None):
def command_qemu(args: PmbArgs, device: str, arch: Arch, img_path, img_path_2nd=None):
"""
Generate the full qemu command with arguments to run postmarketOS
"""
cmdline = pmb.parse.deviceinfo().kernel_cmdline
cmdline = pmb.parse.deviceinfo().kernel_cmdline or ""
if args.cmdline:
cmdline = args.cmdline
@ -126,7 +126,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None):
# QEMU mach-virt's max CPU count is 8, limit it so it will work correctly
# on systems with more than 8 CPUs
if arch != pmb.config.arch_native and ncpus > 8:
if not arch.is_native() and ncpus > 8:
ncpus = 8
if args.host_qemu:
@ -149,7 +149,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None):
])})
command = []
if pmb.config.arch_native in ["aarch64", "armv7"]:
if Arch.native() in [Arch.aarch64, Arch.armv7]:
# Workaround for QEMU failing on aarch64 asymmetric multiprocessor
# arch (big/little architecture
# https://en.wikipedia.org/wiki/ARM_big.LITTLE) see
@ -159,11 +159,11 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None):
ncpus = ncpus_bl
logging.info("QEMU will run on big/little architecture on the"
f" first {ncpus} cores (from /proc/cpuinfo)")
command += [chroot_native / "lib" / f"ld-musl-{pmb.config.arch_native}.so.1"]
command += [chroot_native / "lib" / f"ld-musl-{Arch.native()}.so.1"]
command += [chroot_native / "usr/bin/taskset"]
command += ["-c", "0-" + str(ncpus - 1)]
command += [chroot_native / "lib" / f"ld-musl-{pmb.config.arch_native}.so.1"]
command += [chroot_native / "lib" / f"ld-musl-{Arch.native()}.so.1"]
command += ["--library-path=" + ":".join([
str(chroot_native / "lib"),
str(chroot_native / "usr/lib"),
@ -203,13 +203,13 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None):
command += ["-netdev", f"user,id=net,hostfwd=tcp:127.0.0.1:{port_ssh}-:22"]
command += ["-device", "virtio-net-pci,netdev=net"]
if arch == "x86_64":
if arch == Arch.x86_64:
command += ["-device", "virtio-vga-gl"]
elif arch == "aarch64":
elif arch == Arch.aarch64:
command += ["-M", "virt"]
command += ["-cpu", "cortex-a57"]
command += ["-device", "virtio-gpu-pci"]
elif arch == "riscv64":
elif arch == Arch.riscv64:
command += ["-M", "virt"]
command += ["-device", "virtio-gpu-pci"]
else:
@ -221,7 +221,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None):
"if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF.fd"]
# Kernel Virtual Machine (KVM) support
native = pmb.config.arch_native == pmb.parse.deviceinfo().arch
native = pmb.parse.deviceinfo().arch.is_native()
if args.qemu_kvm and native and os.path.exists("/dev/kvm"):
command += ["-enable-kvm"]
command += ["-cpu", "host"]
@ -246,7 +246,7 @@ def command_qemu(args: PmbArgs, device: str, arch, img_path, img_path_2nd=None):
return (command, env)
def resize_image(args: PmbArgs, img_size_new, img_path):
def resize_image(img_size_new, img_path):
"""
Truncates an image to a specific size. The value must be larger than the
current image size, and it must be specified in MiB or GiB units (powers of
@ -292,7 +292,7 @@ def sigterm_handler(number, frame):
" and killed the QEMU VM it was running.")
def install_depends(args: PmbArgs, arch):
def install_depends(args: PmbArgs, arch: Arch):
"""
Install any necessary qemu dependencies in native chroot
"""
@ -309,7 +309,7 @@ def install_depends(args: PmbArgs, arch):
"qemu-hw-display-virtio-gpu-pci",
"qemu-hw-display-virtio-vga",
"qemu-hw-display-virtio-vga-gl",
"qemu-system-" + arch,
"qemu-system-" + arch.qemu(),
"qemu-ui-gtk",
"qemu-ui-opengl",
"qemu-ui-sdl",
@ -338,7 +338,7 @@ def run(args: PmbArgs):
raise RuntimeError("'pmbootstrap qemu' can be only used with one of "
"the QEMU device packages. Run 'pmbootstrap init' "
"and select the 'qemu' vendor.")
arch = pmb.parse.arch.alpine_to_qemu(pmb.parse.deviceinfo().arch)
arch = pmb.parse.deviceinfo().arch
img_path = system_image(device)
img_path_2nd = None
@ -347,7 +347,7 @@ def run(args: PmbArgs):
if not args.host_qemu:
install_depends(args, arch)
logging.info("Running postmarketOS in QEMU VM (" + arch + ")")
logging.info("Running postmarketOS in QEMU VM (" + arch.qemu() + ")")
qemu, env = command_qemu(args, device, arch, img_path, img_path_2nd)
@ -358,7 +358,7 @@ def run(args: PmbArgs):
# Resize the rootfs (or show hint)
if args.image_size:
resize_image(args, args.image_size, img_path)
resize_image(args.image_size, img_path)
else:
logging.info("NOTE: Run 'pmbootstrap qemu --image-size 2G' to set"
" the rootfs size when you run out of space!")

8
pmb/sideload/__init__.py
View file

@ -2,6 +2,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import os
from typing import List
from pmb.core.arch import Arch
from pmb.helpers import logging
import shlex
@ -9,7 +10,6 @@ from pmb.types import PathString, PmbArgs
import pmb.helpers.run
import pmb.helpers.run_core
import pmb.parse.apkindex
import pmb.parse.arch
import pmb.config.pmaports
import pmb.build
from pmb.core import get_context
@ -39,7 +39,7 @@ def scp_abuild_key(args: PmbArgs, user: str, host: str, port: str):
pmb.helpers.run.user(command, output="tui")
def ssh_find_arch(args: PmbArgs, user: str, host: str, port: str) -> str:
def ssh_find_arch(args: PmbArgs, user: str, host: str, port: str) -> Arch:
"""Connect to a device via ssh and query the architecture."""
logging.info(f"Querying architecture of {user}@{host}")
command = ["ssh", "-p", port, f"{user}@{host}", "uname -m"]
@ -49,7 +49,7 @@ def ssh_find_arch(args: PmbArgs, user: str, host: str, port: str) -> str:
output_lines = output.strip().splitlines()
# Pick out last line which should contain the foreign device's architecture
foreign_machine_type = output_lines[-1]
alpine_architecture = pmb.parse.arch.machine_type_to_alpine(foreign_machine_type)
alpine_architecture = Arch.from_machine_type(foreign_machine_type)
return alpine_architecture
@ -81,7 +81,7 @@ def ssh_install_apks(args: PmbArgs, user, host, port, paths):
pmb.helpers.run.user(command, output="tui")
def sideload(args: PmbArgs, user: str, host: str, port: str, arch: str, copy_key: bool, pkgnames):
def sideload(args: PmbArgs, user: str, host: str, port: str, arch: Arch, copy_key: bool, pkgnames):
""" Build packages if necessary and install them via SSH.
:param user: target device ssh username

4
pmb/types.py
View file

@ -5,6 +5,8 @@ from argparse import Namespace
from pathlib import Path
from typing import Dict, List, Optional, Tuple, TypedDict, Union
from pmb.core.arch import Arch
PathString = Union[Path, str]
Env = Dict[str, PathString]
@ -37,7 +39,7 @@ class PmbArgs(Namespace):
android_recovery_zip: str
aports: Optional[Path]
_aports_real: str
arch: str
arch: Arch
as_root: str
assume_yes: str
auto: str