Mirror/pmbootstrap
1
0
Fork 1
mirror of https://gitlab.postmarketos.org/postmarketOS/pmbootstrap.git synced 2025-07-12 19:09:56 +03:00
Code Issues Wiki Activity

FIXUP: build: abstract CrossCompile type logic

Browse source 
* Make CrossCompile a proper enum type rather than a string literal,
* Introduce methods to get the correct host/build chroots depending on the
  cross compile type and target architecture.
* Remove autodetect.chroot() since it doesn't do what we expect, adjust
  all users to use cross.build_chroot() instead.
* Refactor package building to correctly use cross.host_chroot() and
  cross.build_chroot().

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>
Part-of: https://gitlab.postmarketos.org/postmarketOS/pmbootstrap/-/merge_requests/2568
This commit is contained in:
Caleb Connolly 2025-03-16 15:10:30 +00:00 • committed by Oliver Smith
parent 0eaedba632
commit dcc4137ee8
No known key found for this signature in database
GPG key ID: 5AE7F5513E0885CB
7 changed files with 107 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

46
pmb/build/_package.py
View file

@ -10,7 +10,7 @@ from pmb.core.arch import Arch
from pmb.core.context import Context
from pmb.core.pkgrepo import pkgrepo_relative_path
from pmb.helpers import logging
from pmb.types import Apkbuild, CrossCompileType
from pmb.types import Apkbuild, CrossCompile
from pathlib import Path
import pmb.build
@ -217,8 +217,7 @@ class BuildQueueItem(TypedDict):
output_path: Path
channel: str
depends: list[str]
cross: CrossCompileType
chroot: Chroot
cross: CrossCompile
def has_cyclical_dependency(
@ -493,7 +492,7 @@ def packages(
aports: Path,
apkbuild: dict[str, Any],
depends: list[str],
cross: CrossCompileType = "autodetect",
cross: CrossCompile | None = None,
) -> list[str]:
# Skip if already queued
name = apkbuild["pkgname"]
@ -540,7 +539,6 @@ def packages(
),
"channel": channel,
"depends": depends,
"chroot": chroot,
"cross": cross,
}
)
@ -613,16 +611,19 @@ def packages(
" build the package with --src again."
)
cross = "autodetect"
prev_cross = "autodetect"
cross = None
prev_cross = None
hostchroot = None # buildroot for the architecture we're building for
total_pkgs = len(build_queue)
count = 0
for pkg in build_queue:
count += 1
hostchroot = chroot = pkg["chroot"]
prev_cross = cross
cross = pkg["cross"]
pkg_arch = pkg["arch"]
hostchroot = cross.host_chroot(pkg_arch)
buildchroot = cross.build_chroot(pkg_arch)
apkbuild = pkg["apkbuild"]
channel = pkg["channel"]
@ -648,25 +649,19 @@ def packages(
)
)
# (re)-initialize the cross compiler stuff when cross method changes
prev_cross = cross
cross = pmb.build.autodetect.crosscompile(pkg["apkbuild"], pkg_arch)
if cross == "cross-native2" or cross == "cross-native":
chroot = Chroot.native()
# One time chroot initialization
if hostchroot != chroot:
if hostchroot != buildchroot:
pmb.build.init(hostchroot)
if pmb.build.init(chroot):
pmb.build.other.configure_abuild(chroot)
pmb.build.other.configure_ccache(chroot)
if pmb.build.init(buildchroot):
pmb.build.other.configure_abuild(buildchroot)
pmb.build.other.configure_ccache(buildchroot)
if "rust" in all_dependencies or "cargo" in all_dependencies:
pmb.chroot.apk.install(["sccache"], chroot)
pmb.chroot.apk.install(["sccache"], buildchroot)
if cross != prev_cross and cross not in ["unnecessary", "qemu-only"]:
if cross != prev_cross and cross.enabled():
pmb.build.init_compiler(context, pkg_depends, cross, pkg_arch)
if cross == "crossdirect":
pmb.chroot.mount_native_into_foreign(chroot)
if cross == CrossCompile.CROSSDIRECT:
pmb.chroot.mount_native_into_foreign(buildchroot)
depends_build: list[str] = []
depends_host: list[str] = []
@ -702,11 +697,11 @@ def packages(
logging.info("*** Install build dependencies")
if src:
depends_build.append("rsync")
pmb.chroot.apk.install(depends_build, chroot, build=False)
pmb.chroot.apk.install(depends_build, buildchroot, build=False)
# Build and finish up
msg = f"@YELLOW@=>@END@ @BLUE@{channel}/{pkg['name']}@END@: Building package"
if cross != "unnecessary":
if cross != CrossCompile.UNNECESSARY:
msg += f" (cross compiling: {cross})"
logging.info(msg)
@ -720,13 +715,12 @@ def packages(
cross,
strict,
force,
hostchroot,
src,
bootstrap_stage,
)
except RuntimeError:
raise BuildFailedError(f"Couldn't build {output}!")
finish(pkg["apkbuild"], channel, pkg_arch, output, chroot, strict)
finish(pkg["apkbuild"], channel, pkg_arch, output, buildchroot, strict)
# Clear package cache for the next run
_package_cache = {}

27
pmb/build/autodetect.py
View file

@ -7,10 +7,9 @@ from pmb.helpers import logging
import pmb.config
import pmb.chroot.apk
import pmb.helpers.pmaports
from pmb.core import Chroot
from pmb.core.context import get_context
from pmb.meta import Cache
from pmb.types import Apkbuild, CrossCompileType
from pmb.types import Apkbuild, CrossCompile
def arch_from_deviceinfo(pkgname: str, aport: Path) -> Arch | None:
@ -82,26 +81,16 @@ def arch(package: str | Apkbuild) -> Arch:
return Arch.native()
def chroot(apkbuild: Apkbuild, arch: Arch) -> Chroot:
if arch == Arch.native():
return Chroot.native()
if "pmb:cross-native" in apkbuild["options"]:
return Chroot.native()
return Chroot.buildroot(arch)
def crosscompile(apkbuild: Apkbuild, arch: Arch) -> CrossCompileType:
def crosscompile(apkbuild: Apkbuild, arch: Arch) -> CrossCompile:
"""Decide the type of compilation necessary to build a given APKBUILD."""
if not get_context().cross:
return "qemu-only"
return CrossCompile.QEMU_ONLY
if not arch.cpu_emulation_required():
return "unnecessary"
return CrossCompile.UNNECESSARY
if "pmb:cross-native" in apkbuild["options"]:
return "cross-native"
return CrossCompile.CROSS_NATIVE
if arch.is_native() or "pmb:cross-native2" in apkbuild["options"]:
return "cross-native2"
return CrossCompile.CROSS_NATIVE2
if "!pmb:crossdirect" in apkbuild["options"]:
return "qemu-only"
return "crossdirect"
return CrossCompile.QEMU_ONLY
return CrossCompile.CROSSDIRECT

46
pmb/build/backend.py
View file

@ -11,7 +11,7 @@ from pmb.core import Context
from pmb.core.arch import Arch
from pmb.core.chroot import Chroot
from pmb.helpers import logging
from pmb.types import Apkbuild, CrossCompileType, Env
from pmb.types import Apkbuild, CrossCompile, Env
class BootstrapStage(enum.IntEnum):
@ -190,10 +190,9 @@ def run_abuild(
pkgver: str,
channel: str,
arch: Arch,
cross: CrossCompileType,
cross: CrossCompile,
strict: bool = False,
force: bool = False,
hostchroot: Chroot = Chroot.native(),
src: str | None = None,
bootstrap_stage: int = BootstrapStage.NONE,
) -> None:
@ -210,18 +209,23 @@ def run_abuild(
the environment variables dict generated in this function.
"""
# Sanity check
if cross == "cross-native" and "!tracedeps" not in apkbuild["options"]:
if cross == CrossCompile.CROSS_NATIVE and "!tracedeps" not in apkbuild["options"]:
logging.warning(
"WARNING: Option !tracedeps is not set, but cross compiling with"
" cross-native (version 1). This will probably fail!"
)
hostchroot = cross.host_chroot(arch)
buildchroot = cross.build_chroot(arch)
# For cross-native2 compilation, bindmount the "host" rootfs to /mnt/sysroot
# it will be used as the "sysroot"
if cross == "cross-native2":
pmb.mount.bind(hostchroot.path, Chroot.native() / "/mnt/sysroot", umount=True)
chroot = Chroot.native() if cross == "cross-native2" else hostchroot
if cross == CrossCompile.CROSS_NATIVE2:
if buildchroot != Chroot.native():
raise ValueError(
"Trying to use cross-native2 build buildchroot != native! This is a bug"
)
pmb.mount.bind(hostchroot.path, buildchroot / "/mnt/sysroot", umount=True)
pkgdir = context.config.work / "packages" / channel
if not pkgdir.exists():
@ -241,16 +245,16 @@ def run_abuild(
["rm", "-f", "/home/pmos/packages/pmos"],
["ln", "-sf", f"/mnt/pmbootstrap/packages/{channel}", "/home/pmos/packages/pmos"],
],
chroot,
buildchroot,
)
# Environment variables
env: Env = {"SUDO_APK": "abuild-apk --no-progress"}
if cross == "cross-native":
if cross == CrossCompile.CROSS_NATIVE:
hostspec = arch.alpine_triple()
env["CROSS_COMPILE"] = hostspec + "-"
env["CC"] = hostspec + "-gcc"
if cross == "cross-native2":
if cross == CrossCompile.CROSS_NATIVE2:
env["CHOST"] = str(arch)
env["CBUILDROOT"] = "/mnt/sysroot"
env["CFLAGS"] = "-Wl,-rpath-link=/mnt/sysroot/usr/lib"
@ -261,7 +265,7 @@ def run_abuild(
except ValueError:
logging.debug(f"Not setting $GOARCH for {arch}")
elif cross == "crossdirect":
elif cross == CrossCompile.CROSSDIRECT:
env["PATH"] = ":".join([f"/native/usr/lib/crossdirect/{arch}", pmb.config.chroot_path])
else:
env["CARCH"] = str(arch)
@ -269,7 +273,7 @@ def run_abuild(
env["CCACHE_DISABLE"] = "1"
# Use sccache without crossdirect (crossdirect uses it via rustc.sh)
if context.ccache and cross != "crossdirect":
if context.ccache and cross != CrossCompile.CROSSDIRECT:
env["RUSTC_WRAPPER"] = "/usr/bin/sccache"
# Cache binary objects from go in this path (like ccache)
@ -300,16 +304,16 @@ def run_abuild(
cmd += ["-K"]
# Copy the aport to the chroot and build it
pmb.build.copy_to_buildpath(apkbuild["pkgname"], chroot, no_override=strict)
pmb.build.copy_to_buildpath(apkbuild["pkgname"], buildchroot, no_override=strict)
if src and strict:
logging.debug(f"({chroot}) Ensuring previous build artifacts are removed")
pmb.chroot.root(["rm", "-rf", "/tmp/pmbootstrap-local-source-copy"], chroot)
override_source(apkbuild, pkgver, src, chroot)
link_to_git_dir(chroot)
logging.debug(f"({buildchroot}) Ensuring previous build artifacts are removed")
pmb.chroot.root(["rm", "-rf", "/tmp/pmbootstrap-local-source-copy"], buildchroot)
override_source(apkbuild, pkgver, src, buildchroot)
link_to_git_dir(buildchroot)
try:
pmb.chroot.user(cmd, chroot, Path("/home/pmos/build"), env=env)
pmb.chroot.user(cmd, buildchroot, Path("/home/pmos/build"), env=env)
finally:
handle_csum_failure(apkbuild, chroot)
handle_csum_failure(apkbuild, buildchroot)
pmb.helpers.run.root(["umount", Chroot.native() / "/mnt/sysroot"], output="null", check=False)
pmb.helpers.run.root(["umount", buildchroot / "/mnt/sysroot"], output="null", check=False)

5
pmb/build/envkernel.py
View file

@ -247,10 +247,7 @@ def package_kernel(args: PmbArgs) -> None:
if not kbuild_out:
kbuild_out = ".output"
if "pmb:cross-native" in apkbuild["options"]:
chroot = Chroot.native()
else:
chroot = pmb.build.autodetect.chroot(apkbuild, arch)
chroot = Chroot.native()
# Install package dependencies
depends = pmb.build.get_depends(context, apkbuild)

8
pmb/build/init.py
View file

@ -13,7 +13,7 @@ import pmb.chroot.apk
import pmb.helpers.run
from pmb.core import Chroot
from pmb.core.context import get_context
from pmb.types import CrossCompileType
from pmb.types import CrossCompile
def init_abuild_minimal(chroot: Chroot = Chroot.native(), build_pkgs: list[str] = []) -> None:
@ -112,9 +112,7 @@ def init(chroot: Chroot = Chroot.native()) -> bool:
return True
def init_compiler(
context: Context, depends: list[str], cross: CrossCompileType, arch: Arch
) -> None:
def init_compiler(context: Context, depends: list[str], cross: CrossCompile, arch: Arch) -> None:
arch_str = str(arch)
cross_pkgs = ["ccache-cross-symlinks", "abuild"]
if "gcc4" in depends:
@ -125,7 +123,7 @@ def init_compiler(
cross_pkgs += ["gcc-" + arch_str, "g++-" + arch_str]
if "clang" in depends or "clang-dev" in depends:
cross_pkgs += ["clang"]
if cross == "crossdirect":
if cross == CrossCompile.CROSSDIRECT:
cross_pkgs += ["crossdirect"]
if "rust" in depends or "cargo" in depends:
if context.ccache:

2
pmb/build/kconfig.py
View file

@ -177,8 +177,8 @@ def _init(pkgname: str, arch: Arch | None) -> tuple[str, Arch, Any, Chroot, Env]
if arch is None:
arch = get_arch(apkbuild)
chroot = pmb.build.autodetect.chroot(apkbuild, arch)
cross = pmb.build.autodetect.crosscompile(apkbuild, arch)
chroot = cross.build_chroot(arch)
hostspec = arch.alpine_triple()
# Set up build tools and makedepends

57
pmb/types.py
View file

@ -1,21 +1,62 @@
# Copyright 2024 Caleb Connolly
# SPDX-License-Identifier: GPL-3.0-or-later
import enum
import subprocess
from argparse import Namespace
from pathlib import Path
from typing import Any, Literal, TypedDict
from pmb.core.arch import Arch
from pmb.core.chroot import Chroot
class CrossCompile(enum.Enum):
# Cross compilation isn't needed for this package
UNNECESSARY = "unnecessary"
# Cross compilation disabled, only use QEMU
QEMU_ONLY = "qemu-only"
# Cross compilation will use crossdirect
CROSSDIRECT = "crossdirect"
# Cross compilation will use cross-native
CROSS_NATIVE = "cross-native"
# Cross compilation will use cross-native2
CROSS_NATIVE2 = "cross-native2"
def __str__(self) -> str:
return self.value
def enabled(self) -> bool:
"""Are we cross-compiling for this value of cross?"""
return self not in [CrossCompile.UNNECESSARY, CrossCompile.QEMU_ONLY]
def host_chroot(self, arch: Arch) -> Chroot:
"""Chroot for the package target architecture (the "host" machine).
Cross native (v1) is the exception, since we exclusively use the native
chroot for that."""
if arch == Arch.native():
return Chroot.native()
match self:
case CrossCompile.CROSS_NATIVE:
return Chroot.native()
case _:
return Chroot.buildroot(arch)
def build_chroot(self, arch: Arch) -> Chroot:
"""Chroot for the package build architecture (the "build" machine)."""
if arch == Arch.native():
return Chroot.native()
match self:
case CrossCompile.CROSSDIRECT | CrossCompile.QEMU_ONLY:
return Chroot.buildroot(arch)
# FIXME: are there cases where we're building for a different arch
# but don't need to cross compile?
case CrossCompile.UNNECESSARY | CrossCompile.CROSS_NATIVE | CrossCompile.CROSS_NATIVE2:
return Chroot.native()
CrossCompileType = Literal[
"autodetect",
"unnecessary",
"qemu-only",
"crossdirect",
"cross-native",
"cross-native2",
]
RunOutputTypeDefault = Literal["log", "stdout", "interactive", "tui", "null"]
RunOutputTypePopen = Literal["background", "pipe"]
RunOutputType = RunOutputTypeDefault | RunOutputTypePopen