Administration/pmbootstrap-meow
2
0
Fork 0
forked from Mirror/pmbootstrap
Code Pull requests Activity Actions

build: add pmb:cross-native2 (MR 2474)

Browse source 
Set things up so that we can run abuild on the native chroot and use
it's cross compilation features rather than running it and the build
system through QEMU. This massively speeds up building when it works.

cross-native used to be quite limited in functionality and didn't
integrate into abuild itself, this commit fixes that.

Packages can opt-in to this by adding pmb:cross-native2 to their options
and configuring makedepends_host and makedepends_build.

This also speeds up building packages like postmarketos-initramfs since
it entirely avoids running commands through QEMU (usually abuild itself
would be run through QEMU).

Lastly, we preserve the old pmb:cross-kernel options, this can be used to
enable the old cross compiler behaviour which is used for cross
compiling kernels in pmaports. This allows them to keep being supporting
while we adapt them to the new cross-native2.

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>
This commit is contained in:
Caleb Connolly 2025-01-16 00:07:56 +01:00 • committed by Oliver Smith
parent 5a00964943
commit 2ee916f5d6
No known key found for this signature in database
GPG key ID: 5AE7F5513E0885CB
5 changed files with 106 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

71
pmb/build/_package.py
View file

@ -616,13 +616,15 @@ def packages(
cross = None
prev_cross = None
hostchroot = None # buildroot for the architecture we're building for
total_pkgs = len(build_queue)
count = 0
for pkg in build_queue:
count += 1
chroot = pkg["chroot"]
hostchroot = chroot = pkg["chroot"]
pkg_arch = pkg["arch"]
apkbuild = pkg["apkbuild"]
channel = pkg["channel"]
output = pkg["output_path"]
@ -633,40 +635,87 @@ def packages(
else:
log_callback(pkg)
# FIXME: this is only used to detect special compilers and a workaround for rust
# in pmb.build.init_compiler(), this should all be refactored and enforce correct
# APKBUILDs rather than trying to hack things in here
pkg_depends = list(
set(
[
*pkg["depends"],
*apkbuild.get("makedepends", []),
*apkbuild.get("makedepends_build", []),
*apkbuild.get("makedepends_host", []),
]
)
)
# (re)-initialize the cross compiler stuff when cross method changes
prev_cross = cross
cross = pmb.build.autodetect.crosscompile(pkg["apkbuild"], pkg_arch)
if cross == "native" or cross == "kernel":
chroot = Chroot.native()
# One time chroot initialization
if hostchroot != chroot:
pmb.build.init(hostchroot)
if pmb.build.init(chroot):
pmb.build.other.configure_abuild(chroot)
pmb.build.other.configure_ccache(chroot)
if "rust" in all_dependencies or "cargo" in all_dependencies:
pmb.chroot.apk.install(["sccache"], chroot)
pkg_depends = pkg["depends"]
if src:
pkg_depends.append("rsync")
# (re)-initialize the cross compiler stuff when cross method changes
prev_cross = cross
cross = pmb.build.autodetect.crosscompile(pkg["apkbuild"], pkg_arch)
if cross != prev_cross:
pmb.build.init_compiler(context, pkg_depends, cross, pkg_arch)
if cross == "crossdirect":
pmb.chroot.mount_native_into_foreign(chroot)
if not strict and "pmb:strict" not in pkg["apkbuild"]["options"] and len(pkg_depends):
pmb.chroot.apk.install(pkg_depends, chroot, build=False)
depends_build: list[str] = []
depends_host: list[str] = []
# * makedepends_build are dependencies that should be installed in the native
# chroot (e.g. 'meson').
# * makedepends_host are dependencies that should be
# in the chroot for the architecture we're building for (e.g. 'libevdev-dev').
if apkbuild["makedepends_host"]:
depends_host = apkbuild["makedepends_host"]
else:
depends_host = apkbuild["makedepends"]
if apkbuild["makedepends_build"]:
# If we have makedepends_build but not host then just subtract
# the host depends from the main depends
if "makedepends_host" not in apkbuild:
depends_host = list(set(depends_host) - set(apkbuild["makedepends_build"]))
depends_build = apkbuild["makedepends_build"]
else:
depends_build = apkbuild["makedepends"]
if depends_build and depends_host:
logging.warning(
"WARNING: makedepends not split into _host and _build variants."
" Trying to install all makedepends in both environments, please"
" adjust your APKBUILD to specify host dependencies (e.g. libevdev-dev)"
" and build dependencies (e.g. meson) separately."
)
if depends_host:
logging.info("*** Install host dependencies")
pmb.chroot.apk.install(depends_host, hostchroot, build=False)
if depends_build:
logging.info("*** Install build dependencies")
if src:
depends_build.append("rsync")
pmb.chroot.apk.install(depends_build, chroot, build=False)
# Build and finish up
logging.info(f"@YELLOW@=>@END@ @BLUE@{channel}/{pkg['name']}@END@: Building package")
try:
run_abuild(
context,
pkg["apkbuild"],
apkbuild,
pkg["pkgver"],
channel,
pkg_arch,
strict,
force,
cross,
chroot,
hostchroot,
src,
bootstrap_stage,
)

5
pmb/build/autodetect.py
View file

@ -98,7 +98,10 @@ def crosscompile(apkbuild: Apkbuild, arch: Arch) -> CrossCompileType:
return None
if not arch.cpu_emulation_required():
return None
if arch.is_native() or "pmb:cross-native" in apkbuild["options"]:
# deprecated cross-native environment for building kernels
if "pmb:cross-native" in apkbuild["options"]:
return "kernel"
if arch.is_native() or "pmb:cross-native2" in apkbuild["options"]:
return "native"
if "!pmb:crossdirect" in apkbuild["options"]:
return None

58
pmb/build/backend.py
View file

@ -193,7 +193,7 @@ def run_abuild(
strict: bool = False,
force: bool = False,
cross: CrossCompileType = None,
suffix: Chroot = Chroot.native(),
hostchroot: Chroot = Chroot.native(),
src: str | None = None,
bootstrap_stage: int = BootstrapStage.NONE,
) -> None:
@ -217,6 +217,14 @@ def run_abuild(
" cross-compiling in the native chroot. This will"
" probably fail!"
)
# For cross-native2 compilation, bindmount the "host" rootfs to /mnt/sysroot
# it will be used as the "sysroot"
if cross == "native":
pmb.mount.bind(hostchroot.path, Chroot.native() / "/mnt/sysroot", umount=True)
chroot = Chroot.native() if cross == "native" else hostchroot
pkgdir = context.config.work / "packages" / channel
if not pkgdir.exists():
pmb.helpers.run.root(["mkdir", "-p", pkgdir])
@ -235,17 +243,28 @@ def run_abuild(
["rm", "-f", "/home/pmos/packages/pmos"],
["ln", "-sf", f"/mnt/pmbootstrap/packages/{channel}", "/home/pmos/packages/pmos"],
],
suffix,
chroot,
)
# Environment variables
env: Env = {"CARCH": str(arch), "SUDO_APK": "abuild-apk --no-progress"}
if cross == "native":
env: Env = {"SUDO_APK": "abuild-apk --no-progress"}
if cross == "kernel":
hostspec = arch.alpine_triple()
env["CROSS_COMPILE"] = hostspec + "-"
env["CC"] = hostspec + "-gcc"
if cross == "crossdirect":
if cross == "native":
env["CHOST"] = str(arch)
env["CBUILDROOT"] = "/mnt/sysroot"
env["CFLAGS"] = "-Wl,-rpath-link=/mnt/sysroot/usr/lib"
try:
env["GOARCH"] = arch.go()
except ValueError:
logging.debug(f"Not setting $GOARCH for {arch}")
elif cross == "crossdirect":
env["PATH"] = ":".join([f"/native/usr/lib/crossdirect/{arch}", pmb.config.chroot_path])
else:
env["CARCH"] = str(arch)
if not context.ccache:
env["CCACHE_DISABLE"] = "1"
@ -269,15 +288,10 @@ def run_abuild(
env["BOOTSTRAP"] = str(bootstrap_stage)
# Build the abuild command
cmd = ["abuild", "-D", "postmarketOS"]
if strict or "pmb:strict" in apkbuild["options"]:
if not strict:
logging.debug(
apkbuild["pkgname"] + ": 'pmb:strict' found in options, building in strict mode"
)
cmd += ["-r"] # install depends with abuild
else:
cmd += ["-d"] # do not install depends with abuild
# Since we install dependencies with pmb, disable dependency handling in abuild.
# This is also required so that abuild doesn't try to install base-build-$ARCH packages
# which don't exist
cmd = ["abuild", "-d", "-D", "postmarketOS"]
if force:
cmd += ["-f"]
if src:
@ -286,14 +300,16 @@ def run_abuild(
cmd += ["-K"]
# Copy the aport to the chroot and build it
pmb.build.copy_to_buildpath(apkbuild["pkgname"], suffix, no_override=strict)
pmb.build.copy_to_buildpath(apkbuild["pkgname"], chroot, no_override=strict)
if src and strict:
logging.debug(f"({suffix}) Ensuring previous build artifacts are removed")
pmb.chroot.root(["rm", "-rf", "/tmp/pmbootstrap-local-source-copy"], suffix)
override_source(apkbuild, pkgver, src, suffix)
link_to_git_dir(suffix)
logging.debug(f"({chroot}) Ensuring previous build artifacts are removed")
pmb.chroot.root(["rm", "-rf", "/tmp/pmbootstrap-local-source-copy"], chroot)
override_source(apkbuild, pkgver, src, chroot)
link_to_git_dir(chroot)
try:
pmb.chroot.user(cmd, suffix, Path("/home/pmos/build"), env=env)
pmb.chroot.user(cmd, chroot, Path("/home/pmos/build"), env=env)
finally:
handle_csum_failure(apkbuild, suffix)
handle_csum_failure(apkbuild, chroot)
pmb.helpers.run.root(["umount", Chroot.native() / "/mnt/sysroot"], output="null", check=False)

5
pmb/build/envkernel.py
View file

@ -247,7 +247,10 @@ def package_kernel(args: PmbArgs) -> None:
if not kbuild_out:
kbuild_out = ".output"
chroot = pmb.build.autodetect.chroot(apkbuild, arch)
if "pmb:cross-native" in apkbuild["options"]:
chroot = Chroot.native()
else:
chroot = pmb.build.autodetect.chroot(apkbuild, arch)
# Install package dependencies
depends = pmb.build.get_depends(context, apkbuild)

2
pmb/types.py
View file

@ -8,7 +8,7 @@ from typing import Any, Literal, TypedDict
from pmb.core.arch import Arch
CrossCompileType = Literal["native", "crossdirect"] | None
CrossCompileType = Literal["native", "crossdirect", "kernel"] | None
RunOutputTypeDefault = Literal["log", "stdout", "interactive", "tui", "null"]
RunOutputTypePopen = Literal["background", "pipe"]
RunOutputType = RunOutputTypeDefault | RunOutputTypePopen