ccache: Fix for distcc cross-compiling / various improvements (#1026)

* ccache: Fix for distcc cross-compiling / various improvements

* Make ccache work when cross-compiling with distcc (fix #716)
* Allow to configure the ccache size in "pmbootstrap init"
* Moved ccache stats code from pmb/build/other.py to
  pmb/helpers/frontend.py
* Grouped job count, ccache size and timestamp based rebuilds
  together to "build options" and allow to skip them
* Sorted config options that had to be modified anyway
  alphabetically

* Improve comment in arch-bin-masquerade APKBUILD

aports/cross/arch-bin-masquerade/APKBUILD

@@ -0,0 +1,43 @@
+# This package gets installed in the native and foreign arch chroots.
+# It creates files like /usr/lib/arch-bin-masquerade/armhf/gcc, which
+# point in the native chroot to the armhf-cross-compiler, and in the
+# armhf chroot to /usr/bin/gcc. That way compilation works fine, even
+# when distcc gets the absolute path to the compiler passed (ccache does
+# that).
+
+pkgname=arch-bin-masquerade
+pkgver=1
+pkgrel=0
+pkgdesc="Wrappers for ccache + distcc (native and foreign chroots)"
+url="https://postmarketOS.org"
+arch="all"
+license="MIT"
+options="!check !tracedeps"
+
+package() {
+	# Architectures and binaries
+	_archs="x86_64 armhf aarch64"
+	_bins="c++ cc cpp g++ gcc"
+
+	# Iterate over architectures
+	for _arch in $_archs; do
+		# Create the arch-specific bin folder
+		_hostspec="$(arch_to_hostspec $_arch)"
+		_bindir="$pkgdir/usr/lib/arch-bin-masquerade/$_arch"
+		mkdir -p "$_bindir"
+		cd "$_bindir"
+
+		# Iterate over binaries and create wrappers
+		for _bin in $_bins; do
+			{
+				echo "#!/bin/sh"
+				if [ "$_arch" == "$CARCH" ]; then
+					echo "exec /usr/bin/${_bin} \"\$@\""
+				else
+					echo "exec /usr/bin/${_hostspec}-${_bin} \"\$@\""
+				fi
+			} > "$_bin"
+			chmod +x "$_bin"
+		done
+	done
+}

aports/cross/ccache-cross-symlinks/APKBUILD

@@ -1,16 +1,16 @@
-# Maintainer: Oliver Smith <ollieparanoid@bitmessage.ch>
+# This package gets installed in the native chroot only. When cross-
-# NOTE: This could probably be upstreamed to the official ccache aport.
+# compiling packages in the native chroot (e.g. kernel packages), the
+# cross-compiler does not get called directly, but wrapped through
+# ccache, which can then cache the results.
 
 pkgname=ccache-cross-symlinks
 pkgver=1
-pkgrel=3
+pkgrel=4
 pkgdesc="Enable ccache for cross-compilers with symlinks"
 url="https://ccache.samba.org/"
 arch="noarch"
 license="MIT"
 depends="ccache"
-makedepends=""
-source=""
 options="!check"
 
 package() {

aports/cross/gcc-cross-wrappers/APKBUILD

@@ -1,28 +0,0 @@
-pkgname=gcc-cross-wrappers
-pkgver=1
-pkgrel=1
-pkgdesc="GCC wrappers pointing to cross-compilers (for distcc + ccache)"
-url="https://github.com/postmarketOS"
-arch="noarch"
-license="MIT"
-depends=""
-makedepends=""
-source=""
-options="!check"
-
-package() {
-	local _archs="armhf aarch64"
-	local _bins="c++ cc cpp g++ gcc"
-	for _arch in $_archs; do
-		_bindir="$pkgdir/usr/lib/gcc-cross-wrappers/$_arch/bin"
-		_hostspec="$(arch_to_hostspec $_arch)"
-		mkdir -p "$_bindir"
-		for _bin in $_bins; do
-			{
-				echo "#!/bin/sh"
-				echo "${_hostspec}-${_bin} \"\$@\""
-			} > $_bindir/$_bin
-			chmod +x $_bindir/$_bin
-		done
-	done
-}

pmb/build/__init__.py

@@ -21,6 +21,6 @@ from pmb.build.init import init
 from pmb.build.checksum import checksum
 from pmb.build.menuconfig import menuconfig
 from pmb.build.other import copy_to_buildpath, is_necessary, \
-    find_aport, ccache_stats, index_repo
+    find_aport, index_repo
 from pmb.build._package import package
 from pmb.build.qemu_workaround_aarch64 import qemu_workaround_aarch64

pmb/build/_package.py

@@ -175,10 +175,11 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
     if not is_necessary_warn_depends(args, apkbuild, arch, force, built):
         return False
 
-    # Install and configure abuild, gcc, dependencies
+    # Install and configure abuild, ccache, gcc, dependencies
     if not skip_init_buildenv:
         pmb.build.init(args, suffix)
         pmb.build.other.configure_abuild(args, suffix)
+        pmb.build.other.configure_ccache(args, suffix)
     if not strict and len(depends):
         pmb.chroot.apk.install(args, depends, suffix)
 
@@ -187,13 +188,30 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
         pmb.chroot.apk.install(args, ["gcc-" + arch, "g++-" + arch,
                                       "ccache-cross-symlinks"])
     if cross == "distcc":
-        pmb.chroot.apk.install(args, ["distcc"], suffix=suffix,
+        pmb.chroot.apk.install(args, ["distcc", "arch-bin-masquerade"],
-                               build=False)
+                               suffix=suffix)
         pmb.chroot.distccd.start(args, arch)
 
     return True
 
 
+def get_gcc_version(args, arch):
+    """
+    Get the GCC version for a specific arch from parsing the right APKINDEX.
+    We feed this to ccache, so it knows the right GCC version, when
+    cross-compiling in a foreign arch chroot with distcc. See the "using
+    ccache with other compiler wrappers" section of their man page:
+    <https://linux.die.net/man/1/ccache>
+    :returns: a string like "6.4.0-r5"
+    """
+    repository = args.mirror_alpine + args.alpine_version + "/main"
+    hash = pmb.helpers.repo.hash(repository)
+    index_path = (args.work + "/cache_apk_" + arch + "/APKINDEX." +
+                  hash + ".tar.gz")
+    apkindex = pmb.parse.apkindex.read(args, "gcc", index_path, True)
+    return apkindex["version"]
+
+
 def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
                suffix="native"):
     """
@@ -225,7 +243,9 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
         env["CROSS_COMPILE"] = hostspec + "-"
         env["CC"] = hostspec + "-gcc"
     if cross == "distcc":
-        env["PATH"] = "/usr/lib/distcc/bin:" + pmb.config.chroot_path
+        env["CCACHE_PREFIX"] = "distcc"
+        env["CCACHE_PATH"] = "/usr/lib/arch-bin-masquerade/" + arch + ":/usr/bin"
+        env["CCACHE_COMPILERCHECK"] = "string:" + get_gcc_version(args, arch)
         env["DISTCC_HOSTS"] = "127.0.0.1:" + args.port_distccd
 
     # Build the abuild command

pmb/build/other.py

@@ -264,15 +264,12 @@ def index_repo(args, arch=None):
         pmb.parse.apkindex.clear_cache(args, path + "/APKINDEX.tar.gz")
 
 
-def ccache_stats(args, arch):
-    suffix = "native"
-    if args.arch:
-        suffix = "buildroot_" + arch
-    pmb.chroot.user(args, ["ccache", "-s"], suffix, log=False)
-
-
-# set the correct JOBS count in abuild.conf
 def configure_abuild(args, suffix, verify=False):
+    """
+    Set the correct JOBS count in abuild.conf
+
+    :param verify: internally used to test if changing the config has worked.
+    """
     path = args.work + "/chroot_" + suffix + "/etc/abuild.conf"
     prefix = "export JOBS="
     with open(path, encoding="utf-8") as handle:
@@ -289,3 +286,27 @@ def configure_abuild(args, suffix, verify=False):
                 configure_abuild(args, suffix, True)
             return
     raise RuntimeError("Could not find " + prefix + " line in " + path)
+
+
+def configure_ccache(args, suffix="native", verify=False):
+    """
+    Set the maximum ccache size
+
+    :param verify: internally used to test if changing the config has worked.
+    """
+    # Check if the settings have been set already
+    arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
+    path = args.work + "/cache_ccache_" + arch + "/ccache.conf"
+    if os.path.exists(path):
+        with open(path, encoding="utf-8") as handle:
+            for line in handle:
+                if line == ("max_size = " + args.ccache_size + "\n"):
+                    return
+    if verify:
+        raise RuntimeError("Failed to configure ccache: " + path + "\nTry to"
+                           " delete the file (or zap the chroot).")
+
+    # Set the size and verify
+    pmb.chroot.user(args, ["ccache", "--max-size", args.ccache_size],
+                    suffix)
+    configure_ccache(args, suffix, True)

pmb/chroot/distccd.py

@@ -83,11 +83,16 @@ def is_running(args):
 def generate_cmdline(args, arch):
     """
     :returns: a dictionary suitable for pmb.chroot.user(), to start the distccd
-              with the cross-compiler in the path and all options set.
+              with all options set.
+    NOTE: The distcc client of the foreign arch chroot passes the
+          absolute path to the compiler, which points to
+          "/usr/lib/arch-bin-masquerade/armhf/gcc" for example. This also
+          exists in the native chroot, and points to the armhf cross-
+          compiler there (both the native and foreign chroot have the
+          arch-bin-masquerade package installed, which creates the
+          wrapper scripts).
     """
-    path = "/usr/lib/gcc-cross-wrappers/" + arch + "/bin:" + pmb.config.chroot_path
+    ret = ["distccd",
-    ret = ["PATH=" + path,
-           "distccd",
            "--pid-file", "/home/pmos/distccd.pid",
            "--listen", "127.0.0.1",
            "--allow", "127.0.0.1",
@@ -110,7 +115,7 @@ def start(args, arch):
     if info and info["cmdline"] == " ".join(cmdline):
         return
     stop(args)
-    pmb.chroot.apk.install(args, ["distcc", "gcc-cross-wrappers"])
+    pmb.chroot.apk.install(args, ["distcc", "arch-bin-masquerade"])
 
     # Start daemon with cross-compiler in path
     logging.info("(native) start distccd (" + arch + ") on 127.0.0.1:" +

pmb/config/__init__.py

@@ -45,8 +45,9 @@ apk_tools_static_min_version = "2.7.2-r0"
 work_version = "1"
 
 # Only save keys to the config file, which we ask for in 'pmbootstrap init'.
-config_keys = ["device", "extra_packages", "jobs", "timestamp_based_rebuild",
+config_keys = ["ccache_size", "device", "extra_packages", "jobs", "keymap",
-               "work", "qemu_mesa_driver", "ui", "user", "keymap", "timezone"]
+               "qemu_mesa_driver", "timestamp_based_rebuild", "timezone"
+               "ui", "user", "work"]
 
 # Config file/commandline default values
 # $WORK gets replaced with the actual value for args.work (which may be
@@ -54,29 +55,29 @@ config_keys = ["device", "extra_packages", "jobs", "timestamp_based_rebuild",
 defaults = {
     "alpine_version": "edge",  # alternatively: latest-stable
     "aports": os.path.normpath(pmb_src + "/aports"),
-    "config": os.path.expanduser("~") + "/.config/pmbootstrap.cfg",
+    "ccache_size": "5G",
-    "device": "samsung-i9100",
-    "extra_packages": "none",
-    "jobs": str(multiprocessing.cpu_count() + 1),
-    "timestamp_based_rebuild": True,
-    "log": "$WORK/log.txt",
-    "mirror_alpine": "http://dl-cdn.alpinelinux.org/alpine/",
-    "mirror_postmarketos": "http://postmarketos.brixit.nl",
-    "work": os.path.expanduser("~") + "/.local/var/pmbootstrap",
-    "port_distccd": "33632",
-    "qemu_mesa_driver": "dri-virtio",
-    "ui": "weston",
-    "user": "user",
-    "keymap": "",
-    "timezone": "GMT",
-
     # aes-xts-plain64 would be better, but this is not supported on LineageOS
     # kernel configs
     "cipher": "aes-cbc-plain64",
+    "config": os.path.expanduser("~") + "/.config/pmbootstrap.cfg",
+    "device": "samsung-i9100",
+    "extra_packages": "none",
     # A higher value is typically desired, but this can lead to VERY long open
     # times on slower devices due to host systems being MUCH faster than the
     # target device: <https://github.com/postmarketOS/pmbootstrap/issues/429>
-    "iter_time": "200"
+    "iter_time": "200",
+    "jobs": str(multiprocessing.cpu_count() + 1),
+    "keymap": "",
+    "log": "$WORK/log.txt",
+    "mirror_alpine": "http://dl-cdn.alpinelinux.org/alpine/",
+    "mirror_postmarketos": "http://postmarketos.brixit.nl",
+    "port_distccd": "33632",
+    "qemu_mesa_driver": "dri-virtio",
+    "timestamp_based_rebuild": True,
+    "timezone": "GMT",
+    "ui": "weston",
+    "user": "user",
+    "work": os.path.expanduser("~") + "/.local/var/pmbootstrap",
 }
 
 #
@@ -156,7 +157,7 @@ build_packages = ["abuild", "build-base", "ccache"]
 
 # fnmatch for supported pkgnames, that can be directly compiled inside
 # the native chroot and a cross-compiler, without using distcc
-build_cross_native = ["linux-*"]
+build_cross_native = ["linux-*", "arch-bin-masquerade"]
 
 # Necessary kernel config options
 necessary_kconfig_options = {

pmb/config/init.py

@@ -118,7 +118,8 @@ def ask_for_timezone(args):
             if pmb.helpers.cli.confirm(args, "Use this timezone instead of GMT?",
                                        default="y"):
                 return tz
-    logging.info("WARNING: Unable to determine timezone configuration on host, using GMT.")
+    logging.info("WARNING: Unable to determine timezone configuration on host,"
+                 " using GMT.")
     return "GMT"
 
 
@@ -160,6 +161,43 @@ def ask_for_qemu_mesa_driver(args):
                       " it, see qemu_mesa_drivers in pmb/config/__init__.py.")
 
 
+def ask_for_build_options(args, cfg):
+    # Allow to skip build options
+    ts_rebuild = "True" if args.timestamp_based_rebuild else "False"
+    logging.info("Build options: Parallel jobs: " + args.jobs +
+                 ", ccache per arch: " + args.ccache_size +
+                 ", timestamp based rebuilds: " + ts_rebuild)
+
+    if not pmb.helpers.cli.confirm(args, "Change them?",
+                                   default=False):
+        return
+
+    # Parallel job count
+    logging.info("How many jobs should run parallel on this machine, when"
+                 " compiling?")
+    answer = pmb.helpers.cli.ask(args, "Jobs", None, args.jobs,
+                                 validation_regex="[1-9][0-9]*")
+    cfg["pmbootstrap"]["jobs"] = answer
+
+    # Ccache size
+    logging.info("We use ccache to speed up building the same code multiple"
+                 " times. How much space should the ccache folder take up per"
+                 " architecture? After init is through, you can check the current"
+                 " usage with 'pmbootstrap stats'. Answer with 0 for infinite.")
+    regex = "0|[0-9]+(k|M|G|T|Ki|Mi|Gi|Ti)"
+    answer = pmb.helpers.cli.ask(args, "Ccache size", None, args.ccache_size,
+                                 lowercase_answer=False, validation_regex=regex)
+    cfg["pmbootstrap"]["ccache_size"] = answer
+
+    # Timestamp based rebuilds
+    logging.info("Rebuild packages, when the last modified timestamp changed,"
+                 " even if the version did not change?"
+                 " This makes pmbootstrap behave more like 'make'.")
+    answer = pmb.helpers.cli.confirm(args, "Timestamp based rebuilds",
+                                     default=args.timestamp_based_rebuild)
+    cfg["pmbootstrap"]["timestamp_based_rebuild"] = str(answer)
+
+
 def frontend(args):
     cfg = pmb.config.load(args)
 
@@ -172,7 +210,8 @@ def frontend(args):
 
     # Device keymap
     if device_exists:
-        cfg["pmbootstrap"]["keymap"] = ask_for_keymaps(args, device=cfg["pmbootstrap"]["device"])
+        cfg["pmbootstrap"]["keymap"] = ask_for_keymaps(
+            args, device=cfg["pmbootstrap"]["device"])
 
     # Username
     cfg["pmbootstrap"]["user"] = pmb.helpers.cli.ask(args, "Username", None,
@@ -182,19 +221,8 @@ def frontend(args):
     cfg["pmbootstrap"]["ui"] = ask_for_ui(args)
     cfg["pmbootstrap"]["work"] = ask_for_work_path(args)
 
-    # Parallel job count
+    # Various build options
-    logging.info("How many jobs should run parallel on this machine, when"
+    ask_for_build_options(args, cfg)
-                 " compiling?")
-    cfg["pmbootstrap"]["jobs"] = pmb.helpers.cli.ask(args, "Jobs",
-                                                     None, args.jobs, validation_regex="[1-9][0-9]*")
-
-    # Timestamp based rebuilds
-    logging.info("Rebuild packages, when the last modified timestamp changed,"
-                 " even if the version did not change? This makes pmbootstrap"
-                 " behave more like 'make'.")
-    answer = pmb.helpers.cli.confirm(args, "Timestamp based rebuilds",
-                                     default=args.timestamp_based_rebuild)
-    cfg["pmbootstrap"]["timestamp_based_rebuild"] = str(answer)
 
     # Extra packages to be installed to rootfs
     logging.info("Additional packages that will be installed to rootfs."
@@ -215,7 +243,8 @@ def frontend(args):
     if (device_exists and
             len(glob.glob(args.work + "/chroot_*")) and
             pmb.helpers.cli.confirm(args, "Zap existing chroots to apply configuration?", default=True)):
-        setattr(args, "deviceinfo", pmb.parse.deviceinfo(args, device=cfg["pmbootstrap"]["device"]))
+        setattr(args, "deviceinfo", pmb.parse.deviceinfo(
+            args, device=cfg["pmbootstrap"]["device"]))
         # Do not zap any existing packages or cache_http directories
         pmb.chroot.zap(args, confirm=False)
 

pmb/helpers/frontend.py

@@ -261,7 +261,15 @@ def shutdown(args):
 
 
 def stats(args):
-    pmb.build.ccache_stats(args, args.arch)
+    # Chroot suffix
+    suffix = "native"
+    if args.arch != args.arch_native:
+        suffix = "buildroot_" + args.arch
+
+    # Install ccache and display stats
+    pmb.chroot.apk.install(args, ["ccache"], suffix)
+    logging.info("(" + suffix + ") % ccache -s")
+    pmb.chroot.user(args, ["ccache", "-s"], suffix, log=False)
 
 
 def log(args):

pmb/parse/arguments.py

@@ -207,7 +207,7 @@ def arguments():
 
     # Action: stats
     stats = sub.add_parser("stats", help="show ccache stats")
-    stats.add_argument("--arch")
+    stats.add_argument("--arch", default=arch_native, choices=arch_choices)
 
     # Action: build_init / chroot
     build_init = sub.add_parser("build_init", help="initialize build"

test/test_build_package.py

@@ -229,13 +229,13 @@ def test_run_abuild(args, monkeypatch):
     assert func(args, apkbuild, "armhf", cross="native") == (output, cmd, env)
 
     # cross=distcc
-    env = {"CARCH": "armhf",
+    (output, cmd, env) = func(args, apkbuild, "armhf", cross="distcc")
-           "PATH": "/usr/lib/distcc/bin:" + pmb.config.chroot_path,
+    assert output == "armhf/test-1-r2.apk"
-           "DISTCC_HOSTS": "127.0.0.1:33632"}
+    assert env["CARCH"] == "armhf"
-    cmd = ["CARCH=armhf", "PATH=" + "/usr/lib/distcc/bin:" +
+    assert env["CCACHE_PREFIX"] == "distcc"
-           pmb.config.chroot_path, "DISTCC_HOSTS=127.0.0.1:33632", "abuild",
+    assert env["CCACHE_PATH"] == "/usr/lib/arch-bin-masquerade/armhf:/usr/bin"
-           "-d"]
+    assert env["CCACHE_COMPILERCHECK"].startswith("string:")
-    assert func(args, apkbuild, "armhf", cross="distcc") == (output, cmd, env)
+    assert env["DISTCC_HOSTS"] == "127.0.0.1:33632"
 
 
 def test_finish(args, monkeypatch):

test/test_questions.py

@@ -154,3 +154,21 @@ def test_questions(args, monkeypatch, tmpdir):
     answers = ["/dev/null", os.path.dirname(__file__), pmb.config.pmb_src,
                tmpdir]
     assert pmb.config.init.ask_for_work_path(args) == tmpdir
+
+    #
+    # BUILD OPTIONS
+    #
+    func = pmb.config.init.ask_for_build_options
+    cfg = {"pmbootstrap": {}}
+
+    # Skip changing anything
+    answers = ["n"]
+    func(args, cfg)
+    assert cfg == {"pmbootstrap": {}}
+
+    # Answer everything
+    answers = ["y", "5", "2G", "n"]
+    func(args, cfg)
+    assert cfg == {"pmbootstrap": {"jobs": "5",
+                                   "ccache_size": "2G",
+                                   "timestamp_based_rebuild": "False"}}