pmb: adjust to distcc 3.3 and wrap it with sshd

Overview:
Since Alpine updated to distcc 3.3 last week, pmbootstrap wasn't able to use
distcc for cross compilation anymore. It always falled back to running the
compiler in QEMU (which works, but is a lot slower). The reason for that is,
that distcc requires all compilers that are being used in a whitelist now.

This partially fixes CVE-2004-2687 in distccd, which allowed trivial remote
code execution by any process connecting to the distccd server. We only run
distccd on localhost, but still this can be used for privilege escalation of
sandboxed processes running on the host system (not part of pmbootstrap
chroots).

Because the CVE is only partially fixed (see the comment in
`pmb/chroot/distccd.py` for details), we make sure that only the building
chroots can talk to the distcc server by running distcc over ssh.

Details:
* Completely refactored `pmb/chroot/distccd.py` to run distcc over ssh
  * Store the running distcc server's arguments as JSON now, not as INI
* Make debugging distcc issues easy:
  * Set DISTCC_BACKOFF_PERIOD=0, so the distcc client will not ignore the
    server after errors happened (this masks the original error!)
  * New pmbootstrap parameters:
    * `--distcc-nofallback`: avoids falling back to compiling with QEMU and not
	   throwing an error
	* `--ccache-disable`: avoid ccache (when the compiler output is cached,
	  distcc does not get used)
  * `--verbose` prints verbose output of the distcc too
  * New test case, that uses the new pmbootstrap parameters to force
	compilation through distcc, and shows the output of distcc and distccd in
	verbose mode on error (as well as the log of sshd)

pmb/build/_package.py

@@ -193,8 +193,6 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
         pmb.chroot.apk.install(args, ["gcc-" + arch, "g++-" + arch,
                                       "ccache-cross-symlinks"])
     if cross == "distcc":
-        pmb.chroot.apk.install(args, ["distcc", "arch-bin-masquerade"],
-                               suffix=suffix)
         pmb.chroot.distccd.start(args, arch)
 
     # "native" cross-compile: build and install dependencies (#1061)
@@ -350,7 +348,16 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
         env["CCACHE_PREFIX"] = "distcc"
         env["CCACHE_PATH"] = "/usr/lib/arch-bin-masquerade/" + arch + ":/usr/bin"
         env["CCACHE_COMPILERCHECK"] = "string:" + get_gcc_version(args, arch)
-        env["DISTCC_HOSTS"] = "127.0.0.1:" + args.port_distccd
+        env["DISTCC_HOSTS"] = "@127.0.0.1:/home/pmos/.distcc-sshd/distccd"
+        env["DISTCC_SSH"] = ("ssh -o StrictHostKeyChecking=no -p" +
+                             args.port_distccd)
+        env["DISTCC_BACKOFF_PERIOD"] = "0"
+        if not args.distcc_fallback:
+            env["DISTCC_FALLBACK"] = "0"
+        if args.verbose:
+            env["DISTCC_VERBOSE"] = "1"
+    if not args.ccache:
+        env["CCACHE_DISABLE"] = "1"
 
     # Build the abuild command
     cmd = ["abuild", "-D", "postmarketOS"]

pmb/chroot/distccd.py

@@ -16,123 +16,251 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
-import configparser
 import errno
+import json
 import logging
 import os
 import pmb.chroot
 import pmb.config
 import pmb.chroot.apk
 
+""" Packages for foreign architectures (e.g. armhf) get built in chroots
+    running with QEMU. While this works, it is painfully slow. So we speed it
+    up by using distcc to let cross compilers running in the native chroots do
+    the heavy lifting.
+
+    This file sets up an SSH server in the native chroot, which will then be
+    used by the foreign arch chroot to communicate with the distcc daemon. We
+    make sure that only the foreign arch chroot can connect to the sshd by only
+    listening on localhost, as well as generating dedicated ssh keys.
+
+    Using the SSH server instead of running distccd directly is a security
+    measure. Distccd does not authenticate its clients and would therefore
+    allow any process of the host system (not related to pmbootstrap) to
+    execute compilers in the native chroot. By modifying the compiler's options
+    or sending malicious data to the compiler, it is likely that the process
+    can gain remote code execution [1]. That way, a compromised, but sandboxed
+    process could gain privilege escalation.
+
+    [1]: <https://github.com/distcc/distcc/issues/155#issuecomment-374014645>
+"""
+
+
+def init_server(args):
+    """
+    Install dependencies and generate keys for the server.
+    """
+    # Install dependencies
+    pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
+                                  "openssh-server"])
+
+    # Config folder (nothing to do if existing)
+    dir = "/home/pmos/.distcc-sshd"
+    dir_outside = args.work + "/chroot_native" + dir
+    if os.path.exists(dir_outside):
+        return
+
+    # Generate keys
+    logging.info("(native) generate distcc-sshd server keys")
+    pmb.chroot.user(args, ["mkdir", "-p", dir + "/etc/ssh"])
+    pmb.chroot.user(args, ["ssh-keygen", "-A", "-f", dir])
+
+
+def init_client(args, suffix):
+    """
+    Install dependencies and generate keys for the client.
+    """
+    # Install dependencies
+    pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
+                                  "openssh-client"], suffix)
+
+    # Public key path (nothing to do if existing)
+    pub = "/home/pmos/id_ed25519.pub"
+    pub_outside = args.work + "/chroot_" + suffix + pub
+    if os.path.exists(pub_outside):
+        return
+
+    # Generate keys
+    logging.info("(" + suffix + ") generate distcc-sshd client keys")
+    pmb.chroot.user(args, ["ssh-keygen", "-t", "ed25519", "-N", "",
+                           "-f", "/home/pmos/.ssh/id_ed25519"], suffix)
+    pmb.chroot.user(args, ["cp", "/home/pmos/.ssh/id_ed25519.pub", pub],
+                    suffix)
+
+
+def configure_authorized_keys(args, suffix):
+    """
+    Exclusively allow one foreign arch chroot to access the sshd.
+    """
+    auth = "/home/pmos/.distcc-sshd/authorized_keys"
+    auth_outside = args.work + "/chroot_native/" + auth
+    pub = "/home/pmos/id_ed25519.pub"
+    pub_outside = args.work + "/chroot_" + suffix + pub
+    pmb.helpers.run.root(args, ["cp", pub_outside, auth_outside])
+
+
+def configure_cmdlist(args, arch):
+    """
+    Create a whitelist of all the cross compiler wrappers.
+
+    Distcc 3.3 and above requires such a whitelist, or else it will only run
+    with the --make-me-a-botnet parameter (even in ssh mode).
+    """
+    dir = "/home/pmos/.distcc-sshd"
+    with open(args.work + "/chroot_native/tmp/cmdlist", "w") as handle:
+        for cmd in ["c++", "cc", "cpp", "g++", "gcc"]:
+            cmd_full = "/usr/lib/arch-bin-masquerade/" + arch + "/" + cmd
+            handle.write(cmd_full + "\n")
+    pmb.chroot.root(args, ["mv", "/tmp/cmdlist", dir + "/cmdlist"])
+    pmb.chroot.user(args, ["cat", dir + "/cmdlist"])
+
+
+def configure_distccd_wrapper(args):
+    """
+    Wrap distccd in a shell script, so we can pass the compiler whitelist and
+    set the verbose flag (when pmbootstrap is running with --verbose).
+    """
+    dir = "/home/pmos/.distcc-sshd"
+    with open(args.work + "/chroot_native/tmp/wrapper", "w") as handle:
+        handle.write("#!/bin/sh\n"
+                     "export DISTCC_CMDLIST='" + dir + "/cmdlist'\n"
+                     "distccd --log-file /home/pmos/distccd.log --nice 19")
+        if args.verbose:
+            handle.write(" --verbose")
+        handle.write(" \"$@\"\n")
+    pmb.chroot.root(args, ["mv", "/tmp/wrapper", dir + "/distccd"])
+    pmb.chroot.user(args, ["cat", dir + "/distccd"])
+    pmb.chroot.root(args, ["chmod", "+x", dir + "/distccd"])
+
+
+def configure_sshd(args):
+    """
+    Configure the SSH daemon in the native chroot.
+    """
+    dir = "/home/pmos/.distcc-sshd"
+    config = """AllowAgentForwarding no
+                AllowTcpForwarding no
+                AuthorizedKeysFile /home/pmos/.distcc-sshd/authorized_keys
+                HostKey /home/pmos/.distcc-sshd/etc/ssh/ssh_host_ed25519_key
+                ListenAddress 127.0.0.1
+                PasswordAuthentication no
+                PidFile /home/pmos/.distcc-sshd/sshd.pid
+                Port """ + args.port_distccd + """
+                X11Forwarding no"""
+
+    with open(args.work + "/chroot_native/tmp/cfg", "w") as handle:
+        for line in config.split("\n"):
+            handle.write(line.lstrip() + "\n")
+    pmb.chroot.root(args, ["mv", "/tmp/cfg", dir + "/sshd_config"])
+    pmb.chroot.user(args, ["cat", dir + "/sshd_config"])
+
 
 def get_running_pid(args):
     """
-    :returns: the running distccd's pid as integer or None
+    :returns: the running distcc-sshd's pid as integer or None
     """
-    pidfile = args.work + "/chroot_native/home/pmos/distccd.pid"
+    # PID file must exist
-    if not os.path.exists(pidfile):
+    pidfile = "/home/pmos/.distcc-sshd/sshd.pid"
+    pidfile_outside = args.work + "/chroot_native" + pidfile
+    if not os.path.exists(pidfile_outside):
         return None
-    with open(pidfile, "r") as handle:
-        lines = handle.readlines()
-    return int(lines[0][:-1])
-
-
-def get_running_info(args):
-    """
-    :returns: A dictionary in the form of {"arch": .., "cmdline": "" }. arch is
-              the architecture (e.g. "armhf" or "aarch64"), and "cmdline" is the
-              saved value from the generate_cmdline() list, joined on space.
-              If the information can not be read, "arch" and "cmdline" are set to
-              "unknown".
-    The arch is used to print a nice stop message, the full cmdline is used to
-    check whether distccd needs to be restartet (e.g. because the arch has been
-    changed, or the verbose flag).
-    """
-    info = configparser.ConfigParser()
-    path = args.work + "/chroot_native/tmp/distccd_running_info"
-    if os.path.exists(path):
-        info.read(path)
-    else:
-        info["distccd"] = {}
-        info["distccd"]["arch"] = "unknown"
-        info["distccd"]["cmdline"] = "unknown"
-    return info["distccd"]
-
-
-def is_running(args):
-    """
-    :returns: When not running: None
-              When running: result from get_running_info()
-    """
-    # Get the PID
-    pid = get_running_pid(args)
-    if not pid:
-        return False
 
     # Verify, if it still exists by sending a kill signal
+    with open(pidfile_outside, "r") as handle:
+        pid = int(handle.read()[:-1])
     try:
         os.kill(pid, 0)
     except OSError as err:
         if err.errno == errno.ESRCH:  # no such process
-            pmb.chroot.root(args, ["rm", "/home/pmos/distccd.pid"])
+            pmb.helpers.run.root(args, ["rm", pidfile_outside])
-            return False
+            return None
-        elif err.errno == errno.EPERM:  # access denied
+    return pid
-            return get_running_info(args)
 
 
-def generate_cmdline(args, arch):
+def get_running_parameters(args):
     """
-    :returns: a dictionary suitable for pmb.chroot.user(), to start the distccd
+    Get the parameters of the currently running distcc-sshd instance.
-              with all options set.
+
-    NOTE: The distcc client of the foreign arch chroot passes the
+    :returns: a dictionary in the form of
-          absolute path to the compiler, which points to
+              {"arch": "armhf", "port": 1234, "verbose": False}
-          "/usr/lib/arch-bin-masquerade/armhf/gcc" for example. This also
+              If the information can not be read, "arch" is set to "unknown"
-          exists in the native chroot, and points to the armhf cross-
-          compiler there (both the native and foreign chroot have the
-          arch-bin-masquerade package installed, which creates the
-          wrapper scripts).
     """
-    ret = ["distccd",
+    # Return defaults
-           "--pid-file", "/home/pmos/distccd.pid",
+    path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
-           "--listen", "127.0.0.1",
+    if not os.path.exists(path):
-           "--allow", "127.0.0.1",
+        return {"arch": "unknown", "port": 0, "verbose": False}
-           "--port", args.port_distccd,
+
-           "--log-file", "/home/pmos/distccd.log",
+    # Parse the file as JSON
-           "--jobs", args.jobs,
+    with open(path, "r") as handle:
-           "--nice", "19",
+        return json.loads(handle.read())
-           "--job-lifetime", "60",
-           "--daemon"
-           ]
-    if args.verbose:
-        ret.append("--verbose")
-    return ret
 
 
-def start(args, arch):
+def set_running_parameters(args, arch):
-    # Skip when already running with the same cmdline
+    """
-    cmdline = generate_cmdline(args, arch)
+    Set the parameters of the currently running distcc-sshd instance.
-    info = is_running(args)
+    """
-    if info and info["cmdline"] == " ".join(cmdline):
+    parameters = {"arch": arch,
-        return
+                  "port": args.port_distccd,
-    stop(args)
+                  "verbose": args.verbose}
-    pmb.chroot.apk.install(args, ["distcc", "arch-bin-masquerade"])
 
-    # Start daemon with cross-compiler in path
+    path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
-    logging.info("(native) start distccd (" + arch + ") on 127.0.0.1:" +
+    with open(path, "w") as handle:
-                 args.port_distccd)
+        json.dump(parameters, handle)
-    pmb.chroot.user(args, cmdline)
 
-    # Write down the arch and cmdline
+
-    info = configparser.ConfigParser()
+def is_running_with_same_parameters(args, arch):
-    info["distccd"] = {}
+    """
-    info["distccd"]["arch"] = arch
+    Check whether we can use the already running distcc-sshd instance with our
-    info["distccd"]["cmdline"] = " ".join(cmdline)
+    current set of parameters. In case we can use it directly, we save some
-    with open(args.work + "/chroot_native/tmp/distccd_running_info", "w") as handle:
+    time, otherwise we need to stop it, configure it again, and start it once
-        info.write(handle)
+    more.
+    """
+    if not get_running_pid(args):
+        return False
+
+    parameters = get_running_parameters(args)
+    return (parameters["arch"] == arch and
+            parameters["port"] == args.port_distccd and
+            parameters["verbose"] == args.verbose)
 
 
 def stop(args):
-    info = is_running(args)
+    """
-    if info:
+    Kill the sshd process (by using its pid).
-        logging.info("(native) stop distccd (" + info["arch"] + ")")
+    """
-        pmb.chroot.user(args, ["kill", str(get_running_pid(args))])
+    pid = get_running_pid(args)
+    if not pid:
+        return
+
+    parameters = get_running_parameters(args)
+    logging.info("(native) stop distcc-sshd (" + parameters["arch"] + ")")
+    pmb.chroot.user(args, ["kill", str(pid)])
+
+
+def start(args, arch):
+    """
+    Set up a new distcc-sshd instance or use an already running one.
+    """
+    if is_running_with_same_parameters(args, arch):
+        return
+    stop(args)
+
+    # Initialize server and client
+    suffix = "buildroot_" + arch
+    init_server(args)
+    init_client(args, suffix)
+
+    logging.info("(native) start distcc-sshd (" + arch + ") on 127.0.0.1:" +
+                 args.port_distccd)
+
+    # Configure server parameters (arch, port, verbose)
+    configure_authorized_keys(args, suffix)
+    configure_distccd_wrapper(args)
+    configure_cmdlist(args, arch)
+    configure_sshd(args)
+
+    # Run
+    dir = "/home/pmos/.distcc-sshd"
+    pmb.chroot.user(args, ["/usr/sbin/sshd", "-f", dir + "/sshd_config",
+                           "-E", dir + "/log.txt"])
+    set_running_parameters(args, arch)

pmb/parse/arguments.py

@@ -226,9 +226,6 @@ def arguments():
     # Other
     parser.add_argument("-V", "--version", action="version",
                         version=pmb.config.version)
-    parser.add_argument("--no-cross", action="store_false", dest="cross",
-                        help="disable crosscompiler, build only with qemu + gcc (slower!)")
-
     parser.add_argument("-a", "--alpine-version", dest="alpine_version",
                         help="examples: edge, latest-stable, v3.5")
     parser.add_argument("-c", "--config", dest="config",
@@ -257,6 +254,17 @@ def arguments():
                         " directory permissions!)", dest="as_root",
                         action="store_true")
 
+    # Compiler
+    parser.add_argument("--ccache-disable", action="store_false",
+                        dest="ccache", help="do not cache the compiled output")
+    parser.add_argument("--distcc-nofallback", action="store_false",
+                        help="when using the cross compiler via distcc fails,"
+                             "do not fall back to compiling slowly with QEMU",
+                        dest="distcc_fallback")
+    parser.add_argument("--no-cross", action="store_false", dest="cross",
+                        help="disable cross compiler, build only with QEMU and"
+                             " gcc (slow!)")
+
     # Logging
     parser.add_argument("-l", "--log", dest="log", default=None,
                         help="path to log file")

test/test_build_package.py

@@ -252,7 +252,8 @@ def test_run_abuild(args, monkeypatch):
     assert env["CCACHE_PREFIX"] == "distcc"
     assert env["CCACHE_PATH"] == "/usr/lib/arch-bin-masquerade/armhf:/usr/bin"
     assert env["CCACHE_COMPILERCHECK"].startswith("string:")
-    assert env["DISTCC_HOSTS"] == "127.0.0.1:33632"
+    assert env["DISTCC_HOSTS"] == "@127.0.0.1:/home/pmos/.distcc-sshd/distccd"
+    assert env["DISTCC_BACKOFF_PERIOD"] == "0"
 
 
 def test_finish(args, monkeypatch):

test/test_cross_compile_distcc.py

@@ -0,0 +1,66 @@
+"""
+Copyright 2018 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import os
+import pytest
+import sys
+
+# Import from parent directory
+sys.path.append(os.path.realpath(
+    os.path.join(os.path.dirname(__file__) + "/..")))
+import pmb.build
+import pmb.chroot.distccd
+import pmb.helpers.logging
+
+
+@pytest.fixture
+def args(tmpdir, request):
+    import pmb.parse
+    sys.argv = ["pmbootstrap", "init"]
+    args = pmb.parse.arguments()
+    args.log = args.work + "/log_testsuite.txt"
+    pmb.helpers.logging.init(args)
+    request.addfinalizer(args.logfd.close)
+    return args
+
+
+def test_cross_compile_distcc(args):
+    # Delete old distccd log
+    pmb.chroot.distccd.stop(args)
+    distccd_log = args.work + "/chroot_native/home/pmos/distccd.log"
+    if os.path.exists(distccd_log):
+        pmb.helpers.run.root(args, ["rm", distccd_log])
+
+    # Force usage of distcc (no fallback, no ccache)
+    args.verbose = True
+    args.ccache = False
+    args.distcc_fallback = False
+
+    # Compile, print distccd and sshd logs on error
+    try:
+        pmb.build.package(args, "hello-world", arch="armhf", force=True)
+    except RuntimeError:
+        print("distccd log:")
+        pmb.helpers.run.user(args, ["cat", distccd_log], output="stdout",
+                             check=False)
+        print("sshd log:")
+        sshd_log = args.work + "/chroot_native/home/pmos/.distcc-sshd/log.txt"
+        pmb.helpers.run.root(args, ["cat", sshd_log], output="stdout",
+                             check=False)
+        raise