Administration/pmbootstrap-meow
2
0
Fork 0
forked from Mirror/pmbootstrap
Code Pull requests Activity Actions

pmb: adjust to distcc 3.3 and wrap it with sshd

Browse source 
Overview:
Since Alpine updated to distcc 3.3 last week, pmbootstrap wasn't able to use
distcc for cross compilation anymore. It always falled back to running the
compiler in QEMU (which works, but is a lot slower). The reason for that is,
that distcc requires all compilers that are being used in a whitelist now.

This partially fixes CVE-2004-2687 in distccd, which allowed trivial remote
code execution by any process connecting to the distccd server. We only run
distccd on localhost, but still this can be used for privilege escalation of
sandboxed processes running on the host system (not part of pmbootstrap
chroots).

Because the CVE is only partially fixed (see the comment in
`pmb/chroot/distccd.py` for details), we make sure that only the building
chroots can talk to the distcc server by running distcc over ssh.

Details:
* Completely refactored `pmb/chroot/distccd.py` to run distcc over ssh
  * Store the running distcc server's arguments as JSON now, not as INI
* Make debugging distcc issues easy:
  * Set DISTCC_BACKOFF_PERIOD=0, so the distcc client will not ignore the
    server after errors happened (this masks the original error!)
  * New pmbootstrap parameters:
    * `--distcc-nofallback`: avoids falling back to compiling with QEMU and not
	   throwing an error
	* `--ccache-disable`: avoid ccache (when the compiler output is cached,
	  distcc does not get used)
  * `--verbose` prints verbose output of the distcc too
  * New test case, that uses the new pmbootstrap parameters to force
	compilation through distcc, and shows the output of distcc and distccd in
	verbose mode on error (as well as the log of sshd)
This commit is contained in:
Oliver Smith 2018-07-25 21:09:45 +02:00 • committed by Martijn Braam
parent d648794f7a
commit 4844719b1d
5 changed files with 308 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

13
pmb/build/_package.py
View file

@ -193,8 +193,6 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
pmb.chroot.apk.install(args, ["gcc-" + arch, "g++-" + arch,
"ccache-cross-symlinks"])
if cross == "distcc":
pmb.chroot.apk.install(args, ["distcc", "arch-bin-masquerade"],
suffix=suffix)
pmb.chroot.distccd.start(args, arch)
# "native" cross-compile: build and install dependencies (#1061)
@ -350,7 +348,16 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
env["CCACHE_PREFIX"] = "distcc"
env["CCACHE_PATH"] = "/usr/lib/arch-bin-masquerade/" + arch + ":/usr/bin"
env["CCACHE_COMPILERCHECK"] = "string:" + get_gcc_version(args, arch)
env["DISTCC_HOSTS"] = "127.0.0.1:" + args.port_distccd
env["DISTCC_HOSTS"] = "@127.0.0.1:/home/pmos/.distcc-sshd/distccd"
env["DISTCC_SSH"] = ("ssh -o StrictHostKeyChecking=no -p" +
args.port_distccd)
env["DISTCC_BACKOFF_PERIOD"] = "0"
if not args.distcc_fallback:
env["DISTCC_FALLBACK"] = "0"
if args.verbose:
env["DISTCC_VERBOSE"] = "1"
if not args.ccache:
env["CCACHE_DISABLE"] = "1"
# Build the abuild command
cmd = ["abuild", "-D", "postmarketOS"]

310
pmb/chroot/distccd.py
View file

@ -16,123 +16,251 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import configparser
import errno
import json
import logging
import os
import pmb.chroot
import pmb.config
import pmb.chroot.apk
""" Packages for foreign architectures (e.g. armhf) get built in chroots
running with QEMU. While this works, it is painfully slow. So we speed it
up by using distcc to let cross compilers running in the native chroots do
the heavy lifting.
This file sets up an SSH server in the native chroot, which will then be
used by the foreign arch chroot to communicate with the distcc daemon. We
make sure that only the foreign arch chroot can connect to the sshd by only
listening on localhost, as well as generating dedicated ssh keys.
Using the SSH server instead of running distccd directly is a security
measure. Distccd does not authenticate its clients and would therefore
allow any process of the host system (not related to pmbootstrap) to
execute compilers in the native chroot. By modifying the compiler's options
or sending malicious data to the compiler, it is likely that the process
can gain remote code execution [1]. That way, a compromised, but sandboxed
process could gain privilege escalation.
[1]: <https://github.com/distcc/distcc/issues/155#issuecomment-374014645>
"""
def init_server(args):
"""
Install dependencies and generate keys for the server.
"""
# Install dependencies
pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
"openssh-server"])
# Config folder (nothing to do if existing)
dir = "/home/pmos/.distcc-sshd"
dir_outside = args.work + "/chroot_native" + dir
if os.path.exists(dir_outside):
return
# Generate keys
logging.info("(native) generate distcc-sshd server keys")
pmb.chroot.user(args, ["mkdir", "-p", dir + "/etc/ssh"])
pmb.chroot.user(args, ["ssh-keygen", "-A", "-f", dir])
def init_client(args, suffix):
"""
Install dependencies and generate keys for the client.
"""
# Install dependencies
pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
"openssh-client"], suffix)
# Public key path (nothing to do if existing)
pub = "/home/pmos/id_ed25519.pub"
pub_outside = args.work + "/chroot_" + suffix + pub
if os.path.exists(pub_outside):
return
# Generate keys
logging.info("(" + suffix + ") generate distcc-sshd client keys")
pmb.chroot.user(args, ["ssh-keygen", "-t", "ed25519", "-N", "",
"-f", "/home/pmos/.ssh/id_ed25519"], suffix)
pmb.chroot.user(args, ["cp", "/home/pmos/.ssh/id_ed25519.pub", pub],
suffix)
def configure_authorized_keys(args, suffix):
"""
Exclusively allow one foreign arch chroot to access the sshd.
"""
auth = "/home/pmos/.distcc-sshd/authorized_keys"
auth_outside = args.work + "/chroot_native/" + auth
pub = "/home/pmos/id_ed25519.pub"
pub_outside = args.work + "/chroot_" + suffix + pub
pmb.helpers.run.root(args, ["cp", pub_outside, auth_outside])
def configure_cmdlist(args, arch):
"""
Create a whitelist of all the cross compiler wrappers.
Distcc 3.3 and above requires such a whitelist, or else it will only run
with the --make-me-a-botnet parameter (even in ssh mode).
"""
dir = "/home/pmos/.distcc-sshd"
with open(args.work + "/chroot_native/tmp/cmdlist", "w") as handle:
for cmd in ["c++", "cc", "cpp", "g++", "gcc"]:
cmd_full = "/usr/lib/arch-bin-masquerade/" + arch + "/" + cmd
handle.write(cmd_full + "\n")
pmb.chroot.root(args, ["mv", "/tmp/cmdlist", dir + "/cmdlist"])
pmb.chroot.user(args, ["cat", dir + "/cmdlist"])
def configure_distccd_wrapper(args):
"""
Wrap distccd in a shell script, so we can pass the compiler whitelist and
set the verbose flag (when pmbootstrap is running with --verbose).
"""
dir = "/home/pmos/.distcc-sshd"
with open(args.work + "/chroot_native/tmp/wrapper", "w") as handle:
handle.write("#!/bin/sh\n"
"export DISTCC_CMDLIST='" + dir + "/cmdlist'\n"
"distccd --log-file /home/pmos/distccd.log --nice 19")
if args.verbose:
handle.write(" --verbose")
handle.write(" \"$@\"\n")
pmb.chroot.root(args, ["mv", "/tmp/wrapper", dir + "/distccd"])
pmb.chroot.user(args, ["cat", dir + "/distccd"])
pmb.chroot.root(args, ["chmod", "+x", dir + "/distccd"])
def configure_sshd(args):
"""
Configure the SSH daemon in the native chroot.
"""
dir = "/home/pmos/.distcc-sshd"
config = """AllowAgentForwarding no
AllowTcpForwarding no
AuthorizedKeysFile /home/pmos/.distcc-sshd/authorized_keys
HostKey /home/pmos/.distcc-sshd/etc/ssh/ssh_host_ed25519_key
ListenAddress 127.0.0.1
PasswordAuthentication no
PidFile /home/pmos/.distcc-sshd/sshd.pid
Port """ + args.port_distccd + """
X11Forwarding no"""
with open(args.work + "/chroot_native/tmp/cfg", "w") as handle:
for line in config.split("\n"):
handle.write(line.lstrip() + "\n")
pmb.chroot.root(args, ["mv", "/tmp/cfg", dir + "/sshd_config"])
pmb.chroot.user(args, ["cat", dir + "/sshd_config"])
def get_running_pid(args):
"""
:returns: the running distccd's pid as integer or None
:returns: the running distcc-sshd's pid as integer or None
"""
pidfile = args.work + "/chroot_native/home/pmos/distccd.pid"
if not os.path.exists(pidfile):
# PID file must exist
pidfile = "/home/pmos/.distcc-sshd/sshd.pid"
pidfile_outside = args.work + "/chroot_native" + pidfile
if not os.path.exists(pidfile_outside):
return None
with open(pidfile, "r") as handle:
lines = handle.readlines()
return int(lines[0][:-1])
def get_running_info(args):
"""
:returns: A dictionary in the form of {"arch": .., "cmdline": "" }. arch is
the architecture (e.g. "armhf" or "aarch64"), and "cmdline" is the
saved value from the generate_cmdline() list, joined on space.
If the information can not be read, "arch" and "cmdline" are set to
"unknown".
The arch is used to print a nice stop message, the full cmdline is used to
check whether distccd needs to be restartet (e.g. because the arch has been
changed, or the verbose flag).
"""
info = configparser.ConfigParser()
path = args.work + "/chroot_native/tmp/distccd_running_info"
if os.path.exists(path):
info.read(path)
else:
info["distccd"] = {}
info["distccd"]["arch"] = "unknown"
info["distccd"]["cmdline"] = "unknown"
return info["distccd"]
def is_running(args):
"""
:returns: When not running: None
When running: result from get_running_info()
"""
# Get the PID
pid = get_running_pid(args)
if not pid:
return False
# Verify, if it still exists by sending a kill signal
with open(pidfile_outside, "r") as handle:
pid = int(handle.read()[:-1])
try:
os.kill(pid, 0)
except OSError as err:
if err.errno == errno.ESRCH: # no such process
pmb.chroot.root(args, ["rm", "/home/pmos/distccd.pid"])
return False
elif err.errno == errno.EPERM: # access denied
return get_running_info(args)
pmb.helpers.run.root(args, ["rm", pidfile_outside])
return None
return pid
def generate_cmdline(args, arch):
def get_running_parameters(args):
"""
:returns: a dictionary suitable for pmb.chroot.user(), to start the distccd
with all options set.
NOTE: The distcc client of the foreign arch chroot passes the
absolute path to the compiler, which points to
"/usr/lib/arch-bin-masquerade/armhf/gcc" for example. This also
exists in the native chroot, and points to the armhf cross-
compiler there (both the native and foreign chroot have the
arch-bin-masquerade package installed, which creates the
wrapper scripts).
Get the parameters of the currently running distcc-sshd instance.
:returns: a dictionary in the form of
{"arch": "armhf", "port": 1234, "verbose": False}
If the information can not be read, "arch" is set to "unknown"
"""
ret = ["distccd",
"--pid-file", "/home/pmos/distccd.pid",
"--listen", "127.0.0.1",
"--allow", "127.0.0.1",
"--port", args.port_distccd,
"--log-file", "/home/pmos/distccd.log",
"--jobs", args.jobs,
"--nice", "19",
"--job-lifetime", "60",
"--daemon"
]
if args.verbose:
ret.append("--verbose")
return ret
# Return defaults
path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
if not os.path.exists(path):
return {"arch": "unknown", "port": 0, "verbose": False}
# Parse the file as JSON
with open(path, "r") as handle:
return json.loads(handle.read())
def start(args, arch):
# Skip when already running with the same cmdline
cmdline = generate_cmdline(args, arch)
info = is_running(args)
if info and info["cmdline"] == " ".join(cmdline):
return
stop(args)
pmb.chroot.apk.install(args, ["distcc", "arch-bin-masquerade"])
def set_running_parameters(args, arch):
"""
Set the parameters of the currently running distcc-sshd instance.
"""
parameters = {"arch": arch,
"port": args.port_distccd,
"verbose": args.verbose}
# Start daemon with cross-compiler in path
logging.info("(native) start distccd (" + arch + ") on 127.0.0.1:" +
args.port_distccd)
pmb.chroot.user(args, cmdline)
path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
with open(path, "w") as handle:
json.dump(parameters, handle)
# Write down the arch and cmdline
info = configparser.ConfigParser()
info["distccd"] = {}
info["distccd"]["arch"] = arch
info["distccd"]["cmdline"] = " ".join(cmdline)
with open(args.work + "/chroot_native/tmp/distccd_running_info", "w") as handle:
info.write(handle)
def is_running_with_same_parameters(args, arch):
"""
Check whether we can use the already running distcc-sshd instance with our
current set of parameters. In case we can use it directly, we save some
time, otherwise we need to stop it, configure it again, and start it once
more.
"""
if not get_running_pid(args):
return False
parameters = get_running_parameters(args)
return (parameters["arch"] == arch and
parameters["port"] == args.port_distccd and
parameters["verbose"] == args.verbose)
def stop(args):
info = is_running(args)
if info:
logging.info("(native) stop distccd (" + info["arch"] + ")")
pmb.chroot.user(args, ["kill", str(get_running_pid(args))])
"""
Kill the sshd process (by using its pid).
"""
pid = get_running_pid(args)
if not pid:
return
parameters = get_running_parameters(args)
logging.info("(native) stop distcc-sshd (" + parameters["arch"] + ")")
pmb.chroot.user(args, ["kill", str(pid)])
def start(args, arch):
"""
Set up a new distcc-sshd instance or use an already running one.
"""
if is_running_with_same_parameters(args, arch):
return
stop(args)
# Initialize server and client
suffix = "buildroot_" + arch
init_server(args)
init_client(args, suffix)
logging.info("(native) start distcc-sshd (" + arch + ") on 127.0.0.1:" +
args.port_distccd)
# Configure server parameters (arch, port, verbose)
configure_authorized_keys(args, suffix)
configure_distccd_wrapper(args)
configure_cmdlist(args, arch)
configure_sshd(args)
# Run
dir = "/home/pmos/.distcc-sshd"
pmb.chroot.user(args, ["/usr/sbin/sshd", "-f", dir + "/sshd_config",
"-E", dir + "/log.txt"])
set_running_parameters(args, arch)

14
pmb/parse/arguments.py
View file

@ -226,9 +226,6 @@ def arguments():
# Other
parser.add_argument("-V", "--version", action="version",
version=pmb.config.version)
parser.add_argument("--no-cross", action="store_false", dest="cross",
help="disable crosscompiler, build only with qemu + gcc (slower!)")
parser.add_argument("-a", "--alpine-version", dest="alpine_version",
help="examples: edge, latest-stable, v3.5")
parser.add_argument("-c", "--config", dest="config",
@ -257,6 +254,17 @@ def arguments():
" directory permissions!)", dest="as_root",
action="store_true")
# Compiler
parser.add_argument("--ccache-disable", action="store_false",
dest="ccache", help="do not cache the compiled output")
parser.add_argument("--distcc-nofallback", action="store_false",
help="when using the cross compiler via distcc fails,"
"do not fall back to compiling slowly with QEMU",
dest="distcc_fallback")
parser.add_argument("--no-cross", action="store_false", dest="cross",
help="disable cross compiler, build only with QEMU and"
" gcc (slow!)")
# Logging
parser.add_argument("-l", "--log", dest="log", default=None,
help="path to log file")

3
test/test_build_package.py
View file

@ -252,7 +252,8 @@ def test_run_abuild(args, monkeypatch):
assert env["CCACHE_PREFIX"] == "distcc"
assert env["CCACHE_PATH"] == "/usr/lib/arch-bin-masquerade/armhf:/usr/bin"
assert env["CCACHE_COMPILERCHECK"].startswith("string:")
assert env["DISTCC_HOSTS"] == "127.0.0.1:33632"
assert env["DISTCC_HOSTS"] == "@127.0.0.1:/home/pmos/.distcc-sshd/distccd"
assert env["DISTCC_BACKOFF_PERIOD"] == "0"
def test_finish(args, monkeypatch):

66
test/test_cross_compile_distcc.py Normal file
View file

@ -0,0 +1,66 @@
"""
Copyright 2018 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import pytest
import sys
# Import from parent directory
sys.path.append(os.path.realpath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.build
import pmb.chroot.distccd
import pmb.helpers.logging
@pytest.fixture
def args(tmpdir, request):
import pmb.parse
sys.argv = ["pmbootstrap", "init"]
args = pmb.parse.arguments()
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args
def test_cross_compile_distcc(args):
# Delete old distccd log
pmb.chroot.distccd.stop(args)
distccd_log = args.work + "/chroot_native/home/pmos/distccd.log"
if os.path.exists(distccd_log):
pmb.helpers.run.root(args, ["rm", distccd_log])
# Force usage of distcc (no fallback, no ccache)
args.verbose = True
args.ccache = False
args.distcc_fallback = False
# Compile, print distccd and sshd logs on error
try:
pmb.build.package(args, "hello-world", arch="armhf", force=True)
except RuntimeError:
print("distccd log:")
pmb.helpers.run.user(args, ["cat", distccd_log], output="stdout",
check=False)
print("sshd log:")
sshd_log = args.work + "/chroot_native/home/pmos/.distcc-sshd/log.txt"
pmb.helpers.run.root(args, ["cat", sshd_log], output="stdout",
check=False)
raise