diff --git a/keys/README b/keys/README new file mode 100644 index 00000000..da673d1b --- /dev/null +++ b/keys/README @@ -0,0 +1,6 @@ +All Alpine Linux keys are stored here, so we can verify the downloaded files with pmbootstrap before APK itself is verified. + +Sources for the keys (must be identical, there's a testcase that verifies this): + https://github.com/alpinelinux/aports/tree/master/main/alpine-keys + http://git.alpinelinux.org/cgit/aports/tree/main/alpine-keys?h=master + alpine-keys package diff --git a/keys/alpine-devel@lists.alpinelinux.org-4a6a0840.rsa.pub b/keys/alpine-devel@lists.alpinelinux.org-4a6a0840.rsa.pub new file mode 100644 index 00000000..bb4bdc80 --- /dev/null +++ b/keys/alpine-devel@lists.alpinelinux.org-4a6a0840.rsa.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1yHJxQgsHQREclQu4Ohe +qxTxd1tHcNnvnQTu/UrTky8wWvgXT+jpveroeWWnzmsYlDI93eLI2ORakxb3gA2O +Q0Ry4ws8vhaxLQGC74uQR5+/yYrLuTKydFzuPaS1dK19qJPXB8GMdmFOijnXX4SA +jixuHLe1WW7kZVtjL7nufvpXkWBGjsfrvskdNA/5MfxAeBbqPgaq0QMEfxMAn6/R +L5kNepi/Vr4S39Xvf2DzWkTLEK8pcnjNkt9/aafhWqFVW7m3HCAII6h/qlQNQKSo +GuH34Q8GsFG30izUENV9avY7hSLq7nggsvknlNBZtFUcmGoQrtx3FmyYsIC8/R+B +ywIDAQAB +-----END PUBLIC KEY----- diff --git a/keys/alpine-devel@lists.alpinelinux.org-5243ef4b.rsa.pub b/keys/alpine-devel@lists.alpinelinux.org-5243ef4b.rsa.pub new file mode 100644 index 00000000..6cbfad74 --- /dev/null +++ b/keys/alpine-devel@lists.alpinelinux.org-5243ef4b.rsa.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvNijDxJ8kloskKQpJdx+ +mTMVFFUGDoDCbulnhZMJoKNkSuZOzBoFC94omYPtxnIcBdWBGnrm6ncbKRlR+6oy +DO0W7c44uHKCFGFqBhDasdI4RCYP+fcIX/lyMh6MLbOxqS22TwSLhCVjTyJeeH7K +aA7vqk+QSsF4TGbYzQDDpg7+6aAcNzg6InNePaywA6hbT0JXbxnDWsB+2/LLSF2G +mnhJlJrWB1WGjkz23ONIWk85W4S0XB/ewDefd4Ly/zyIciastA7Zqnh7p3Ody6Q0 +sS2MJzo7p3os1smGjUF158s6m/JbVh4DN6YIsxwl2OjDOz9R0OycfJSDaBVIGZzg +cQIDAQAB +-----END PUBLIC KEY----- diff --git a/keys/alpine-devel@lists.alpinelinux.org-524d27bb.rsa.pub b/keys/alpine-devel@lists.alpinelinux.org-524d27bb.rsa.pub new file mode 100644 index 00000000..1d34c93e --- /dev/null +++ b/keys/alpine-devel@lists.alpinelinux.org-524d27bb.rsa.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr8s1q88XpuJWLCZALdKj +lN8wg2ePB2T9aIcaxryYE/Jkmtu+ZQ5zKq6BT3y/udt5jAsMrhHTwroOjIsF9DeG +e8Y3vjz+Hh4L8a7hZDaw8jy3CPag47L7nsZFwQOIo2Cl1SnzUc6/owoyjRU7ab0p +iWG5HK8IfiybRbZxnEbNAfT4R53hyI6z5FhyXGS2Ld8zCoU/R4E1P0CUuXKEN4p0 +64dyeUoOLXEWHjgKiU1mElIQj3k/IF02W89gDj285YgwqA49deLUM7QOd53QLnx+ +xrIrPv3A+eyXMFgexNwCKQU9ZdmWa00MjjHlegSGK8Y2NPnRoXhzqSP9T9i2HiXL +VQIDAQAB +-----END PUBLIC KEY----- diff --git a/keys/alpine-devel@lists.alpinelinux.org-5261cecb.rsa.pub b/keys/alpine-devel@lists.alpinelinux.org-5261cecb.rsa.pub new file mode 100644 index 00000000..83f0658e --- /dev/null +++ b/keys/alpine-devel@lists.alpinelinux.org-5261cecb.rsa.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwlzMkl7b5PBdfMzGdCT0 +cGloRr5xGgVmsdq5EtJvFkFAiN8Ac9MCFy/vAFmS8/7ZaGOXoCDWbYVLTLOO2qtX +yHRl+7fJVh2N6qrDDFPmdgCi8NaE+3rITWXGrrQ1spJ0B6HIzTDNEjRKnD4xyg4j +g01FMcJTU6E+V2JBY45CKN9dWr1JDM/nei/Pf0byBJlMp/mSSfjodykmz4Oe13xB +Ca1WTwgFykKYthoLGYrmo+LKIGpMoeEbY1kuUe04UiDe47l6Oggwnl+8XD1MeRWY +sWgj8sF4dTcSfCMavK4zHRFFQbGp/YFJ/Ww6U9lA3Vq0wyEI6MCMQnoSMFwrbgZw +wwIDAQAB +-----END PUBLIC KEY----- diff --git a/keys/alpine-devel@lists.alpinelinux.org-58199dcc.rsa.pub b/keys/alpine-devel@lists.alpinelinux.org-58199dcc.rsa.pub new file mode 100644 index 00000000..2b99a0d1 --- /dev/null +++ b/keys/alpine-devel@lists.alpinelinux.org-58199dcc.rsa.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3v8/ye/V/t5xf4JiXLXa +hWFRozsnmn3hobON20GdmkrzKzO/eUqPOKTpg2GtvBhK30fu5oY5uN2ORiv2Y2ht +eLiZ9HVz3XP8Fm9frha60B7KNu66FO5P2o3i+E+DWTPqqPcCG6t4Znk2BypILcit +wiPKTsgbBQR2qo/cO01eLLdt6oOzAaF94NH0656kvRewdo6HG4urbO46tCAizvCR +CA7KGFMyad8WdKkTjxh8YLDLoOCtoZmXmQAiwfRe9pKXRH/XXGop8SYptLqyVVQ+ +tegOD9wRs2tOlgcLx4F/uMzHN7uoho6okBPiifRX+Pf38Vx+ozXh056tjmdZkCaV +aQIDAQAB +-----END PUBLIC KEY----- diff --git a/keys/alpine-devel@lists.alpinelinux.org-58cbb476.rsa.pub b/keys/alpine-devel@lists.alpinelinux.org-58cbb476.rsa.pub new file mode 100644 index 00000000..a9ead55e --- /dev/null +++ b/keys/alpine-devel@lists.alpinelinux.org-58cbb476.rsa.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoSPnuAGKtRIS5fEgYPXD +8pSGvKAmIv3A08LBViDUe+YwhilSHbYXUEAcSH1KZvOo1WT1x2FNEPBEFEFU1Eyc ++qGzbA03UFgBNvArurHQ5Z/GngGqE7IarSQFSoqewYRtFSfp+TL9CUNBvM0rT7vz +2eMu3/wWG+CBmb92lkmyWwC1WSWFKO3x8w+Br2IFWvAZqHRt8oiG5QtYvcZL6jym +Y8T6sgdDlj+Y+wWaLHs9Fc+7vBuyK9C4O1ORdMPW15qVSl4Lc2Wu1QVwRiKnmA+c +DsH/m7kDNRHM7TjWnuj+nrBOKAHzYquiu5iB3Qmx+0gwnrSVf27Arc3ozUmmJbLj +zQIDAQAB +-----END PUBLIC KEY----- diff --git a/keys/alpine-devel@lists.alpinelinux.org-58e4f17d.rsa.pub b/keys/alpine-devel@lists.alpinelinux.org-58e4f17d.rsa.pub new file mode 100644 index 00000000..8f990949 --- /dev/null +++ b/keys/alpine-devel@lists.alpinelinux.org-58e4f17d.rsa.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvBxJN9ErBgdRcPr5g4hV +qyUSGZEKuvQliq2Z9SRHLh2J43+EdB6A+yzVvLnzcHVpBJ+BZ9RV30EM9guck9sh +r+bryZcRHyjG2wiIEoduxF2a8KeWeQH7QlpwGhuobo1+gA8L0AGImiA6UP3LOirl +I0G2+iaKZowME8/tydww4jx5vG132JCOScMjTalRsYZYJcjFbebQQolpqRaGB4iG +WqhytWQGWuKiB1A22wjmIYf3t96l1Mp+FmM2URPxD1gk/BIBnX7ew+2gWppXOK9j +1BJpo0/HaX5XoZ/uMqISAAtgHZAqq+g3IUPouxTphgYQRTRYpz2COw3NF43VYQrR +bQIDAQAB +-----END PUBLIC KEY----- diff --git a/pmb/__init__.py b/pmb/__init__.py new file mode 100644 index 00000000..84978349 --- /dev/null +++ b/pmb/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" diff --git a/pmb/aportgen/__init__.py b/pmb/aportgen/__init__.py new file mode 100644 index 00000000..0ee4bf81 --- /dev/null +++ b/pmb/aportgen/__init__.py @@ -0,0 +1,49 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import logging +import pmb.aportgen.binutils +import pmb.aportgen.musl +import pmb.aportgen.gcc +import pmb.helpers.git + + +def generate(args, pkgname): + # Prepare git repo and temp folder + pmb.helpers.git.clone(args, "aports_upstream") + logging.info("(native) generate " + pkgname + " aport") + if os.path.exists(args.work + "/aportgen"): + pmb.helpers.run.user(args, ["rm", "-r", args.work + "/aportgen"]) + + # Choose generator based on the name + if pkgname.startswith("binutils-"): + pmb.aportgen.binutils.generate(args, pkgname) + elif pkgname.startswith("musl-"): + pmb.aportgen.musl.generate(args, pkgname) + elif pkgname.startswith("gcc-"): + pmb.aportgen.gcc.generate(args, pkgname) + else: + raise ValueError("No generator available for " + pkgname + "!") + + # Move to the aports folder + path_target = args.aports + "/" + pkgname + if os.path.exists(path_target): + pmb.helpers.run.user(args, ["rm", "-r", path_target]) + pmb.helpers.run.user( + args, ["mv", args.work + "/aportgen", path_target]) diff --git a/pmb/aportgen/binutils.py b/pmb/aportgen/binutils.py new file mode 100644 index 00000000..55db5c04 --- /dev/null +++ b/pmb/aportgen/binutils.py @@ -0,0 +1,71 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import pmb.helpers.run +import pmb.aportgen.core + + +def generate(args, pkgname): + # Copy original aport + arch = pkgname.split("-")[1] + path_original = "main/binutils" + upstream = (args.work + "/cache_git/aports_upstream/" + path_original) + pmb.helpers.run.user(args, ["cp", "-r", upstream, args.work + "/aportgen"]) + + # Rewrite APKBUILD + fields = { + "pkgname": pkgname, + "pkgdesc": "Tools necessary to build programs for " + arch + " targets", + "makedepends_build": "", + "makedepends_host": "", + "makedepends": "gettext libtool autoconf automake bison", + "subpackages": "", + } + + replace_functions = { + "build": """ + _target="$(arch_to_hostspec armhf)" + cd "$builddir" + "$builddir"/configure \\ + --build="$CBUILD" \\ + --target=$_target \\ + --with-lib-path=/usr/lib \\ + --prefix=/usr \\ + --with-sysroot=/usr/$_target \\ + --enable-ld=default \\ + --enable-gold=yes \\ + --enable-plugins \\ + --disable-multilib \\ + --disable-werror \\ + --disable-nls \\ + || return 1 + make + """, + "package": """ + cd "$builddir" + make install DESTDIR="$pkgdir" || return 1 + + # remove man, info folders + rm -rf "$pkgdir"/usr/share + """, + "libs": None, + "gold": None, + } + + pmb.aportgen.core.rewrite(args, pkgname, path_original, fields, "binutils", + replace_functions) diff --git a/pmb/aportgen/core.py b/pmb/aportgen/core.py new file mode 100644 index 00000000..eb0b0b25 --- /dev/null +++ b/pmb/aportgen/core.py @@ -0,0 +1,117 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import fnmatch + + +def format_function(name, body, remove_indent=4): + """ + Format the body of a shell function passed to rewrite() below, so it fits + the format of the original APKBUILD. + """ + ret = "" + lines = body.split("\n") + for i in range(len(lines)): + line = lines[i] + if not line.strip(): + if not ret or i == len(lines) - 1: + continue + ret += line[remove_indent:] + "\n" + return name + "() {\n" + ret + "}\n" + + +def rewrite(args, pkgname, path_original, fields={}, replace_pkgname=None, + replace_functions={}, replace_simple={}, below_header=""): + """ + Append a header to $WORK/aportgen/APKBUILD, delete maintainer/contributor + lines (so they won't be bugged with issues regarding our generated aports), + and add reference to the original aport. + + :param fields: key-value pairs of fields, that shall be changed in the + APKBUILD. For example: {"pkgdesc": "my new package", "subpkgs": ""} + :param replace_pkgname: When set, $pkgname gets replaced with that string in + every line. + :param replace_functions: Function names and new bodies, for example: + {"build": "return 0"} + The body can also be None (deletes the function) + :param replace_simple: Lines, that fnmatch the pattern, get + replaced/deleted. Example: {"*test*": "# test", "*mv test.bin*": None} + :param below_header: String, that gets directly placed below the header. + + """ + # Header + lines_new = [ + "# Automatically generated aport, do not edit!\n", + "# Generator: pmbootstrap aportgen " + pkgname + "\n", + "# Based on: " + path_original + "\n", + "\n", + ] + for line in below_header.split("\n"): + lines_new += line.strip() + "\n" + + # Copy/modify lines, skip Maintainer/Contributor + path = args.work + "/aportgen/APKBUILD" + with open(path, "r+", encoding="utf-8") as handle: + skip_in_func = False + for line in handle.readlines(): + # Skip maintainer/contributor + if line.startswith("# Maintainer") or line.startswith( + "# Contributor"): + continue + + # Replace functions + if skip_in_func: + if line.startswith("}"): + skip_in_func = False + continue + else: + for func, body in replace_functions.items(): + if line.startswith(func + "() {"): + skip_in_func = True + if body: + lines_new += format_function(func, body) + break + if skip_in_func: + continue + + # Replace fields + for key, value in fields.items(): + if line.startswith(key + "="): + line = key + "=\"" + value + "\"\n" + break + + # Replace $pkgname + if replace_pkgname and "$pkgname" in line: + line = line.replace("$pkgname", replace_pkgname) + + # Replace simple + for pattern, replacement in replace_simple.items(): + if fnmatch.fnmatch(line, pattern): + line = replacement + if replacement: + line += "\n" + break + if line is None: + continue + + lines_new.append(line) + + # Write back + handle.seek(0) + handle.write("".join(lines_new)) + handle.truncate() diff --git a/pmb/aportgen/gcc.py b/pmb/aportgen/gcc.py new file mode 100644 index 00000000..d4e3515e --- /dev/null +++ b/pmb/aportgen/gcc.py @@ -0,0 +1,72 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import pmb.helpers.run +import pmb.aportgen.core + + +def generate(args, pkgname): + # Copy original aport + arch = pkgname.split("-")[1] + path_original = "main/gcc" + upstream = (args.work + "/cache_git/aports_upstream/" + path_original) + pmb.helpers.run.user(args, ["cp", "-r", upstream, args.work + "/aportgen"]) + + # Rewrite APKBUILD + fields = { + "pkgname": pkgname, + "pkgdesc": "Stage2 cross-compiler for " + arch, + "depends": "isl binutils-" + arch, + "makedepends_build": "gcc g++ paxmark bison flex texinfo gawk zip gmp-dev mpfr-dev mpc1-dev zlib-dev", + "makedepends_host": "linux-headers gmp-dev mpfr-dev mpc1-dev isl-dev zlib-dev musl-dev-" + arch + " binutils-" + arch, + "subpackages": "", + + "LIBGOMP": "false", + "LIBGCC": "false", + "LIBATOMIC": "false", + "LIBITM": "false", + } + + below_header = "CTARGET_ARCH=" + arch + """ + CTARGET="$(arch_to_hostspec ${CTARGET_ARCH})" + CBUILDROOT="/usr/$CTARGET" + LANG_OBJC=false + LANG_JAVA=false + LANG_GO=false + LANG_FORTRAN=false + LANG_ADA=false + options="!strip !tracedeps" + """ + + replace_simple = { + # Do not package libstdc++ + '*subpackages="$subpackages libstdc++:libcxx:*': + ' subpackages="$subpackages g++$_target:gpp"', + + # Do not move gdb.py + '*-gdb.py*': None, + '*/usr/share/gdb/python/auto-load/usr/lib/*': None, + } + + pmb.aportgen.core.rewrite( + args, + pkgname, + path_original, + fields, + replace_simple=replace_simple, + below_header=below_header) diff --git a/pmb/aportgen/musl.py b/pmb/aportgen/musl.py new file mode 100644 index 00000000..5678e367 --- /dev/null +++ b/pmb/aportgen/musl.py @@ -0,0 +1,113 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import glob +import os +import pmb.helpers.run +import pmb.aportgen.core +import pmb.parse.apkindex +import pmb.chroot.apk +import pmb.chroot.apk_static + + +def generate(args, pkgname): + # Install musl in chroot (so we have the APKINDEX and verified musl apks) + arch = pkgname.split("-")[1] + apkindex = pmb.chroot.apk_static.download(args, "APKINDEX.tar.gz") + pmb.chroot.apk.install(args, ["musl-dev"], "buildroot_" + arch) + + # Parse musl version from APKINDEX + package_data = pmb.parse.apkindex.read(args, "musl", apkindex) + version = package_data["version"] + pkgver = version.split("-r")[0] + pkgrel = version.split("-r")[1] + + # Copy the apk files to the distfiles cache + for subpkgname in ["musl", "musl-dev"]: + path = glob.glob(args.work + "/cache_apk_" + arch + "/" + subpkgname + + "-" + version + ".*.apk")[0] + path_target = (args.work + "/cache_distfiles/" + subpkgname + "-" + + version + "-" + arch + ".apk") + if not os.path.exists(path_target): + pmb.helpers.run.user(args, ["cp", path, path_target]) + + # Hash the distfiles + hashes = pmb.chroot.user(args, ["sha512sum", + "musl-" + version + "-" + arch + ".apk", + "musl-dev-" + version + "-" + arch + ".apk"], "buildroot_" + arch, + working_dir="/var/cache/distfiles", return_stdout=True) + + # Write the APKBUILD + pmb.helpers.run.user(args, ["mkdir", "-p", args.work + "/aportgen"]) + with open(args.work + "/aportgen/APKBUILD", "w", encoding="utf-8") as handle: + # Variables + handle.write("# Automatically generated aport, do not edit!\n" + "# Generator: pmbootstrap aportgen " + pkgname + "\n" + "\n" + "pkgname=" + pkgname + "\n" + "pkgver=" + pkgver + "\n" + "pkgrel=" + pkgrel + "\n" + "subpackages=\"musl-dev-" + arch + ":package_dev\"\n" + "\n" + "_arch=\"" + arch + "\"\n" + "_mirror=\"" + args.mirror_alpine + "\"\n" + ) + # Static part + static = """ + url="https://musl-libc.org" + license="MIT" + arch="all" + options="!check !strip" + pkgdesc="the musl library (lib c) implementation for $_arch" + + _target="$(arch_to_hostspec $_arch)" + + source=" + musl-$pkgver-r$pkgrel-$_arch.apk::$_mirror/edge/main/$_arch/musl-$pkgver-r$pkgrel.apk + musl-dev-$pkgver-r$pkgrel-$_arch.apk::$_mirror/edge/main/$_arch/musl-dev-$pkgver-r$pkgrel.apk + " + + package() { + mkdir -p "$pkgdir/usr/$_target" + cd "$pkgdir/usr/$_target" + tar -xf $srcdir/musl-$pkgver-r$pkgrel-$_arch.apk + rm .PKGINFO .SIGN.* + } + package_dev() { + mkdir -p "$subpkgdir/usr/$_target" + cd "$subpkgdir/usr/$_target" + tar -xf $srcdir/musl-dev-$pkgver-r$pkgrel-$_arch.apk + rm .PKGINFO .SIGN.* + + # symlink everything from /usr/$_target/usr/* to /usr/$_target/* + # so the cross-compiler gcc does not fail to build. + for _dir in include lib; do + mkdir -p "$subpkgdir/usr/$_target/$_dir" + cd "$subpkgdir/usr/$_target/usr/$_dir" + for i in *; do + cd "$subpkgdir/usr/$_target/$_dir" + ln -s /usr/$_target/usr/$_dir/$i $i + done + done + } + """ + for line in static.split("\n"): + handle.write(line[12:] + "\n") + + # Hashes + handle.write("sha512sums=\"" + hashes.rstrip() + "\"") diff --git a/pmb/chroot/__init__.py b/pmb/chroot/__init__.py new file mode 100644 index 00000000..ba4e740c --- /dev/null +++ b/pmb/chroot/__init__.py @@ -0,0 +1,24 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +from pmb.chroot.init import init +from pmb.chroot.mount import mount +from pmb.chroot.root import root +from pmb.chroot.user import user +from pmb.chroot.shutdown import shutdown +from pmb.chroot.zap import zap diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py new file mode 100644 index 00000000..879a3e15 --- /dev/null +++ b/pmb/chroot/apk.py @@ -0,0 +1,76 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os +import pmb.chroot +import pmb.parse.apkindex + + +def install(args, packages, suffix="native", build=True): + """ + :param build: automatically build the package, when it does not exist yet + and it is inside the pm-aports folder. Checking this is expensive - if + you know, that all packages are provides by upstream repos, set this to + False! + """ + # Initialize chroot + pmb.chroot.init(args, suffix) + + # Filter already installed packages + packages_installed = installed(args, suffix) + packages_todo = [] + for package in packages: + if package not in packages_installed: + packages_todo.append(package) + if not len(packages_todo): + return + + # Build packages if necessary + arch = pmb.parse.arch.from_chroot_suffix(args, suffix) + if build: + for package in packages_todo: + pmb.build.package(args, package, arch) + + # Sanitize packages: don't allow '--allow-untrusted' and other options + # to be passed to apk! + for package in packages_todo: + if package.startswith("-"): + raise ValueError("Invalid package name: " + package) + + # Install everything + logging.info("(" + suffix + ") install " + " ".join(packages_todo)) + pmb.chroot.root(args, ["apk", "--no-progress", "add"] + packages_todo, + suffix) + +# Update all packages installed in a chroot + + +def update(args, suffix="native"): + pmb.chroot.init(args, suffix) + pmb.chroot.root(args, ["apk", "update"], suffix) + +# Get all explicitly installed packages + + +def installed(args, suffix="native"): + world = args.work + "/chroot_" + suffix + "/etc/apk/world" + if not os.path.exists(world): + return [] + with open(world, encoding="utf-8") as handle: + return handle.read().splitlines() diff --git a/pmb/chroot/apk_static.py b/pmb/chroot/apk_static.py new file mode 100644 index 00000000..d9b052e7 --- /dev/null +++ b/pmb/chroot/apk_static.py @@ -0,0 +1,179 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import urllib.request +import os +import logging +import shutil +import tarfile +import tempfile +import stat + +import pmb.helpers.run +import pmb.config +import pmb.config.load +import pmb.parse.apkindex +import pmb.helpers.http + + +def read_signature_info(tar): + """ + Find various information about the signature, that was used to sign + /sbin/apk.static inside the archive (not to be confused with the normal apk + archive signature!) + + :returns: (sigfilename, sigkey_path) + """ + # Get signature filename and key + prefix = "sbin/apk.static.SIGN.RSA." + sigfilename = None + for filename in tar.getnames(): + if filename.startswith(prefix): + sigfilename = filename + break + if not sigfilename: + raise RuntimeError("Could not find signature filename in apk." + + " This means, that your apk file is damaged. Delete it" + + " and try again. If the problem persists, fill out a bug" + + " report.") + sigkey = sigfilename[len(prefix):] + logging.debug("sigfilename: " + sigfilename) + logging.debug("sigkey: " + sigkey) + + # Get path to keyfile on disk + sigkey_path = pmb.config.pmb_src + "/keys/" + sigkey + if "/" in sigkey or not os.path.exists(sigkey_path): + raise RuntimeError("Invalid signature key: " + sigkey) + + return (sigfilename, sigkey_path) + + +def extract_temp(tar, sigfilename): + """ + Extract apk.static and signature as temporary files. + """ + ret = { + "apk": { + "filename": "sbin/apk.static", + "temp_path": None + }, + "sig": { + "filename": sigfilename, + "temp_path": None + } + } + for ftype in ret.keys(): + member = tar.getmember(ret[ftype]["filename"]) + + handle, path = tempfile.mkstemp(ftype, "pmbootstrap") + handle = open(handle, "wb") + ret[ftype]["temp_path"] = path + shutil.copyfileobj(tar.extractfile(member), handle) + + logging.debug("extracted: " + path) + handle.close() + return ret + + +def verify_signature(args, files, sigkey_path): + """ + Verify the signature with openssl. + + :param files: return value from extract_temp() + :raises RuntimeError: when verification failed and removes temp files + """ + logging.debug("Verify apk.static signature with " + sigkey_path) + try: + pmb.helpers.run.user(args, ["openssl", "dgst", "-sha1", "-verify", + sigkey_path, "-signature", files[ + "sig"]["temp_path"], + files["apk"]["temp_path"]], check=True) + except: + os.unlink(files["sig"]["temp_path"]) + os.unlink(files["apk"]["temp_path"]) + raise RuntimeError("Failed to validate signature of apk.static." + " There's something wrong with the archive - run 'pmbootstrap" + " zap -a' and try again!") + + +def extract(args, version, apk_path): + """ + Extract everything to temporary locations, verify signatures and reported + versions. When everything is right, move the extracted apk.static to the + final location. + """ + # Extract to a temporary path + with tarfile.open(apk_path, "r:gz") as tar: + sigfilename, sigkey_path = read_signature_info(tar) + files = extract_temp(tar, sigfilename) + + # Verify signature + verify_signature(args, files, sigkey_path) + os.unlink(files["sig"]["temp_path"]) + temp_path = files["apk"]["temp_path"] + + # Verify the version, that the extracted binary reports + logging.debug("Verify the version reported by the apk.static binary" + + " (must match the package version " + version + ")") + os.chmod(temp_path, os.stat(temp_path).st_mode | stat.S_IEXEC) + version_bin = pmb.helpers.run.user(args, [temp_path, "--version"], + check=True, return_stdout=True) + version_bin = version_bin.split(" ")[1].split(",")[0] + if not version.startswith(version_bin + "-r"): + os.unlink(temp_path) + raise RuntimeError("Downloaded apk-tools-static-" + version + ".apk," + " but the apk binary inside that package reports to be" + " version: " + version_bin + "! Looks like a downgrade attack" + " from a malicious server! Switch the server (-m) and try again.") + + # Move it to the right path + target_path = args.work + "/apk.static" + shutil.move(temp_path, target_path) + + +def download(args, file): + """ + Download a single file from an Alpine mirror. + """ + base_url = args.mirror_alpine + "edge/main/" + args.arch_native + return pmb.helpers.http.download(args, base_url + "/" + file, file) + + +def init(args): + """ + Download, verify, extract $WORK/apk.static. + """ + base_url = args.mirror_alpine + "edge/main/" + args.arch_native + apkindex = download(args, "APKINDEX.tar.gz") + index_data = pmb.parse.apkindex.read(args, "apk-tools-static", apkindex) + version = index_data["version"] + version_min = pmb.config.apk_tools_static_min_version + apk_name = "apk-tools-static-" + version + ".apk" + if pmb.parse.apkindex.compare_version(version, version_min) == -1: + raise RuntimeError("Server provides an outdated version of" + " apk-tools-static: " + version + + " (expected at least " + version_min + + "). Looks like a downgrade attack from a" + " malicious server! Switch the server (-m) and try again!") + apk_static = download(args, apk_name) + extract(args, version, apk_static) + + +def run(args, parameters, check): + pmb.helpers.run.root( + args, [args.work + "/apk.static"] + parameters, check=check) diff --git a/pmb/chroot/binfmt.py b/pmb/chroot/binfmt.py new file mode 100644 index 00000000..227f5225 --- /dev/null +++ b/pmb/chroot/binfmt.py @@ -0,0 +1,68 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import logging + +import pmb.helpers.run +import pmb.parse +import pmb.parse.arch + + +def is_registered(arch_debian): + return os.path.exists("/proc/sys/fs/binfmt_misc/qemu-" + arch_debian) + + +def register(args, arch): + """ + Get arch, magic, mask. + """ + arch_debian = pmb.parse.arch.alpine_to_debian(arch) + if is_registered(arch_debian): + return + pmb.chroot.apk.install(args, ["qemu-user-static-repack", + "qemu-user-static-repack-binfmt"]) + info = pmb.parse.binfmt_info(args, arch_debian) + + # Build registration string + # https://en.wikipedia.org/wiki/Binfmt_misc + # :name:type:offset:magic:mask:interpreter:flags + name = "qemu-" + arch_debian + type = "M" + offset = "" + magic = info["magic"] + mask = info["mask"] + interpreter = "/usr/bin/qemu-" + arch_debian + "-static" + flags = "C" + code = ":".join(["", name, type, offset, magic, mask, interpreter, + flags]) + + # Register in binfmt_misc + logging.info("Register qemu binfmt (" + arch_debian + ")") + register = "/proc/sys/fs/binfmt_misc/register" + pmb.helpers.run.root( + args, ["sh", "-c", 'echo "' + code + '" > ' + register]) + + +def unregister(args, arch): + arch_debian = pmb.parse.arch.alpine_to_debian(arch) + binfmt_file = "/proc/sys/fs/binfmt_misc/qemu-" + arch_debian + if not os.path.exists(binfmt_file): + return + logging.info("Unregister qemu binfmt (" + arch_debian + ")") + pmb.helpers.run.root(args, ["sh", "-c", "echo -1 > " + binfmt_file]) diff --git a/pmb/chroot/distccd.py b/pmb/chroot/distccd.py new file mode 100644 index 00000000..5db9eb6d --- /dev/null +++ b/pmb/chroot/distccd.py @@ -0,0 +1,81 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os +import errno +import pmb.chroot +import pmb.config +import pmb.chroot.apk + + +def get_pid(args): + pidfile = args.work + "/chroot_native/home/user/distccd.pid" + if not os.path.exists(pidfile): + return None + with open(pidfile, "r") as handle: + lines = handle.readlines() + return int(lines[0][:-1]) + + +def is_running(args): + # Get the PID + pid = get_pid(args) + if not pid: + return False + + # Verify, if it still exists by sending a kill signal + try: + os.kill(pid, 0) + except OSError as err: + if err.errno == errno.ESRCH: # no such process + pmb.chroot.root(args, ["rm", "/home/user/distccd.pid"]) + return False + elif err.errno == errno.EPERM: # access denied + return True + + +def start(args): + if is_running(args): + return + pmb.chroot.apk.install(args, ["distcc", "gcc-cross-wrappers"]) + + # Start daemon with cross-compiler in path + arch = args.deviceinfo["arch"] + path = "/usr/lib/gcc-cross-wrappers/" + arch + "/bin:" + pmb.config.chroot_path + daemon = ["PATH=" + path, + "distccd", + "--pid-file", "/home/user/distccd.pid", + "--listen", "127.0.0.1", + "--allow", "127.0.0.1", + "--port", args.port_distccd, + "--log-file", "/home/user/distccd.log", + "--jobs", args.jobs, + "--nice", "19", + "--job-lifetime", "60", + "--daemon" + ] + logging.info("(native) start distccd (listen on 127.0.0.1:" + + args.port_distccd + ")") + pmb.chroot.user(args, daemon) + + +def stop(args): + if is_running(args): + logging.info("(native) stop distccd") + pmb.chroot.user(args, ["kill", str(get_pid(args))]) diff --git a/pmb/chroot/init.py b/pmb/chroot/init.py new file mode 100644 index 00000000..0f631c94 --- /dev/null +++ b/pmb/chroot/init.py @@ -0,0 +1,123 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os +import shlex +import glob +import filecmp + +import pmb.chroot +import pmb.chroot.apk_static +import pmb.config +import pmb.helpers.run +import pmb.parse.arch + + +def copy_resolv_conf(args, suffix="native"): + """ + Use pythons super fast file compare function (due to caching) + and copy the /etc/resolv.conf to the chroot, in case it is + different from the host. + """ + host = "/etc/resolv.conf" + chroot = args.work + "/chroot_" + suffix + host + if not os.path.exists(chroot) or not filecmp.cmp(host, chroot): + pmb.helpers.run.root(args, ["cp", host, chroot]) + + +def init(args, suffix="native"): + # When already initialized: just prepare the chroot + chroot = args.work + "/chroot_" + suffix + arch = pmb.parse.arch.from_chroot_suffix(args, suffix) + pmb.chroot.mount(args, suffix) + if os.path.islink(chroot + "/bin/sh"): + if suffix != "native": + pmb.chroot.binfmt.register(args, arch) + copy_resolv_conf(args, suffix) + return + + # Require apk-tools-static + pmb.chroot.apk_static.init(args) + + # Non-native chroot: require qemu-user-static + if suffix != "native": + pmb.chroot.apk.install(args, ["qemu-user-static-repack", + "qemu-user-static-repack-binfmt"]) + pmb.chroot.binfmt.register(args, arch) + + logging.info("(" + suffix + ") install alpine-base") + + # Initialize cache + apk_cache = args.work + "/cache_apk_" + arch + pmb.helpers.run.root(args, ["ln", "-s", "/var/cache/apk", chroot + + "/etc/apk/cache"]) + + # Copy /etc/apk/keys/ and resolv.conf + logging.debug(pmb.config.apk_keys_path) + for key in glob.glob(pmb.config.apk_keys_path + "/*.pub"): + pmb.helpers.run.root(args, ["cp", key, args.work + + "/config_apk_keys/"]) + copy_resolv_conf(args, suffix) + + # Write /etc/apk/repositories + repos_path = chroot + "/etc/apk/repositories" + if not os.path.exists(repos_path): + lines = ["/home/user/packages/user"] + directories = ["main", "community"] + if args.alpine_version == "edge": + directories.append("testing") + for dir in directories: + lines.append(args.mirror_alpine + args.alpine_version + + "/" + dir) + for line in lines: + pmb.helpers.run.root(args, ["sh", "-c", + "echo " + shlex.quote(line) + " >> " + repos_path]) + + # Install alpine-base (no clean exit for non-native chroot!) + pmb.chroot.apk_static.run(args, ["-U", "--root", chroot, + "--cache-dir", apk_cache, "--initdb", "--arch", arch, + "add", "alpine-base"], check=(suffix == "native")) + + # Create device nodes + for dev in pmb.config.chroot_device_nodes: + path = chroot + "/dev/" + str(dev[4]) + if not os.path.exists(path): + pmb.helpers.run.root(args, ["mknod", + "-m", str(dev[0]), # permissions + path, # name + str(dev[1]), # type + str(dev[2]), # major + str(dev[3]), # minor + ]) + + # Non-native chroot: install qemu-user-binary, run apk fix + if suffix != "native": + arch_debian = pmb.parse.arch.alpine_to_debian(arch) + pmb.helpers.run.root(args, ["cp", args.work + + "/chroot_native/usr/bin/qemu-" + arch_debian + "-static", + chroot + "/usr/bin/qemu-" + arch_debian + "-static"]) + pmb.chroot.root(args, ["apk", "fix"], suffix, + auto_init=False) + + # Add user (-D: don't assign password) + logging.debug("Add user") + pmb.chroot.root(args, ["adduser", "-D", "user", "-u", pmb.config.chroot_uid_user], + suffix, auto_init=False) + pmb.chroot.root(args, ["chown", "-R", "user:user", "/home/user"], + suffix) diff --git a/pmb/chroot/mount.py b/pmb/chroot/mount.py new file mode 100644 index 00000000..d3ccaaa4 --- /dev/null +++ b/pmb/chroot/mount.py @@ -0,0 +1,37 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import pmb.config +import pmb.parse +import pmb.helpers.mount + + +def mount(args, suffix="native"): + arch = pmb.parse.arch.from_chroot_suffix(args, suffix) + + # get all mountpoints + mountpoints = {} + for source, target in pmb.config.chroot_mount_bind.items(): + source = source.replace("$WORK", args.work) + source = source.replace("$ARCH", arch) + mountpoints[source] = target + + # mount if necessary + for source, target in mountpoints.items(): + target_full = args.work + "/chroot_" + suffix + target + pmb.helpers.mount.bind(args, source, target_full) diff --git a/pmb/chroot/other.py b/pmb/chroot/other.py new file mode 100644 index 00000000..9e641c6d --- /dev/null +++ b/pmb/chroot/other.py @@ -0,0 +1,30 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import glob + + +def installed_kernel_flavors(args, suffix): + prefix = "vmlinuz-" + prefix_len = len(prefix) + pattern = args.work + "/chroot_" + suffix + "/boot/" + prefix + "*" + ret = [] + for file in glob.glob(pattern): + ret.append(os.path.basename(file)[prefix_len:]) + return ret diff --git a/pmb/chroot/root.py b/pmb/chroot/root.py new file mode 100644 index 00000000..d12f66af --- /dev/null +++ b/pmb/chroot/root.py @@ -0,0 +1,72 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import shutil +import shlex + +import pmb.config +import pmb.chroot +import pmb.chroot.binfmt +import pmb.helpers.run + + +def root(args, cmd, suffix="native", working_dir="/", log=True, + auto_init=True, return_stdout=False, check=True): + """ + Run a command inside a chroot as root. + + :param log: When set to true, redirect all output to the logfile + :param auto_init: Automatically initialize the chroot + """ + # Get and verify chroot folder + chroot = args.work + "/chroot_" + suffix + if not auto_init and not os.path.islink(chroot + "/bin/sh"): + raise RuntimeError("Chroot does not exist: " + chroot) + + pmb.chroot.init(args, suffix) + + # Run the args with sudo chroot, and with cleaned environment + # variables + sh_bin = shutil.which("sh") + chroot_bin = shutil.which("chroot") + for i in range(len(cmd)): + cmd[i] = shlex.quote(cmd[i]) + + cmd_inner_shell = ("cd " + shlex.quote(working_dir) + ";" + + " ".join(cmd)) + cmd_full = ["sudo", sh_bin, "-c", + "unset $(env | cut -d= -f1);" + # unset all + " CHARSET=UTF-8" + + " PATH=" + pmb.config.chroot_path + + " SHELL=/bin/ash" + + " HISTFILE=~/.ash_history" + + " " + chroot_bin + + " " + chroot + + " sh -c " + shlex.quote(cmd_inner_shell) + ] + + # Generate log message + log_message = "(" + suffix + ") % " + if working_dir != "/": + log_message += "cd " + working_dir + " && " + log_message += " ".join(cmd) + + # Run the command + return pmb.helpers.run.core(args, cmd_full, log_message, log, + return_stdout, check) diff --git a/pmb/chroot/shutdown.py b/pmb/chroot/shutdown.py new file mode 100644 index 00000000..3e14192c --- /dev/null +++ b/pmb/chroot/shutdown.py @@ -0,0 +1,53 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import glob +import os + +import pmb.install.losetup +import pmb.helpers.mount +import pmb.chroot +import pmb.chroot.distccd + + +def shutdown(args, only_install_related=False): + pmb.chroot.distccd.stop(args) + + # Umount installation-related paths (order is important!) + pmb.helpers.mount.umount_all(args, args.work + + "/chroot_native/mnt/install/boot") + pmb.helpers.mount.umount_all(args, args.work + + "/chroot_native/mnt/install") + if os.path.exists(args.work + "/chroot_native/dev/mapper/pm_crypt"): + pmb.chroot.root(args, ["cryptsetup", "luksClose", "pm_crypt"]) + + # Umount all losetup mounted images + chroot = args.work + "/chroot_native" + if pmb.helpers.mount.ismount(chroot + "/dev/loop-control"): + pattern = chroot + "/home/user/rootfs/*.img" + for path_outside in glob.glob(pattern): + path = path_outside[len(chroot):] + pmb.install.losetup.umount(args, path) + + if not only_install_related: + # Clean up the rest + pmb.helpers.mount.umount_all(args, args.work) + pmb.helpers.mount.umount_all(args, args.work) + pmb.chroot.binfmt.unregister(args, args.deviceinfo["arch"]) + logging.info("Shutdown complete") diff --git a/pmb/chroot/user.py b/pmb/chroot/user.py new file mode 100644 index 00000000..edf8d907 --- /dev/null +++ b/pmb/chroot/user.py @@ -0,0 +1,32 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import pmb.chroot.root + + +def user(args, cmd, suffix="native", working_dir="/", log=True, + auto_init=True, return_stdout=False, check=True): + """ + Run a command inside a chroot as "user" + + :param log: When set to true, redirect all output to the logfile + :param auto_init: Automatically initialize the chroot + """ + cmd = ["su", "user", "-c", " ".join(cmd)] + return pmb.chroot.root(args, cmd, suffix, working_dir, log, + auto_init, return_stdout, check) diff --git a/pmb/chroot/zap.py b/pmb/chroot/zap.py new file mode 100644 index 00000000..7c9264e1 --- /dev/null +++ b/pmb/chroot/zap.py @@ -0,0 +1,46 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import glob + +import pmb.chroot +import pmb.helpers.run + + +def zap(args): + pmb.chroot.shutdown(args) + patterns = [ + "chroot_native", + "chroot_buildroot_" + args.deviceinfo["arch"], + "chroot_rootfs_" + args.device, + ] + + # Only ask for removal, if the user specificed the extra '-p' switch. + # Deleting the packages by accident is really annoying. + if args.packages: + patterns += ["packages"] + if args.http: + patterns += ["cache_http"] + + for pattern in patterns: + pattern = os.path.abspath(args.work + "/" + pattern) + matches = glob.glob(pattern) + for match in matches: + if pmb.helpers.cli.ask(args, "Remove " + match + "?") == "y": + pmb.helpers.run.root(args, ["rm", "-rf", match]) diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py new file mode 100644 index 00000000..24a97045 --- /dev/null +++ b/pmb/config/__init__.py @@ -0,0 +1,225 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os + +# +# Exported functions +# +from pmb.config.init import init +from pmb.config.load import load +from pmb.config.save import save + + +# +# Exported variables (internal configuration) +# +version = "0.1.0" +pmb_src = os.path.normpath(os.path.realpath(__file__) + "/../../..") +apk_keys_path = pmb_src + "/keys" + +# Update this frequently to prevent a MITM attack with an outdated version +# (which may contain a vulnerable apk/libressl, and allows and attacker to +# exploit the system!) +apk_tools_static_min_version = "2.7.1-r0" + +# Config file/commandline default values +# $WORK gets replaced with the actual value for args.work (which may be +# overriden on the commandline) +defaults = { + "alpine_version": "edge", # alternatively: latest-stable + "aports": os.path.normpath(pmb_src + "/../aports"), + "config": os.path.expanduser("~") + "/.config/pmbootstrap.cfg", + "device": "samsung-i9100", + "log": "$WORK/log.txt", + "mirror_alpine": "https://nl.alpinelinux.org/alpine/", + "work": os.path.expanduser("~") + "/.local/var/pmbootstrap", + "port_distccd": "33632", + + # aes-xts-plain64 would be better, but this is not supported on LineageOS + # kernel configs + "cipher": "aes-cbc-plain64" +} + +# +# CHROOT +# + +# Usually the ID for the first user created is 1000. However, we want +# pmbootstrap to work even if the 'user' account inside the chroots has +# another UID, so we force it to be different. +chroot_uid_user = "12345" + +# The PATH variable used inside all chroots +chroot_path = ":".join([ + "/usr/lib/ccache/bin", + "/usr/local/sbin", + "/usr/local/bin", + "/usr/sbin:/usr/bin", + "/sbin", + "/bin" +]) + +# Folders, that get mounted inside the chroot +# $WORK gets replaced with args.work +# $ARCH gets replaced with the chroot architecture (eg. x86_64, armhf) +chroot_mount_bind = { + "/proc": "/proc", + "$WORK/cache_apk_$ARCH": "/var/cache/apk", + "$WORK/cache_ccache_$ARCH": "/home/user/.ccache", + "$WORK/cache_distfiles": "/var/cache/distfiles", + "$WORK/cache_git": "/home/user/git", + "$WORK/config_abuild": "/home/user/.abuild", + "$WORK/config_apk_keys": "/etc/apk/keys", + "$WORK/packages": "/home/user/packages/user", +} + +# The package alpine-base only creates some device nodes. Specify here, which +# additional nodes will get created during initialization of the chroot. +# Syntax for each entry: [permissions, type, major, minor, name] +chroot_device_nodes = [ + [666, "c", 1, 5, "zero"], + [666, "c", 1, 7, "full"], + [644, "c", 1, 8, "random"], + [644, "c", 1, 9, "urandom"], +] + + +# +# BUILD +# + +# Packages, that will be installed in a chroot before it build packages +# for the first time +build_packages = ["abuild", "build-base", "ccache"] + +# fnmatch for supported pkgnames, that can be directly compiled inside +# the native chroot and a cross-compiler, without using distcc +build_cross_native = ["linux-*"] + +# Variables in APKBUILD files, that get parsed +apkbuild_attributes = { + "arch": {"array": True}, + "depends": {"array": True}, + "makedepends": {"array": True}, + "options": {"array": True}, + "pkgname": {"array": False}, + "pkgrel": {"array": False}, + "pkgver": {"array": False}, + "subpackages": {"array": True}, + + # cross-compilers + "makedepends_build": {"array": True}, + "makedepends_host": {"array": True}, + + # kernels + "_flavor": {"array": False}, + "_device": {"array": False}, + "_kernver": {"array": False}, + "_pmb_build_in_native_chroot": {"array": False}, + + # mesa + "_llvmver": {"array": False}, +} + +# +# INSTALL +# + +# Packages, that will be installed inside the native chroot to perform +# the installation to the device. +# util-linux: losetup, fallocate +install_native_packages = ["cryptsetup", "util-linux", "e2fsprogs", "parted"] +install_device_packages = [ + + # postmarketos + "postmarketos-base", "postmarketos-demos", + + # weston + "weston", "weston-shell-desktop", "weston-backend-fbdev", "weston-backend-drm", + "weston-backend-x11", "weston-clients", "weston-terminal", + "weston-xwayland", "xorg-server-xwayland", + + # other + "ttf-droid" +] +install_size_image = "835M" +install_size_boot = "100M" + +# fnmatch-patterns, that the sdcard patch must match. Otherwise the +# installer will refuse to format the device. +install_valid_sdcard_devices = ["/dev/mmcblk*", "/dev/loop*"] + + +# +# FLASH +# + +# These folders will be mounted at the same location into the native +# chroot, before the flash programs get started. +flash_mount_bind = [ + "/sys/bus/usb/devices/", + "/sys/devices/", + "/dev/bus/usb/" +] + +# Allowed variables: +# $KERNEL, $RAMDISK, $IMAGE (system partition image), $BOOTPARAM +flashers = { + "fastboot": { + "depends": ["android-tools"], + "actions": + { + "list_devices": [["fastboot", "devices", "-l"]], + "flash_system": [["fastboot", "flash", "system", "$IMAGE"]], + "flash_kernel": [["fastboot", + "--base", "$OFFSET_BASE", + "--kernel-offset", "$OFFSET_KERNEL", + "--ramdisk-offset", "$OFFSET_RAMDISK", + "--tags-offset", "$OFFSET_TAGS", + "--page-size", "$PAGE_SIZE", + "flash:raw", "$KERNEL", "$RAMDISK"]], + "boot": [["fastboot", + "--base", "$OFFSET_BASE", + "--kernel-offset", "$OFFSET_KERNEL", + "--ramdisk-offset", "$OFFSET_RAMDISK", + "--tags-offset", "$OFFSET_TAGS", + "--page-size", "$PAGE_SIZE", + "boot", "$KERNEL", "$RAMDISK"]], + } + }, + "heimdall": { + "depends": ["heimdall"], + "actions": + { + "list_devices": [["heimdall", "detect"]], + "flash_system": [ + ["heimdall_wait_for_device.sh"], + ["heimdall", "flash", "--SYSTEM", "$IMAGE"]], + "flash_kernel": [["heimdall_flash_kernel.sh", "$RAMDISK", "$KERNEL"]] + }, + }, +} + +# +# GIT +# +git_repos = { + "aports_upstream": "https://github.com/alpinelinux/aports", + "apk-tools": "https://github.com/alpinelinux/apk-tools", +} diff --git a/pmb/config/init.py b/pmb/config/init.py new file mode 100644 index 00000000..333604ae --- /dev/null +++ b/pmb/config/init.py @@ -0,0 +1,63 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os +import multiprocessing + +import pmb.config +import pmb.helpers.cli +import pmb.helpers.devices + + +def init(args): + cfg = pmb.config.load(args) + + # Device + devices = sorted(pmb.helpers.devices.list(args)) + logging.info("Target device (either an existing one, or a new one for" + " porting). Available: " + ", ".join(devices)) + cfg["pmbootstrap"]["device"] = pmb.helpers.cli.ask(args, "Device", + None, args.device) + + # Work folder + logging.info("Location of the 'work' path. Multiple chroots (native," + " device arch, device rootfs) will be created in there.") + cfg["pmbootstrap"]["work"] = pmb.helpers.cli.ask(args, "Work path", + None, args.work) + os.makedirs(cfg["pmbootstrap"]["work"], 0o700, True) + + # Parallel job count + default = args.jobs + if not default: + default = multiprocessing.cpu_count() + 1 + logging.info("How many jobs should run parallel on this machine, when" + " compiling?") + cfg["pmbootstrap"]["jobs"] = pmb.helpers.cli.ask(args, "Jobs", + None, default) + + # Save config + pmb.config.save(args, cfg) + + logging.info( + "WARNING: The applications in the chroots do not get updated automatically.") + logging.info("Run 'pmbootstrap zap' to delete all chroots once a day before" + " working with pmbootstrap!") + logging.info("It only takes a few seconds, and all packages are cached.") + + logging.info("Done!") diff --git a/pmb/config/load.py b/pmb/config/load.py new file mode 100644 index 00000000..4deb6e20 --- /dev/null +++ b/pmb/config/load.py @@ -0,0 +1,36 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import configparser +import os +import pmb.config + + +def load(args): + cfg = configparser.ConfigParser() + if os.path.isfile(args.config): + cfg.read(args.config) + + if "pmbootstrap" not in cfg: + cfg["pmbootstrap"] = {} + + for key in pmb.config.defaults: + if key not in cfg["pmbootstrap"]: + cfg["pmbootstrap"][key] = pmb.config.defaults[key] + + return cfg diff --git a/pmb/config/save.py b/pmb/config/save.py new file mode 100644 index 00000000..dd40c969 --- /dev/null +++ b/pmb/config/save.py @@ -0,0 +1,27 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import logging + + +def save(args, cfg): + logging.debug("save config: " + args.config) + os.makedirs(os.path.dirname(args.config), 0o700, True) + with open(args.config, "w") as handle: + cfg.write(handle) diff --git a/pmb/flasher/__init__.py b/pmb/flasher/__init__.py new file mode 100644 index 00000000..fed79cef --- /dev/null +++ b/pmb/flasher/__init__.py @@ -0,0 +1,21 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +from pmb.flasher.init import init +from pmb.flasher.run import run +from pmb.flasher.frontend import frontend diff --git a/pmb/flasher/frontend.py b/pmb/flasher/frontend.py new file mode 100644 index 00000000..3842d8f7 --- /dev/null +++ b/pmb/flasher/frontend.py @@ -0,0 +1,88 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os + +import pmb.flasher +import pmb.install +import pmb.chroot.other + + +def kernel(args): + # Parse the kernel flavor + suffix = "rootfs_" + args.device + flavor = args.flavor + flavors = pmb.chroot.other.installed_kernel_flavors(args, suffix) + if flavor: + if flavor not in flavors: + raise RuntimeError("No kernel installed with flavor " + flavor + "!" + + " Run 'pmbootstrap flasher list_flavors' to get a list.") + elif not len(flavors): + raise RuntimeError( + "No kernel flavors installed in chroot " + suffix + "!") + else: + flavor = flavors[0] + + # Generate the paths and run the flasher + pmb.flasher.init(args) + mnt = "/mnt/rootfs_" + args.device + kernel = mnt + "/boot/vmlinuz-" + flavor + ramdisk = mnt + "/boot/initramfs-" + flavor + if args.action_flasher == "boot": + logging.info("(native) boot " + flavor + " kernel") + pmb.flasher.run(args, "boot", kernel, ramdisk) + else: + logging.info("(native) flash kernel '" + flavor + "'") + pmb.flasher.run(args, "flash_kernel", kernel, ramdisk) + + +def list_flavors(args): + suffix = "rootfs_" + args.device + logging.info("(" + suffix + ") installed kernel flavors:") + for flavor in pmb.chroot.other.installed_kernel_flavors(args, suffix): + logging.info("* " + flavor) + + +def system(args): + # Generate system image, install flasher + img_path = "/home/user/rootfs/" + args.device + ".img" + if not os.path.exists(args.work + "/chroot_native" + img_path): + setattr(args, "sdcard", None) + pmb.install.install(args, False) + pmb.flasher.init(args) + + # Run the flasher + logging.info("(native) flash system image") + pmb.flasher.run(args, "flash_system", image=img_path) + + +def list_devices(args): + pmb.flasher.run(args, "list_devices") + + +def frontend(args): + action = args.action_flasher + if action in ["boot", "flash_kernel"]: + kernel(args) + if action == "flash_system": + system(args) + if action == "list_flavors": + list_flavors(args) + if action == "list_devices": + list_devices(args) diff --git a/pmb/flasher/init.py b/pmb/flasher/init.py new file mode 100644 index 00000000..6b1aba12 --- /dev/null +++ b/pmb/flasher/init.py @@ -0,0 +1,46 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import pmb.config +import pmb.chroot.apk +import pmb.helpers.mount + + +def init(args): + # Validate method + method = args.deviceinfo["flash_methods"] + if method not in pmb.config.flashers: + raise RuntimeError("Flash method " + method + " is not supported by the" + " current configuration. However, adding a new flash method is " + " not that hard, when the flashing application already exists.\n" + "Make sure, it is packaged for Alpine Linux, or package it " + " yourself, and then add it to pmb/config/__init__.py.") + cfg = pmb.config.flashers[method] + + # Install depends + pmb.chroot.apk.install(args, cfg["depends"]) + + # Mount folders from host system + for folder in pmb.config.flash_mount_bind: + pmb.helpers.mount.bind(args, folder, args.work + + "/chroot_native" + folder) + + # Mount device chroot inside native chroot (required for kernel/ramdisk) + mountpoint = "/mnt/rootfs_" + args.device + pmb.helpers.mount.bind(args, args.work + "/chroot_rootfs_" + args.device, + args.work + "/chroot_native" + mountpoint) diff --git a/pmb/flasher/run.py b/pmb/flasher/run.py new file mode 100644 index 00000000..22bb7e5b --- /dev/null +++ b/pmb/flasher/run.py @@ -0,0 +1,58 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import pmb.flasher + + +def run(args, action, kernel=None, ramdisk=None, image=None): + pmb.flasher.init(args) + + # Verify action + method = args.deviceinfo["flash_methods"] + cfg = pmb.config.flashers[method] + if action not in cfg["actions"]: + raise RuntimeError("action " + action + " is not" + " configured for method " + method + "!") + + # Variable setup + vars = { + "$KERNEL": kernel, + "$RAMDISK": ramdisk, + "$IMAGE": image, + "$OFFSET_BASE": args.deviceinfo["flash_offset_base"], + "$OFFSET_KERNEL": args.deviceinfo["flash_offset_kernel"], + "$OFFSET_RAMDISK": args.deviceinfo["flash_offset_ramdisk"], + "$OFFSET_SECOND": args.deviceinfo["flash_offset_second"], + "$OFFSET_TAGS": args.deviceinfo["flash_offset_tags"], + "$PAGE_SIZE": args.deviceinfo["flash_pagesize"], + } + + # Each action has multiple commands + for command in cfg["actions"][action]: + # Variable replacement + for key, value in vars.items(): + for i in range(len(command)): + if key in command[i]: + if not value: + raise RuntimeError("Variable " + key + " found in" + " action " + action + " for method " + method + "," + " but the value for this variable is None!") + command[i] = command[i].replace(key, value) + + # Run the action + pmb.chroot.root(args, command, log=False) diff --git a/pmb/helpers/__init__.py b/pmb/helpers/__init__.py new file mode 100644 index 00000000..84978349 --- /dev/null +++ b/pmb/helpers/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" diff --git a/pmb/helpers/cli.py b/pmb/helpers/cli.py new file mode 100644 index 00000000..6d0a280e --- /dev/null +++ b/pmb/helpers/cli.py @@ -0,0 +1,39 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import datetime + + +def ask(args, question="Continue?", choices=['y', 'n'], default='n', + lowercase_answer=True): + date = datetime.datetime.now().strftime("%H:%M:%S") + question = "[" + date + "] " + question + if choices: + question += " (" + str.join("/", choices) + ")" + if default: + question += " [" + str(default) + "]" + + ret = input(question + ": ") + if lowercase_answer: + ret = ret.lower() + if ret == "": + ret = str(default) + + args.logfd.write(question + " " + ret + "\n") + args.logfd.flush() + return ret diff --git a/pmb/helpers/devices.py b/pmb/helpers/devices.py new file mode 100644 index 00000000..0c523570 --- /dev/null +++ b/pmb/helpers/devices.py @@ -0,0 +1,44 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import glob +import pmb.parse + + +def list(args): + """ + Get all devices, for which aports are available + :returns: ["first-device", "second-device", ...] + """ + ret = [] + for path in glob.glob(args.aports + "/device-*"): + device = os.path.basename(path).split("-", 1)[1] + ret += [device] + return ret + + +def list_apkbuilds(args): + """ + :returns: { "first-device": {"pkgname": ..., "pkgver": ...}, ... } + """ + ret = {} + for device in list(args): + apkbuild_path = args.aports + "/device-" + device + "/APKBUILD" + ret[device] = pmb.parse.apkbuild(apkbuild_path) + return ret diff --git a/pmb/helpers/file.py b/pmb/helpers/file.py new file mode 100644 index 00000000..e49ab0e9 --- /dev/null +++ b/pmb/helpers/file.py @@ -0,0 +1,27 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +def replace(path, old, new): + text = "" + with open(path, 'r') as handle: + text = handle.read() + + text = text.replace(old, new) + + with open(path, 'w') as handle: + handle.write(text) diff --git a/pmb/helpers/git.py b/pmb/helpers/git.py new file mode 100644 index 00000000..4eed7b1e --- /dev/null +++ b/pmb/helpers/git.py @@ -0,0 +1,35 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os + +import pmb.build +import pmb.chroot.apk +import pmb.config + + +def clone(args, repo_name): + if repo_name not in pmb.config.git_repos: + raise ValueError("No git repository configured for " + repo_name) + + if not os.path.exists(args.work + "/cache_git/" + repo_name): + pmb.chroot.apk.install(args, ["git"]) + logging.info("(native) git clone " + pmb.config.git_repos[repo_name]) + pmb.chroot.user(args, ["git", "clone", "--depth=1", + pmb.config.git_repos[repo_name], repo_name], working_dir="/home/user/git/") diff --git a/pmb/helpers/http.py b/pmb/helpers/http.py new file mode 100644 index 00000000..2abf0a21 --- /dev/null +++ b/pmb/helpers/http.py @@ -0,0 +1,49 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import hashlib +import shutil +import logging +import urllib.request +import pmb.helpers.run + + +def download(args, url, prefix, cache=True): + """ + Download a file to disk. + """ + # Create cache folder + if not os.path.exists(args.work + "/cache_http"): + pmb.helpers.run.user(args, ["mkdir", "-p", args.work + "/cache_http"]) + + # Check if file exists in cache + prefix = prefix.replace("/", "_") + path = (args.work + "/cache_http/" + prefix + "_" + + hashlib.sha512(url.encode("utf-8")).hexdigest()) + if os.path.exists(path): + if cache: + return path + pmb.helpers.run.user(args, ["rm", path]) + + # Download the file + logging.info("Download " + url) + with urllib.request.urlopen(url) as response: + with open(path, "wb") as handle: + shutil.copyfileobj(response, handle) + return path diff --git a/pmb/helpers/logging.py b/pmb/helpers/logging.py new file mode 100644 index 00000000..bf7de538 --- /dev/null +++ b/pmb/helpers/logging.py @@ -0,0 +1,76 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os + + +class log_handler(logging.StreamHandler): + """ + Write to stdout and to the already opened log file. + """ + _args = None + + def emit(self, record): + try: + msg = self.format(record) + + # INFO or higher: Write to stdout + if not self._args.quiet and record.levelno >= logging.INFO: + stream = self.stream + stream.write(msg) + stream.write(self.terminator) + self.flush() + + # Everything: Write to logfd + msg = "(" + str(os.getpid()).zfill(6) + ") " + msg + self._args.logfd.write(msg + "\n") + self._args.logfd.flush() + + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) + + +def init(args): + """ + Set log format and add the log file descriptor to args.logfd. + """ + if not os.path.exists(args.work): + os.makedirs(args.work) + + date_format = "%H:%M:%S" + setattr(args, "logfd", open(args.log, "a+")) + + root_logger = logging.getLogger() + root_logger.handlers = [] + + formatter = None + root_logger.setLevel(logging.DEBUG) + if args.verbose: + formatter = logging.Formatter("[%(asctime)s %(module)s]" + " %(message)s", datefmt=date_format) + else: + formatter = logging.Formatter("[%(asctime)s] %(message)s", + datefmt=date_format) + + handler = log_handler() + log_handler._args = args + handler.setFormatter(formatter) + root_logger.addHandler(handler) diff --git a/pmb/helpers/mount.py b/pmb/helpers/mount.py new file mode 100644 index 00000000..008bfff0 --- /dev/null +++ b/pmb/helpers/mount.py @@ -0,0 +1,90 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import pmb.helpers.run + + +def ismount(folder): + """ + Ismount() implementation, that works for mount --bind. + Workaround for: https://bugs.python.org/issue29707 + """ + folder = os.path.abspath(folder) + with open("/proc/mounts", "r") as handle: + for line in handle: + words = line.split() + if len(words) >= 2 and words[1] == folder: + return True + return False + + +def bind(args, source, destination, create_folders=True): + """ + Mount --bind a folder and create necessary directory structure. + """ + if ismount(destination): + return + + # Check/create folders + for path in [source, destination]: + if os.path.exists(path): + continue + if create_folders: + pmb.helpers.run.root(args, ["mkdir", "-p", path]) + else: + raise RuntimeError("Mount failed, folder does not exist: " + + path) + + # Actually mount the folder + pmb.helpers.run.root(args, ["mount", "--bind", source, destination]) + + # Verify, that it has worked + if not ismount(destination): + raise RuntimeError("Mount failed: " + source + " -> " + destination) + +# Mount a blockdevice + + +def bind_blockdevice(args, source, destination): + # Skip existing mountpoint + if ismount(destination): + return + + # Create empty file + if not os.path.exists(destination): + pmb.helpers.run.root(args, ["touch", destination]) + + # Mount + pmb.helpers.run.root(args, ["mount", "--bind", source, + destination]) + + +def umount_all(args, folder): + """ + Umount all folders, that are mounted inside a given folder. + """ + folder = os.path.abspath(folder) + with open("/proc/mounts", "r") as handle: + for line in handle: + words = line.split() + if len(words) < 2 or not words[1].startswith(folder): + continue + pmb.helpers.run.root(args, ["umount", words[1]]) + if ismount(words[1]): + raise RuntimeError("Failed to umount: " + words[1]) diff --git a/pmb/helpers/run.py b/pmb/helpers/run.py new file mode 100644 index 00000000..6bfb97f3 --- /dev/null +++ b/pmb/helpers/run.py @@ -0,0 +1,70 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import subprocess +import logging + + +def core(args, cmd, log_message, log, return_stdout, check=True): + logging.debug(log_message) + """ + Run the command and write the output to the log. + + :param check: raise an exception, when the command fails + """ + + try: + ret = None + if log: + if return_stdout: + ret = subprocess.run(cmd, stdout=subprocess.PIPE, + check=check).stdout.decode('utf-8') + args.logfd.write(ret) + else: + subprocess.run(cmd, stdout=args.logfd, stderr=args.logfd, + check=check) + args.logfd.flush() + else: + logging.debug("*** output passed to pmbootstrap stdout, not" + + " to this log ***") + subprocess.run(cmd, check=check) + + except subprocess.CalledProcessError as exc: + raise RuntimeError("Command failed: " + log_message) from exc + return ret + + +def user(args, cmd, log=True, working_dir=None, return_stdout=False, + check=True): + """ + :param working_dir: defaults to args.work + """ + if not working_dir: + working_dir = args.work + + # TODO: maintain and check against a whitelist + return core(args, cmd, "% " + " ".join(cmd), log, return_stdout, check) + + +def root(args, cmd, log=True, working_dir=None, return_stdout=False, + check=True): + """ + :param working_dir: defaults to args.work + """ + cmd = ["sudo"] + cmd + return user(args, cmd, log, working_dir, return_stdout, check) diff --git a/pmb/install/__init__.py b/pmb/install/__init__.py new file mode 100644 index 00000000..f9277d82 --- /dev/null +++ b/pmb/install/__init__.py @@ -0,0 +1,21 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +from pmb.install.install import install +from pmb.install.partition import partition +from pmb.install.format import format diff --git a/pmb/install/blockdevice.py b/pmb/install/blockdevice.py new file mode 100644 index 00000000..91e672c5 --- /dev/null +++ b/pmb/install/blockdevice.py @@ -0,0 +1,98 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os +import pmb.helpers.mount +import pmb.install.losetup +import pmb.helpers.cli +import pmb.config +import fnmatch + + +def sdcard_validate_path(args): + for pattern in pmb.config.install_valid_sdcard_devices: + if fnmatch.fnmatch(args.sdcard, pattern): + return True + return False + + +def mount_sdcard(args): + # Sanity checks + if args.deviceinfo["external_disk_install"] != "true": + raise RuntimeError("According to the deviceinfo, this device does" + " not support a sdcard installation.") + if not os.path.exists(args.sdcard): + raise RuntimeError("The sdcard device does not exist: " + + args.sdcard) + if not sdcard_validate_path(args): + raise RuntimeError("The sdcard path does not look valid. We will" + " not attempt to format this!") + if pmb.helpers.cli.ask(args, "EVERYTHING ON " + args.sdcard + " WILL BE" + " ERASED! CONTINUE?") != "y": + raise RuntimeError("Aborted.") + + logging.info("(native) mount /dev/install (host: " + args.sdcard + ")") + pmb.helpers.mount.bind_blockdevice(args, args.sdcard, + args.work + "/chroot_native/dev/install") + + +def create_and_mount_image(args): + # Short variables for paths + chroot = args.work + "/chroot_native" + img_path = "/home/user/rootfs/" + args.device + ".img" + img_path_outside = chroot + img_path + + # Umount and delete existing image + if os.path.exists(img_path_outside): + pmb.helpers.mount.umount_all(args, chroot + "/mnt") + pmb.install.losetup.umount(args, img_path) + pmb.chroot.root(args, ["rm", img_path]) + if os.path.exists(img_path_outside): + raise RuntimeError("Failed to remove old image file: " + + img_path_outside) + + # Create empty image file + size = pmb.config.install_size_image + logging.info("(native) create " + args.device + ".img (" + size + ")") + logging.info("WARNING: Make sure, that your target device's partition" + " table has allocated at least " + size + " as system partition!") + if pmb.helpers.cli.ask(args) != "y": + raise RuntimeError("Aborted.") + + pmb.chroot.user(args, ["mkdir", "-p", "/home/user/rootfs"]) + pmb.chroot.root(args, ["fallocate", "-l", size, img_path]) + + # Mount to /dev/install + logging.info("(native) mount /dev/install (" + args.device + ".img)") + pmb.install.losetup.mount(args, img_path) + device = pmb.install.losetup.device_by_back_file(args, img_path) + pmb.helpers.mount.bind_blockdevice(args, device, args.work + + "/chroot_native/dev/install") + + +def create(args): + """ + Create /dev/install (the "install blockdevice"). + """ + pmb.helpers.mount.umount_all( + args, args.work + "/chroot_native/dev/install") + if args.sdcard: + mount_sdcard(args) + else: + create_and_mount_image(args) diff --git a/pmb/install/format.py b/pmb/install/format.py new file mode 100644 index 00000000..45056aa6 --- /dev/null +++ b/pmb/install/format.py @@ -0,0 +1,58 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import logging +import pmb.chroot + + +def format_and_mount_boot(args): + mountpoint = "/mnt/install/boot" + logging.info("(native) format /dev/installp1 (boot, ext2), mount to " + + mountpoint) + pmb.chroot.root(args, ["mkfs.ext2", "-F", "-q", "/dev/installp1"]) + pmb.chroot.root(args, ["mkdir", "-p", mountpoint]) + pmb.chroot.root(args, ["mount", "/dev/installp1", mountpoint]) + + +def format_and_mount_root(args): + mountpoint = "/dev/mapper/pm_crypt" + logging.info("(native) format /dev/installp2 (root, luks), mount to " + + mountpoint) + pmb.chroot.root(args, ["cryptsetup", "luksFormat", "--use-urandom", + "--cipher", args.cipher, "-q", "/dev/installp2"], log=False) + pmb.chroot.root(args, ["cryptsetup", "luksOpen", "/dev/installp2", + "pm_crypt"], log=False) + if not os.path.exists(args.work + "/chroot_native" + mountpoint): + raise RuntimeError("Failed to open cryptdevice!") + + +def format_and_mount_pm_crypt(args): + cryptdevice = "/dev/mapper/pm_crypt" + mountpoint = "/mnt/install" + logging.info("(native) format " + cryptdevice + " (ext4), mount to " + + mountpoint) + pmb.chroot.root(args, ["mkfs.ext4", "-F", "-q", cryptdevice]) + pmb.chroot.root(args, ["mkdir", "-p", mountpoint]) + pmb.chroot.root(args, ["mount", cryptdevice, mountpoint]) + + +def format(args): + format_and_mount_root(args) + format_and_mount_pm_crypt(args) + format_and_mount_boot(args) diff --git a/pmb/install/install.py b/pmb/install/install.py new file mode 100644 index 00000000..2cf1685d --- /dev/null +++ b/pmb/install/install.py @@ -0,0 +1,113 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os +import glob + +import pmb.chroot +import pmb.chroot.apk +import pmb.config +import pmb.helpers.run +import pmb.install.blockdevice +import pmb.install + + +def copy_files(args): + # Mount the device rootfs + logging.info("(native) copy rootfs_" + args.device + " to" + + " /mnt/install/") + mountpoint = "/mnt/rootfs_" + args.device + pmb.helpers.mount.bind(args, args.work + "/chroot_rootfs_" + args.device, + args.work + "/chroot_native" + mountpoint) + + # Get all folders inside the device rootfs + folders = [] + for path in glob.glob(args.work + "/chroot_native" + mountpoint + "/*"): + folders += [os.path.basename(path)] + + # Run the copy command + pmb.chroot.root(args, ["cp", "-a"] + folders + ["/mnt/install/"], + working_dir=mountpoint) + +# copy over keys and delete unneded mount folders + + +def fix_mount_folders(args): + # copy over keys + rootfs = args.work + "/chroot_native/mnt/install/" + for key in glob.glob(args.work + "/config_apk_keys/*.pub"): + pmb.helpers.run.root(args, ["cp", key, rootfs + "/etc/apk/keys/"]) + + # delete everything (-> empty mount folders) in /home/user + pmb.helpers.run.root(args, ["rm", "-r", rootfs + "/home/user"]) + pmb.helpers.run.root(args, ["mkdir", rootfs + "/home/user"]) + pmb.helpers.run.root(args, ["chown", pmb.config.chroot_uid_user, + rootfs + "/home/user"]) + + +def set_user_password(args): + """ + Loop until the passwords for user and root have been changed successfully. + """ + suffix = "rootfs_" + args.device + while True: + try: + pmb.chroot.root(args, ["passwd", "user"], suffix, log=False) + break + except RuntimeError: + logging.info("WARNING: Failed to set the password. Try it" + " one more time.") + pass + + +def install(args, show_flash_msg=True): + # Install required programs in native chroot + logging.info("*** (1/5) PREPARE NATIVE CHROOT ***") + pmb.chroot.apk.install(args, pmb.config.install_native_packages, + build=False) + + # Install all packages to device rootfs chroot + logging.info("*** (2/5) CREATE DEVICE ROOTFS (" + args.device + ") ***") + suffix = "rootfs_" + args.device + pmb.chroot.apk.install(args, pmb.config.install_device_packages + + ["device-" + args.device], suffix) + pmb.chroot.apk.update(args, suffix) + set_user_password(args) + + # Partition and fill image/sdcard + logging.info("*** (3/5) PREPARE INSTALL BLOCKDEVICE ***") + pmb.chroot.shutdown(args, True) + pmb.install.blockdevice.create(args) + pmb.install.partition(args) + pmb.install.format(args) + + # Just copy all the files + logging.info("*** (4/5) FILL INSTALL BLOCKDEVICE ***") + copy_files(args) + fix_mount_folders(args) + pmb.chroot.shutdown(args, True) + + # Flash to target device + logging.info("*** (5/5) FLASHING TO DEVICE ***") + if show_flash_msg: + logging.info("Run the following to flash your installation to the" + " target device:") + logging.info("* pmbootstrap flasher flash_kernel") + if not args.sdcard: + logging.info("* pmbootstrap flasher flash_system") diff --git a/pmb/install/losetup.py b/pmb/install/losetup.py new file mode 100644 index 00000000..96221c27 --- /dev/null +++ b/pmb/install/losetup.py @@ -0,0 +1,71 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import glob +import json +import logging + +import pmb.helpers.mount +import pmb.helpers.run +import pmb.chroot + + +def init(args): + pmb.helpers.run.root(args, ["modprobe", "loop"]) + for loopdev in glob.glob("/dev/loop*"): + pmb.helpers.mount.bind_blockdevice(args, loopdev, + args.work + "/chroot_native/" + loopdev) + + +def mount(args, img_path): + """ + :param img_path: Path to the img file inside native chroot. + """ + logging.debug("(native) mount " + img_path + " (loop)") + init(args) + pmb.chroot.root(args, ["losetup", "-f", img_path]) + + +def device_by_back_file(args, back_file): + """ + Get the /dev/loopX device, that points to a specific image file. + """ + + # Get list from losetup + losetup_output = pmb.chroot.root(args, ["losetup", "--json", + "--list"], return_stdout=True) + if not losetup_output: + return None + + # Find the back_file + losetup = json.loads(losetup_output) + for loopdevice in losetup["loopdevices"]: + if loopdevice["back-file"] == back_file: + return loopdevice["name"] + return None + + +def umount(args, img_path): + """ + :param img_path: Path to the img file inside native chroot. + """ + device = device_by_back_file(args, img_path) + if not device: + return + logging.debug("(native) umount " + device) + pmb.chroot.root(args, ["losetup", "-d", device]) diff --git a/pmb/install/partition.py b/pmb/install/partition.py new file mode 100644 index 00000000..48c2abea --- /dev/null +++ b/pmb/install/partition.py @@ -0,0 +1,57 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import pmb.chroot +import pmb.config +import pmb.install.losetup + + +def partitions_mount(args): + """ + Mount blockdevices of partitions inside native chroot + """ + prefix = args.sdcard + if not args.sdcard: + img_path = "/home/user/rootfs/" + args.device + ".img" + prefix = pmb.install.losetup.device_by_back_file(args, img_path) + for suffix in ["p1", "p2"]: + pmb.helpers.mount.bind_blockdevice(args, prefix + suffix, + args.work + "/chroot_native/dev/install" + suffix) + + +def partition(args): + """ + Partition /dev/install and create /dev/install{p1,p2} + """ + + size_boot = pmb.config.install_size_boot + logging.info("(native) partition /dev/install (boot: " + size_boot + + ", root: the rest)") + commands = [ + ["mktable", "msdos"], + ["mkpart", "primary", "ext2", "2048s", size_boot], + ["mkpart", "primary", size_boot, "100%"], + ["set", "1", "boot", "on"] + ] + for command in commands: + pmb.chroot.root(args, ["parted", "-s", "/dev/install"] + + command) + + # Mount new partitions + partitions_mount(args) diff --git a/pmb/parse/__init__.py b/pmb/parse/__init__.py new file mode 100644 index 00000000..e71c5ece --- /dev/null +++ b/pmb/parse/__init__.py @@ -0,0 +1,23 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +from pmb.parse.arguments import arguments +from pmb.parse.apkbuild import apkbuild +from pmb.parse.deviceinfo import deviceinfo +from pmb.parse.binfmt_info import binfmt_info +import pmb.parse.arch diff --git a/pmb/parse/apkbuild.py b/pmb/parse/apkbuild.py new file mode 100644 index 00000000..2e0488e8 --- /dev/null +++ b/pmb/parse/apkbuild.py @@ -0,0 +1,125 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import pmb.config + + +def replace_variables(apkbuild): + """ + Replace a hardcoded list of variables inside the APKBUILD. + """ + ret = apkbuild + # _flavor: ${_device} (lineageos kernel packages) + ret["_flavor"] = ret["_flavor"].replace("${_device}", + ret["_device"]) + + # pkgname: $_flavor + ret["pkgname"] = ret["pkgname"].replace("${_flavor}", ret["_flavor"]) + + # subpackages: $pkgname + replaced = [] + for subpackage in ret["subpackages"]: + replaced.append(subpackage.replace("$pkgname", ret["pkgname"])) + ret["subpackages"] = replaced + + # makedepend: $makedepends_host, $makedepends_build, $_llvmver + replaced = [] + for makedepend in ret["makedepends"]: + if makedepend.startswith("$"): + key = makedepend[1:] + if key in ret: + replaced += ret[key] + else: + raise RuntimeError("Could not resolve variable " + + makedepend + " in APKBUILD of " + + apkbuild["pkgname"]) + else: + # replace in the middle of the string + for var in ["_llvmver"]: + makedepend = makedepend.replace("$" + var, ret[var]) + replaced += [makedepend] + ret["makedepends"] = replaced + return ret + + +def cut_off_function_names(apkbuild): + """ + For subpackages: only keep the subpackage name, without the internal + function name, that tells how to build the subpackage. + """ + sub = apkbuild["subpackages"] + for i in range(len(sub)): + sub[i] = sub[i].split(":", 1)[0] + apkbuild["subpackages"] = sub + return apkbuild + + +def apkbuild(path): + """ + Parse relevant information out of the APKBUILD file. This is not meant + to be perfect and catch every edge case (for that, a full shell parser + would be necessary!). Instead, it should just work with the use-cases + covered by pmbootstrap and not take too long. + + :param path: Full path to the APKBUILD + :returns: Relevant variables from the APKBUILD. Arrays get returned as + arrays. + """ + with open(path, encoding="utf-8") as handle: + lines = handle.readlines() + + # Parse all attributes from the config + ret = {} + for i in range(len(lines)): + for attribute, options in pmb.config.apkbuild_attributes.items(): + if not lines[i].startswith(attribute + "="): + continue + + # Extend the line value until we reach the ending quote sign + line_value = lines[i][len(attribute + "="):-1] + end_char = None + if line_value.startswith("\""): + end_char = "\"" + value = "" + while i < len(lines) - 1: + value += line_value.replace("\"", "").strip() + if not end_char or line_value.endswith(end_char): + break + value += " " + i += 1 + line_value = lines[i][:-1] + + # Split up arrays, delete empty strings inside the list + if options["array"]: + if value: + value = list(filter(None, value.split(" "))) + else: + value = [] + ret[attribute] = value + + # Add missing entries + for attribute, options in pmb.config.apkbuild_attributes.items(): + if attribute not in ret: + if options["array"]: + ret[attribute] = [] + else: + ret[attribute] = "" + + ret = replace_variables(ret) + ret = cut_off_function_names(ret) + return ret diff --git a/pmb/parse/apkindex.py b/pmb/parse/apkindex.py new file mode 100644 index 00000000..ead1a85d --- /dev/null +++ b/pmb/parse/apkindex.py @@ -0,0 +1,109 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import distutils.version +import glob +import os +import tarfile + + +def compare_version(a_str, b_str): + """ + http://stackoverflow.com/a/11887885 + LooseVersion behaves just like apk's version check, at least + for all package versions, that have "-r". + + :returns: + (a < b): -1 + (a == b): 0 + (a > b): 1 + """ + if a_str is None: + a_str = "0" + if b_str is None: + b_str = "0" + a = distutils.version.LooseVersion(a_str) + b = distutils.version.LooseVersion(b_str) + if a < b: + return -1 + if a == b: + return 0 + return 1 + + +def read(args, package, path, must_exist=True): + """ + :param path: Path to APKINDEX.tar.gz, defaults to $WORK/APKINDEX.tar.gz + :param package: The package of which you want to read the properties. + :param must_exist: When set to true, raise an exception when the package is + missing in the index, or the index file was not found. + :returns: {"pkgname": ..., "version": ..., "depends": [...]} + When the package appears multiple times in the APKINDEX, this + function returns the attributes of the latest version. + """ + # Verify APKINDEX path + if not os.path.exists(path): + if not must_exist: + return None + raise RuntimeError("File not found: " + path) + + # Read the tarfile + ret = None + with tarfile.open(path, "r:gz") as tar: + with tar.extractfile(tar.getmember("APKINDEX")) as handle: + current = {} + for line in handle: + line = line.decode() + if line == "\n": # end of package + if current["pkgname"] == package: + if not ret or compare_version(current["version"], + ret["version"]) == 1: + ret = current + current = {} + if line.startswith("P:"): # package + current["pkgname"] = line[2:-1] + if line.startswith("V:"): # version + current["version"] = line[2:-1] + if line.startswith("D:"): # depends + depends = line[2:-1] + if depends: + current["depends"] = depends.split(" ") + else: + current["depends"] = [] + if not ret and must_exist: + raise RuntimeError("Package " + package + " not found in " + path) + return ret + + +def read_any_index(args, package, arch=None): + """ + Check if *any* APKINDEX has a specific package. + + :param arch: defaults to native architecture + """ + if not arch: + arch = args.arch_native + indexes = [args.work + "/packages/" + arch + "/APKINDEX.tar.gz"] + pattern = args.work + "/cache_apk_" + arch + "/APKINDEX.*.tar.gz" + indexes += glob.glob(pattern) + + for index in indexes: + index_data = read(args, package, index, False) + if index_data: + return index_data + return None diff --git a/pmb/parse/arch.py b/pmb/parse/arch.py new file mode 100644 index 00000000..666b30db --- /dev/null +++ b/pmb/parse/arch.py @@ -0,0 +1,103 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import platform +import logging +import fnmatch + + +def alpine_native(): + machine = platform.machine() + ret = "" + + if machine == "x86_64": + ret = "x86_64" + else: + raise ValueError("Can not map platform.machine " + machine + + " to the right Alpine Linux architecture") + + logging.debug("(native) Alpine architecture: " + ret) + return ret + + +def from_chroot_suffix(args, suffix): + if suffix == "native": + return args.arch_native + if suffix == "rootfs_" + args.device: + return args.deviceinfo["arch"] + if suffix.startswith("buildroot_"): + return suffix.split("_", 2)[1] + + raise ValueError("Invalid chroot suffix: " + suffix + + " (wrong device chosen in 'init' step?)") + + +def alpine_to_debian(arch): + """ + Convert the architecture to the string used in the binfmt info + (aka. the Debian architecture format). + """ + + mapping = { + "x86_64": "amd64", + "armhf": "arm", + } + for pattern, arch_debian in mapping.items(): + if fnmatch.fnmatch(arch, pattern): + return arch_debian + raise ValueError("Can not map Alpine architecture " + arch + + " to the right Debian architecture.") + + +def alpine_to_kernel(arch): + """ + Convert the architecture to the string used inside the kernel sources. + You can read the mapping from the linux-vanilla APKBUILD for example. + """ + mapping = { + "aarch64*": "arm64", + "arm*": "arm", + "ppc*": "powerpc", + "s390*": "s390" + } + for pattern, arch_kernel in mapping.items(): + if fnmatch.fnmatch(arch, pattern): + return arch_kernel + return arch + + +def alpine_to_hostspec(arch): + """ + See: abuild source code/functions.sh.in: arch_to_hostspec() + """ + mapping = { + "aarch64": "aarch64-alpine-linux-musl", + "armhf": "armv6-alpine-linux-muslgnueabihf", + "armv7": "armv7-alpine-linux-musleabihf", + "ppc": "powerpc-alpine-linux-musl", + "ppc64": "powerpc64-alpine-linux-musl", + "ppc64le": "powerpc64le-alpine-linux-musl", + "s390x": "s390x-alpine-linux-musl", + "x86": "i586-alpine-linux-musl", + "x86_66": "x86_64-alpine-linux-musl", + } + if arch in mapping: + return mapping[arch] + + raise ValueError("Can not map Alpine architecture " + arch + + " to the right hostspec value") diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py new file mode 100644 index 00000000..f6c98ad5 --- /dev/null +++ b/pmb/parse/arguments.py @@ -0,0 +1,145 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import argparse +import pmb.config +import pmb.parse.arch + + +def arguments_flasher(subparser): + ret = subparser.add_parser("flasher", help="flash something to the" + " target device") + sub = ret.add_subparsers(dest="action_flasher") + + # Other + sub.add_parser("flash_system", help="flash the system partition") + sub.add_parser("list_flavors", help="list installed kernel flavors" + + " inside the device rootfs chroot on this computer") + sub.add_parser("list_devices", help="show connected devices") + + # Boot, flash kernel + boot = sub.add_parser("boot", help="boot a kernel once") + flash_kernel = sub.add_parser("flash_kernel", help="flash a kernel") + for action in [boot, flash_kernel]: + action.add_argument("--flavor", default=None) + + return ret + + +def arguments(): + parser = argparse.ArgumentParser(prog="pmbootstrap") + + # Other + parser.add_argument("-V", "--version", action="version", + version=pmb.config.version) + parser.add_argument("--no-cross", action="store_false", dest="cross", + help="disable crosscompiler, build only with qemu + gcc (slower!)") + + parser.add_argument("-a", "--alpine-version", dest="alpine_version", + help="examples: edge, latest-stable, v3.5") + parser.add_argument("-c", "--config", dest="config", + default=pmb.config.defaults["config"]) + parser.add_argument("-d", "--port-distccd", dest="port_distccd") + parser.add_argument("-m", "--mirror-alpine", dest="mirror_alpine") + parser.add_argument("-j", "--jobs", help="parallel jobs when compiling") + parser.add_argument("-p", "--aports", + help="postmarketos aports paths") + parser.add_argument("-w", "--work", help="folder where all data" + " gets stored (chroots, caches, built packages)") + + # Logging + parser.add_argument("-l", "--log", dest="log", default=None) + parser.add_argument("-v", "--verbose", dest="verbose", + action="store_true", help="output the source file, where the log" + " message originated from with each log message") + parser.add_argument("-q", "--quiet", dest="quiet", + action="store_true", help="do not output any log messages") + + # Actions + sub = parser.add_subparsers(title="action", dest="action") + sub.add_parser("init", help="initialize config file") + sub.add_parser("log", help="follow the pmbootstrap logfile") + sub.add_parser("log_distccd", help="follow the distccd logfile") + sub.add_parser("shutdown", help="umount, unregister binfmt") + sub.add_parser("index", help="re-index all repositories with custom built" + " packages (do this after manually removing package files)") + arguments_flasher(sub) + + # Action: zap + zap = sub.add_parser("zap", help="safely delete chroot" + "folders") + zap.add_argument("-p", "--packages", action="store_true", help="also delete" + " the precious, self-compiled packages") + zap.add_argument("-hc", "--http", action="store_true", help="also delete http" + "cache") + + # Action: stats + stats = sub.add_parser("stats", help="show ccache stats") + stats.add_argument("--arch") + + # Action: chroot / build_init / kernel + build_init = sub.add_parser("build_init", help="initialize build" + " environment (usually you do not need to call this)") + chroot = sub.add_parser("chroot", help="start shell in chroot") + chroot.add_argument("command", default=["sh"], help="command" + " to execute inside the chroot. default: sh", nargs='*') + for action in [build_init, chroot]: + action.add_argument("--suffix", default="native") + + # Action: install + install = sub.add_parser("install", help="set up device specific" + + " chroot and install to sdcard or image file") + install.add_argument("--sdcard", help="path to the sdcard device," + " eg. /dev/mmcblk0") + install.add_argument("--cipher", help="cryptsetup cipher used to" + " encrypt the system partition, eg. aes-xts-plain64") + + # Action: build / checksum / menuconfig / parse_apkbuild / aportgen + menuconfig = sub.add_parser("menuconfig", help="run menuconfig on" + " a kernel aport") + checksum = sub.add_parser("checksum", help="update aport checksums") + parse_apkbuild = sub.add_parser("parse_apkbuild") + aportgen = sub.add_parser("aportgen", help="generate a package build recipe" + " (aport/APKBUILD) based on an upstream aport from Alpine") + build = sub.add_parser("build", help="create a package for a" + " specific architecture") + build.add_argument("--arch") + build.add_argument("--force", action="store_true") + for action in [checksum, build, menuconfig, parse_apkbuild, aportgen]: + action.add_argument("package") + + # Use defaults from the user's config file + args = parser.parse_args() + cfg = pmb.config.load(args) + for varname in cfg["pmbootstrap"]: + if varname not in args or not getattr(args, varname): + setattr(args, varname, cfg["pmbootstrap"][varname]) + + # Replace $WORK in variables from user's config + for varname in cfg["pmbootstrap"]: + old = getattr(args, varname) + setattr(args, varname, old.replace("$WORK", args.work)) + + # Add convinience shortcuts + setattr(args, "arch_native", pmb.parse.arch.alpine_native()) + + # Add the deviceinfo (only after initialization) + if args.action != "init": + setattr(args, "deviceinfo", pmb.parse.deviceinfo(args)) + + return args diff --git a/pmb/parse/binfmt_info.py b/pmb/parse/binfmt_info.py new file mode 100644 index 00000000..57c0ef7c --- /dev/null +++ b/pmb/parse/binfmt_info.py @@ -0,0 +1,48 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging + +# Get magic and mask from binfmt info file +# Return: {magic: ..., mask: ...} + + +def binfmt_info(args, arch_debian): + # Parse the info file + full = {} + info = args.work + "/chroot_native/usr/share/qemu-user-binfmt.txt" + logging.debug("parsing: " + info) + with open(info, "r") as handle: + for line in handle: + if line.startswith('#') or "=" not in line: + continue + splitted = line.split("=") + key = splitted[0].strip() + value = splitted[1] + full[key] = value[1:-2] + + ret = {} + logging.debug("filtering by architecture: " + arch_debian) + for type in ["mask", "magic"]: + key = arch_debian + "_" + type + if key not in full: + raise RuntimeError("Could not find key " + key + " in binfmt info file:" + + info) + ret[type] = full[key] + logging.debug("=> " + str(ret)) + return ret diff --git a/pmb/parse/deviceinfo.py b/pmb/parse/deviceinfo.py new file mode 100644 index 00000000..d54fdf46 --- /dev/null +++ b/pmb/parse/deviceinfo.py @@ -0,0 +1,51 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os + + +def deviceinfo(args, device=None): + """ + :param device: defaults to args.device + """ + if not device: + device = args.device + + aport = args.aports + "/device-" + device + if not os.path.exists(aport) or not os.path.exists(aport + "/deviceinfo"): + logging.fatal("You will need to create a device-specific package") + logging.fatal("before you can continue. Please create at least the") + logging.fatal("following files:") + logging.fatal(aport + "/APKBUILD") + logging.fatal(aport + "/deviceinfo") + raise RuntimeError("Incomplete device information") + + ret = {} + path = aport + "/deviceinfo" + with open(path) as handle: + for line in handle: + if not line.startswith("deviceinfo_"): + continue + if "=" not in line: + raise SyntaxError(path + ": No '=' found:\n\t" + line) + split = line.split("=", 1) + key = split[0][len("deviceinfo_"):] + value = split[1].replace("\"", "").replace("\n", "") + ret[key] = value + return ret diff --git a/pmbootstrap.py b/pmbootstrap.py new file mode 100755 index 00000000..1d1c7e48 --- /dev/null +++ b/pmbootstrap.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" + +import sys +import logging +import os +import json +import traceback + +import pmb.aportgen +import pmb.build +import pmb.config +import pmb.chroot +import pmb.chroot.other +import pmb.flasher +import pmb.helpers.logging +import pmb.helpers.run +import pmb.parse +import pmb.install + + +def main(): + try: + # Parse arguments + args = pmb.parse.arguments() + pmb.helpers.logging.init(args) + + # Initialize or require config + if args.action == "init": + return pmb.config.init(args) + if not os.path.exists(args.config): + logging.critical("Please specify a config file, or run" + " 'pmbootstrap init' to generate one.") + return 1 + + # All other actions + if args.action == "aportgen": + pmb.aportgen.generate(args, args.package) + elif args.action == "build": + pmb.build.package(args, args.package, args.arch, args.force, False) + elif args.action == "build_init": + pmb.build.init(args, args.suffix) + elif args.action == "checksum": + pmb.build.checksum(args, args.package) + elif args.action == "chroot": + pmb.chroot.root(args, args.command, args.suffix, log=False) + elif args.action == "index": + pmb.build.index_repo(args) + elif args.action == "install": + pmb.install.install(args) + elif args.action == "flasher": + pmb.flasher.frontend(args) + elif args.action == "menuconfig": + pmb.build.menuconfig(args, args.package, args.deviceinfo["arch"]) + elif args.action == "parse_apkbuild": + print(json.dumps(pmb.parse.apkbuild(args.aports + "/" + + args.package + "/APKBUILD"), indent=4)) + elif args.action == "shutdown": + pmb.chroot.shutdown(args) + elif args.action == "stats": + pmb.build.ccache_stats(args, args.arch) + elif args.action == "log": + pmb.helpers.run.user(args, ["tail", "-f", args.log], log=False) + elif args.action == "log_distccd": + pmb.chroot.user(args, ["tail", "-f", "/home/user/distccd.log"], + log=False) + elif args.action == "zap": + pmb.chroot.zap(args) + else: + logging.info("Run pmbootstrap -h for usage information.") + + # Print finish timestamp + logging.info("Done") + + except Exception as e: + logging.info("ERROR: " + str(e)) + logging.info("Run 'pmbootstrap log' for details.") + logging.debug(traceback.format_exc()) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/test_apk_static.py b/test/test_apk_static.py new file mode 100644 index 00000000..a3ea8300 --- /dev/null +++ b/test/test_apk_static.py @@ -0,0 +1,126 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +#!/usr/bin/env python3 +import os +import sys +import tarfile +import glob +import pytest + +# Import from parent directory +pmb_src = os.path.abspath(os.path.join(os.path.dirname(__file__) + "/..")) +sys.path.append(pmb_src) +import pmb.chroot.apk_static +import pmb.parse.apkindex + + +@pytest.fixture +def args(): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + setattr(args, "logfd", open("/dev/null", "a+")) + yield args + args.logfd.close() + + +def test_read_signature_info(tmpdir): + with tarfile.open(tmpdir + "/test.apk", "w:gz") as tar: + # No signature found + with pytest.raises(RuntimeError) as e: + pmb.chroot.apk_static.read_signature_info(tar) + assert "Could not find signature" in str(e.value) + + # Add signature file with invalid name + tar.add(__file__, "sbin/apk.static.SIGN.RSA.invalid.pub") + with pytest.raises(RuntimeError) as e: + pmb.chroot.apk_static.read_signature_info(tar) + assert "Invalid signature key" in str(e.value) + + # Add signature file with realistic name + path = glob.glob(pmb_src + "/keys/*.pub")[0] + name = os.path.basename(path) + path_archive = "sbin/apk.static.SIGN.RSA." + name + with tarfile.open(tmpdir + "/test2.apk", "w:gz") as tar: + tar.add(__file__, path_archive) + sigfilename, sigkey_path = pmb.chroot.apk_static.read_signature_info( + tar) + assert sigfilename == path_archive + assert sigkey_path == path + + +def test_successful_extraction(args, tmpdir): + if os.path.exists(args.work + "/apk.static"): + os.remove(args.work + "/apk.static") + + pmb.chroot.apk_static.init(args) + assert os.path.exists(args.work + "/apk.static") + os.remove(args.work + "/apk.static") + + +def test_signature_verification(args, tmpdir): + if os.path.exists(args.work + "/apk.static"): + os.remove(args.work + "/apk.static") + + apk_index = pmb.chroot.apk_static.download(args, "APKINDEX.tar.gz") + version = pmb.parse.apkindex.read(args, "apk-tools-static", + apk_index)["version"] + apk_path = pmb.chroot.apk_static.download(args, + "apk-tools-static-" + version + ".apk") + + # Extract to temporary folder + with tarfile.open(apk_path, "r:gz") as tar: + sigfilename, sigkey_path = pmb.chroot.apk_static.read_signature_info( + tar) + files = pmb.chroot.apk_static.extract_temp(tar, sigfilename) + + # Verify signature (successful) + pmb.chroot.apk_static.verify_signature(args, files, sigkey_path) + + # Append data to extracted apk.static + with open(files["apk"]["temp_path"], "ab") as handle: + handle.write("appended something".encode()) + + # Verify signature again (fail) (this deletes the tempfiles) + with pytest.raises(RuntimeError) as e: + pmb.chroot.apk_static.verify_signature(args, files, sigkey_path) + assert "Failed to validate signature" in str(e.value) + + # + # Test "apk.static --version" check + # + with pytest.raises(RuntimeError) as e: + pmb.chroot.apk_static.extract(args, "99.1.2-r1", apk_path) + assert "downgrade attack" in str(e.value) + + +def test_outdated_version(args): + if os.path.exists(args.work + "/apk.static"): + os.remove(args.work + "/apk.static") + + # change min version + min = pmb.config.apk_tools_static_min_version + pmb.config.apk_tools_static_min_version = "99.1.2-r1" + + with pytest.raises(RuntimeError) as e: + pmb.chroot.apk_static.init(args) + assert "outdated version" in str(e.value) + + # reset min version + pmb.config.apk_tools_static_min_version = min diff --git a/test/test_aportgen.py b/test/test_aportgen.py new file mode 100644 index 00000000..c702f840 --- /dev/null +++ b/test/test_aportgen.py @@ -0,0 +1,60 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +#!/usr/bin/env python3 +import os +import sys +import pytest +import filecmp + +# Import from parent directory +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__) + "/.."))) +import pmb.aportgen + + +@pytest.fixture +def args(tmpdir): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + setattr(args, "logfd", open("/dev/null", "a+")) + setattr(args, "_aports_real", args.aports) + args.aports = str(tmpdir) + yield args + args.logfd.close() + + +def test_aportgen(args): + # Create aportgen folder -> code path where it still exists + pmb.helpers.run.user(args, ["mkdir", "-p", args.work + "/aportgen"]) + + # Generate all valid packages (gcc-armhf twice, so the output folder + # exists) + for pkgname in ["binutils-armhf", "musl-armhf", "gcc-armhf", "gcc-armhf"]: + pmb.aportgen.generate(args, pkgname) + path_new = args.aports + "/" + pkgname + "/APKBUILD" + path_old = args._aports_real + "/" + pkgname + "/APKBUILD" + assert os.path.exists(path_new) + assert filecmp.cmp(path_new, path_old, False) + + +def test_aportgen_invalid_generator(args): + with pytest.raises(ValueError) as e: + pmb.aportgen.generate(args, "pkgname-with-no-generator") + assert "No generator available" in str(e.value) diff --git a/test/test_build.py b/test/test_build.py new file mode 100644 index 00000000..3a22c7d8 --- /dev/null +++ b/test/test_build.py @@ -0,0 +1,48 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +#!/usr/bin/env python3 +import os +import sys +import pytest + +# Import from parent directory +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__) + "/.."))) +import pmb.aportgen + + +@pytest.fixture +def args(tmpdir): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + setattr(args, "logfd", open("/dev/null", "a+")) + yield args + args.logfd.close() + + +def test_build(args): + pmb.build.package(args, "hello-world", args.arch_native, True) + + +def test_build_armhf(args): + """ + Build in armhf chroot, with cross-compiler through distcc. + """ + pmb.build.package(args, "hello-world", "armhf", True) diff --git a/test/test_keys.py b/test/test_keys.py new file mode 100644 index 00000000..6d8a3525 --- /dev/null +++ b/test/test_keys.py @@ -0,0 +1,57 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +#!/usr/bin/env python3 +import os +import sys +import pytest +import glob +import filecmp + +# Import from parent directory +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__) + "/.."))) +import pmb.parse.apkindex +import pmb.helpers.git + + +@pytest.fixture +def args(): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + setattr(args, "logfd", open("/dev/null", "a+")) + yield args + args.logfd.close() + return args + + +def test_keys(args): + mirror_path = os.path.join(os.path.dirname(__file__) + "../keys") + original_path = args.work + "/cache_git/aports_upstream/main/alpine-keys" + pmb.helpers.git.clone(args, "aports_upstream") + + # Check if original keys are mirrored correctly + for path in glob.glob(original_path + "/*.key"): + key = os.path.basename(path) + assert filecmp.cmp(original_path + "/" + key, mirror_path + "/" + key, + False) + + # Find outdated keys, which need to be removed + for path in glob.glob(mirror_path + "/*.key"): + assert os.path.exists(original_path + "/" + os.path.basename(path)) diff --git a/test/test_shell_escape.py b/test/test_shell_escape.py new file mode 100644 index 00000000..aa72c257 --- /dev/null +++ b/test/test_shell_escape.py @@ -0,0 +1,65 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +#!/usr/bin/env python3 +import os +import sys +import pytest + +# Import from parent directory +pmb_src = os.path.abspath(os.path.join(os.path.dirname(__file__) + "/..")) +sys.path.append(pmb_src) +import pmb.helpers.run +import pmb.chroot.root +import pmb.chroot.user + + +@pytest.fixture +def args(): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + setattr(args, "logfd", open("/dev/null", "a+")) + yield args + args.logfd.close() + + +def test_shell_escape(args): + cmds = { + "test\n": ["echo", "test"], + "test && test\n": ["echo", "test", "&&", "test"], + "test ; test\n": ["echo", "test", ";", "test"], + "'test\"test\\'\n": ["echo", "'test\"test\\'"], + "*\n": ["echo", "*"], + "$PWD\n": ["echo", "$PWD"], + } + for expected, cmd in cmds.items(): + core = pmb.helpers.run.core(args, cmd, "test", True, True) + assert expected == core + + user = pmb.helpers.run.user(args, cmd, return_stdout=True) + assert expected == user + + root = pmb.helpers.run.root(args, cmd, return_stdout=True) + assert expected == root + + chroot_root = pmb.chroot.root(args, cmd, return_stdout=True) + assert expected == chroot_root + + chroot_user = pmb.chroot.user(args, cmd, return_stdout=True) + assert expected == chroot_user diff --git a/test/test_subprocess.py b/test/test_subprocess.py new file mode 100644 index 00000000..3a26a742 --- /dev/null +++ b/test/test_subprocess.py @@ -0,0 +1,33 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import glob + + +def test_use_pmb_helpers_run_instead_of_subprocess_run(): + src = os.path.abspath(os.path.dirname(__file__) + "/..") + files = glob.glob(src + "/pmb/**/*.py", + recursive=True) + glob.glob(src + "*.py") + okay = os.path.abspath(src + "/pmb/helpers/run.py") + for file in files: + with open(file, "r") as handle: + source = handle.read() + if file != okay and "subprocess.run" in source: + raise RuntimeError("File " + file + " use pmb.helpers.run.user()" + " instead of subprocess.run()!") diff --git a/test/test_version.py b/test/test_version.py new file mode 100644 index 00000000..c9489058 --- /dev/null +++ b/test/test_version.py @@ -0,0 +1,69 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +#!/usr/bin/env python3 +import os +import sys +import pytest + +# Import from parent directory +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__) + "/.."))) +import pmb.parse.apkindex +import pmb.helpers.git + + +@pytest.fixture +def args(): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + setattr(args, "logfd", open("/dev/null", "a+")) + yield args + args.logfd.close() + return args + + +def test_version(args): + # clone official test file from apk-tools + pmb.helpers.git.clone(args, "apk-tools") + path = args.work + "/cache_git/apk-tools/test/version.data" + + mapping = {-1: "<", 0: "=", 1: ">"} + with open(path) as handle: + for line in handle: + split = line.split(" ") + a = split[0] + b = split[2].rstrip() + expected = split[1] + + # Alpine packages nowadays always have '-r' in their version + if "-r" not in a or "-r" not in b: + continue + + print(line.rstrip()) + try: + result = pmb.parse.apkindex.compare_version(a, b) + real = mapping[result] + except TypeError: + # FIXME: Bug in Python: + # https://bugs.python.org/issue14894 + # When this happens in pmbootstrap, it will also raise the + # TypeError exception. + continue + assert(real == expected)