pmb: qemu-user-static from aport, not from Debian

Overview:
In order to execute foreign arch binaries on the host system, we are
using the Linux kernel's binfmt_misc feature in combination with
static builds of QEMU. Before this patch, the statically compiled
QEMU binaries were taken from Debian (mostly because I did not realize
that Alpine ships them as well). Now we can use the ones from the aport.

Benefits:
This allows us to easily update and patch the QEMU executables, we
don't need to be in sync with Debian's versions anymore.

Alpine's package is more modular, so we can save some download,
install, zap time, as well as disk space: setting up an armhf chroot
with pmbootstrap took ~102 MB before, now it's ~18 MB.

Detailed changes:
* Remove `cross/qemu-user-static-repack` aport
* Add `data/qemu-user-binfmt.txt` with the binfmt_misc flags for ELF
  binaries of various arches (extracted from Debian's packaging)
* When parsing that file, don't write verbose messages to
  `pmbootstrap log` anymore, only to the verbose log (can be enabled
  with `pmbootstrap -v`)
* Rename `pmb.parse.arch.alpine_to_debian()` to ...`alpine_to_qemu()`
* Rename `arch_debian` to `arch_qemu`

aports/cross/qemu-user-static-repack/APKBUILD

@@ -1,62 +0,0 @@
-pkgname="qemu-user-static-repack"
-pkgver=2.8
-pkgrel=11
-pkgdesc="QEMU user mode emulation binaries (static version)"
-arch="all"
-url="https://wiki.debian.org/DebianKernel/ARMMP"
-license="GPL2"
-_debver="${pkgver}+dfsg-6+deb9u3"
-_deburl="https://deb.debian.org/debian/pool/main/q/qemu/qemu-user-static_${_debver}"
-source="
-	qemu-user-static_${_debver}_x86.deb::${_deburl}_i386.deb
-	qemu-user-static_${_debver}_x86_64.deb::${_deburl}_amd64.deb
-	qemu-user-static_${_debver}_armhf.deb::${_deburl}_armhf.deb
-	qemu-user-static_${_debver}_aarch64.deb::${_deburl}_arm64.deb
-	"
-makedepends="tar xz"
-subpackages="$pkgname-doc $pkgname-binfmt:binfmt:noarch"
-options="!check"
-
-unpack() {
-	cd "$srcdir"
-	for i in $source; do
-		case ${i%::*} in
-			*${CARCH}.deb) ar x ${i%::*} ;;
-		esac
-	done
-
-	# postinst in this archive contains the binfmt information
-	tar -xf "$srcdir/control.tar.gz"
-}
-
-package() {
-	mkdir -p "$pkgdir"
-	tar -xJf "$srcdir"/data.tar.xz -C $pkgdir
-	return 0
-}
-
-_binfmtout="$srcdir/qemu-user-binfmt.txt"
-build() {
-	cd "$srcdir"
-	for line in \
-		"# Non-standard file format with grepped binfmt information" \
-		"# from Debians postinst script. Used in pmbootstrap."
-	do
-		echo "$line" >> $_binfmtout
-	done
-
-	for suffix in mask magic; do
-		grep "_${suffix}=" postinst >> $_binfmtout
-	done
-}
-
-binfmt() {
-	mkdir -p "$pkgdir-binfmt"
-	install -Dm644 $_binfmtout \
-		"$pkgdir-binfmt/usr/share/qemu-user-binfmt.txt"
-}
-
-sha512sums="466544a6e2b2ee4b0a25d290d9a95dd9082131118ee2818436e0a1ab4835ed5eaed0d4479dd392c2285dcb70e572a3323da970ff625706b9d72164992e8dd4a6  qemu-user-static_2.8+dfsg-6+deb9u3_x86.deb
-96ffac25444f7fd3275dfb5c00b73a033b4668572aac3448e77e8d87ca27698f9b32fc7d57f1b311ee8b0662a7d7cb39c716bbdb433e0bd798e7b6d2d8380c2a  qemu-user-static_2.8+dfsg-6+deb9u3_x86_64.deb
-d7aac0a40e3795a3cc83131f95d5e5c8e6164d29fc58a412ee5b5306017f9d47b3c659504bea93b87dbbe86ee07b60284c38e58476a6d7142f3cfde392f75502  qemu-user-static_2.8+dfsg-6+deb9u3_armhf.deb
-cb46e7ce7091b8d032a1a2fc7238ecce0238494517a23d64ad92dc50dc5ba5acdfb9e0d907db3c46d4c23d6c63612aa07d610ed3e11c4664c61d6bafe5b117ed  qemu-user-static_2.8+dfsg-6+deb9u3_aarch64.deb"

data/qemu-user-binfmt.txt

@@ -0,0 +1,61 @@
+# Magic and mask parameters of Linux ELF binaries of various CPU architectures,
+# required to configure binfmt_misc to run foreign arch binaries with QEMU.
+# Extracted from Debian's QEMU packaging ("binfmt-update-in").
+# See also: <https://en.wikipedia.org/wiki/Binfmt_misc>
+
+    aarch64_magic='\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00'
+     aarch64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+      alpha_magic='\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x26\x90'
+       alpha_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+        arm_magic='\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00'
+         arm_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+      armeb_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28'
+       armeb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+       cris_magic='\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x4c\x00'
+        cris_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+       hppa_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x0f'
+        hppa_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+       i386_magic='\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00'
+        i386_mask='\xff\xff\xff\xff\xff\xfe\xfe\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+       m68k_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x04'
+        m68k_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+ microblaze_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\xba\xab'
+  microblaze_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+       mips_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
+        mips_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+     mipsel_magic='\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
+      mipsel_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xfe\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+     mips64_magic='\x7f\x45\x4c\x46\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
+      mips64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+   mips64el_magic='\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
+    mips64el_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xfe\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+        ppc_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x14'
+         ppc_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+      ppc64_magic='\x7f\x45\x4c\x46\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15'
+       ppc64_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+ ppc64abi32_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15'
+  ppc64abi32_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+    ppc64le_magic='\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15\x00'
+     ppc64le_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\x00'
+    riscv32_magic='\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xf3\x00'
+     riscv32_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+    riscv64_magic='\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xf3\x00'
+     riscv64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+      s390x_magic='\x7f\x45\x4c\x46\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x16'
+       s390x_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+        sh4_magic='\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a\x00'
+         sh4_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+      sh4eb_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a'
+       sh4eb_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+      sparc_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x02'
+       sparc_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+sparc32plus_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x12'
+ sparc32plus_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+    sparc64_magic='\x7f\x45\x4c\x46\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2b'
+     sparc64_mask='\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+     x86_64_magic='\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00'
+      x86_64_mask='\xff\xff\xff\xff\xff\xfe\xfe\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+     xtensa_magic='\x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x5e\x00'
+      xtensa_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+   xtensaeb_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x5e'
+    xtensaeb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'

pmb/chroot/binfmt.py

@@ -25,46 +25,45 @@ import pmb.parse
 import pmb.parse.arch
 
 
-def is_registered(arch_debian):
+def is_registered(arch_qemu):
-    return os.path.exists("/proc/sys/fs/binfmt_misc/qemu-" + arch_debian)
+    return os.path.exists("/proc/sys/fs/binfmt_misc/qemu-" + arch_qemu)
 
 
 def register(args, arch):
     """
     Get arch, magic, mask.
     """
-    arch_debian = pmb.parse.arch.alpine_to_debian(arch)
+    arch_qemu = pmb.parse.arch.alpine_to_qemu(arch)
-    if is_registered(arch_debian):
+    if is_registered(arch_qemu):
         return
     pmb.helpers.other.check_binfmt_misc(args)
-    pmb.chroot.apk.install(args, ["qemu-user-static-repack",
+    pmb.chroot.apk.install(args, ["qemu-" + arch_qemu])
-                                  "qemu-user-static-repack-binfmt"])
+    info = pmb.parse.binfmt_info(args, arch_qemu)
-    info = pmb.parse.binfmt_info(args, arch_debian)
 
     # Build registration string
     # https://en.wikipedia.org/wiki/Binfmt_misc
     # :name:type:offset:magic:mask:interpreter:flags
-    name = "qemu-" + arch_debian
+    name = "qemu-" + arch_qemu
     type = "M"
     offset = ""
     magic = info["magic"]
     mask = info["mask"]
-    interpreter = "/usr/bin/qemu-" + arch_debian + "-static"
+    interpreter = "/usr/bin/qemu-" + arch_qemu + "-static"
     flags = "C"
     code = ":".join(["", name, type, offset, magic, mask, interpreter,
                      flags])
 
     # Register in binfmt_misc
-    logging.info("Register qemu binfmt (" + arch_debian + ")")
+    logging.info("Register qemu binfmt (" + arch_qemu + ")")
     register = "/proc/sys/fs/binfmt_misc/register"
     pmb.helpers.run.root(
         args, ["sh", "-c", 'echo "' + code + '" > ' + register])
 
 
 def unregister(args, arch):
-    arch_debian = pmb.parse.arch.alpine_to_debian(arch)
+    arch_qemu = pmb.parse.arch.alpine_to_qemu(arch)
-    binfmt_file = "/proc/sys/fs/binfmt_misc/qemu-" + arch_debian
+    binfmt_file = "/proc/sys/fs/binfmt_misc/qemu-" + arch_qemu
     if not os.path.exists(binfmt_file):
         return
-    logging.info("Unregister qemu binfmt (" + arch_debian + ")")
+    logging.info("Unregister qemu binfmt (" + arch_qemu + ")")
     pmb.helpers.run.root(args, ["sh", "-c", "echo -1 > " + binfmt_file])

pmb/chroot/init.py

@@ -62,10 +62,8 @@ def init(args, suffix="native"):
     # Require apk-tools-static
     pmb.chroot.apk_static.init(args)
 
-    # Non-native chroot: require qemu-user-static
+    # Non-native chroot: set up QEMU with binfmt_misc
     if emulate:
-        pmb.chroot.apk.install(args, ["qemu-user-static-repack",
-                                      "qemu-user-static-repack-binfmt"])
         pmb.chroot.binfmt.register(args, arch)
 
     logging.info("(" + suffix + ") install alpine-base")
@@ -84,11 +82,11 @@ def init(args, suffix="native"):
 
     # Non-native chroot: install qemu-user-binary
     if emulate:
-        arch_debian = pmb.parse.arch.alpine_to_debian(arch)
+        arch_qemu = pmb.parse.arch.alpine_to_qemu(arch)
         pmb.helpers.run.root(args, ["mkdir", "-p", chroot + "/usr/bin"])
         pmb.helpers.run.root(args, ["cp", args.work +
-                                    "/chroot_native/usr/bin/qemu-" + arch_debian + "-static",
+                                    "/chroot_native/usr/bin/qemu-" + arch_qemu,
-                                    chroot + "/usr/bin/qemu-" + arch_debian + "-static"])
+                                    chroot + "/usr/bin/qemu-" + arch_qemu + "-static"])
 
     # Install alpine-base
     pmb.helpers.repo.update(args, arch)

pmb/parse/arch.py

@@ -49,10 +49,9 @@ def from_chroot_suffix(args, suffix):
                      " (wrong device chosen in 'init' step?)")
 
 
-def alpine_to_debian(arch):
+def alpine_to_qemu(arch):
     """
-    Convert the architecture to the string used in the binfmt info
+    Convert the architecture to the string used in the QEMU packaging.
-    (aka. the Debian architecture format).
     """
 
     mapping = {
@@ -61,9 +60,9 @@ def alpine_to_debian(arch):
         "armhf": "arm",
         "aarch64": "aarch64",
     }
-    for pattern, arch_debian in mapping.items():
+    for pattern, arch_qemu in mapping.items():
         if fnmatch.fnmatch(arch, pattern):
-            return arch_debian
+            return arch_qemu
     raise ValueError("Can not map Alpine architecture '" + arch + "'"
                      " to the right Debian architecture.")
 

pmb/parse/binfmt_info.py

@@ -17,16 +17,17 @@ You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
 import logging
+import pmb.config
 
 # Get magic and mask from binfmt info file
 # Return: {magic: ..., mask: ...}
 
 
-def binfmt_info(args, arch_debian):
+def binfmt_info(args, arch_qemu):
     # Parse the info file
     full = {}
-    info = args.work + "/chroot_native/usr/share/qemu-user-binfmt.txt"
+    info = pmb.config.pmb_src + "/data/qemu-user-binfmt.txt"
-    logging.debug("parsing: " + info)
+    logging.verbose("parsing: " + info)
     with open(info, "r") as handle:
         for line in handle:
             if line.startswith('#') or "=" not in line:
@@ -37,12 +38,12 @@ def binfmt_info(args, arch_debian):
             full[key] = value[1:-2]
 
     ret = {}
-    logging.debug("filtering by architecture: " + arch_debian)
+    logging.verbose("filtering by architecture: " + arch_qemu)
     for type in ["mask", "magic"]:
-        key = arch_debian + "_" + type
+        key = arch_qemu + "_" + type
         if key not in full:
             raise RuntimeError("Could not find key " + key + " in binfmt info file: " +
                                info)
         ret[type] = full[key]
-    logging.debug("=> " + str(ret))
+    logging.verbose("=> " + str(ret))
     return ret