Close #226: Launch postmarketOS in a qemu virtual machine (#350)

Thanks to Pablo Castellano and Martijn Braam!
In postmarketOS we are now able to generate system images with the
correct configuration so that they can boot already using qemu

This commit brings the `pmbootstrap qemu` action.
This command is very handy because you don't have to set all the
qemu parameters, pmbootstrap does it for you.

* device-qemu-vexpress: Added kernel command line according to wiki
* qemu: Added workaround for image writing permissions
* qemu: Added support to launch postmarketOS in a QEMU virtual machine

- Support for emulating these architectures in QEMU: arm, aarch64, x86_84
- Generate QEMU command correctly depending no guest architecture (arm/x86)
- Run QEMU in the same architecture as the host by default
- Refactoring in pmb.parse.arch and pmb.qemu.run
- Raise exception if DTB file or system image are not present
- Display more useful information when something fails (e.g. image not found)
- Run qemu version depending on arch (host or argument), not device configured

* device-qemu-amd64: set deviceinfo_kernel_cmdline to "PMOS_NO_OUTPUT_REDIRECT"
* qemu: added --memory argument to specific guest RAM
* device-qemu-amd64: adjusted deviceinfo_kernel_cmdline (console=tty1)
* Added /etc/network/interfaces for qemu-amd64
* qemu: Added KVM support if /dev/kvm if present
* Specify separate machines for architecture
* qemu: Check if QEMU is installed instead of crashing
* Added graphics driver to qemu-aarch64

- Use arm (as used in qemu) instead of armhf (used in Alpine)
- qemu argument is -dtb
- Follow same style to build the command + arguments

* qemu: Added SSH port redirection: ./pmbootstrap.py qemu -p 2222

aports/device/device-qemu-aarch64/APKBUILD

@@ -0,0 +1,23 @@
+pkgname=device-qemu-aarch64
+pkgver=1
+pkgrel=1
+pkgdesc="Simulated device in qemu with vexpress soc"
+url="https://github.com/postmarketOS"
+arch="noarch"
+license="MIT"
+depends="linux-postmarketos"
+makedepends=""
+subpackages=""
+source="deviceinfo"
+options="!check"
+
+build() {
+	return 0
+}
+
+package() {
+	install -D -m644 "$srcdir"/deviceinfo \
+		"$pkgdir"/etc/deviceinfo
+}
+
+sha512sums="8df46383685a40a96b7c604c29d413ecdd4b4a9b92cb61f452283042f189b1ca01cb70632fec0711dd67229e0501729762c6841359ac7ea455f516b512abce9c  deviceinfo"

aports/device/device-qemu-aarch64/deviceinfo

@@ -0,0 +1,21 @@
+# Reference: <https://postmarketos.org/deviceinfo>
+# Please use double quotes only. You can source this file in shell scripts.
+
+deviceinfo_format_version="0"
+deviceinfo_name="Qemu aarch64 virt"
+deviceinfo_manufacturer="Qemu"
+deviceinfo_date=""
+deviceinfo_keyboard="true"
+deviceinfo_nonfree="????"
+deviceinfo_dtb=""
+deviceinfo_modules_initfs="virtio-gpu"
+deviceinfo_external_disk="true"
+deviceinfo_external_disk_install="true"
+deviceinfo_flash_methods="none"
+deviceinfo_generate_legacy_uboot_initfs="false"
+deviceinfo_arch="aarch64"
+deviceinfo_kernel_cmdline="console=ttyAMA0,38400n8 console=tty1 PMOS_NO_OUTPUT_REDIRECT"
+
+# Splash
+deviceinfo_screen_width="640"
+deviceinfo_screen_height="480"

aports/device/device-qemu-amd64/APKBUILD

@@ -1,6 +1,6 @@
 pkgname=device-qemu-amd64
 pkgver=1
-pkgrel=1
+pkgrel=2
 pkgdesc="Simulated device in qemu with an x86 platform"
 url="https://github.com/postmarketOS"
 arch="noarch"
@@ -8,7 +8,7 @@ license="MIT"
 depends="linux-postmarketos"
 makedepends=""
 subpackages=""
-source="deviceinfo"
+source="deviceinfo interfaces"
 options="!check"
 
 build() {
@@ -18,6 +18,9 @@ build() {
 package() {
 	install -D -m644 "$srcdir"/deviceinfo \
 		"$pkgdir"/etc/deviceinfo
+	install -D -m644 "$srcdir"/interfaces \
+		"$pkgdir"/etc/network/interfaces
 }
 
-sha512sums="6d8c51d5b3d8e59640bdb4272fcd70fb8d262d6233133725b49e3e832ee713710377761456684223d9f4d311c2285bf41e92d88b68ce01963fe5c8ca132117db  deviceinfo"
+sha512sums="b73fe8c04f482fe63b31b80ad901d8e5349fc713e99137a273b0cfbae25e4c7d23bff8e5aac582ab4a3b87d0039aa166dd5b9827c1afc0551cef800916749697  deviceinfo
+d510ca304066840aa5e6c4fc71ded1b7e83012c93837fa39e37bdb873b3636230030d56f7aa50c93fc688f563cb4cb96c02ad333bbc45a400c1ebee1792a9dd4  interfaces"

aports/device/device-qemu-amd64/deviceinfo

@@ -14,6 +14,7 @@ deviceinfo_external_disk_install="true"
 deviceinfo_flash_methods="none"
 deviceinfo_generate_legacy_uboot_initfs="false"
 deviceinfo_arch="x86_64"
+deviceinfo_kernel_cmdline="console=tty1 console=ttyS0 PMOS_NO_OUTPUT_REDIRECT"
 
 # Splash
 deviceinfo_screen_width="640"

aports/device/device-qemu-amd64/interfaces

@@ -0,0 +1,5 @@
+auto lo
+iface lo inet loopback
+
+auto eth0
+iface eth0 inet dhcp

aports/device/device-qemu-vexpress/APKBUILD

@@ -1,6 +1,6 @@
 pkgname=device-qemu-vexpress
 pkgver=1
-pkgrel=7
+pkgrel=8
 pkgdesc="Simulated device in qemu with vexpress soc"
 url="https://github.com/postmarketOS"
 arch="noarch"
@@ -20,4 +20,4 @@ package() {
 		"$pkgdir"/etc/deviceinfo
 }
 
-sha512sums="cd0caab21677b5abfc4d58565831e3f1bcca87dd32626625a2ba9bbe53650aa7557d56a46a8cf1557725ecbe43a38e9e9c4637b8d9e925b1a8fbcb0710f7d521  deviceinfo"
+sha512sums="a18ab789c19d8802be6a9e9c842f4c304f267bd78f46b096b4ba6609430d1544e0a9eca9bc22db67a0728efae3aad6c2def9f7e64157d31aeb31b4bb839076d5  deviceinfo"

aports/device/device-qemu-vexpress/deviceinfo

@@ -14,6 +14,7 @@ deviceinfo_external_disk_install="true"
 deviceinfo_flash_methods="none"
 deviceinfo_generate_legacy_uboot_initfs="false"
 deviceinfo_arch="armhf"
+deviceinfo_kernel_cmdline="console=ttyAMA0,38400n8 console=tty1 PMOS_NO_OUTPUT_REDIRECT"
 
 # Splash
 deviceinfo_screen_width="640"

aports/main/linux-postmarketos/APKBUILD

@@ -30,6 +30,7 @@ install=
 source="
 	https://cdn.kernel.org/pub/linux/kernel/v${pkgver%%.*}.x/linux-$_kernver.tar.xz
 	config-${_flavor}.armhf
+	config-${_flavor}.aarch64
 	config-${_flavor}.x86_64
        0001-rx51-drm-regression-workaround.patch
 "
@@ -109,8 +110,13 @@ build() {
 package() {
 	cd "$srcdir/build/arch/${_carch}/boot"
 	
+	if [ "$CARCH" == "aarch64" ]; then
+		install -Dm644 "$srcdir/build/arch/${_carch}/boot/Image" \
+			"$pkgdir/boot/vmlinuz-$_flavor"
+	else
 		install -Dm644 "$srcdir/build/arch/${_carch}/boot/"*zImage \
 			"$pkgdir/boot/vmlinuz-$_flavor"
+	fi
 
 	install -D "$srcdir/build/include/config/kernel.release" \
 		"$pkgdir/usr/share/kernel/$_flavor/kernel.release"
@@ -185,6 +191,7 @@ dev() {
 
 sha512sums="8e81b41b253e63233e92948941f44c6482acb52aa3a3fd172f03a38a86f2c35b2ad4fd407acd1bc3964673eba344fe104d3a03e3ff4bf9cd1f22bd44263bd728  linux-4.12.tar.xz
 3f4435ce0cf227e4ba64aa7153c913370c53069e34a376e453786d601b5a73706a3f44b002fab1b10a72dc94e896c1d8fb462c2b78e62ed69c468d98aa7220da  config-postmarketos.armhf
-c2bdaf8885dab4c3d3f96d856179138fed2d33cadcb9b33997238c8395a13eb8cb2b35f5f887339c5b47b34c8aa1bcd121059987601c1908103b66c23da478f0  config-postmarketos.x86_64
+7a8f407fe7eec177deae1ecab923d97847e47fbdab646f0f6a21135eb228123c4f3c21cbcc8837506ea36d6f35ad8983daa43514c6da037459321ae7fa4efa00  config-postmarketos.aarch64
+df0420c373bbc5d52957b504ce154c0a3ef062069da8a163e23aea455879b7c23a12ab518bfae5a6fd245204c2e9460edac6ef3e0d415740b921510e75f5b23b  config-postmarketos.x86_64
 17c48bb7b4218297bd2be6faa5b6570ce1560a33385237a9962c0884d782c9a722a25a30077b6721d2943a9b98c29dcad6adfef718b0163c559c19a79519319b  0001-rx51-drm-regression-workaround.patch
 8a6b72524050733c166524230d85f808275a65c28f06444350ebb8c64dd4cab666f8629ef1d1d2b6c25c1f36820a1fd114510af5a38509df55f9c3071543e647  patch-4.12.4.xz"

aports/main/linux-postmarketos/config-postmarketos.x86_64

@@ -1,6 +1,6 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86_64 4.12.0 Kernel Configuration
+# Linux/x86_64 4.12.4 Kernel Configuration
 #
 CONFIG_64BIT=y
 CONFIG_X86_64=y
@@ -1673,6 +1673,7 @@ CONFIG_VIRTIO_NET=m
 # Distributed Switch Architecture drivers
 #
 CONFIG_ETHERNET=y
+CONFIG_MDIO=y
 CONFIG_NET_VENDOR_3COM=y
 # CONFIG_VORTEX is not set
 # CONFIG_TYPHOON is not set
@@ -1733,7 +1734,18 @@ CONFIG_NET_VENDOR_EXAR=y
 # CONFIG_VXGE is not set
 CONFIG_NET_VENDOR_HP=y
 # CONFIG_HP100 is not set
-# CONFIG_NET_VENDOR_INTEL is not set
+CONFIG_NET_VENDOR_INTEL=y
+# CONFIG_E100 is not set
+CONFIG_E1000=y
+CONFIG_E1000E=y
+CONFIG_E1000E_HWTS=y
+# CONFIG_IGB is not set
+# CONFIG_IGBVF is not set
+CONFIG_IXGB=y
+CONFIG_IXGBE=y
+CONFIG_IXGBE_HWMON=y
+# CONFIG_I40E is not set
+CONFIG_NET_VENDOR_I825XX=y
 # CONFIG_JME is not set
 # CONFIG_NET_VENDOR_MARVELL is not set
 CONFIG_NET_VENDOR_MELLANOX=y

pmb/helpers/frontend.py

@@ -34,6 +34,7 @@ import pmb.helpers.other
 import pmb.helpers.run
 import pmb.install
 import pmb.parse
+import pmb.qemu
 
 
 def _parse_suffix(args):
@@ -115,6 +116,10 @@ def parse_apkindex(args):
     print(json.dumps(result, indent=4))
 
 
+def qemu(args):
+    pmb.qemu.run(args)
+
+
 def shutdown(args):
     pmb.chroot.shutdown(args)
 

pmb/helpers/run.py

@@ -34,8 +34,8 @@ def core(args, cmd, log_message, log, return_stdout, check=True,
         working_dir_old = os.getcwd()
         os.chdir(working_dir)
 
-    try:
     ret = None
+    try:
         if log:
             if return_stdout:
                 ret = subprocess.check_output(cmd).decode("utf-8")

pmb/parse/arch.py

@@ -120,3 +120,41 @@ def cpu_emulation_required(args, arch):
 
     # No match: then it's required
     return True
+
+
+def uname_to_qemu(arch):
+    """
+    Convert the most common architectures returned by 'uname' to those
+    used by the QEMU binary
+    """
+    mapping = {
+        "aarch64": "aarch64",
+        "arm": "arm",
+        "armeb": "arm",
+        "armel": "arm",
+        "armhf": "arm",
+        "x86_64": "x86_64",
+        "amd64": "x86_64",
+    }
+    if arch in mapping:
+        return mapping[arch]
+
+    raise ValueError("Can not map host architecture '" + arch + "'"
+                     " to the right QEMU value")
+
+
+def qemu_to_pmos_device(arch):
+    """
+    Convert the architecture used in the QEMU binary to the aport name in
+    postmarketOS defining the device
+    """
+    mapping = {
+        "arm": "qemu-vexpress",
+        "aarch64": "qemu-aarch64",
+        "x86_64": "qemu-amd64",
+    }
+    if arch in mapping:
+        return mapping[arch]
+
+    raise ValueError("Can not map QEMU value '" + arch + "'"
+                     " to the right postmarketOS device")

pmb/parse/arguments.py

@@ -218,6 +218,16 @@ def arguments():
     parse_apkindex.add_argument("apkindex_path")
     parse_apkindex.add_argument("package", default=None, nargs="?")
 
+    # Action: qemu
+    qemu = sub.add_parser("qemu")
+    qemu.add_argument("--arch", choices=["aarch64", "arm", "x86_64"],
+                      help="emulate a different architecture")
+    qemu.add_argument("--cmdline", help="override kernel commandline")
+    qemu.add_argument("-m", "--memory", type=int, default=1024,
+                      help="guest RAM (default: 1024)")
+    qemu.add_argument("-p", "--port", type=int, default=2222,
+                      help="ssh port (default: 2222)")
+
     # Use defaults from the user's config file
     args = parser.parse_args()
     cfg = pmb.config.load(args)

pmb/qemu/__init__.py

@@ -0,0 +1,19 @@
+"""
+Copyright 2017 Pablo Castellano
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+from pmb.qemu.run import run

pmb/qemu/run.py

@@ -0,0 +1,144 @@
+"""
+Copyright 2017 Pablo Castellano
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import logging
+import os
+
+import pmb.build
+import pmb.chroot
+import pmb.chroot.apk
+import pmb.chroot.other
+import pmb.chroot.initfs
+import pmb.helpers.devices
+import pmb.helpers.run
+import pmb.parse.arch
+
+
+def system_image(args, device):
+    """
+    Returns path to system image for specified device. In case that it doesn't
+    exist, raise and exception explaining how to generate it.
+    """
+    path = args.work + "/chroot_native/home/user/rootfs/" + device + ".img"
+    if not os.path.exists(path):
+        logging.debug("Could not find system image: " + path)
+        img_command = "pmbootstrap install"
+        if device != args.device:
+            img_command = "pmbootstrap init' and '" + img_command
+        message = "The system image '{0}' has not been generated yet, please" \
+                  " run '{1}' first.".format(device, img_command)
+        raise RuntimeError(message)
+    return path
+
+
+def which_qemu(args, arch):
+    """
+    Finds the qemu executable or raises and exception otherwise
+    """
+    executable = "qemu-system-" + arch
+    try:
+        pmb.helpers.run.user(args, ["which", executable])
+        return executable
+    except RuntimeError:
+        raise RuntimeError("Could not find the '" + executable + "' executable"
+                           " in your PATH. Please install it in order to"
+                           " run qemu.")
+
+
+def qemu_command(args, arch, device, img_path):
+    """
+    Generate the full qemu command with arguments to run postmarketOS
+    """
+    qemu_bin = which_qemu(args, arch)
+    deviceinfo = pmb.parse.deviceinfo(args, device=device)
+    cmdline = deviceinfo["kernel_cmdline"]
+    if args.cmdline:
+        cmdline = args.cmdline
+    logging.info("cmdline: " + cmdline)
+
+    rootfs = args.work + "/chroot_rootfs_" + device
+    command = [qemu_bin]
+    command += ["-kernel", rootfs + "/boot/vmlinuz-postmarketos"]
+    command += ["-initrd", rootfs + "/boot/initramfs-postmarketos"]
+    command += ["-append", '"' + cmdline + '"']
+    command += ["-m", str(args.memory)]
+    command += ["-redir", "tcp:" + str(args.port) + "::22"]
+
+    if deviceinfo["dtb"] != "":
+        dtb_image = rootfs + "/usr/share/dtb/" + deviceinfo["dtb"] + ".dtb"
+        if not os.path.exists(dtb_image):
+            raise RuntimeError("DTB file not found: " + dtb_image)
+        command += ["-dtb", dtb_image]
+
+    if arch == "x86_64":
+        command += ["-serial", "stdio"]
+        command += ["-drive", "file=" + img_path + ",format=raw"]
+
+    elif arch == "arm":
+        command += ["-M", "vexpress-a9"]
+        command += ["-sd", img_path]
+
+    elif arch == "aarch64":
+        command += ["-M", "virt"]
+        command += ["-cpu", "cortex-a57"]
+        command += ["-device", "virtio-gpu-pci"]
+
+        # add storage
+        command += ["-device", "virtio-blk-device,drive=system"]
+        command += ["-drive", "if=none,id=system,file={},id=hd0".format(img_path)]
+
+    else:
+        raise RuntimeError("Architecture {} not supported by this command yet.".format(arch))
+
+    # Kernel Virtual Machine (KVM) support
+    enable_kvm = True
+    if args.arch:
+        arch1 = pmb.parse.arch.uname_to_qemu(args.arch_native)
+        arch2 = pmb.parse.arch.uname_to_qemu(args.arch)
+        enable_kvm = (arch1 == arch2)
+    if enable_kvm and os.path.exists("/dev/kvm"):
+        command += ["-enable-kvm"]
+    else:
+        logging.info("Warning: qemu is not using KVM and will run slower!")
+
+    return command
+
+
+def run(args):
+    """
+    Run a postmarketOS image in qemu
+    """
+    arch = pmb.parse.arch.uname_to_qemu(args.arch_native)
+    if args.arch:
+        arch = pmb.parse.arch.uname_to_qemu(args.arch)
+    logging.info("Running postmarketOS in QEMU VM (" + arch + ")")
+
+    device = pmb.parse.arch.qemu_to_pmos_device(arch)
+    img_path = system_image(args, device)
+
+    # Workaround: qemu runs as local user and needs write permissions in the
+    # system image, which is owned by root
+    if not os.access(img_path, os.W_OK):
+        pmb.helpers.run.root(args, ["chmod", "666", img_path])
+
+    command = qemu_command(args, arch, device, img_path)
+
+    logging.info("Command: " + " ".join(command))
+    logging.info("You can login to postmarketOS using SSH:")
+    logging.info("ssh -p " + str(args.port) + " user@localhost")
+    pmb.helpers.run.user(args, command)