pmb: Look for device/* files in device/*/... instead (!1879)

In the future, device ports will be located in a subdirectory
below device/... (e.g. device/testing/device-...).
Replace all occurrences of device/* with a glob that checks the
subdirectories instead.

Note: To ensure that this always works properly we should also add some
checks that all devices are indeed located under one of the supported
subdirectories (i.e. testing/community/main).

Change the glob for pmaports to <aports>/**/APKBUILD.
This allows using subdirectories for organization outside of device/
as well.

pmb/config/__init__.py

@@ -24,7 +24,7 @@ apk_keys_path = pmb_src + "/pmb/data/keys"
 apk_tools_static_min_version = "2.10.5-r0"
 
 # postmarketOS aports compatibility (checked against "version" in pmaports.cfg)
-pmaports_min_version = "4"
+pmaports_min_version = "5"
 
 # Version of the work folder (as asked during 'pmbootstrap init'). Increase
 # this number, whenever migration is required and provide the migration code,
@@ -450,7 +450,7 @@ aportgen = {
         "prefixes": ["binutils", "busybox-static", "gcc", "musl", "grub-efi"],
         "confirm_overwrite": False,
     },
-    "device": {
+    "device/testing": {
         "prefixes": ["device", "linux"],
         "confirm_overwrite": True,
     }

pmb/config/init.py

@@ -5,6 +5,7 @@ import glob
 import os
 import shutil
 
+import pmb.aportgen
 import pmb.config
 import pmb.config.pmaports
 import pmb.helpers.cli
@@ -185,10 +186,10 @@ def ask_for_device_nonfree(args, device):
     :returns: answers as dict, e.g. {"firmware": True, "userland": False}
     """
     # Parse existing APKBUILD or return defaults (when called from test case)
-    apkbuild_path = args.aports + "/device/device-" + device + "/APKBUILD"
+    apkbuild_path = pmb.helpers.devices.find_path(args, device, 'APKBUILD')
     ret = {"firmware": args.nonfree_firmware,
            "userland": args.nonfree_userland}
-    if not os.path.exists(apkbuild_path):
+    if not apkbuild_path:
         return ret
     apkbuild = pmb.parse.apkbuild(args, apkbuild_path)
 
@@ -259,8 +260,7 @@ def ask_for_device(args):
                                        codenames)
 
         device = vendor + '-' + codename
-        device_exists = os.path.exists(args.aports + "/device/device-" +
+        device_exists = pmb.helpers.devices.find_path(args, device, 'deviceinfo') is not None
-                                       device + "/deviceinfo")
         if not device_exists:
             if device == args.device:
                 raise RuntimeError(

pmb/helpers/devices.py

@@ -5,6 +5,24 @@ import glob
 import pmb.parse
 
 
+def find_path(args, codename, file=''):
+    """
+    Find path to device APKBUILD under `device/*/device-`.
+    :param codename: device codename
+    :param file: file to look for (e.g. APKBUILD or deviceinfo), may be empty
+    :returns: path to APKBUILD
+    """
+    g = glob.glob(args.aports + "/device/*/device-" + codename + '/' + file)
+    if not g:
+        return None
+
+    if len(g) != 1:
+        raise RuntimeError(codename + " found multiple times in the device"
+                           " subdirectory of pmaports")
+
+    return g[0]
+
+
 def list_codenames(args, vendor=None):
     """
     Get all devices, for which aports are available
@@ -12,7 +30,7 @@ def list_codenames(args, vendor=None):
     :returns: ["first-device", "second-device", ...]
     """
     ret = []
-    for path in glob.glob(args.aports + "/device/device-*"):
+    for path in glob.glob(args.aports + "/device/*/device-*"):
         device = os.path.basename(path).split("-", 1)[1]
         if (vendor is None) or device.startswith(vendor + '-'):
             ret.append(device)
@@ -25,7 +43,7 @@ def list_vendors(args):
     :returns: {"vendor1", "vendor2", ...}
     """
     ret = set()
-    for path in glob.glob(args.aports + "/device/device-*"):
+    for path in glob.glob(args.aports + "/device/*/device-*"):
         vendor = os.path.basename(path).split("-", 2)[1]
         ret.add(vendor)
     return ret
@@ -37,7 +55,7 @@ def list_apkbuilds(args):
     """
     ret = {}
     for device in list_codenames(args):
-        apkbuild_path = args.aports + "/device/device-" + device + "/APKBUILD"
+        apkbuild_path = args.aports + "/device/*/device-" + device + "/APKBUILD"
         ret[device] = pmb.parse.apkbuild(args, apkbuild_path)
     return ret
 

pmb/helpers/frontend.py

@@ -1,6 +1,5 @@
 # Copyright 2020 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-import glob
 import json
 import logging
 import os
@@ -249,8 +248,8 @@ def kconfig(args):
         # Default to all kernel packages
         packages = []
         if args.package == "" or args.package is None:
-            for aport in glob.glob(args.aports + "/*/linux-*"):
+            for aport in pmb.helpers.pmaports.get_list(args, "linux-*"):
-                packages.append(os.path.basename(aport).split("linux-")[1])
+                packages.append(aport.split("linux-")[1])
         else:
             packages = [args.package]
 

pmb/helpers/pmaports.py

@@ -12,10 +12,14 @@ import os
 import pmb.parse
 
 
-def get_list(args):
+def _glob_apkbuilds(args, pkgname='*'):
+    return glob.glob(args.aports + "/**/" + pkgname + "/APKBUILD", recursive=True)
+
+
+def get_list(args, pkgname='*'):
     """ :returns: list of all pmaport pkgnames (["hello-world", ...]) """
     ret = []
-    for apkbuild in glob.glob(args.aports + "/*/*/APKBUILD"):
+    for apkbuild in _glob_apkbuilds(args, pkgname):
         ret.append(os.path.basename(os.path.dirname(apkbuild)))
     ret.sort()
     return ret
@@ -31,11 +35,11 @@ def guess_main_dev(args, subpkgname):
     :returns: full path to the pmaport or None
     """
     pkgname = subpkgname[:-4]
-    paths = glob.glob(args.aports + "/*/" + pkgname)
+    paths = _glob_apkbuilds(args, pkgname)
     if paths:
         logging.debug(subpkgname + ": guessed to be a subpackage of " +
                       pkgname + " (just removed '-dev')")
-        return paths[0]
+        return os.path.dirname(paths[0])
 
     logging.debug(subpkgname + ": guessed to be a subpackage of " + pkgname +
                   ", which we can't find in pmaports, so it's probably in"
@@ -73,11 +77,11 @@ def guess_main(args, subpkgname):
         pkgname = "-".join(words)
 
         # Look in pmaports
-        paths = glob.glob(args.aports + "/*/" + pkgname)
+        paths = _glob_apkbuilds(args, pkgname)
         if paths:
             logging.debug(subpkgname + ": guessed to be a subpackage of " +
                           pkgname)
-            return paths[0]
+            return os.path.dirname(paths[0])
 
 
 def find(args, package, must_exist=True):
@@ -99,17 +103,17 @@ def find(args, package, must_exist=True):
             raise RuntimeError("Invalid pkgname: " + package)
 
         # Search in packages
-        paths = glob.glob(args.aports + "/*/" + package)
+        paths = _glob_apkbuilds(args, package)
         if len(paths) > 1:
             raise RuntimeError("Package " + package + " found in multiple"
                                " aports subfolders. Please put it only in one"
                                " folder.")
         elif len(paths) == 1:
-            ret = paths[0]
+            ret = os.path.dirname(paths[0])
 
         # Search in subpackages and provides
         if not ret:
-            for path_current in glob.glob(args.aports + "/*/*/APKBUILD"):
+            for path_current in _glob_apkbuilds(args):
                 apkbuild = pmb.parse.apkbuild(args, path_current)
                 found = False
 

pmb/install/_install.py

@@ -11,6 +11,7 @@ import pmb.chroot.apk
 import pmb.chroot.other
 import pmb.chroot.initfs
 import pmb.config
+import pmb.helpers.devices
 import pmb.helpers.run
 import pmb.install.blockdevice
 import pmb.install.file
@@ -62,8 +63,7 @@ def get_nonfree_packages(args, device):
               ["device-nokia-n900-nonfree-firmware"]
     """
     # Read subpackages
-    apkbuild_path = args.aports + "/device/device-" + device + "/APKBUILD"
+    apkbuild = pmb.parse.apkbuild(args, pmb.helpers.devices.find_path(args, device, 'APKBUILD'))
-    apkbuild = pmb.parse.apkbuild(args, apkbuild_path)
     subpackages = apkbuild["subpackages"]
 
     # Check for firmware and userland

pmb/parse/_apkbuild.py

@@ -6,6 +6,7 @@ import re
 from collections import OrderedDict
 
 import pmb.config
+import pmb.helpers.devices
 import pmb.parse.version
 
 # sh variable name regex: https://stackoverflow.com/a/2821201/3527128
@@ -340,8 +341,8 @@ def kernels(args, device):
                         "downstream": "Downstream description"}
     """
     # Read the APKBUILD
-    apkbuild_path = args.aports + "/device/device-" + device + "/APKBUILD"
+    apkbuild_path = pmb.helpers.devices.find_path(args, device, 'APKBUILD')
-    if not os.path.exists(apkbuild_path):
+    if apkbuild_path is None:
         return None
     subpackages = apkbuild(args, apkbuild_path)["subpackages"]
 

pmb/parse/deviceinfo.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import pmb.config
+import pmb.helpers.devices
 
 
 def sanity_check(info, path):
@@ -62,15 +63,14 @@ def deviceinfo(args, device=None):
         logging.fatal("Please provide a path to the aports directory using the -p flag")
         raise RuntimeError("Aports directory missing")
 
-    aport = args.aports + "/device/device-" + device
+    path = pmb.helpers.devices.find_path(args, device, 'deviceinfo')
-    if not os.path.exists(aport) or not os.path.exists(aport + "/deviceinfo"):
+    if not path:
         raise RuntimeError(
             "Device '" + device + "' not found. Run 'pmbootstrap init' to"
             " start a new device port or to choose another device. It may have"
             " been renamed, see <https://postmarketos.org/renamed>")
 
     ret = {}
-    path = aport + "/deviceinfo"
     with open(path) as handle:
         for line in handle:
             if not line.startswith("deviceinfo_"):

test/test_aportgen_device_wizard.py

@@ -32,9 +32,9 @@ def args(tmpdir, request):
     pmb.helpers.run.user(args, ["cp", "-r", path_dev, tmpdir + "/main"])
 
     # Copy the linux-lg-mako aport (we currently copy patches from there)
-    pmb.helpers.run.user(args, ["mkdir", "-p", tmpdir + "/device"])
+    pmb.helpers.run.user(args, ["mkdir", "-p", tmpdir + "/device/testing"])
-    path_mako = args._aports_real + "/device/linux-lg-mako"
+    path_mako = args._aports_real + "/device/testing/linux-lg-mako"
-    pmb.helpers.run.user(args, ["cp", "-r", path_mako, tmpdir + "/device"])
+    pmb.helpers.run.user(args, ["cp", "-r", path_mako, tmpdir + "/device/testing"])
     return args
 
 
@@ -63,9 +63,9 @@ def generate(args, monkeypatch, answers):
     pmb.aportgen.generate(args, "linux-testsuite-testdevice")
     monkeypatch.undo()
 
-    apkbuild_path = (args.aports + "/device/device-testsuite-testdevice/"
+    apkbuild_path = (args.aports + "/device/testing/device-testsuite-testdevice/"
                      "APKBUILD")
-    apkbuild_path_linux = (args.aports + "/device/"
+    apkbuild_path_linux = (args.aports + "/device/testing/"
                            "linux-testsuite-testdevice/APKBUILD")
 
     # The build fails if the email is not a valid email, so remove them just for tests

test/test_build_package.py

@@ -352,9 +352,9 @@ def test_build_local_source_high_level(args, tmpdir):
     # aports: Add deviceinfo (required by pmbootstrap to start)
     tmpdir = str(tmpdir)
     aports = tmpdir + "/aports"
-    aport = aports + "/device/device-" + args.device
+    aport = aports + "/device/testing/device-" + args.device
     os.makedirs(aport)
-    shutil.copy(args.aports + "/device/device-" + args.device + "/deviceinfo",
+    shutil.copy(args.aports + "/device/testing/device-" + args.device + "/deviceinfo",
                 aport)
 
     # aports: Add modified hello-world aport (source="", uses $builddir)

test/test_parse_apkbuild.py

@@ -34,7 +34,7 @@ def test_subpackages(args):
     assert subpkg["depends"] == ["postmarketos-base", "glibc"]
 
     # Successful extraction
-    path = (testdata + "/init_questions_device/aports/device/"
+    path = (testdata + "/init_questions_device/aports/device/testing/"
             "device-nonfree-firmware/APKBUILD")
     apkbuild = pmb.parse.apkbuild(args, path)
     subpkg = apkbuild["subpackages"]["device-nonfree-firmware-nonfree-firmware"]

test/test_pkgrel_bump.py

@@ -77,11 +77,11 @@ def setup_work(args, tmpdir):
             pmb.helpers.run.user(args, ["ln", "-s", path, tmpdir + "/"])
 
     # Copy testdata and selected device aport
-    for folder in ["device", "main"]:
+    for folder in ["device/testing", "main"]:
         pmb.helpers.run.user(args, ["mkdir", "-p", args.aports, tmpdir +
                                     "/_aports/" + folder])
-    pmb.helpers.run.user(args, ["cp", "-r", args.aports + "/device/device-" +
+    pmb.helpers.run.user(args, ["cp", "-r", args.aports + "/device/testing/device-" +
-                                args.device, tmpdir + "/_aports/device"])
+                                args.device, tmpdir + "/_aports/device/testing"])
     for pkgname in ["testlib", "testapp", "testsubpkg"]:
         pmb.helpers.run.user(args, ["cp", "-r",
                                     "test/testdata/pkgrel_bump/aports/" + pkgname,

test/testcases_fast.sh

@@ -21,7 +21,7 @@ cd "$(dirname "$0")/.."
 # Make sure we have a valid device (#1128)
 device="$(./pmbootstrap.py config device)"
 work="$(./pmbootstrap.py config work)"
-deviceinfo="$work/cache_git/pmaports/device/device-$device/deviceinfo"
+deviceinfo="$work/cache_git/pmaports/device/testing/device-$device/deviceinfo"
 if ! [ -e "$deviceinfo" ]; then
 	echo "ERROR: Could not find deviceinfo file for selected device '$device'."
 	echo "Expected path: $deviceinfo"