Fix pmbootstrap zap -m / various zap improvements (#1166)

zap -m:
* APKINDEX parsing: parse the "origin" field as well, so we know
  where a subpackage comes from
* pmbootstrap zap -m: properly delete all packages, that do not
  have an aport or where the aport has another version. This also
  works with subpackages now,
  we use the origin field to resolve it.
* Only reindex when packages have been deleted in "zap -m"

zap in general:
* Show the amount of cleared up space after the deletion instead
  of "Done"
* Print "Shutdown complete" to "pmbootstrap log" instead of stdout
  (we need to call it twice during zap now to get the space
  calculation right)
* Add `--dry` argument to `pmbootstrap zap` (this was very useful
  for debugging) to list the packages/chroots that would get
  deleted
* Roughly output the command that would get executed to delete
  files, so it's obvious what's going on in --dry mode. (% rm ...)

pmb/chroot/shutdown.py

@@ -94,4 +94,4 @@ def shutdown(args, only_install_related=False):
         for arch in pmb.config.build_device_architectures:
             if pmb.parse.arch.cpu_emulation_required(args, arch):
                 pmb.chroot.binfmt.unregister(args, arch)
-        logging.info("Shutdown complete")
+        logging.debug("Shutdown complete")

pmb/chroot/zap.py

@@ -16,39 +16,46 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
-import os
 import glob
 import logging
+import math
+import os
 
 import pmb.chroot
 import pmb.helpers.run
 
 
-def zap(args, confirm=True, packages=False, http=False, mismatch_bins=False,
+def zap(args, confirm=True, dry=False, packages=False, http=False,
-        old_bins=False, distfiles=False):
+        mismatch_bins=False, old_bins=False, distfiles=False):
     """
     Shutdown everything inside the chroots (e.g. distccd, adb), umount
     everything and then safely remove folders from the work-directory.
 
-    :arg packages: Remove *all* self-compiled packages (!)
+    :param dry: Only show what would be deleted, do not delete for real
-    :arg http: Clear the http cache (used e.g. for the initial apk download)
+    :param packages: Remove *all* self-compiled packages (!)
-    :arg mismatch_bins: Remove the packages, that have a different version
+    :param http: Clear the http cache (used e.g. for the initial apk download)
-                        compared to what is in the abuilds folder.
+    :param mismatch_bins: Remove the packages, that have a different version
-    :arg old_bins: Clean out outdated binary packages downloaded from
+                          compared to what is in the aports folder.
+    :param old_bins: Clean out outdated binary packages downloaded from
                      mirrors (e.g. from Alpine)
-    :arg distfiles: Clear the downloaded files cache
+    :param distfiles: Clear the downloaded files cache
 
     NOTE: This function gets called in pmb/config/init.py, with only args.work
     and args.device set!
     """
+    # Get current work folder size
+    if not dry:
+        pmb.chroot.shutdown(args)
+        logging.debug("Calculate work folder size")
+        size_old = pmb.helpers.other.folder_size(args, args.work)
 
     # Delete packages with a different version compared to aports, then re-index
     if mismatch_bins:
-        zap_mismatch_bins(args, confirm)
+        zap_mismatch_bins(args, confirm, dry)
 
     # Delete outdated binary packages
     if old_bins:
-        zap_old_bins(args, confirm)
+        zap_old_bins(args, confirm, dry)
 
     pmb.chroot.shutdown(args)
 
@@ -71,52 +78,87 @@ def zap(args, confirm=True, packages=False, http=False, mismatch_bins=False,
         matches = glob.glob(pattern)
         for match in matches:
             if not confirm or pmb.helpers.cli.confirm(args, "Remove " + match + "?"):
+                logging.info("% rm -rf " + match)
+                if not dry:
                     pmb.helpers.run.root(args, ["rm", "-rf", match])
 
     # Chroots were zapped, so no repo lists exist anymore
     args.cache["apk_repository_list_updated"].clear()
 
+    # Print amount of cleaned up space
+    if dry:
+        logging.info("Dry run: nothing has been deleted")
+    else:
+        size_new = pmb.helpers.other.folder_size(args, args.work)
+        mb = (size_old - size_new) / 1024 / 1024
+        logging.info("Cleared up ~" + str(math.ceil(mb)) + " MB of space")
 
-def zap_mismatch_bins(args, confirm=True):
+
+def zap_mismatch_bins(args, confirm=True, dry=False):
     if not os.path.exists(args.work + "/packages/"):
         return
     if confirm and not pmb.helpers.cli.confirm(args, "Remove packages that are newer than"
                                                " the corresponding package in aports?"):
         return
-    for arch in os.listdir(os.path.realpath(args.work + "/packages/")):
-        arch_pkg_path = os.path.realpath(args.work) + "/packages/" + arch
-        # Delete all broken symbolic links
-        pmb.helpers.run.root(args, ["find", "-L", arch_pkg_path, "-maxdepth", "1",
-                             "-type", "l", "-delete"])
-        bin_apks = pmb.parse.apkindex.parse(args, arch_pkg_path + "/APKINDEX.tar.gz")
-        for bin_apk in bin_apks:
-            bin_pkgname = bin_apks[bin_apk]["pkgname"]
-            bin_version = bin_apks[bin_apk]["version"]
-            bin_apk_path = arch_pkg_path + "/" + bin_pkgname + "-" + bin_version + ".apk"
 
-            # Do not fail if unable to find aport
+    reindex = False
-            aport = pmb.build.other.find_aport(args, bin_pkgname, False)
+    for apkindex_path in glob.glob(args.work + "/packages/*/APKINDEX.tar.gz"):
-            if not aport:
+        apkindex = pmb.parse.apkindex.parse(args, apkindex_path)
-                logging.warning("WARNING: Could not resolve aport for package " + bin_apk_path)
+        for pkgname, bin_data in apkindex.items():
+            # Only real packages have apks, provided packages do not exist
+            # (e.g. "so:libtest.so.1.2")
+            if pkgname != bin_data["pkgname"]:
+                continue
+            origin = bin_data["origin"]
+            version = bin_data["version"]
+            arch = bin_data["arch"]
+
+            # Apk path
+            apk_path_short = arch + "/" + pkgname + "-" + version + ".apk"
+            apk_path = args.work + "/packages/" + apk_path_short
+            if not os.path.exists(apk_path):
+                logging.info("WARNING: Package mentioned in index not"
+                             " found: " + apk_path_short)
+                continue
+
+            # Aport path
+            aport_path = pmb.build.other.find_aport(args, origin, False)
+            if not aport_path:
+                logging.info("% rm " + apk_path_short + " (" + origin +
+                             " aport not found)")
+                if not dry:
+                    pmb.helpers.run.root(args, ["rm", apk_path])
+                    reindex = True
                 continue
-            apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
-            aport_version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
 
             # Clear out any binary apks that do not match what is in aports
-            if pmb.parse.version.compare(bin_version, aport_version) and os.path.exists(bin_apk_path):
+            apkbuild = pmb.parse.apkbuild(args, aport_path + "/APKBUILD")
-                logging.info("Remove mismatched binary package (aports version: " +
+            version_aport = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
-                             aport_version + "): " + arch + "/" + bin_pkgname + "-" +
+            if version != version_aport:
-                             bin_version + ".apk")
+                logging.info("% rm " + apk_path_short + " (" + origin +
-                pmb.helpers.run.root(args, ["rm", bin_apk_path])
+                             " aport: " + version_aport + ")")
+                if not dry:
+                    pmb.helpers.run.root(args, ["rm", apk_path])
+                    reindex = True
+
+    if reindex:
         pmb.build.other.index_repo(args)
 
 
-def zap_old_bins(args, confirm=True):
+def zap_old_bins(args, confirm=True, dry=False):
+    # Check whether we need to do anything
+    paths = glob.glob(args.work + "/cache_apk_*")
+    if not len(paths):
+        return
     if confirm and not pmb.helpers.cli.confirm(args, "Remove outdated binary packages?"):
         return
-    arch_native = pmb.parse.arch.alpine_native()
+
-    if os.path.exists(args.work + "/cache_apk_" + arch_native):
+    # Iterate over existing apk caches
-        pmb.chroot.root(args, ["apk", "-v", "cache", "clean"])
+    for path in paths:
-    for arch in pmb.config.build_device_architectures:
+        arch = os.path.basename(path).split("_", 2)[2]
-        if arch != arch_native and os.path.exists(args.work + "/cache_apk_" + arch):
+        chroot = "native" if arch == args.arch_native else "buildroot_" + arch
-            pmb.chroot.root(args, ["apk", "-v", "cache", "clean"], "buildroot_" + arch)
+
+        # Clean the cache with apk
+        logging.info("(" + chroot + ") apk -v cache clean")
+        if not dry:
+            pmb.chroot.root(args, ["apk", "-v", "cache", "clean"], chroot)

pmb/helpers/frontend.py

@@ -329,10 +329,13 @@ def log_distccd(args):
 
 
 def zap(args):
-    pmb.chroot.zap(args, packages=args.packages, http=args.http,
+    pmb.chroot.zap(args, dry=args.dry, packages=args.packages, http=args.http,
                    mismatch_bins=args.mismatch_bins, old_bins=args.old_bins,
                    distfiles=args.distfiles)
 
+    # Don't write the "Done" message
+    pmb.helpers.logging.disable()
+
 
 def bootimg_analyze(args):
     bootimg = pmb.parse.bootimg(args, args.path)

pmb/parse/apkindex.py

@@ -47,11 +47,12 @@ def parse_next_block(args, path, lines, start):
     ret = {}
     mapping = {
         "A": "arch",
-        "P": "pkgname",
-        "V": "version",
         "D": "depends",
+        "o": "origin",
+        "P": "pkgname",
         "p": "provides",
-        "t": "timestamp"
+        "t": "timestamp",
+        "V": "version",
     }
     end_of_block_found = False
     for i in range(start[0], len(lines)):

pmb/parse/arguments.py

@@ -226,6 +226,9 @@ def arguments():
 
     # Action: zap
     zap = sub.add_parser("zap", help="safely delete chroot folders")
+    zap.add_argument("--dry", action="store_true", help="instead of actually"
+                     " deleting anything, print out what would have been"
+                     " deleted")
     zap.add_argument("-p", "--packages", action="store_true", help="also delete"
                      " the precious, self-compiled packages")
     zap.add_argument("-hc", "--http", action="store_true", help="also delete http"