pmbootstrap challenge: subpackages, list of changed files

* Two new functions for getting a list of files and their timestamps
  in the repo, and diffing that information to get a list of changed
  files: pmb.helpers.repo.files() and pmb.helpers.repo.diff().
  (I've put it in the helpers folder, because it is not specific to
  one chroot, but to all chroots at once.)
* pmbootstrap challenge (new command introduced a few commits back to
  verify, that the contents of an APK file are deterministic) uses
  these functions to a) support subpackages and b) optionally
  output a list of changed files (this gets used in the pmbuilder
  script, which lives outside of this repository).

This commit is progress for #64 again.

pmb/build/challenge.py

@@ -26,6 +26,7 @@ import shutil
 import pmb.build
 import pmb.parse.apkbuild
 import pmb.parse.other
+import pmb.helpers.repo
 
 
 def diff_files(tar_a, tar_b, member_a, member_b, name):
@@ -51,7 +52,6 @@ def diff_files(tar_a, tar_b, member_a, member_b, name):
 
 
 def diff(args, apk_a, apk_b):
-    logging.info("Challenge " + apk_a)
     with tarfile.open(apk_a, "r:gz") as tar_a:
         with tarfile.open(apk_b, "r:gz") as tar_b:
             # List of files must be the same
@@ -128,8 +128,27 @@ def challenge(args, apk_path):
                                " (installed: " + package_installed + ","
                                " buildinfo: " + package_buildinfo + ")!")
     # Build the package
-    output = pmb.build.package(args, buildinfo["pkgname"], buildinfo["carch"],
+    repo_before = pmb.helpers.repo.files(args)
-                               force=True)
+    pmb.build.package(args, buildinfo["pkgname"], buildinfo["carch"],
+                      force=True)
+    repo_diff = pmb.helpers.repo.diff(args, repo_before)
 
     # Diff the apk contents
-    diff(args, apk_path, args.work + "/packages/" + output)
+    staging_path = os.path.abspath(os.path.dirname(apk_path) + "/../")
+    for file in repo_diff:
+        if file.endswith(".apk"):
+            logging.info("Verify " + file)
+            diff(
+                args,
+                staging_path +
+                "/" +
+                file,
+                args.work +
+                "/packages/" +
+                file)
+
+    # Output the changed files from the repository
+    if args.output_repo_changes:
+        with open(args.output_repo_changes, "w") as handler:
+            for file in repo_diff:
+                handler.write(file + "\n")

pmb/helpers/repo.py

@@ -0,0 +1,60 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import glob
+import os
+
+
+def files(args):
+    """
+    Returns all files (apk/buildinfo) with their last modification timestamp
+    inside the package repository, sorted by architecture.
+
+    :returns: {"x86_64": {"first.apk": last_modified_timestamp, ... }, ... }
+    """
+    ret = {}
+    for arch_folder in glob.glob(args.work + "/packages/*"):
+        arch = os.path.basename(arch_folder)
+        ret[arch] = {}
+        for file in glob.glob(arch_folder + "/*"):
+            basename = os.path.basename(file)
+            ret[arch][basename] = os.path.getmtime(file)
+    return ret
+
+
+def diff(args, files_a, files_b=None):
+    """
+    Returns a list of files, that have been added or modified inside the
+    package repository.
+
+    :param files_a: return value from pmb.helpers.repo.files()
+    :param files_b: defaults to creating a new list
+    :returns: ["x86_64/APKINDEX.tar.gz", "x86_64/package.apk",
+               "x86_64/package.buildinfo", ...]
+    """
+    if not files_b:
+        files_b = files(args)
+
+    ret = []
+    for arch in files_b.keys():
+        for file, timestamp in files_b[arch].items():
+            if (arch not in files_a or file not in files_a[arch] or
+                    timestamp is not files_a[arch][file]):
+                ret.append(arch + "/" + file)
+
+    return sorted(ret)

pmb/parse/arguments.py

@@ -173,6 +173,10 @@ def arguments():
     # Action: challenge
     challenge = sub.add_parser("challenge",
                                help="rebuild a package and diff its contents")
+    challenge.add_argument("--output-repo-changes", dest="output_repo_changes",
+                           help="pass the path to a file here, to store a list"
+                                " of apk- and APKINDEX-files that have been"
+                                " changed during the build", default=None)
     challenge.add_argument("apk")
 
     # Use defaults from the user's config file