Close #871: Enable binary repository (#887)

* add my own build key
* enable the repo in the config
* update the README file
* Adjust testcase, that validates the keys and enable it in testcases_fast.sh
* Only save/load keys to/from the config file, which we ask for during
  'pmbootstrap init', so the binary repo gets used even if a config file
  already exists (this also removes a workaround, that deletes the work
  folder path from the config dictionary before writing it)
* Download missing APKINDEX.tar.gz files with Python code, before
  attempting to build packages (so we know which ones aleady exist in
  the binary packages repository)
* Consider APKINDEX files older than 4 hours as outdated and download
  them again (also in Python code)
* Provide 'pmbootstrap update' to force-update the APKINDEX files
* Travis: more logging output on failure
* Only allow keys from config_keys to be used by "pmbootstrap config"

pmb/helpers/repo.py

@@ -19,6 +19,9 @@ along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 import glob
 import os
 import hashlib
+import logging
+import pmb.helpers.http
+import pmb.helpers.run
 
 
 def files(args):
@@ -148,3 +151,48 @@ def apkindex_files(args, arch=None):
                    hash(url) + ".tar.gz")
 
     return ret
+
+
+def update(args, force=False):
+    """
+    Download the APKINDEX files for all URLs and architectures.
+    :arg force: even update when the APKINDEX file is fairly recent
+    """
+
+    architectures = [args.arch_native] + pmb.config.build_device_architectures
+    retention_hours = pmb.config.apkindex_retention_time
+    retention_seconds = retention_hours * 3600
+
+    outdated = {}
+    for url in urls(args, False):
+        for arch in architectures:
+            url_full = url + "/" + arch + "/APKINDEX.tar.gz"
+            cache_apk_outside = args.work + "/cache_apk_" + arch
+            apkindex = cache_apk_outside + "/APKINDEX." + hash(url) + ".tar.gz"
+
+            reason = None
+            if not os.path.exists(apkindex):
+                reason = "file does not exist yet"
+            elif force:
+                reason = "forced update"
+            elif pmb.helpers.file.is_older_than(apkindex, retention_seconds):
+                reason = "older than " + str(retention_hours) + "h"
+            if not reason:
+                continue
+
+            logging.debug("APKINDEX outdated (" + reason + "): " + url_full)
+            outdated[url_full] = apkindex
+
+    if not len(outdated):
+        return
+
+    # Show one message only
+    logging.info("Update package index (" + str(len(outdated)) + "x)")
+    for url, target in outdated.items():
+        # Download and move to right location
+        temp = pmb.helpers.http.download(args, url, "APKINDEX", False,
+                                         logging.DEBUG)
+        target_folder = os.path.dirname(target)
+        if not os.path.exists(target_folder):
+            pmb.helpers.run.root(args, ["mkdir", "-p", target_folder])
+        pmb.helpers.run.root(args, ["cp", temp, target])