Close #871: Enable binary repository (#887)

* add my own build key
* enable the repo in the config
* update the README file
* Adjust testcase, that validates the keys and enable it in testcases_fast.sh
* Only save/load keys to/from the config file, which we ask for during
  'pmbootstrap init', so the binary repo gets used even if a config file
  already exists (this also removes a workaround, that deletes the work
  folder path from the config dictionary before writing it)
* Download missing APKINDEX.tar.gz files with Python code, before
  attempting to build packages (so we know which ones aleady exist in
  the binary packages repository)
* Consider APKINDEX files older than 4 hours as outdated and download
  them again (also in Python code)
* Provide 'pmbootstrap update' to force-update the APKINDEX files
* Travis: more logging output on failure
* Only allow keys from config_keys to be used by "pmbootstrap config"

.travis.yml

@@ -19,5 +19,6 @@ after_success:
   - coveralls
 after_failure:
   - cat ~/.local/var/pmbootstrap/log.txt
+  - cat ~/.local/var/pmbootstrap/log_testsuite.txt
 notifications:
   - email: false

keys/README

@@ -4,3 +4,6 @@ Sources for the keys (must be identical, there's a testcase that verifies this):
 	https://github.com/alpinelinux/aports/tree/master/main/alpine-keys
 	http://git.alpinelinux.org/cgit/aports/tree/main/alpine-keys?h=master
 	alpine-keys package
+
+In addition, this key holds keys for the official postmarketOS repository:
+	https://github.com/postmarketOS/pmbootstrap/issues/871

keys/pmos-5a03a13a.rsa.pub

@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtfF2BObXL387N/ypUtK6
+2cEyOEjLqW9qjCvUxpPRowy6/7y/7Bi3eaERfmjl8FWaxnCNWJRBlMNGGmsz4A/G
+5hXMWj4PmnXZlaczXXbZCyeOxrswF7lnN0tntJr8F0CBh+WLK1il5n35udG+ahYH
+CBUM5lzKDK833zF9u55eX3zieHXcge0lfL82Ks4nzBMQeLD39RhzgQGOOADFIm7G
+NEmLm+0AGTvfCwUvbnfv29hyZAo4LYsfHQK8Vun0llvjFGHBh5tCmOSNmfCLu7NK
+rfx+COLlDiQML6DLmenZZqQIGu1fxfjGK3eqpiTu1t3w/OHay/mftYJDmUK3vlW/
+iwIDAQAB
+-----END PUBLIC KEY-----

pmb/build/package.py

@@ -25,6 +25,7 @@ import pmb.build.buildinfo
 import pmb.chroot
 import pmb.chroot.apk
 import pmb.chroot.distccd
+import pmb.helpers.repo
 import pmb.parse
 import pmb.parse.arch
 
@@ -37,6 +38,9 @@ def package(args, pkgname, carch, force=False, buildinfo=False, strict=False,
     :param force: even build, if not necessary
     :returns: output path relative to the packages folder
     """
+    # Get existing binary package indexes
+    pmb.helpers.repo.update(args)
+
     # Get aport, skip upstream only packages
     aport = pmb.build.find_aport(args, pkgname, False)
     if not aport:

pmb/config/__init__.py

@@ -24,6 +24,7 @@ import os
 #
 from pmb.config.load import load
 from pmb.config.save import save
+from pmb.config.merge_with_args import merge_with_args
 
 
 #
@@ -43,6 +44,10 @@ apk_tools_static_min_version = "2.7.2-r0"
 # see migrate_work_folder()).
 work_version = "1"
 
+# Only save keys to the config file, which we ask for in 'pmbootstrap init'.
+config_keys = ["device", "extra_packages", "jobs", "timestamp_based_rebuild",
+               "work", "qemu_mesa_driver", "ui", "user", "keymap", "timezone"]
+
 # Config file/commandline default values
 # $WORK gets replaced with the actual value for args.work (which may be
 # overriden on the commandline)
@@ -56,7 +61,7 @@ defaults = {
     "timestamp_based_rebuild": True,
     "log": "$WORK/log.txt",
     "mirror_alpine": "http://dl-cdn.alpinelinux.org/alpine/",
-    "mirror_postmarketos": "",
+    "mirror_postmarketos": "http://postmarketos.brixit.nl",
     "work": os.path.expanduser("~") + "/.local/var/pmbootstrap",
     "port_distccd": "33632",
     "qemu_mesa_driver": "dri-virtio",
@@ -131,6 +136,9 @@ chroot_device_nodes = [
     [644, "c", 1, 9, "urandom"],
 ]
 
+# Age in hours that we keep the APKINDEXes before downloading them again.
+# You can force-update them with 'pmbootstrap update'.
+apkindex_retention_time = 4
 
 #
 # BUILD

pmb/config/init.py

@@ -207,9 +207,6 @@ def frontend(args):
     # Configure timezone info
     cfg["pmbootstrap"]["timezone"] = ask_for_timezone(args)
 
-    # Do not save aports location to config file
-    del cfg["pmbootstrap"]["aports"]
-
     # Save config
     pmb.config.save(args, cfg)
 

pmb/config/load.py

@@ -16,6 +16,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
+import logging
 import configparser
 import os
 import pmb.config
@@ -30,7 +31,17 @@ def load(args):
         cfg["pmbootstrap"] = {}
 
     for key in pmb.config.defaults:
-        if key not in cfg["pmbootstrap"]:
+        if key in pmb.config.config_keys and key not in cfg["pmbootstrap"]:
             cfg["pmbootstrap"][key] = str(pmb.config.defaults[key])
 
+        # We used to save default values in the config, which can *not* be
+        # configured in "pmbootstrap init". That doesn't make sense, we always
+        # want to use the defaults from pmb/config/__init__.py in that case, not
+        # some outdated version we saved some time back (eg. aports folder,
+        # postmarketOS binary packages mirror).
+        if key not in pmb.config.config_keys and key in cfg["pmbootstrap"]:
+            logging.debug("Ignored unconfigurable and possibly outdated default"
+                          " value from config: " + str(cfg["pmbootstrap"][key]))
+            del cfg["pmbootstrap"][key]
+
     return cfg

pmb/config/merge_with_args.py

@@ -0,0 +1,51 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import pmb.config
+
+
+def merge_with_args(args):
+    """
+    We have the internal config (pmb/config/__init__.py) and the user config
+    (usually ~/.config/pmbootstrap.cfg, can be changed with the '-c' parameter).
+
+    Args holds the variables parsed from the commandline (e.g. -j fills out
+    args.jobs), and values specified on the commandline count the most.
+
+    In case it is not specified on the commandline, for the keys in
+    pmb.config.config_keys, we look into the value set in the the user config.
+
+    When that is empty as well (e.g. just before pmbootstrap init), or the key
+    is not in pmb.config_keys, we use the default value from the internal
+    config.
+    """
+    # Use defaults from the user's config file
+    cfg = pmb.config.load(args)
+    for key in cfg["pmbootstrap"]:
+        if key not in args or getattr(args, key) is None:
+            value = cfg["pmbootstrap"][key]
+            if key in pmb.config.defaults:
+                default = pmb.config.defaults[key]
+                if isinstance(default, bool):
+                    value = (value.lower() == "true")
+            setattr(args, key, value)
+
+    # Use defaults from pmb.config.defaults
+    for key, value in pmb.config.defaults.items():
+        if key not in args or getattr(args, key) is None:
+            setattr(args, key, value)

pmb/config/save.py

@@ -21,7 +21,7 @@ import logging
 
 
 def save(args, cfg):
-    logging.debug("save config: " + args.config)
+    logging.debug("Save config: " + args.config)
     os.makedirs(os.path.dirname(args.config), 0o700, True)
     with open(args.config, "w") as handle:
         cfg.write(handle)

pmb/helpers/file.py

@@ -18,7 +18,7 @@ along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
 
 import os
-
+import time
 import pmb.helpers.run
 
 
@@ -60,6 +60,16 @@ def is_up_to_date(path_sources, path_target=None, lastmod_target=None):
     return lastmod_target >= lastmod_source
 
 
+def is_older_than(path, seconds):
+    """
+    Check if a single file is older than a given amount of seconds.
+    """
+    if not os.path.exists(path):
+        return True
+    lastmod = os.path.getmtime(path)
+    return lastmod + seconds < time.time()
+
+
 def symlink(args, file, link):
     """
     Checks if the symlink is already present, otherwise create it.

pmb/helpers/frontend.py

@@ -35,6 +35,7 @@ import pmb.chroot.other
 import pmb.flasher
 import pmb.helpers.logging
 import pmb.helpers.other
+import pmb.helpers.repo
 import pmb.helpers.run
 import pmb.install
 import pmb.parse
@@ -114,16 +115,15 @@ def chroot(args):
 
 
 def config(args):
-    pmb.helpers.logging.disable()
+    keys = pmb.config.config_keys
-    if args.name and args.name not in pmb.config.defaults:
+    if args.name and args.name not in keys:
-        valid_keys = ", ".join(sorted(pmb.config.defaults.keys()))
+        logging.info("NOTE: Valid config keys: " + ", ".join(keys))
-        print("The variable name you have specified is invalid.")
+        raise RuntimeError("Invalid config key: " + args.name)
-        print("The following are supported: " + valid_keys)
-        sys.exit(1)
 
     cfg = pmb.config.load(args)
     if args.value:
         cfg["pmbootstrap"][args.name] = args.value
+        logging.info("Config changed: " + args.name + "='" + args.value + "'")
         pmb.config.save(args, cfg)
     elif args.name:
         value = cfg["pmbootstrap"].get(args.name, "")
@@ -131,6 +131,9 @@ def config(args):
     else:
         cfg.write(sys.stdout)
 
+    # Don't write the "Done" message
+    pmb.helpers.logging.disable()
+
 
 def index(args):
     pmb.build.index_repo(args, args.arch_native)
@@ -157,6 +160,10 @@ def menuconfig(args):
     pmb.build.menuconfig(args, args.package)
 
 
+def update(args):
+    pmb.helpers.repo.update(args, True)
+
+
 def kconfig_check(args):
     # Default to all kernel packages
     packages = args.packages

pmb/helpers/http.py

@@ -24,7 +24,7 @@ import urllib.request
 import pmb.helpers.run
 
 
-def download(args, url, prefix, cache=True):
+def download(args, url, prefix, cache=True, loglevel=logging.INFO):
     """
     Download a file to disk.
     """
@@ -42,7 +42,7 @@ def download(args, url, prefix, cache=True):
         pmb.helpers.run.user(args, ["rm", path])
 
     # Download the file
-    logging.info("Download " + url)
+    logging.log(loglevel, "Download " + url)
     with urllib.request.urlopen(url) as response:
         with open(path, "wb") as handle:
             shutil.copyfileobj(response, handle)

pmb/helpers/repo.py

@@ -19,6 +19,9 @@ along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 import glob
 import os
 import hashlib
+import logging
+import pmb.helpers.http
+import pmb.helpers.run
 
 
 def files(args):
@@ -148,3 +151,48 @@ def apkindex_files(args, arch=None):
                    hash(url) + ".tar.gz")
 
     return ret
+
+
+def update(args, force=False):
+    """
+    Download the APKINDEX files for all URLs and architectures.
+    :arg force: even update when the APKINDEX file is fairly recent
+    """
+
+    architectures = [args.arch_native] + pmb.config.build_device_architectures
+    retention_hours = pmb.config.apkindex_retention_time
+    retention_seconds = retention_hours * 3600
+
+    outdated = {}
+    for url in urls(args, False):
+        for arch in architectures:
+            url_full = url + "/" + arch + "/APKINDEX.tar.gz"
+            cache_apk_outside = args.work + "/cache_apk_" + arch
+            apkindex = cache_apk_outside + "/APKINDEX." + hash(url) + ".tar.gz"
+
+            reason = None
+            if not os.path.exists(apkindex):
+                reason = "file does not exist yet"
+            elif force:
+                reason = "forced update"
+            elif pmb.helpers.file.is_older_than(apkindex, retention_seconds):
+                reason = "older than " + str(retention_hours) + "h"
+            if not reason:
+                continue
+
+            logging.debug("APKINDEX outdated (" + reason + "): " + url_full)
+            outdated[url_full] = apkindex
+
+    if not len(outdated):
+        return
+
+    # Show one message only
+    logging.info("Update package index (" + str(len(outdated)) + "x)")
+    for url, target in outdated.items():
+        # Download and move to right location
+        temp = pmb.helpers.http.download(args, url, "APKINDEX", False,
+                                         logging.DEBUG)
+        target_folder = os.path.dirname(target)
+        if not os.path.exists(target_folder):
+            pmb.helpers.run.root(args, ["mkdir", "-p", target_folder])
+        pmb.helpers.run.root(args, ["cp", temp, target])

pmb/parse/arguments.py

@@ -174,6 +174,7 @@ def arguments():
     sub.add_parser("shutdown", help="umount, unregister binfmt")
     sub.add_parser("index", help="re-index all repositories with custom built"
                    " packages (do this after manually removing package files)")
+    sub.add_parser("update", help="update all APKINDEX files")
     arguments_export(sub)
     arguments_flasher(sub)
     arguments_initfs(sub)
@@ -323,21 +324,15 @@ def arguments():
 
     # Use defaults from the user's config file
     args = parser.parse_args()
-    cfg = pmb.config.load(args)
+    pmb.config.merge_with_args(args)
-    for varname in cfg["pmbootstrap"]:
-        if varname not in args or not getattr(args, varname):
-            value = cfg["pmbootstrap"][varname]
-            if varname in pmb.config.defaults:
-                default = pmb.config.defaults[varname]
-                if isinstance(default, bool):
-                    value = (value.lower() == "true")
-            setattr(args, varname, value)
 
-    # Replace $WORK in variables from user's config
+    # Replace $WORK in variables from any config
-    for varname in cfg["pmbootstrap"]:
+    for key, value in pmb.config.defaults.items():
-        old = getattr(args, varname)
+        if key not in args:
+            continue
+        old = getattr(args, key)
         if isinstance(old, str):
-            setattr(args, varname, old.replace("$WORK", args.work))
+            setattr(args, key, old.replace("$WORK", args.work))
 
     # Add convenience shortcuts
     setattr(args, "arch_native", arch_native)

test/test_config_user.py

@@ -0,0 +1,87 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import os
+import sys
+import pytest
+
+# Import from parent directory
+sys.path.append(os.path.realpath(
+    os.path.join(os.path.dirname(__file__) + "/..")))
+import pmb.aportgen
+import pmb.config
+import pmb.helpers.frontend
+import pmb.helpers.logging
+
+
+@pytest.fixture
+def args(tmpdir, request):
+    import pmb.parse
+    sys.argv = ["pmbootstrap.py", "chroot"]
+    args = pmb.parse.arguments()
+    args.log = args.work + "/log_testsuite.txt"
+    pmb.helpers.logging.init(args)
+    request.addfinalizer(args.logfd.close)
+    return args
+
+
+def change_config(monkeypatch, path_config, key, value):
+    args = args_patched(monkeypatch, ["pmbootstrap.py", "-c", path_config, "config",
+                                      key, value])
+    pmb.helpers.frontend.config(args)
+
+
+def args_patched(monkeypatch, argv):
+    monkeypatch.setattr(sys, "argv", argv)
+    return pmb.parse.arguments()
+
+
+def test_config_user(args, tmpdir, monkeypatch):
+    # Temporary paths
+    tmpdir = str(tmpdir)
+    path_work = tmpdir + "/work"
+    path_config = tmpdir + "/pmbootstrap.cfg"
+
+    # Generate default config (only uses tmpdir)
+    os.chdir(pmb.config.pmb_src)
+    pmb.helpers.run.user(args, ["sh", "-c", "yes '' | ./pmbootstrap.py -c '" +
+                                path_config + "' -w '" + path_work + "' init"])
+
+    # Load and verify default config
+    argv = ["pmbootstrap.py", "-c", path_config, "config"]
+    args_default = args_patched(monkeypatch, argv)
+    assert args_default.work == path_work
+    assert args_default.timestamp_based_rebuild is True
+
+    # Modify timestamp_based_rebuild
+    change_config(monkeypatch, path_config, "timestamp_based_rebuild", "false")
+    assert args_patched(monkeypatch, argv).timestamp_based_rebuild is False
+    change_config(monkeypatch, path_config, "timestamp_based_rebuild", "true")
+    assert args_patched(monkeypatch, argv).timestamp_based_rebuild is True
+
+    # Modify jobs count
+    change_config(monkeypatch, path_config, "jobs", "9000")
+    assert args_patched(monkeypatch, argv).jobs == "9000"
+
+    # Override jobs count via commandline (-j)
+    argv_jobs = ["pmbootstrap.py", "-c", path_config, "-j", "1000", "config"]
+    assert args_patched(monkeypatch, argv_jobs).jobs == "1000"
+
+    # Override a config option with something that evaluates to false
+    argv_empty = ["pmbootstrap.py", "-c", path_config, "-w", "", "config"]
+    assert args_patched(monkeypatch, argv_empty).work == ""

test/test_file.py

@@ -0,0 +1,54 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import os
+import sys
+import time
+import pytest
+
+# Import from parent directory
+sys.path.append(os.path.realpath(
+    os.path.join(os.path.dirname(__file__) + "/..")))
+import pmb.helpers.git
+import pmb.helpers.logging
+import pmb.parse.version
+
+
+@pytest.fixture
+def args(request):
+    import pmb.parse
+    sys.argv = ["pmbootstrap.py", "chroot"]
+    args = pmb.parse.arguments()
+    args.log = args.work + "/log_testsuite.txt"
+    pmb.helpers.logging.init(args)
+    request.addfinalizer(args.logfd.close)
+    return args
+
+
+def test_file_is_older_than(args, tmpdir):
+    # Create a file last modified 10s ago
+    tempfile = str(tmpdir) + "/test"
+    pmb.helpers.run.user(args, ["touch", tempfile])
+    past = time.time() - 10
+    os.utime(tempfile, (-1, past))
+
+    # Check the bounds
+    func = pmb.helpers.file.is_older_than
+    assert func(tempfile, 9) is True
+    assert func(tempfile, 10) is True
+    assert func(tempfile, 11) is False

test/test_keys.py

@@ -26,7 +26,6 @@ import filecmp
 sys.path.append(os.path.realpath(
     os.path.join(os.path.dirname(__file__) + "/..")))
 import pmb.parse.apkindex
-import pmb.helpers.git
 import pmb.helpers.logging
 
 
@@ -42,16 +41,45 @@ def args(request):
 
 
 def test_keys(args):
-    mirror_path = os.path.join(os.path.dirname(__file__) + "../keys")
+    # Get the alpine-keys apk filename
-    original_path = args.work + "/cache_git/aports_upstream/main/alpine-keys"
+    pmb.chroot.init(args)
-    pmb.helpers.git.clone(args, "aports_upstream")
+    info = pmb.parse.apkindex.read_any_index(args, "alpine-keys")
+    version = info["version"]
+    pattern = (args.work + "/cache_apk_" + args.arch_native + "/alpine-keys-" +
+               version + ".*.apk")
+    filename = os.path.basename(glob.glob(pattern)[0])
 
-    # Check if original keys are mirrored correctly
+    # Extract it to a temporary folder
-    for path in glob.glob(original_path + "/*.key"):
+    temp = "/tmp/test_keys_extract"
-        key = os.path.basename(path)
+    temp_outside = args.work + "/chroot_native" + temp
-        assert filecmp.cmp(original_path + "/" + key, mirror_path + "/" + key,
+    if os.path.exists(temp_outside):
-                           False)
+        pmb.chroot.root(args, ["rm", "-r", temp])
+    pmb.chroot.user(args, ["mkdir", "-p", temp])
+    pmb.chroot.user(args, ["tar", "xvf", "/var/cache/apk/" + filename],
+                    working_dir=temp)
+
+    # Get all relevant key file names as {"filename": "full_outside_path"}
+    keys_upstream = {}
+    for arch in pmb.config.build_device_architectures + ["x86_64"]:
+        pattern = temp_outside + "/usr/share/apk/keys/" + arch + "/*.pub"
+        for path in glob.glob(pattern):
+            keys_upstream[os.path.basename(path)] = path
+    assert len(keys_upstream)
+
+    # Check if the keys are mirrored correctly
+    mirror_path_keys = os.path.dirname(__file__) + "/../keys"
+    for key, original_path in keys_upstream.items():
+        mirror_path = mirror_path_keys + "/" + key
+        assert filecmp.cmp(mirror_path, original_path, False)
+
+    # Find postmarketOS keys
+    keys_pmos = ["pmos-5a03a13a.rsa.pub"]
+    for key in keys_pmos:
+        assert os.path.exists(mirror_path_keys + "/" + key)
 
     # Find outdated keys, which need to be removed
-    for path in glob.glob(mirror_path + "/*.key"):
+    glob_result = glob.glob(mirror_path_keys + "/*.pub")
-        assert os.path.exists(original_path + "/" + os.path.basename(path))
+    assert len(glob_result)
+    for path in glob_result:
+        key = os.path.basename(key)
+        assert key in keys_pmos or key in keys_upstream

test/testcases_fast.sh

@@ -4,13 +4,11 @@
 # aport_in_sync_with_git: clones Alpine's aports repo
 # aportgen: clones Alpine's aports repo
 # build: builds cross-compilers for aarch64 and armhf
-# keys: clones Alpine's aports repo
 # version: clones Alpine's apk repo
 disabled="
 	aport_in_sync_with_git
 	aportgen
 	build
-	keys
 	version
 "