Add support for the binary repository, inactive by default (#64)

* New commandline parameter --mirror-pmOS, where the binary repository
  URL for postmarketOS can be specified (empty by default as of now,
  this will be filled with the real URL once the repo works)
* Do not build packages, when they are in the binary repository and
  the version of the package in the binary repository is up-to-date.
* Add a testcase for pmb.build.is_necessary().

pmb/build/other.py

@@ -78,38 +78,39 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None):
     :param apkindex_path: override the APKINDEX.tar.gz path
     :returns: boolean
     """
-
     # Get new version from APKBUILD
     package = apkbuild["pkgname"]
     version_new = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
 
     # Get old version from APKINDEX
-    if not apkindex_path:
+    if apkindex_path:
-        apkindex_path = (args.work + "/packages/" + arch +
+        index_data = pmb.parse.apkindex.read(
-                         "/APKINDEX.tar.gz")
+            args, package, apkindex_path, False)
-    version_old = None
+    else:
-    index_data = pmb.parse.apkindex.read(args, package, apkindex_path,
+        index_data = pmb.parse.apkindex.read_any_index(args, package, arch)
-                                         False)
+    if not index_data:
-    if index_data:
+        return True
-        version_old = index_data["version"]
 
+    # a) Binary repo has a newer version
+    version_old = index_data["version"]
     if pmb.parse.apkindex.compare_version(version_old,
                                           version_new) == 1:
-        logging.warning("WARNING: Package " + package + "-" + version_old +
+        logging.warning("WARNING: Package '" + package + "' in your aports folder"
-                        " in your binary repository is higher than the version defined" +
+                        " has version " + version_old + ", but the binary package"
-                        " in the APKBUILD. Consider cleaning your package cache" +
+                        " repositories already have version " + version_new + "!")
-                        " (pmbootstrap zap -p) or removing that file and running" +
+        return False
-                        " 'pmbootstrap index'!")
 
+    # b) Aports folder has a newer version
     if version_new != version_old:
         return True
 
-    # Check if all files in the aport folder have an older timestamp,
+    # c) The version is the same. Check if all files in the aport folder have an
-    # than the package.
+    # older timestamp, than the package. This way the pkgrel doesn't need to be
-    path_target = (os.path.dirname(apkindex_path) + "/" + package + "-" +
+    # increased while developing locally.
-                   version_new + ".apk")
+    lastmod_target = float(index_data["timestamp"])
     path_sources = glob.glob(args.aports + "/" + package + "/*")
-    if pmb.helpers.file.is_up_to_date(path_target, path_sources):
+    if pmb.helpers.file.is_up_to_date(
+            path_sources, lastmod_target=lastmod_target):
         return False
     return True
 

pmb/chroot/init.py

@@ -79,6 +79,8 @@ def init(args, suffix="native"):
     repos_path = chroot + "/etc/apk/repositories"
     if not os.path.exists(repos_path):
         lines = ["/home/user/packages/user"]
+        if args.mirror_postmarketos:
+            lines.append(args.mirror_postmarketos)
         directories = ["main", "community"]
         if args.alpine_version == "edge":
             directories.append("testing")

pmb/config/__init__.py

@@ -48,6 +48,7 @@ defaults = {
     "device": "samsung-i9100",
     "log": "$WORK/log.txt",
     "mirror_alpine": "https://nl.alpinelinux.org/alpine/",
+    "mirror_postmarketos": "",
     "work": os.path.expanduser("~") + "/.local/var/pmbootstrap",
     "port_distccd": "33632",
 

pmb/helpers/file.py

@@ -31,21 +31,28 @@ def replace(path, old, new):
         handle.write(text)
 
 
-def is_up_to_date(path_target, path_sources):
+def is_up_to_date(path_sources, path_target=None, lastmod_target=None):
     """
     Check if a file is up-to-date by comparing the last modified timestamps
     (just like make does it).
 
-    :param path_target: full path to the target file
     :param path_sources: list of full paths to the source files
+    :param path_target: full path to the target file
+    :param lastmod_target: the timestamp of the target file. specify this as
+                           alternative to specifying path_target.
     """
 
+    if path_target and lastmod_target:
+        raise RuntimeError(
+            "Specify path_target *or* lastmod_target, not both!")
+
     lastmod_source = None
     for path_source in path_sources:
         lastmod = os.path.getmtime(path_source)
         if not lastmod_source or lastmod > lastmod_source:
             lastmod_source = lastmod
 
-    lastmod_target = os.path.getmtime(path_target)
+    if path_target:
+        lastmod_target = os.path.getmtime(path_target)
 
     return lastmod_target >= lastmod_source

pmb/parse/apkindex.py

@@ -70,7 +70,8 @@ def parse_next_block(args, path, lines, start):
         "P": "pkgname",
         "V": "version",
         "D": "depends",
-        "p": "provides"
+        "p": "provides",
+        "t": "timestamp"
     }
     end_of_block_found = False
     for i in range(start[0], len(lines)):
@@ -93,7 +94,7 @@ def parse_next_block(args, path, lines, start):
     # Format and return the block
     if end_of_block_found:
         # Check for required keys
-        for key in ["pkgname", "version"]:
+        for key in ["pkgname", "version", "timestamp"]:
             if key not in ret:
                 raise RuntimeError("Missing required key '" + key +
                                    "' in block " + str(ret) + ", file: " + path)

pmb/parse/arguments.py

@@ -88,6 +88,7 @@ def arguments():
     parser.add_argument("-c", "--config", dest="config",
                         default=pmb.config.defaults["config"])
     parser.add_argument("-d", "--port-distccd", dest="port_distccd")
+    parser.add_argument("-mp", "--mirror-pmOS", dest="mirror_postmarketos")
     parser.add_argument("-m", "--mirror-alpine", dest="mirror_alpine")
     parser.add_argument("-j", "--jobs", help="parallel jobs when compiling")
     parser.add_argument("-p", "--aports",

test/test_build_is_necessary.py

@@ -0,0 +1,79 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import os
+import sys
+import pytest
+
+# Import from parent directory
+sys.path.append(os.path.abspath(
+    os.path.join(os.path.dirname(__file__) + "/..")))
+import pmb.build.other
+
+
+@pytest.fixture
+def args(request, tmpdir):
+    import pmb.parse
+    sys.argv = ["pmbootstrap.py", "chroot"]
+    args = pmb.parse.arguments()
+    setattr(args, "logfd", open("/dev/null", "a+"))
+    request.addfinalizer(args.logfd.close)
+    request.addfinalizer(args.logfd.close)
+
+    # Create an empty APKINDEX.tar.gz file, so we can use its path and
+    # timestamp to put test information in the cache.
+    apkindex_path = str(tmpdir) + "/APKINDEX.tar.gz"
+    open(apkindex_path, "a").close()
+    lastmod = os.path.getmtime(apkindex_path)
+    args.cache["apkindex"][apkindex_path] = {"lastmod": lastmod, "ret": {}}
+    return args
+
+
+def test_build_is_necessary(args):
+    # Prepare APKBUILD and APKINDEX data
+    apkbuild = pmb.parse.apkbuild(args.aports + "/hello-world/APKBUILD")
+    apkbuild["pkgver"] = "1"
+    apkbuild["pkgrel"] = "2"
+    apkindex_path = list(args.cache["apkindex"].keys())[0]
+    args.cache["apkindex"][apkindex_path]["ret"] = {
+        "hello-world": {"pkgname": "hello-world", "version": "1-r2"}
+    }
+
+    # a) Binary repo has a newer version
+    args.cache["apkindex"][apkindex_path]["ret"][
+        "hello-world"]["version"] = "999-r1"
+    assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
+
+    # b) Aports folder has a newer version
+    args.cache["apkindex"][apkindex_path][
+        "ret"]["hello-world"]["version"] = "0-r0"
+    assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True
+
+    # c) Same version
+    args.cache["apkindex"][apkindex_path][
+        "ret"]["hello-world"]["version"] = "1-r2"
+
+    # c.1) Newer timestamp in aport (timestamp in repo: 1970-01-01)
+    args.cache["apkindex"][apkindex_path][
+        "ret"]["hello-world"]["timestamp"] = "0"
+    assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True
+
+    # c.2) Newer timestamp in binary repo (timestamp in repo: 3000-01-01)
+    args.cache["apkindex"][apkindex_path]["ret"][
+        "hello-world"]["timestamp"] = "32503680000"
+    assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False