diff --git a/pmb/build/__init__.py b/pmb/build/__init__.py
index c89d01c7..5b6baa8b 100644
--- a/pmb/build/__init__.py
+++ b/pmb/build/__init__.py
@@ -23,3 +23,4 @@ from pmb.build.other import copy_to_buildpath, is_necessary, \
symlink_noarch_package, find_aport, ccache_stats, index_repo
from pmb.build.package import package
from pmb.build.menuconfig import menuconfig
+from pmb.build.challenge import challenge
diff --git a/pmb/build/buildinfo.py b/pmb/build/buildinfo.py
new file mode 100644
index 00000000..1ab3ca74
--- /dev/null
+++ b/pmb/build/buildinfo.py
@@ -0,0 +1,100 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap. If not, see .
+"""
+import os
+import json
+import pmb.chroot
+import pmb.chroot.apk
+import pmb.parse.apkindex
+
+
+def get_depends_recursively(args, pkgnames, arch=None):
+ """
+ :param pkgnames: List of pkgnames, for which the dependencies shall be
+ retrieved.
+ """
+ todo = list(pkgnames)
+ ret = []
+ seen = []
+ while len(todo):
+ pkgname = todo.pop(0)
+ index_data = pmb.parse.apkindex.read_any_index(args, pkgname, arch)
+ if not index_data:
+ raise RuntimeError("Could not find dependency " + pkgname +
+ " of packages " + str(pkgnames) + " in any APKINDEX")
+ pkgname = index_data["pkgname"]
+ if pkgname not in pkgnames and pkgname not in ret:
+ ret.append(pkgname)
+ for depend in index_data["depends"]:
+ if depend not in ret:
+ if depend.startswith("!"):
+ continue
+ for operator in [">", "="]:
+ if operator in depend:
+ depend = depend.split(operator)[0]
+ if depend not in seen:
+ todo.append(depend)
+ seen.append(depend)
+ return ret
+
+
+def generate(args, apk_path, carch, suffix, apkbuild):
+ """
+ :param apk_path: Path to the .apk file, relative to the packages cache.
+ :param carch: Architecture, that the package has been built for.
+ :apkbuild: Return from pmb.parse.apkbuild().
+ """
+ ret = {"pkgname": apkbuild["pkgname"],
+ "pkgver": apkbuild["pkgver"],
+ "pkgrel": apkbuild["pkgrel"],
+ "carch": carch,
+ "versions": []}
+
+ # Add makedepends versions
+ installed = pmb.chroot.apk.installed(args, suffix)
+ relevant = (apkbuild["makedepends"] +
+ get_depends_recursively(args, [apkbuild["pkgname"], "abuild", "build-base"]))
+ for pkgname in relevant:
+ if pkgname in installed:
+ ret["versions"].append(installed[pkgname]["package"])
+ ret["versions"].sort()
+ return ret
+
+
+def write(args, apk_path, carch, suffix, apkbuild):
+ """
+ Write a .buildinfo.json file for a package, right after building it.
+ It stores all information required to rebuild the package, very similar
+ to how they do it in Debian (but as JSON file, so it's easier to parse in
+ Python): https://wiki.debian.org/ReproducibleBuilds/BuildinfoFiles
+
+ :param apk_path: Path to the .apk file, relative to the packages cache.
+ :param carch: Architecture, that the package has been built for.
+ :apkbuild: Return from pmb.parse.apkbuild().
+ """
+ # Write to temp
+ if os.path.exists(args.work + "/chroot_native/tmp/buildinfo"):
+ pmb.chroot.root(args, ["rm", "/tmp/buildinfo"])
+ buildinfo = generate(args, apk_path, carch, suffix, apkbuild)
+ with open(args.work + "/chroot_native/tmp/buildinfo", "w") as handle:
+ handle.write(json.dumps(buildinfo, indent=4, sort_keys=True) + "\n")
+
+ # Move to packages
+ pmb.chroot.root(args, ["chown", "user:user", "/tmp/buildinfo"])
+ pmb.chroot.user(args, ["mv", "/tmp/buildinfo", "/home/user/packages/user/" +
+ apk_path + ".buildinfo.json"])
diff --git a/pmb/build/challenge.py b/pmb/build/challenge.py
new file mode 100644
index 00000000..ed5bc5be
--- /dev/null
+++ b/pmb/build/challenge.py
@@ -0,0 +1,109 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap. If not, see .
+"""
+import logging
+import json
+import os
+import tarfile
+import tempfile
+import filecmp
+import shutil
+import pmb.build
+import pmb.parse.apkbuild
+
+
+def diff(args, apk_a, apk_b):
+ logging.info("Challenge " + apk_a)
+ with tarfile.open(apk_a, "r:gz") as tar_a:
+ with tarfile.open(apk_b, "r:gz") as tar_b:
+ # List of files must be the same
+ list_a = sorted(tar_a.getnames())
+ list_b = tar_b.getnames()
+ list_b.sort()
+ if list_a != list_b:
+ raise RuntimeError(
+ "Both APKs do not contain the same file names!")
+
+ # Iterate through the list
+ for name in list_a:
+ logging.debug("Compare: " + name)
+ if name == ".PKGINFO" or name.startswith(".SIGN.RSA."):
+ logging.debug(
+ "=> Skipping, this is expected to be different")
+ continue
+ temp_files = []
+
+ # Extract
+ for tar in [tar_a, tar_b]:
+ member = tar.getmember(name)
+ if member.isdir():
+ continue
+ handle, path = tempfile.mkstemp("pmbootstrap")
+ handle = open(handle, "wb")
+ shutil.copyfileobj(tar.extractfile(member), handle)
+ handle.close()
+ temp_files.append(path)
+ if not len(temp_files):
+ logging.debug("=> Skipping, this is a directory")
+ continue
+
+ # Compare and delete
+ equal = filecmp.cmp(
+ temp_files[0], temp_files[1], shallow=False)
+ for temp_file in temp_files:
+ os.remove(temp_file)
+ if equal:
+ logging.debug("=> Equal")
+ else:
+ raise RuntimeError("File '" + name + "' is different!")
+
+
+def challenge(args, apk_path):
+ # Parse buildinfo
+ buildinfo_path = apk_path + ".buildinfo.json"
+ if not os.path.exists(buildinfo_path):
+ logging.info("NOTE: To create a .buildinfo.json file, use the"
+ " --buildinfo command while building: 'pmbootstrap build"
+ " --buildinfo '")
+ raise RuntimeError("Missing file: " + buildinfo_path)
+ with open(buildinfo_path) as handle:
+ buildinfo = json.load(handle)
+
+ # Parse and install all packages listed in versions
+ versions = {}
+ for package in buildinfo["versions"]:
+ split = pmb.chroot.apk.package_split(package)
+ pkgname = split["pkgname"]
+ versions[pkgname] = split
+ pmb.chroot.apk.install(args, versions.keys())
+
+ # Verify the installed versions
+ installed = pmb.chroot.apk.installed(args)
+ for pkgname, split in versions.items():
+ package_installed = installed[pkgname]["package"]
+ package_buildinfo = split["package"]
+ if package_installed != package_buildinfo:
+ raise RuntimeError("Dependency " + pkgname + " version is different"
+ " (installed: " + package_installed + ","
+ " buildinfo: " + package_buildinfo + ")!")
+ # Build the package
+ output = pmb.build.package(args, buildinfo["pkgname"], buildinfo["carch"],
+ force=True)
+
+ # Diff the apk contents
+ diff(args, apk_path, args.work + "/packages/" + output)
diff --git a/pmb/build/package.py b/pmb/build/package.py
index 6b8de703..7f600227 100644
--- a/pmb/build/package.py
+++ b/pmb/build/package.py
@@ -21,6 +21,7 @@ import logging
import pmb.build
import pmb.build.autodetect
+import pmb.build.buildinfo
import pmb.build.crosscompiler
import pmb.chroot
import pmb.chroot.apk
@@ -29,11 +30,12 @@ import pmb.parse
import pmb.parse.arch
-def package(args, pkgname, carch, force=False, recurse=True):
+def package(args, pkgname, carch, force=False, recurse=True, buildinfo=False):
"""
Build a package with Alpine Linux' abuild.
:param force: even build, if not necessary
+ :returns: output path relative to the packages folder
"""
# Get aport, skip upstream only packages
aport = pmb.build.find_aport(args, pkgname, False)
@@ -109,6 +111,14 @@ def package(args, pkgname, carch, force=False, recurse=True):
if not os.path.exists(path):
raise RuntimeError("Package not found after build: " + path)
+ # Create .buildinfo.json file
+ if buildinfo:
+ logging.info("(" + suffix + ") generate " + output + ".buildinfo.json")
+ pmb.build.buildinfo.write(args, output, carch_buildenv, suffix,
+ apkbuild)
+
# Symlink noarch packages
if "noarch" in apkbuild["arch"]:
pmb.build.symlink_noarch_package(args, output)
+
+ return output
diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py
index 879a3e15..f4b19429 100644
--- a/pmb/chroot/apk.py
+++ b/pmb/chroot/apk.py
@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see .
"""
import logging
-import os
import pmb.chroot
import pmb.parse.apkindex
@@ -58,19 +57,43 @@ def install(args, packages, suffix="native", build=True):
pmb.chroot.root(args, ["apk", "--no-progress", "add"] + packages_todo,
suffix)
-# Update all packages installed in a chroot
-
def update(args, suffix="native"):
+ """
+ Update all packages installed in a chroot
+ """
pmb.chroot.init(args, suffix)
pmb.chroot.root(args, ["apk", "update"], suffix)
-# Get all explicitly installed packages
+
+def package_split(package):
+ """
+ FIXME: move to pmb.parse
+ """
+ split = package.split("-")
+ pkgrel = split[-1][1:]
+ pkgver = split[-2]
+ version = "-" + pkgver + "-r" + pkgrel
+ pkgname = package[:-1 * len(version)]
+ return {"pkgname": pkgname,
+ "pkgrel": pkgrel,
+ "pkgver": pkgver,
+ "package": package}
def installed(args, suffix="native"):
- world = args.work + "/chroot_" + suffix + "/etc/apk/world"
- if not os.path.exists(world):
- return []
- with open(world, encoding="utf-8") as handle:
- return handle.read().splitlines()
+ """
+ Get all installed packages and their versions.
+ :returns: { "hello-world": {"package": "hello-world-1-r2", "pkgrel": "2",
+ "pkgver": "1", "pkgname": "hello-world"}, ...}
+ """
+ ret = {}
+ list = pmb.chroot.user(args, ["apk", "info", "-vv"], suffix,
+ return_stdout=True)
+ for line in list.split("\n"):
+ if not line.rstrip():
+ continue
+ package = line.split(" - ")[0]
+ split = package_split(package)
+ ret[split["pkgname"]] = split
+ return ret
diff --git a/pmb/helpers/run.py.orig b/pmb/helpers/run.py.orig
new file mode 100644
index 00000000..e9b419a9
--- /dev/null
+++ b/pmb/helpers/run.py.orig
@@ -0,0 +1,76 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap. If not, see .
+"""
+import subprocess
+import logging
+
+
+def core(args, cmd, log_message, log, return_stdout, check=True):
+ logging.debug(log_message)
+ """
+ Run the command and write the output to the log.
+
+ :param check: raise an exception, when the command fails
+ """
+
+ try:
+ ret = None
+ if log:
+ if return_stdout:
+ ret = subprocess.check_output(cmd).decode("utf-8")
+ args.logfd.write(ret)
+ else:
+ subprocess.check_call(cmd, stdout=args.logfd,
+ stderr=args.logfd)
+ args.logfd.flush()
+ else:
+ logging.debug("*** output passed to pmbootstrap stdout, not" +
+ " to this log ***")
+ subprocess.check_call(cmd)
+
+ except subprocess.CalledProcessError as exc:
+ if check:
+ if log:
+ logging.debug("^" * 70)
+ logging.info("NOTE: The failed command's output is above"
+ " the ^^^ line in the logfile: " + args.log)
+ raise RuntimeError("Command failed: " + log_message) from exc
+ else:
+ pass
+ return ret
+
+
+def user(args, cmd, log=True, working_dir=None, return_stdout=False,
+ check=True):
+ """
+ :param working_dir: defaults to args.work
+ """
+ if not working_dir:
+ working_dir = args.work
+
+ # TODO: maintain and check against a whitelist
+ return core(args, cmd, "% " + " ".join(cmd), log, return_stdout, check)
+
+
+def root(args, cmd, log=True, working_dir=None, return_stdout=False,
+ check=True):
+ """
+ :param working_dir: defaults to args.work
+ """
+ cmd = ["sudo"] + cmd
+ return user(args, cmd, log, working_dir, return_stdout, check)
diff --git a/pmb/parse/apkbuild.py b/pmb/parse/apkbuild.py
index 2e0488e8..9af6fe89 100644
--- a/pmb/parse/apkbuild.py
+++ b/pmb/parse/apkbuild.py
@@ -37,7 +37,7 @@ def replace_variables(apkbuild):
replaced.append(subpackage.replace("$pkgname", ret["pkgname"]))
ret["subpackages"] = replaced
- # makedepend: $makedepends_host, $makedepends_build, $_llvmver
+ # makedepends: $makedepends_host, $makedepends_build, $_llvmver
replaced = []
for makedepend in ret["makedepends"]:
if makedepend.startswith("$"):
diff --git a/pmb/parse/apkindex.py b/pmb/parse/apkindex.py
index ead1a85d..6281d3a0 100644
--- a/pmb/parse/apkindex.py
+++ b/pmb/parse/apkindex.py
@@ -74,6 +74,17 @@ def read(args, package, path, must_exist=True):
if not ret or compare_version(current["version"],
ret["version"]) == 1:
ret = current
+ if "provides" in current:
+ for alias in current["provides"]:
+ split = alias.split("=")
+ if len(split) == 1:
+ continue
+ name = split[0]
+ version = split[1]
+ if name == package:
+ if not ret or compare_version(current["version"],
+ version) == 1:
+ ret = current
current = {}
if line.startswith("P:"): # package
current["pkgname"] = line[2:-1]
@@ -85,8 +96,17 @@ def read(args, package, path, must_exist=True):
current["depends"] = depends.split(" ")
else:
current["depends"] = []
+ if line.startswith("p:"): # provides
+ provides = line[2:-1]
+ current["provides"] = provides.split(" ")
if not ret and must_exist:
raise RuntimeError("Package " + package + " not found in " + path)
+
+ if ret:
+ for key in ["depends", "provides"]:
+ if key not in ret:
+ ret[key] = []
+
return ret
diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py
index f6c98ad5..37638550 100644
--- a/pmb/parse/arguments.py
+++ b/pmb/parse/arguments.py
@@ -120,9 +120,15 @@ def arguments():
" specific architecture")
build.add_argument("--arch")
build.add_argument("--force", action="store_true")
+ build.add_argument("--buildinfo", action="store_true")
for action in [checksum, build, menuconfig, parse_apkbuild, aportgen]:
action.add_argument("package")
+ # Action: challenge
+ challenge = sub.add_parser("challenge",
+ help="rebuild a package and diff its contents")
+ challenge.add_argument("apk")
+
# Use defaults from the user's config file
args = parser.parse_args()
cfg = pmb.config.load(args)
diff --git a/pmbootstrap.py b/pmbootstrap.py
index 1d1c7e48..bd140ae5 100755
--- a/pmbootstrap.py
+++ b/pmbootstrap.py
@@ -55,9 +55,12 @@ def main():
if args.action == "aportgen":
pmb.aportgen.generate(args, args.package)
elif args.action == "build":
- pmb.build.package(args, args.package, args.arch, args.force, False)
+ pmb.build.package(args, args.package, args.arch, args.force, False,
+ args.buildinfo)
elif args.action == "build_init":
pmb.build.init(args, args.suffix)
+ elif args.action == "challenge":
+ pmb.build.challenge(args, args.apk)
elif args.action == "checksum":
pmb.build.checksum(args, args.package)
elif args.action == "chroot":