diff --git a/aports/main/libsamsung-ipc/APKBUILD b/aports/main/libsamsung-ipc/APKBUILD index bd54cd85..80a4f06b 100644 --- a/aports/main/libsamsung-ipc/APKBUILD +++ b/aports/main/libsamsung-ipc/APKBUILD @@ -1,7 +1,7 @@ pkgname=libsamsung-ipc pkgver=6.0_0002 _pkgver=${pkgver/_/-} -pkgrel=1 +pkgrel=2 pkgdesc="Implementation of Samsung modem protocol" url="https://redmine.replicant.us/projects/replicant/wiki/Libsamsung-ipc" arch="all" diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 4f369bc7..b003a2c1 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -36,6 +36,7 @@ import pmb.chroot.other import pmb.flasher import pmb.helpers.logging import pmb.helpers.other +import pmb.helpers.pkgrel_bump import pmb.helpers.repo import pmb.helpers.run import pmb.install @@ -252,6 +253,24 @@ def parse_apkindex(args): print(json.dumps(result, indent=4)) +def pkgrel_bump(args): + would_bump = True + if args.auto: + would_bump = pmb.helpers.pkgrel_bump.auto(args, args.dry) + else: + # Each package must exist + for package in args.packages: + pmb.build.other.find_aport(args, package) + + # Increase pkgrel + for package in args.packages: + pmb.helpers.pkgrel_bump.package(args, package, dry=args.dry) + + if args.dry and would_bump: + logging.info("Pkgrels of package(s) would have been bumped!") + sys.exit(1) + + def qemu(args): pmb.qemu.run(args) diff --git a/pmb/helpers/pkgrel_bump.py b/pmb/helpers/pkgrel_bump.py new file mode 100644 index 00000000..2335368a --- /dev/null +++ b/pmb/helpers/pkgrel_bump.py @@ -0,0 +1,162 @@ +""" +Copyright 2018 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import glob +import logging +import os + +import pmb.build.other +import pmb.helpers.file +import pmb.helpers.repo +import pmb.parse + + +def package(args, pkgname, reason="", dry=False): + """ + Increase the pkgrel in the APKBUILD of a specific package. + + :param pkgname: name of the package + :param reason: string to display as reason why it was increased + :param dry: don't modify the APKBUILD, just print the message + """ + # Current and new pkgrel + path = pmb.build.other.find_aport(args, pkgname) + "/APKBUILD" + apkbuild = pmb.parse.apkbuild(args, path) + pkgrel = int(apkbuild["pkgrel"]) + pkgrel_new = pkgrel + 1 + + # Display the message, bail out in dry mode + logging.info("Increase '" + pkgname + "' pkgrel (" + str(pkgrel) + " -> " + + str(pkgrel_new) + ")" + reason) + if dry: + return + + # Increase + old = "\npkgrel=" + str(pkgrel) + "\n" + new = "\npkgrel=" + str(pkgrel_new) + "\n" + pmb.helpers.file.replace(path, old, new) + + # Verify + del(args.cache["apkbuild"][path]) + apkbuild = pmb.parse.apkbuild(args, path) + if int(apkbuild["pkgrel"]) != pkgrel_new: + raise RuntimeError("Failed to bump pkgrel for package '" + pkgname + + "'. Make sure that there's a line with exactly the" + " string '" + old + "' and nothing else in: " + + path) + + +def auto_apkindex_files(args): + """ + Get the paths to the APKINDEX files, that need to be analyzed, sorted by + arch. Relevant are the local pmbootstrap generated APKINDEX as well as the + APKINDEX from the pmOS binary repo. +. + :returns: {"armhf": "...../APKINDEX.tar.gz", ...} + """ + pmb.helpers.repo.update(args) + ret = {} + for arch in pmb.config.build_device_architectures: + ret[arch] = [] + local = args.work + "/packages/" + arch + "/APKINDEX.tar.gz" + if os.path.exists(local): + ret[arch].append(local) + + if args.mirror_postmarketos: + path = (args.work + "/cache_apk_" + arch + "/APKINDEX." + + pmb.helpers.repo.hash(args.mirror_postmarketos) + ".tar.gz") + ret[arch].append(path) + return ret + + +def auto_apkindex_package(args, pkgname, aport_version, apkindex, arch, + dry=False): + """ + Bump the pkgrel of a specific package if it is outdated in the given + APKINDEX. + + :param pkgname: name of the package + :param aport_version: combination of pkgver and pkgrel (e.g. "1.23-r1") + :param apkindex: path to the APKINDEX.tar.gz file + :param arch: the architecture, e.g. "armhf" + :param dry: don't modify the APKBUILD, just print the message + :returns: True when there was an APKBUILD that needed to be changed. + """ + # Binary package + binary = pmb.parse.apkindex.read(args, pkgname, apkindex, + False) + if not binary: + return + + # Skip when aport version != binary package version + compare = pmb.parse.version.compare(aport_version, + binary["version"]) + if compare == -1: + logging.warning("WARNING: Skipping '" + pkgname + + "' in index " + apkindex + ", because the" + " binary version " + binary["version"] + + " is higher than the aport version " + + aport_version) + return + if compare == 1: + logging.verbose(pkgname + ": aport version bigger than the" + " one in the APKINDEX, skipping:" + + apkindex) + return + + # Find missing depends + logging.verbose(pkgname + ": checking depends: " + + ",".join(binary["depends"])) + missing = [] + for depend in binary["depends"]: + if not pmb.parse.apkindex.read_any_index(args, depend, + arch): + # We're only interested in missing depends starting with "so:" + # (which means dynamic libraries that the package was linked + # against) and packages for which no aport exists. + if (depend.startswith("so:") or + not pmb.build.other.find_aport(args, depend)): + missing.append(depend) + + # Increase pkgrel + if len(missing): + package(args, pkgname, reason=", missing depend(s): " + + ", ".join(missing), dry=dry) + return True + + +def auto(args, dry=False): + """ + :returns: True when there was an APKBUILD that needed to be changed. + """ + # Get APKINDEX files + arch_apkindexes = auto_apkindex_files(args) + + # Iterate over aports + ret = False + for aport in glob.glob(args.aports + "/*/*"): + pkgname = os.path.basename(aport) + aport = pmb.parse.apkbuild(args, aport + "/APKBUILD") + aport_version = aport["pkgver"] + "-r" + aport["pkgrel"] + + for arch, apkindexes in arch_apkindexes.items(): + for apkindex in apkindexes: + if auto_apkindex_package(args, pkgname, aport_version, apkindex, + arch, dry): + ret = True + return ret diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 8e1b3ac8..26f45296 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -123,6 +123,23 @@ def arguments_qemu(subparser): return ret +def arguments_pkgrel_bump(subparser): + ret = subparser.add_parser("pkgrel_bump", help="increase the pkgrel to" + " indicate that a package must be rebuilt" + " because of a dependency change") + ret.add_argument("--dry", action="store_true", help="instead of modifying" + " APKBUILDs, exit with >0 when a package would have been" + " bumped") + + # Mutually exclusive: "--auto" or package names + mode = ret.add_mutually_exclusive_group(required=True) + mode.add_argument("--auto", action="store_true", help="all packages which" + " depend on a library which had an incompatible update" + " (libraries with a soname bump)") + mode.add_argument("packages", nargs="*", default=[]) + return ret + + def arguments(): parser = argparse.ArgumentParser(prog="pmbootstrap") arch_native = pmb.parse.arch.alpine_native() @@ -179,6 +196,7 @@ def arguments(): arguments_flasher(sub) arguments_initfs(sub) arguments_qemu(sub) + arguments_pkgrel_bump(sub) # Action: log log = sub.add_parser("log", help="follow the pmbootstrap logfile") diff --git a/pmb/parse/depends.py b/pmb/parse/depends.py index 56ca1819..c2bab5f0 100644 --- a/pmb/parse/depends.py +++ b/pmb/parse/depends.py @@ -30,7 +30,7 @@ def recurse_error_message(pkgname, in_aports, in_apkindexes): ret += " and could not find it" if in_apkindexes: ret += " in any APKINDEX" - return ret + return ret + "." def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True, @@ -64,27 +64,47 @@ def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True, # Get depends and pkgname from aports depends = None pkgname = None + version = None if in_aports: aport = pmb.build.find_aport(args, pkgname_depend, False) if aport: - logging.verbose(pkgname_depend + ": found aport: " + aport) apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD") depends = apkbuild["depends"] + version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] + logging.verbose(pkgname_depend + ": " + version + + " found in " + aport) if pkgname_depend in apkbuild["subpackages"]: pkgname = pkgname_depend else: pkgname = apkbuild["pkgname"] # Get depends and pkgname from APKINDEX - if depends is None and in_apkindexes: + if in_apkindexes: index_data = pmb.parse.apkindex.read_any_index(args, pkgname_depend, arch) if index_data: - depends = index_data["depends"] - pkgname = index_data["pkgname"] + # The binary package's depends override the aport's depends in + # case it has the same or a higher version. Binary packages have + # sonames in their dependencies, which we need to detect + # breakage (#893). + outdated = (version and pmb.parse.version.compare(version, + index_data["version"]) == 1) + if not outdated: + if version: + logging.verbose(pkgname_depend + ": binary package is" + " up to date, using binary dependencies" + " instead of the ones from the aport") + depends = index_data["depends"] + pkgname = index_data["pkgname"] # Nothing found if pkgname is None and strict: + logging.info("NOTE: Run 'pmbootstrap pkgrel_bump --auto' to mark" + " packages with outdated dependencies for rebuild." + " This will most likely fix this issue (soname" + " bump?).") + logging.info("NOTE: More dependency calculation logging with" + " 'pmbootstrap -v'.") raise RuntimeError( recurse_error_message( pkgname_depend, diff --git a/test/test_pkgrel_bump.py b/test/test_pkgrel_bump.py new file mode 100644 index 00000000..738b011b --- /dev/null +++ b/test/test_pkgrel_bump.py @@ -0,0 +1,172 @@ +""" +Copyright 2018 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" + +""" +This file tests pmb.helper.pkgrel_bump +""" + +import glob +import os +import pytest +import sys + +# Import from parent directory +pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/..")) +sys.path.append(pmb_src) +import pmb.helpers.pkgrel_bump +import pmb.helpers.logging + + +@pytest.fixture +def args(request): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def pmbootstrap(args, tmpdir, parameters, zero_exit=True): + """ + Helper function for running pmbootstrap inside the fake work folder (created + by setup() below) with the binary repo disabled and with the testdata + configured as aports. + + :param parameters: what to pass to pmbootstrap, e.g. ["build", "testlib"] + :param zero_exit: expect pmbootstrap to exit with 0 (no error) + """ + # Run pmbootstrap + aports = tmpdir + "/_aports" + config = tmpdir + "/_pmbootstrap.cfg" + + try: + pmb.helpers.run.user(args, ["./pmbootstrap.py", "--work=" + tmpdir, + "--mirror-pmOS=", "--aports=" + aports, + "--config=" + config] + parameters, + working_dir=pmb_src) + + # Verify that it exits as desired + except Exception as exc: + if zero_exit: + raise RuntimeError("pmbootstrap failed") from exc + else: + return + if not zero_exit: + raise RuntimeError("Expected pmbootstrap to fail, but it did not!") + + +def setup_work(args, tmpdir): + """ + Create fake work folder in tmpdir with everything symlinked except for the + built packages. The aports testdata gets copied to the tempfolder as + well, so it can be modified during testing. + """ + # Clean the chroots, and initialize the build chroot in the native chroot. + # We do this before creating the fake work folder, because then all packages + # are still present. + os.chdir(pmb_src) + pmb.helpers.run.user(args, ["./pmbootstrap.py", "-y", "zap"]) + pmb.helpers.run.user(args, ["./pmbootstrap.py", "build_init"]) + pmb.helpers.run.user(args, ["./pmbootstrap.py", "shutdown"]) + + # Link everything from work (except for "packages") to the tmpdir + for path in glob.glob(args.work + "/*"): + if os.path.basename(path) != "packages": + pmb.helpers.run.user(args, ["ln", "-s", path, tmpdir + "/"]) + + # Copy testdata and selected device aport + for folder in ["device", "main"]: + pmb.helpers.run.user(args, ["mkdir", "-p", args.aports, tmpdir + + "/_aports/" + folder]) + pmb.helpers.run.user(args, ["cp", "-r", args.aports + "/device/device-" + + args.device, tmpdir + "/_aports/device"]) + for pkgname in ["testlib", "testapp"]: + pmb.helpers.run.user(args, ["cp", "-r", + "test/testdata/pkgrel_bump/aports/" + pkgname, + tmpdir + "/_aports/main/" + pkgname]) + + # Empty packages folder + pmb.helpers.run.user(args, ["mkdir", "-p", tmpdir + "/packages"]) + pmb.helpers.run.user(args, ["chmod", "777", tmpdir + "/packages"]) + + # Copy over the pmbootstrap config, disable timestamp based rebuilds + pmb.helpers.run.user(args, ["cp", args.config, tmpdir + + "/_pmbootstrap.cfg"]) + pmbootstrap(args, tmpdir, ["config", "timestamp_based_rebuild", "false"]) + + +def verify_pkgrels(args, tmpdir, pkgrel_testlib, pkgrel_testapp): + """ + Verify the pkgrels of the two test APKBUILDs "testlib" and "testapp". + """ + args.cache["apkbuild"] = {} + mapping = {"testlib": pkgrel_testlib, "testapp": pkgrel_testapp} + for pkgname, pkgrel in mapping.items(): + # APKBUILD path + path = tmpdir + "/_aports/main/" + pkgname + "/APKBUILD" + + # Parse and verify + apkbuild = pmb.parse.apkbuild(args, path) + assert pkgrel == int(apkbuild["pkgrel"]) + + +def test_pkgrel_bump_high_level(args, tmpdir): + # Tempdir setup + tmpdir = str(tmpdir) + setup_work(args, tmpdir) + + # Let pkgrel_bump exit normally + pmbootstrap(args, tmpdir, ["build", "testlib"]) + pmbootstrap(args, tmpdir, ["build", "testapp"]) + pmbootstrap(args, tmpdir, ["pkgrel_bump", "--dry", "--auto"]) + verify_pkgrels(args, tmpdir, 0, 0) + + # Increase soname (testlib soname changes with the pkgrel) + pmbootstrap(args, tmpdir, ["pkgrel_bump", "testlib"]) + verify_pkgrels(args, tmpdir, 1, 0) + pmbootstrap(args, tmpdir, ["build", "testlib"]) + pmbootstrap(args, tmpdir, ["pkgrel_bump", "--dry", "--auto"]) + verify_pkgrels(args, tmpdir, 1, 0) + + # Delete package with previous soname (--auto-dry exits with >0 now) + pmb.helpers.run.root(args, ["rm", tmpdir + "/packages/" + + args.arch_native + "/testlib-1.0-r0.apk"]) + pmbootstrap(args, tmpdir, ["index"]) + pmbootstrap(args, tmpdir, ["pkgrel_bump", "--dry", "--auto"], False) + verify_pkgrels(args, tmpdir, 1, 0) + + # Bump the pkgrel of testapp and build it + pmbootstrap(args, tmpdir, ["pkgrel_bump", "--auto"]) + verify_pkgrels(args, tmpdir, 1, 1) + pmbootstrap(args, tmpdir, ["build", "testapp"]) + + # After rebuilding, pkgrel_bump --auto-dry exits with 0 + pmbootstrap(args, tmpdir, ["pkgrel_bump", "--dry", "--auto"]) + verify_pkgrels(args, tmpdir, 1, 1) + + # Test running with specific package names + pmbootstrap(args, tmpdir, ["pkgrel_bump", "invalid_package_name"], False) + pmbootstrap(args, tmpdir, ["pkgrel_bump", "--dry", "testlib"], False) + verify_pkgrels(args, tmpdir, 1, 1) + + # Clean up + pmbootstrap(args, tmpdir, ["shutdown"]) + pmb.helpers.run.root(args, ["rm", "-rf", tmpdir]) diff --git a/test/test_soname_bump.py b/test/test_soname_bump.py new file mode 100644 index 00000000..a14236c5 --- /dev/null +++ b/test/test_soname_bump.py @@ -0,0 +1,58 @@ +""" +Copyright 2018 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" + +""" +This file uses pmb.helper.pkgrel_bump to check if the aports need a pkgrel bump +for any package, caused by a soname bump. Example: A new libressl/openssl +version was released, which increased the soname version, and now all packages +that link against it, need to be rebuilt. +""" + +import os +import pytest +import sys + +# Import from parent directory +pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/..")) +sys.path.append(pmb_src) +import pmb.helpers.pkgrel_bump +import pmb.helpers.logging + + +@pytest.fixture +def args(request): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def test_soname_bump(args): + if pmb.helpers.pkgrel_bump.auto(args, True): + raise RuntimeError("One or more packages need to be rebuilt, because" + " a library they link against had an incompatible" + " upgrade (soname bump). Run 'pmbootstrap" + " pkgrel_bump --auto' to automatically increase the" + " pkgrel in order to trigger a rebuild. If this" + " test case failed during a pull request, the issue" + " needs to be fixed on the 'master' branch first," + " then rebase your PR on 'master' afterwards.") diff --git a/test/testdata/pkgrel_bump/aports/testapp/APKBUILD b/test/testdata/pkgrel_bump/aports/testapp/APKBUILD new file mode 100644 index 00000000..7cf4ce88 --- /dev/null +++ b/test/testdata/pkgrel_bump/aports/testapp/APKBUILD @@ -0,0 +1,29 @@ +pkgname=testapp +pkgver=1.0 +pkgrel=0 +pkgdesc="program using the testlib (for testing soname bumps)" +url="https://postmarketos.org" +arch="all" +license="MIT" +depends="testlib" +makedepends="" +subpackages="" +source="testapp.c" +options="" + +build() { + cd "$srcdir" + $CC testapp.c -o testapp -L/usr/lib/ -ltestlib +} + +check() { + cd "$srcdir" + printf 'hello, world from testlib!\n' > expected + ./testapp > real + diff -q expected real +} + +package() { + install -Dm755 "$srcdir/testapp" "$pkgdir/usr/bin/testapp" +} +sha512sums="73b167575dc0082a1277b0430f095509885c7aaf55e59bad148825a9879f91fe41c6479bb7f34c0cdd15284b0aadd904a5ba2c1ea85fb8bfb061e1cbf4322d76 testapp.c" diff --git a/test/testdata/pkgrel_bump/aports/testapp/testapp.c b/test/testdata/pkgrel_bump/aports/testapp/testapp.c new file mode 100644 index 00000000..581fe8c7 --- /dev/null +++ b/test/testdata/pkgrel_bump/aports/testapp/testapp.c @@ -0,0 +1,7 @@ +#include +#include + +int main(int argc, char **argv) { + testlib_hello(); + return 0; +} diff --git a/test/testdata/pkgrel_bump/aports/testlib/APKBUILD b/test/testdata/pkgrel_bump/aports/testlib/APKBUILD new file mode 100644 index 00000000..a2f38edb --- /dev/null +++ b/test/testdata/pkgrel_bump/aports/testlib/APKBUILD @@ -0,0 +1,38 @@ +pkgname=testlib +pkgver=1.0 +pkgrel=0 +pkgdesc="testing soname bumps (soname changes with pkgrel!)" +url="https://postmarketos.org" +arch="all" +license="MIT" +depends="" +makedepends="" +subpackages="" +source="testlib.c testlib.h" +options="!check" + +build() { + cd "$srcdir" + local major="$pkgrel" + local minor="0" + local soname="libtestlib.so.$major" + local realname="libtestlib.so.$minor.$major" + + $CC -fPIC -c -g -Wall testlib.c -o libtestlib.o + $CC -shared -Wl,-soname,$soname -o $realname libtestlib.o + ln -sf $realname $soname + ln -sf $soname "libtestlib.so" +} + +package() { + cd "$srcdir" + install -Dm755 testlib.h "$pkgdir/usr/include/testlib.h" + + mkdir -p "$pkgdir/usr/lib/" + local i + for i in *.so*; do + cp -a "$i" "$pkgdir/usr/lib/$i" + done +} +sha512sums="15c671462a2f043e798b2998e8706f3ac119648c3d3ae946a0115c1f1aec567537f44e7e778bc77d3af4cd05a2d684677dabd56bb35799fca5939c6c087b4e27 testlib.c +16be61567995052e20f9436c6834c2ca2afcfb04fea15c5d02eb576ecfdc9ef4fed8d977468b2564bbe934d098d111837d96cc323dae3f4dd033aa1d061063ee testlib.h" diff --git a/test/testdata/pkgrel_bump/aports/testlib/testlib.c b/test/testdata/pkgrel_bump/aports/testlib/testlib.c new file mode 100644 index 00000000..cdfd63e8 --- /dev/null +++ b/test/testdata/pkgrel_bump/aports/testlib/testlib.c @@ -0,0 +1,5 @@ +#include + +void testlib_hello() { + printf("hello, world from testlib!\n"); +} diff --git a/test/testdata/pkgrel_bump/aports/testlib/testlib.h b/test/testdata/pkgrel_bump/aports/testlib/testlib.h new file mode 100644 index 00000000..92ba5e0c --- /dev/null +++ b/test/testdata/pkgrel_bump/aports/testlib/testlib.h @@ -0,0 +1,3 @@ +#pragma once + +void testlib_hello();