diff --git a/pmb/__init__.py b/pmb/__init__.py index 2ee0f121..8deed1dd 100644 --- a/pmb/__init__.py +++ b/pmb/__init__.py @@ -16,8 +16,6 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with pmbootstrap. If not, see . """ - - import sys import logging import os diff --git a/pmb/build/__init__.py b/pmb/build/__init__.py index 69e1b861..dab04fe3 100644 --- a/pmb/build/__init__.py +++ b/pmb/build/__init__.py @@ -19,8 +19,8 @@ along with pmbootstrap. If not, see . # Exported functions from pmb.build.init import init from pmb.build.checksum import checksum -from pmb.build.other import copy_to_buildpath, is_necessary, \ - symlink_noarch_packages, find_aport, ccache_stats, index_repo -from pmb.build.package import package from pmb.build.menuconfig import menuconfig +from pmb.build.other import copy_to_buildpath, is_necessary, \ + find_aport, ccache_stats, index_repo +from pmb.build._package import package from pmb.build.qemu_workaround_aarch64 import qemu_workaround_aarch64 diff --git a/pmb/build/_package.py b/pmb/build/_package.py new file mode 100644 index 00000000..b1239141 --- /dev/null +++ b/pmb/build/_package.py @@ -0,0 +1,289 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import logging + +import pmb.build +import pmb.build.autodetect +import pmb.build.buildinfo +import pmb.chroot +import pmb.chroot.apk +import pmb.chroot.distccd +import pmb.helpers.repo +import pmb.parse +import pmb.parse.arch + + +def get_apkbuild(args, pkgname, arch): + """ + Find the APKBUILD path for pkgname. When there is none, try to find it in + the binary package APKINDEX files or raise an exception. + + :param pkgname: package name to be built, as specified in the APKBUILD + :returns: None or full path to APKBUILD + """ + # Get existing binary package indexes + pmb.helpers.repo.update(args) + + # Get aport, skip upstream only packages + aport = pmb.build.find_aport(args, pkgname, False) + if aport: + return pmb.parse.apkbuild(args, aport + "/APKBUILD") + if pmb.parse.apkindex.read_any_index(args, pkgname, arch): + return None + raise RuntimeError("Package '" + pkgname + "': Could not find aport, and" + " could not find this package in any APKINDEX!") + + +def check_arch(args, apkbuild, arch): + """ + Check if the APKBUILD can be built for a specific architecture and abort + with a helpful message if it is not the case. + """ + for value in [arch, "all", "noarch"]: + if value in apkbuild["arch"]: + return + + pkgname = apkbuild["pkgname"] + logging.info("NOTE: You can edit the 'arch=' line inside the APKBUILD") + if args.action == "build": + logging.info("NOTE: Alternatively, use --arch to build for another" + "architecture ('pmbootstrap build --arch=armhf " + + pkgname + "')") + raise RuntimeError("Can't build '" + pkgname + "' for architecture " + + arch) + + +def get_depends(args, apkbuild): + """ + Alpine's abuild always builds/installs the "depends" and "makedepends" + of a package before building it. We used to only care about "makedepends" + and it's still possible to ignore the depends with --ignore-depends. + + :returns: list of dependency pkgnames (eg. ["sdl2", "sdl2_net"]) + """ + ret = list(apkbuild["makedepends"]) + if "ignore_depends" not in args or not args.ignore_depends: + ret += apkbuild["depends"] + + return sorted(set(ret)) + + +def build_depends(args, apkbuild, arch, strict): + """ + Get and build dependencies with verbose logging messages. + + :returns: (depends, depends_built) + """ + # Get dependencies + pkgname = apkbuild["pkgname"] + depends = get_depends(args, apkbuild) + logging.verbose(pkgname + ": build/install dependencies: " + + ", ".join(depends)) + + # Build them + depends_built = [] + for depend in depends: + if package(args, depend, arch, strict=strict): + depends_built += [depend] + logging.verbose(pkgname + ": build dependencies: done, built: " + + ", ".join(depends_built)) + + return (depends, depends_built) + + +def is_necessary_warn_depends(args, apkbuild, arch, force, depends_built): + """ + Check if a build is necessary, and warn if it is not, but there were + dependencies built. + + :returns: True or False + """ + pkgname = apkbuild["pkgname"] + ret = True if force else pmb.build.is_necessary(args, arch, apkbuild) + + if not ret and len(depends_built): + # Warn of potentially outdated package + logging.warning("WARNING: " + pkgname + " depends on rebuilt" + + " package(s) " + ",".join(depends_built) + " (use" + + " 'pmbootstrap build " + pkgname + " --force' if" + + " necessary!)") + + logging.verbose(pkgname + ": build necessary: " + str(ret)) + return ret + + +def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None, + suffix="native", skip_init_buildenv=False): + """ + Build all dependencies, check if we need to build at all (otherwise we've + just initialized the build environment for nothing) and then setup the + whole build environment (abuild, gcc, dependencies, cross-compiler). + + :param cross: None, "native" or "distcc" + :param skip_init_buildenv: can be set to False to avoid initializing the + build environment. Use this when building + something during initialization of the build + environment (e.g. qemu aarch64 bug workaround) + :returns: True when the build is necessary (otherwise False) + """ + # Build dependencies + depends, built = build_depends(args, apkbuild, arch, strict) + + # Check if build is necessary + if not is_necessary_warn_depends(args, apkbuild, arch, force, built): + return False + + # Install and configure abuild, gcc, dependencies + if not skip_init_buildenv: + pmb.build.init(args, suffix) + pmb.build.other.configure_abuild(args, suffix) + if not strict and len(depends): + pmb.chroot.apk.install(args, depends, suffix) + + # Cross-compiler init + if cross: + pmb.chroot.apk.install(args, ["gcc-" + arch, "g++-" + arch, + "ccache-cross-symlinks"]) + if cross == "distcc": + pmb.chroot.apk.install(args, ["distcc"], suffix=suffix, + build=False) + pmb.chroot.distccd.start(args, arch) + + return True + + +def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None, + suffix="native"): + """ + Set up all environment variables and construct the abuild command (all + depending on the cross-compiler method and target architecture), copy + the aport to the chroot and execute abuild. + + :param cross: None, "native" or "distcc" + :returns: (output, cmd, env), output is the destination apk path relative + to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are + used by the test case, and they are the full abuild command and + the environment variables dict generated in this function. + """ + # Sanity check + if cross == "native" and "!tracedeps" not in apkbuild["options"]: + logging.info("WARNING: Option !tracedeps is not set, but we're" + " cross-compiling in the native chroot. This will" + " probably fail!") + + # Pretty log message + output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] + + "-r" + apkbuild["pkgrel"] + ".apk") + logging.info("(" + suffix + ") build " + output) + + # Environment variables + env = {"CARCH": arch} + if cross == "native": + hostspec = pmb.parse.arch.alpine_to_hostspec(arch) + env["CROSS_COMPILE"] = hostspec + "-" + env["CC"] = hostspec + "-gcc" + if cross == "distcc": + env["PATH"] = "/usr/lib/distcc/bin:" + pmb.config.chroot_path + env["DISTCC_HOSTS"] = "127.0.0.1:" + args.port_distccd + + # Build the abuild command + cmd = [] + for key, value in env.items(): + cmd += [key + "=" + value] + cmd += ["abuild"] + if strict: + cmd += ["-r"] # install depends with abuild + else: + cmd += ["-d"] # do not install depends with abuild + if force: + cmd += ["-f"] + + # Copy the aport to the chroot and build it + pmb.build.copy_to_buildpath(args, apkbuild["pkgname"], suffix) + pmb.chroot.user(args, cmd, suffix, "/home/pmos/build") + return (output, cmd, env) + + +def finish(args, apkbuild, arch, output, strict=False, suffix="native", + buildinfo=False): + """ + Various finishing tasks that need to be done after a build. + """ + # Verify output file + path = args.work + "/packages/" + output + if not os.path.exists(path): + raise RuntimeError("Package not found after build: " + path) + + # Create .buildinfo.json file (captures the build environment, from the + # reproducible builds approach in #64 that we aren't using anymore, but it + # might still be useful) + if buildinfo: + logging.info("(" + suffix + ") generate " + output + ".buildinfo.json") + pmb.build.buildinfo.write(args, output, arch, suffix, apkbuild) + + # Clear APKINDEX cache (we only parse APKINDEX files once per session and + # cache the result for faster dependency resolving, but after we built a + # package we need to parse it again) + pmb.parse.apkindex.clear_cache(args, args.work + "/packages/" + + arch + "/APKINDEX.tar.gz") + + # Uninstall build dependencies (strict mode) + if strict: + logging.info("(" + suffix + ") uninstall build dependencies") + pmb.chroot.user(args, ["abuild", "undeps"], suffix, "/home/pmos/build") + + +def package(args, pkgname, arch=None, force=False, buildinfo=False, + strict=False, skip_init_buildenv=False): + """ + Build a package and its dependencies with Alpine Linux' abuild. + + :param pkgname: package name to be built, as specified in the APKBUILD + :param arch: architecture we're building for (default: native) + :param force: even build, if not necessary + :param buildinfo: record the build environment in a .buildinfo.json file + :param strict: avoid building with irrelevant dependencies installed by + letting abuild install and uninstall all dependencies. + :param skip_init_buildenv: can be set to False to avoid initializing the + build environment. Use this when building + something during initialization of the build + environment (e.g. qemu aarch64 bug workaround) + :returns: None if the build was not necessary + output path relative to the packages folder ("armhf/ab-1-r2.apk") + """ + # Only build when APKBUILD exists + arch = arch or args.arch_native + apkbuild = get_apkbuild(args, pkgname, arch) + if not apkbuild: + return + + # Detect the build environment (skip unnecessary builds) + check_arch(args, apkbuild, arch) + suffix = pmb.build.autodetect.suffix(args, apkbuild, arch) + cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix) + if not init_buildenv(args, apkbuild, arch, strict, force, cross, suffix, + skip_init_buildenv): + return + + # Build and finish up + (output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross, + suffix) + finish(args, apkbuild, arch, output, strict, suffix, buildinfo) + return output diff --git a/pmb/build/autodetect.py b/pmb/build/autodetect.py index 26c7245a..0dd95896 100644 --- a/pmb/build/autodetect.py +++ b/pmb/build/autodetect.py @@ -22,28 +22,8 @@ import pmb.chroot.apk import pmb.parse.arch -def carch(args, apkbuild, carch, strict=False): - if "noarch" in apkbuild["arch"]: - if "noarch_arch" in args and args.noarch_arch: - return args.noarch_arch - if strict: - return args.deviceinfo["arch"] - return args.arch_native - if carch: - if "all" not in apkbuild["arch"] and carch not in apkbuild["arch"]: - raise RuntimeError("Architecture '" + carch + "' is not supported" - " for this package. Please add it to the" - " 'arch=' line inside the APKBUILD and try" - " again: " + apkbuild["pkgname"]) - return carch - if ("all" in apkbuild["arch"] or - args.arch_native in apkbuild["arch"]): - return args.arch_native - return apkbuild["arch"][0] - - -def suffix(args, apkbuild, carch): - if carch == args.arch_native: +def suffix(args, apkbuild, arch): + if arch == args.arch_native: return "native" pkgname = apkbuild["pkgname"] @@ -54,10 +34,10 @@ def suffix(args, apkbuild, carch): if fnmatch.fnmatch(pkgname, pattern): return "native" - return "buildroot_" + carch + return "buildroot_" + arch -def crosscompile(args, apkbuild, carch, suffix): +def crosscompile(args, apkbuild, arch, suffix): """ :returns: None, "native" or "distcc" """ @@ -65,7 +45,7 @@ def crosscompile(args, apkbuild, carch, suffix): return None if apkbuild["pkgname"].endswith("-repack"): return None - if not pmb.parse.arch.cpu_emulation_required(args, carch): + if not pmb.parse.arch.cpu_emulation_required(args, arch): return None if suffix == "native": return "native" diff --git a/pmb/build/other.py b/pmb/build/other.py index 3b622d4f..547e82c0 100644 --- a/pmb/build/other.py +++ b/pmb/build/other.py @@ -246,60 +246,22 @@ def index_repo(args, arch=None): paths = glob.glob(args.work + "/packages/*") for path in paths: - path_arch = os.path.basename(path) - path_repo_chroot = "/home/pmos/packages/pmos/" + path_arch - logging.debug("(native) index " + path_arch + " repository") - commands = [ - ["apk", "-q", "index", "--output", "APKINDEX.tar.gz_", - "--rewrite-arch", path_arch, "*.apk"], - ["abuild-sign", "APKINDEX.tar.gz_"], - ["mv", "APKINDEX.tar.gz_", "APKINDEX.tar.gz"] - ] - for command in commands: - pmb.chroot.user(args, command, working_dir=path_repo_chroot) - pmb.parse.apkindex.clear_cache(args, args.work + path + - "/APKINDEX.tar.gz") - - -def symlink_noarch_packages(args): - """ - All noarch packages from the native architecture folder (x86_64 usually) - get symlinked to all other architectures. - """ - # Create the arch folders - architectures = pmb.config.build_device_architectures - logging.debug("Symlink noarch-packages to " + ", ".join(architectures)) - for arch in architectures: - arch_folder = "/mnt/pmbootstrap-packages/" + arch - arch_folder_outside = args.work + "/packages/" + arch - if not os.path.exists(arch_folder_outside): - pmb.chroot.user(args, ["mkdir", "-p", arch_folder]) - - # Create an APKINDEX *without* replaced architectures (that is much - # faster than reading each apk file with Python!) - index = "/tmp/APKINDEX_without_replaced_archs" - index_outside = args.work + "/chroot_native" + index - pmb.chroot.user(args, ["apk", "-q", "index", "--output", index, "*.apk"], - working_dir="/mnt/pmbootstrap-packages/" + args.arch_native) - - # Iterate over noarch packages - for package, data in pmb.parse.apkindex.parse(args, index_outside).items(): - if data["arch"] != "noarch": - continue - - # Create missing symlinks - apk_file = data["pkgname"] + "-" + data["version"] + ".apk" - for arch in architectures: - if os.path.exists(args.work + "/packages/" + arch + "/" + apk_file): - continue - arch_folder = "/mnt/pmbootstrap-packages/" + arch - source = "../" + args.arch_native + "/" + apk_file - pmb.chroot.user(args, ["ln", "-sf", source, "."], - working_dir=arch_folder) - - # Rewrite indexes - for arch in architectures: - index_repo(args, arch) + if os.path.exists(path): + path_arch = os.path.basename(path) + path_repo_chroot = "/home/pmos/packages/pmos/" + path_arch + logging.debug("(native) index " + path_arch + " repository") + commands = [ + ["apk", "-q", "index", "--output", "APKINDEX.tar.gz_", + "--rewrite-arch", path_arch, "*.apk"], + ["abuild-sign", "APKINDEX.tar.gz_"], + ["mv", "APKINDEX.tar.gz_", "APKINDEX.tar.gz"] + ] + for command in commands: + pmb.chroot.user(args, command, working_dir=path_repo_chroot) + else: + logging.debug("NOTE: Can't build index for non-existing path: " + + path) + pmb.parse.apkindex.clear_cache(args, path + "/APKINDEX.tar.gz") def ccache_stats(args, arch): diff --git a/pmb/build/package.py b/pmb/build/package.py deleted file mode 100644 index a1ab1e42..00000000 --- a/pmb/build/package.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Copyright 2017 Oliver Smith - -This file is part of pmbootstrap. - -pmbootstrap is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -pmbootstrap is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with pmbootstrap. If not, see . -""" -import os -import logging - -import pmb.build -import pmb.build.autodetect -import pmb.build.buildinfo -import pmb.chroot -import pmb.chroot.apk -import pmb.chroot.distccd -import pmb.helpers.repo -import pmb.parse -import pmb.parse.arch - - -def package(args, pkgname, carch, force=False, buildinfo=False, strict=False, - init_buildenv=True): - """ - Build a package with Alpine Linux' abuild. - - :param force: even build, if not necessary - :returns: output path relative to the packages folder - """ - # Get existing binary package indexes - pmb.helpers.repo.update(args) - - # Get aport, skip upstream only packages - aport = pmb.build.find_aport(args, pkgname, False) - if not aport: - if pmb.parse.apkindex.read_any_index(args, pkgname, carch): - return - raise RuntimeError("Package " + pkgname + ": Could not find aport," - " and could not find this package in any APKINDEX!") - - # Autodetect the build environment - apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD") - pkgname = apkbuild["pkgname"] - carch_buildenv = pmb.build.autodetect.carch(args, apkbuild, carch, strict) - suffix = pmb.build.autodetect.suffix(args, apkbuild, carch_buildenv) - cross = pmb.build.autodetect.crosscompile(args, apkbuild, carch_buildenv, - suffix) - - # Skip already built versions - if not force and not pmb.build.is_necessary(args, carch_buildenv, apkbuild): - return - - # Initialize build environment, install/build makedepends - if init_buildenv: - pmb.build.init(args, suffix) - if len(apkbuild["makedepends"]): - if strict: - for makedepend in apkbuild["makedepends"]: - package(args, makedepend, carch_buildenv, strict=True) - else: - pmb.chroot.apk.install(args, apkbuild["makedepends"], suffix) - if cross: - pmb.chroot.apk.install(args, ["gcc-" + carch_buildenv, - "g++-" + carch_buildenv, - "ccache-cross-symlinks"]) - if cross == "distcc": - pmb.chroot.apk.install(args, ["distcc"], suffix=suffix, - build=False) - pmb.chroot.distccd.start(args, carch_buildenv) - - # Avoid re-building for circular dependencies - if not force and not pmb.build.is_necessary(args, carch, apkbuild): - return - - # Configure abuild.conf - pmb.build.other.configure_abuild(args, suffix) - - # Generate output name, log build message - output = (carch_buildenv + "/" + apkbuild["pkgname"] + "-" + - apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] + ".apk") - logging.info("(" + suffix + ") build " + output) - - # Sanity check - if cross == "native" and "!tracedeps" not in apkbuild["options"]: - logging.info("WARNING: Option !tracedeps is not set, but we're" - " cross-compiling in the native chroot. This will probably" - " fail!") - - # Run abuild - pmb.build.copy_to_buildpath(args, pkgname, suffix) - cmd = [] - env = {"CARCH": carch_buildenv} - if cross == "native": - hostspec = pmb.parse.arch.alpine_to_hostspec(carch_buildenv) - env["CROSS_COMPILE"] = hostspec + "-" - env["CC"] = hostspec + "-gcc" - if cross == "distcc": - env["PATH"] = "/usr/lib/distcc/bin:" + pmb.config.chroot_path - env["DISTCC_HOSTS"] = "127.0.0.1:" + args.port_distccd - for key, value in env.items(): - cmd += [key + "=" + value] - cmd += ["abuild"] - if strict: - cmd += ["-r"] # install depends with abuild - else: - cmd += ["-d"] # do not install depends with abuild - if force: - cmd += ["-f"] - pmb.chroot.user(args, cmd, suffix, "/home/pmos/build") - - # Verify output file - path = args.work + "/packages/" + output - if not os.path.exists(path): - raise RuntimeError("Package not found after build: " + path) - - # Create .buildinfo.json file - if buildinfo: - logging.info("(" + suffix + ") generate " + output + ".buildinfo.json") - pmb.build.buildinfo.write(args, output, carch_buildenv, suffix, - apkbuild) - - # Symlink noarch package (and subpackages) - if "noarch" in apkbuild["arch"]: - pmb.build.symlink_noarch_packages(args) - - # Clean up (APKINDEX cache, depends when strict) - pmb.parse.apkindex.clear_cache(args, args.work + "/packages/" + - carch_buildenv + "/APKINDEX.tar.gz") - if strict: - logging.info("(" + suffix + ") uninstall makedepends") - pmb.chroot.user(args, ["abuild", "undeps"], suffix, "/home/pmos/build") - - return output diff --git a/pmb/build/qemu_workaround_aarch64.py b/pmb/build/qemu_workaround_aarch64.py index 11618590..80298ede 100644 --- a/pmb/build/qemu_workaround_aarch64.py +++ b/pmb/build/qemu_workaround_aarch64.py @@ -16,7 +16,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with pmbootstrap. If not, see . """ -import pmb.build.package +import pmb.build import pmb.chroot.apk @@ -30,5 +30,5 @@ def qemu_workaround_aarch64(args, suffix="buildroot_aarch64"): """ pkgname = "abuild-aarch64-qemu-workaround" pmb.build.package(args, pkgname, "aarch64", True, - init_buildenv=False) + skip_init_buildenv=True) pmb.chroot.apk.install(args, [pkgname], suffix, False) diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index a9060702..605d23e0 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -196,7 +196,7 @@ def install(args, packages, suffix="native", build=True): packages_with_depends = pmb.parse.depends.recurse(args, packages, arch, strict=True) - # Filter out up-to-date packages + # Filter outdated packages (build them if required) packages_installed = installed(args, suffix) packages_todo = [] for package in packages_with_depends: diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 525089e7..5edd5401 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -42,6 +42,50 @@ import pmb.parse import pmb.qemu +def _build_verify_usage_device_package(args, pkgname): + """ + Detect if the user is about to build a device- package for the wrong + architecture. The package is noarch, but the dependencies (kernel!) will get + pulled in with the same arch as dependency. + """ + # Skip non-device-packages + if not pkgname.startswith("device-"): + return + + # Only continue when the --arch parameter is *not* the device architecture + deviceinfo = args.aports + "/device/" + pkgname + "/deviceinfo" + if not os.path.exists(deviceinfo): + return + device = pkgname.split("-", 1)[1] + arch = pmb.parse.deviceinfo(args, device)["arch"] + if args.arch == arch: + return + + # Abort with a big note + logging.info("Dependency handling in 'pmbootstrap build' has been" + " changed.") + logging.info("Previously we only built and installed the 'makedepends'" + " from the APKBUILDs, now we use the 'depends', too.") + logging.info("") + logging.info("Your options:") + logging.info("* Ignore depends (fast, old behavior, may cause problems" + " with some packages):") + logging.info(" pmbootstrap build " + pkgname + " -i") + logging.info("* Build with depends (kernel!) and specify the right" + " architecture:") + logging.info(" pmbootstrap build " + pkgname + " --arch=" + arch) + logging.info("") + logging.info("This change was necessary to be more compatible with Alpine's" + " abuild.") + logging.info("The default architecture is the native one (" + + args.arch_native + " in your case), so you need to overwrite") + logging.info("it now to get the kernel dependency of your device package" + " for the right architecture.") + logging.info("Sorry for the inconvenience.") + logging.info("") + raise RuntimeError("Missing -i or --arch parameter") + + def _parse_flavor(args): """ Verify the flavor argument if specified, or return a default value. @@ -86,8 +130,16 @@ def aportgen(args): def build(args): + # Strict mode: zap everything if args.strict: pmb.chroot.zap(args, False) + + # Detect wrong usage for device- packages + if not args.ignore_depends: + for package in args.packages: + _build_verify_usage_device_package(args, package) + + # Build all packages for package in args.packages: pmb.build.package(args, package, args.arch, args.force, args.buildinfo, args.strict) @@ -136,8 +188,7 @@ def config(args): def index(args): - pmb.build.index_repo(args, args.arch_native) - pmb.build.symlink_noarch_packages(args) + pmb.build.index_repo(args) def initfs(args): diff --git a/pmb/parse/apkindex.py b/pmb/parse/apkindex.py index 35ab2942..5365f26d 100644 --- a/pmb/parse/apkindex.py +++ b/pmb/parse/apkindex.py @@ -251,10 +251,9 @@ def read_any_index(args, package, arch=None): # Return first match for index in pmb.helpers.repo.apkindex_files(args, arch): index_data = read(args, package, index, False) - logging.verbose("Search for " + package + " in " + index + - " - result: " + str(index_data)) if index_data: + logging.verbose(package + ": found in " + index) return index_data - logging.verbose("No match found in any APKINDEX.tar.gz!") + logging.verbose(package + ": no match found in any APKINDEX.tar.gz!") return None diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index f4cfa4e6..96ff9be9 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -270,11 +270,21 @@ def arguments(): " (aport/APKBUILD) based on an upstream aport from Alpine") build = sub.add_parser("build", help="create a package for a" " specific architecture") - build.add_argument("--arch", choices=arch_choices) - build.add_argument("--force", action="store_true") + build.add_argument("--arch", choices=arch_choices, default=arch_native, + help="CPU architecture to build for (default: " + + arch_native + ")") + build.add_argument("--force", action="store_true", help="even build if not" + " necessary") build.add_argument("--buildinfo", action="store_true") build.add_argument("--strict", action="store_true", help="(slower) zap and install only" " required depends when building, to detect dependency errors") + build.add_argument("-i", "--ignore-depends", action="store_true", + help="only build and install makedepends from an" + " APKBUILD, ignore the depends (old behavior). This is" + " faster for device packages for example, because then" + " you don't need to build and install the kernel. But it" + " is incompatible with how Alpine's abuild handles it.", + dest="ignore_depends") build.add_argument("--noarch-arch", dest="noarch_arch", default=None, help="which architecture to use to build 'noarch'" " packages. Defaults to the native arch normally," diff --git a/pmb/parse/depends.py b/pmb/parse/depends.py index c74909d3..e82327d8 100644 --- a/pmb/parse/depends.py +++ b/pmb/parse/depends.py @@ -62,13 +62,12 @@ def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True, continue # Get depends and pkgname from aports - logging.verbose("Get dependencies of: " + pkgname_depend) depends = None pkgname = None if in_aports: aport = pmb.build.find_aport(args, pkgname_depend, False) if aport: - logging.verbose("-> Found aport: " + aport) + logging.verbose(pkgname_depend + ": found aport: " + aport) apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD") depends = apkbuild["depends"] if pkgname_depend in apkbuild["subpackages"]: @@ -78,7 +77,6 @@ def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True, # Get depends and pkgname from APKINDEX if depends is None and in_apkindexes: - logging.verbose("-> Search through APKINDEX files") index_data = pmb.parse.apkindex.read_any_index(args, pkgname_depend, arch) if index_data: @@ -95,12 +93,11 @@ def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True, # Append to todo/ret (unless it is a duplicate) if pkgname != pkgname_depend: - logging.verbose("-> '" + pkgname_depend + "' is provided by '" + - pkgname + "'") + logging.verbose(pkgname_depend + ": provided by '" + pkgname + "'") if pkgname in ret: - logging.verbose("-> '" + pkgname + "' already found") + logging.verbose(pkgname + ": already found") else: - logging.verbose("-> '" + pkgname + "' depends on: " + str(depends)) + logging.verbose(pkgname + ": depends on: " + ",".join(depends)) if depends: todo += depends ret.append(pkgname) diff --git a/test/test_aportgen_device_wizard.py b/test/test_aportgen_device_wizard.py index 774ff310..fc9c77a4 100644 --- a/test/test_aportgen_device_wizard.py +++ b/test/test_aportgen_device_wizard.py @@ -32,7 +32,7 @@ import pmb.parse @pytest.fixture def args(tmpdir, request): - sys.argv = ["pmbootstrap.py", "chroot"] + sys.argv = ["pmbootstrap.py", "build", "-i", "device-testsuite-testdevice"] args = pmb.parse.arguments() args.log = args.work + "/log_testsuite.txt" pmb.helpers.logging.init(args) diff --git a/test/test_build.py b/test/test_build.py deleted file mode 100644 index 30cd0c5d..00000000 --- a/test/test_build.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Copyright 2017 Oliver Smith - -This file is part of pmbootstrap. - -pmbootstrap is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -pmbootstrap is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with pmbootstrap. If not, see . -""" -import os -import sys -import pytest - -# Import from parent directory -sys.path.append(os.path.realpath( - os.path.join(os.path.dirname(__file__) + "/.."))) -import pmb.aportgen -import pmb.config -import pmb.helpers.logging - - -@pytest.fixture -def args(tmpdir, request): - import pmb.parse - sys.argv = ["pmbootstrap.py", "chroot"] - args = pmb.parse.arguments() - args.log = args.work + "/log_testsuite.txt" - pmb.helpers.logging.init(args) - request.addfinalizer(args.logfd.close) - return args - - -def test_build(args): - pmb.build.package(args, "hello-world", args.arch_native, True) - - -def test_build_cross(args): - """ - Build in non-native chroot, with cross-compiler through distcc. - """ - for arch in pmb.config.build_device_architectures: - pmb.build.package(args, "hello-world", arch, True) diff --git a/test/test_build_package.py b/test/test_build_package.py new file mode 100644 index 00000000..d2ed2d58 --- /dev/null +++ b/test/test_build_package.py @@ -0,0 +1,289 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" + +""" +This file tests all functions from pmb.build._package. +""" + +import os +import pytest +import sys + +# Import from parent directory +sys.path.append(os.path.realpath( + os.path.join(os.path.dirname(__file__) + "/.."))) +import pmb.build +import pmb.build._package +import pmb.config +import pmb.config.init +import pmb.helpers.logging + + +@pytest.fixture +def args(tmpdir, request): + import pmb.parse + sys.argv = ["pmbootstrap", "init"] + args = pmb.parse.arguments() + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def return_none(*args, **kwargs): + return None + + +def return_string(*args, **kwargs): + return "some/random/path.apk" + + +def return_true(*args, **kwargs): + return True + + +def return_false(*args, **kwargs): + return False + + +def return_fake_build_depends(*args, **kwargs): + """ + Fake return value for pmb.build._package.build_depends: + depends: ["alpine-base"], depends_built: [] + """ + return (["alpine-base"], []) + + +def args_patched(monkeypatch, argv): + monkeypatch.setattr(sys, "argv", argv) + return pmb.parse.arguments() + + +def test_get_apkbuild(args): + func = pmb.build._package.get_apkbuild + + # Valid aport + pkgname = "postmarketos-base" + assert func(args, pkgname, "x86_64")["pkgname"] == pkgname + + # Valid binary package + assert func(args, "alpine-base", "x86_64") is None + + # Invalid package + with pytest.raises(RuntimeError) as e: + func(args, "invalid-package-name", "x86_64") + assert "Could not find" in str(e.value) + + +def test_check_arch(args): + func = pmb.build._package.check_arch + apkbuild = {"pkgname": "test"} + + # Arch is right + apkbuild["arch"] = ["armhf"] + func(args, apkbuild, "armhf") + apkbuild["arch"] = ["noarch"] + func(args, apkbuild, "armhf") + apkbuild["arch"] = ["all"] + func(args, apkbuild, "armhf") + + # Arch is wrong + apkbuild["arch"] = ["x86_64"] + with pytest.raises(RuntimeError) as e: + func(args, apkbuild, "armhf") + assert "Can't build" in str(e.value) + + +def test_get_depends(monkeypatch): + func = pmb.build._package.get_depends + apkbuild = {"depends": ["a"], "makedepends": ["c", "b"]} + + # Depends + makedepends + args = args_patched(monkeypatch, ["pmbootstrap", "build", "test"]) + assert func(args, apkbuild) == ["a", "b", "c"] + args = args_patched(monkeypatch, ["pmbootstrap", "install"]) + assert func(args, apkbuild) == ["a", "b", "c"] + + # Ignore depends (-i) + args = args_patched(monkeypatch, ["pmbootstrap", "build", "-i", "test"]) + assert func(args, apkbuild) == ["b", "c"] + + +def test_build_depends(args, monkeypatch): + # Shortcut and fake apkbuild + func = pmb.build._package.build_depends + apkbuild = {"pkgname": "test", "depends": ["a"], "makedepends": ["b"]} + + # No depends built (first makedepends + depends, then only makedepends) + monkeypatch.setattr(pmb.build._package, "package", return_none) + assert func(args, apkbuild, "armhf", True) == (["a", "b"], []) + + # All depends built (makedepends only) + monkeypatch.setattr(pmb.build._package, "package", return_string) + assert func(args, apkbuild, "armhf", False) == (["a", "b"], ["a", "b"]) + + +def test_is_necessary_warn_depends(args, monkeypatch): + # Shortcut and fake apkbuild + func = pmb.build._package.is_necessary_warn_depends + apkbuild = {"pkgname": "test"} + + # Necessary + monkeypatch.setattr(pmb.build, "is_necessary", return_true) + assert func(args, apkbuild, "armhf", False, []) is True + + # Necessary (strict=True overrides is_necessary()) + monkeypatch.setattr(pmb.build, "is_necessary", return_false) + assert func(args, apkbuild, "armhf", True, []) is True + + # Not necessary (with depends: different code path that prints a warning) + assert func(args, apkbuild, "armhf", False, []) is False + assert func(args, apkbuild, "armhf", False, ["first", "second"]) is False + + +def test_init_buildenv(args, monkeypatch): + # Disable effects of functions we don't want to test here + monkeypatch.setattr(pmb.build._package, "build_depends", + return_fake_build_depends) + monkeypatch.setattr(pmb.build._package, "is_necessary_warn_depends", + return_true) + monkeypatch.setattr(pmb.chroot.apk, "install", return_none) + monkeypatch.setattr(pmb.chroot.distccd, "start", return_none) + + # Shortcut and fake apkbuild + func = pmb.build._package.init_buildenv + apkbuild = {"pkgname": "test", "depends": ["a"], "makedepends": ["b"]} + + # Build is necessary (various code paths) + assert func(args, apkbuild, "armhf", strict=True) is True + assert func(args, apkbuild, "armhf", cross="native") is True + assert func(args, apkbuild, "armhf", cross="distcc") is True + + # Build is not necessary (only builds dependencies) + monkeypatch.setattr(pmb.build._package, "is_necessary_warn_depends", + return_false) + assert func(args, apkbuild, "armhf") is False + + +def test_run_abuild(args, monkeypatch): + # Disable effects of functions we don't want to test here + monkeypatch.setattr(pmb.build, "copy_to_buildpath", return_none) + monkeypatch.setattr(pmb.chroot, "user", return_none) + + # Shortcut and fake apkbuild + func = pmb.build._package.run_abuild + apkbuild = {"pkgname": "test", "pkgver": "1", "pkgrel": "2", "options": []} + + # Normal run + output = "armhf/test-1-r2.apk" + env = {"CARCH": "armhf"} + cmd = ["CARCH=armhf", "abuild", "-d"] + assert func(args, apkbuild, "armhf") == (output, cmd, env) + + # Force and strict + cmd = ["CARCH=armhf", "abuild", "-r", "-f"] + assert func(args, apkbuild, "armhf", True, True) == (output, cmd, env) + + # cross=native + env = {"CARCH": "armhf", + "CROSS_COMPILE": "armv6-alpine-linux-muslgnueabihf-", + "CC": "armv6-alpine-linux-muslgnueabihf-gcc"} + cmd = ["CARCH=armhf", "CROSS_COMPILE=armv6-alpine-linux-muslgnueabihf-", + "CC=armv6-alpine-linux-muslgnueabihf-gcc", "abuild", "-d"] + assert func(args, apkbuild, "armhf", cross="native") == (output, cmd, env) + + # cross=distcc + env = {"CARCH": "armhf", + "PATH": "/usr/lib/distcc/bin:" + pmb.config.chroot_path, + "DISTCC_HOSTS": "127.0.0.1:33632"} + cmd = ["CARCH=armhf", "PATH=" + "/usr/lib/distcc/bin:" + + pmb.config.chroot_path, "DISTCC_HOSTS=127.0.0.1:33632", "abuild", + "-d"] + assert func(args, apkbuild, "armhf", cross="distcc") == (output, cmd, env) + + +def test_finish(args, monkeypatch): + # Real output path + output = pmb.build.package(args, "hello-world", force=True) + + # Disable effects of functions we don't want to test below + monkeypatch.setattr(pmb.build.buildinfo, "write", return_none) + monkeypatch.setattr(pmb.chroot, "user", return_none) + + # Shortcut and fake apkbuild + func = pmb.build._package.finish + apkbuild = {} + + # Non-existing output path + with pytest.raises(RuntimeError) as e: + func(args, apkbuild, "armhf", "/invalid/path") + assert "Package not found" in str(e.value) + + # Existing output path + func(args, apkbuild, args.arch_native, output) + + +def test_package(args): + # First build + assert pmb.build.package(args, "hello-world", force=True) + + # Package exists + assert pmb.build.package(args, "hello-world") is None + + # Force building again + assert pmb.build.package(args, "hello-world", force=True) + + # Build for another architecture + assert pmb.build.package(args, "hello-world", "armhf", force=True) + + # Upstream package, for which we don't have an aport + assert pmb.build.package(args, "alpine-base") is None + + +def test_build_depends_high_level(args, monkeypatch): + """ + "hello-world-wrapper" depends on "hello-world". We build both, then delete + "hello-world" and check that it gets rebuilt correctly again. + """ + # Patch pmb.build.is_necessary() to always build the hello-world package + def fake_build_is_necessary(args, arch, apkbuild, apkindex_path=None): + if apkbuild["pkgname"] == "hello-world": + return True + return pmb.build.other.is_necessary(args, arch, apkbuild, + apkindex_path) + monkeypatch.setattr(pmb.build, "is_necessary", + fake_build_is_necessary) + + # Build hello-world to get its full output path + output_hello = pmb.build.package(args, "hello-world") + output_hello_outside = args.work + "/packages/" + output_hello + assert os.path.exists(output_hello_outside) + + # Make sure the wrapper exists + pmb.build.package(args, "hello-world-wrapper") + + # Remove hello-world + pmb.helpers.run.root(args, ["rm", output_hello_outside]) + pmb.build.index_repo(args, args.arch_native) + + # Ask to build the wrapper. It should not build the wrapper (it exists, not + # using force), but build/update its missing dependency "hello-world" + # instead. + assert pmb.build.package(args, "hello-world-wrapper") is None + assert os.path.exists(output_hello_outside) diff --git a/test/test_buildroot_aarch64_init.py b/test/test_buildroot_aarch64_init.py new file mode 100644 index 00000000..6975e456 --- /dev/null +++ b/test/test_buildroot_aarch64_init.py @@ -0,0 +1,76 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" + +""" +This file tests all functions from pmb.build._package. +""" + +import glob +import os +import pytest +import sys + +# Import from parent directory +sys.path.append(os.path.realpath( + os.path.join(os.path.dirname(__file__) + "/.."))) +import pmb.build +import pmb.helpers.logging + + +@pytest.fixture +def args(tmpdir, request): + import pmb.parse + sys.argv = ["pmbootstrap", "init"] + args = pmb.parse.arguments() + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def test_buildroot_aarch64_init(args, monkeypatch): + # Patch pmb.build.is_necessary() to always build the workaround package + def fake_build_is_necessary(args, arch, apkbuild, apkindex_path=None): + if apkbuild["pkgname"] == "abuild-aarch64-qemu-workaround": + return True + return pmb.build.other.is_necessary(args, arch, apkbuild, + apkindex_path) + monkeypatch.setattr(pmb.build, "is_necessary", + fake_build_is_necessary) + + # Remove aarch64 chroot + pmb.chroot.shutdown(args) + path = args.work + "/chroot_buildroot_aarch64" + if os.path.exists(path): + pmb.helpers.run.root(args, ["rm", "-rf", path]) + + # Remove existing workaround packages + pattern_workaround_apk = (args.work + "/packages/aarch64/" + "abuild-aarch64-qemu-workaround-*") + for match in glob.glob(pattern_workaround_apk): + pmb.helpers.run.root(args, ["rm", match]) + pmb.build.index_repo(args, "aarch64") + + # Build hello-world for aarch64, causing the chroot to initialize properly + pmb.build.package(args, "hello-world", "aarch64", force=True) + + # Verify that the workaround was built and installed + assert len(glob.glob(pattern_workaround_apk)) + assert os.path.exists(args.work + "/chroot_buildroot_aarch64/usr/bin" + "/abuild-tar-patched") diff --git a/test/test_challenge_build.py b/test/test_challenge_build.py index 4c1fbfb7..7132f611 100644 --- a/test/test_challenge_build.py +++ b/test/test_challenge_build.py @@ -23,7 +23,7 @@ import pytest # Import from parent directory sys.path.append(os.path.realpath( os.path.join(os.path.dirname(__file__) + "/.."))) -import pmb.build.package +import pmb.build import pmb.challenge.build import pmb.config import pmb.helpers.logging diff --git a/test/test_repo.py b/test/test_repo.py index 4dae0b7a..b3ac2ae6 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -25,7 +25,7 @@ import time # Import from parent directory pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/..")) sys.path.append(pmb_src) -import pmb.build.package +import pmb.build import pmb.helpers.logging import pmb.helpers.repo diff --git a/test/testcases_fast.sh b/test/testcases_fast.sh index 7defb1da..78c4bcb9 100755 --- a/test/testcases_fast.sh +++ b/test/testcases_fast.sh @@ -3,12 +3,10 @@ # Disable slow testcases # aport_in_sync_with_git: clones Alpine's aports repo # aportgen: clones Alpine's aports repo -# build: builds cross-compilers for aarch64 and armhf # version: clones Alpine's apk repo disabled=" aport_in_sync_with_git aportgen - build version "