diff --git a/pmb/build/__init__.py b/pmb/build/__init__.py index 8546fa70..f3d7f0e1 100644 --- a/pmb/build/__init__.py +++ b/pmb/build/__init__.py @@ -19,6 +19,7 @@ along with pmbootstrap. If not, see . # Exported functions from pmb.build.init import init from pmb.build.checksum import checksum +from pmb.build.envkernel import package_kernel from pmb.build.menuconfig import menuconfig from pmb.build.newapkbuild import newapkbuild from pmb.build.other import copy_to_buildpath, is_necessary, \ diff --git a/pmb/build/envkernel.py b/pmb/build/envkernel.py new file mode 100644 index 00000000..48a7a686 --- /dev/null +++ b/pmb/build/envkernel.py @@ -0,0 +1,199 @@ +""" +Copyright 2019 Robert Yang + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import os +import re + +import pmb.aportgen +import pmb.build +import pmb.chroot +import pmb.helpers +import pmb.helpers.pmaports +import pmb.parse + + +def match_kbuild_out(args, word): + """ + Look for paths in the following formats: + "//arch//boot" + "//include/config/kernel.release" + + :param word: space separated string cut out from a line from an APKBUILD + function body, that might be the kbuild output path + :returns: kernel build output directory. + empty string when a separate build output directory isn't used. + None, when no output directory is found. + """ + prefix = "^\\\"?\\$({?builddir}?|{?srcdir}?)\\\"?/" + kbuild_out = "(.*\\/)*" + + postfix = "(arch\\/.*\\/boot.*)\\\"?$" + match = re.match(prefix + kbuild_out + postfix, word) + + if match is None: + postfix = "(include\\/config\\/kernel\\.release)\\\"?$" + match = re.match(prefix + kbuild_out + postfix, word) + + if match is None: + return None + + groups = match.groups() + if groups is None or len(groups) != 3: + return None + + logging.debug("word = " + str(word)) + logging.debug("regex match groups = " + str(groups)) + out_dir = groups[1] + return "" if out_dir is None else out_dir.strip("/") + + +def find_kbuild_output_dir(args, function_body): + """ + Guess what the kernel build output directory is. Parses each line of the + function word by word, looking for paths which contain the kbuild output + directory. + + :param function_body: contents of a function from the kernel APKBUILD + :returns: kbuild output dir + None, when output dir is not found + """ + + guesses = [] + for line in function_body: + for item in line.split(): + kbuild_out = match_kbuild_out(args, item) + if kbuild_out is not None: + guesses.append(kbuild_out) + break + + # Check if guesses are all the same + it = iter(guesses) + first = next(it, None) + if first is None: + raise RuntimeError("Couldn't find a kbuild out directory. Is your " + "APKBUILD messed up? If not, then consider " + "adjusting the patterns in pmb/build/envkernel.py " + "to work with your APKBUILD, or submit an issue.") + if all(first == rest for rest in it): + return first + raise RuntimeError("Multiple kbuild out directories found. Can you modify " + "your APKBUILD so it only has one output path? If you " + "can't resolve it, please open an issue.") + + +def modify_apkbuild(args, pkgname, aport): + """ + Modify kernel APKBUILD to package build output from envkernel.sh + """ + apkbuild_path = aport + "/APKBUILD" + apkbuild = pmb.parse.apkbuild(args, apkbuild_path) + if os.path.exists(args.work + "/aportgen"): + pmb.helpers.run.user(args, ["rm", "-r", args.work + "/aportgen"]) + + pmb.helpers.run.user(args, ["mkdir", args.work + "/aportgen"]) + pmb.helpers.run.user(args, ["cp", "-r", apkbuild_path, + args.work + "/aportgen"]) + + pkgver = pmb.build._package.get_pkgver(apkbuild["pkgver"], + original_source=False) + fields = {"pkgver": pkgver, + "pkgrel": "0", + "subpackages": "", + "builddir": "/home/pmos/build/src"} + + pmb.aportgen.core.rewrite(args, pkgname, apkbuild_path, fields=fields) + + +def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out): + """ + Prepare build environment and run abuild. + + :param pkgname: package name of a linux kernel aport + :param arch: architecture for the kernel + :param apkbuild_path: path to APKBUILD of the kernel aport + :param kbuild_out: kernel build system output sub-directory + """ + chroot = args.work + "/chroot_native" + build_path = "/home/pmos/build" + kbuild_out_source = "/mnt/linux/.output" + + if not os.path.exists(chroot + kbuild_out_source): + raise RuntimeError("No '.output' dir found in your kernel source dir." + "Compile the " + args.device + " kernel with " + "envkernel.sh first, then try again.") + + # Create working directory for abuild + pmb.build.copy_to_buildpath(args, pkgname) + + # Create symlink from abuild working directory to envkernel build directory + build_output = "" if kbuild_out is None else "/" + kbuild_out + if build_output != "": + if os.path.islink(chroot + "/mnt/linux/" + build_output) and \ + os.path.lexists(chroot + "/mnt/linux/" + build_output): + pmb.chroot.root(args, ["rm", "/mnt/linux/" + build_output]) + pmb.chroot.root(args, ["ln", "-s", "/mnt/linux", + build_path + "/src"]) + pmb.chroot.root(args, ["ln", "-s", kbuild_out_source, + build_path + "/src" + build_output]) + + cmd = ["cp", apkbuild_path, chroot + build_path + "/APKBUILD"] + pmb.helpers.run.root(args, cmd) + + # Create the apk package + env = {"CARCH": arch, + "CHOST": arch, + "SUDO_APK": "abuild-apk --no-progress"} + cmd = ["abuild", "rootpkg"] + pmb.chroot.user(args, cmd, working_dir=build_path, env=env) + + # Clean up symlinks + if build_output != "": + if os.path.islink(chroot + "/mnt/linux/" + build_output) and \ + os.path.lexists(chroot + "/mnt/linux/" + build_output): + pmb.chroot.root(args, ["rm", "/mnt/linux/" + build_output]) + pmb.chroot.root(args, ["rm", build_path + "/src"]) + + +def package_kernel(args): + """ + Frontend for 'pmbootstrap build --envkernel': creates a package from + envkernel output. + """ + pkgname = args.packages[0] + if len(args.packages) > 1 or not pkgname.startswith("linux-"): + raise RuntimeError("--envkernel needs exactly one linux-* package as " + "argument.") + + aport = pmb.helpers.pmaports.find(args, pkgname) + function_body = pmb.parse.function_body(aport + "/APKBUILD", "package") + kbuild_out = find_kbuild_output_dir(args, function_body) + + modify_apkbuild(args, pkgname, aport) + apkbuild_path = args.work + "/aportgen/APKBUILD" + + arch = args.deviceinfo["arch"] + apkbuild = pmb.parse.apkbuild(args, apkbuild_path, check_pkgname=False) + suffix = pmb.build.autodetect.suffix(args, apkbuild, arch) + output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] + + "-r" + apkbuild["pkgrel"] + ".apk") + message = "(" + suffix + ") build " + output + logging.info(message) + + run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out) + pmb.build.other.index_repo(args, arch) diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 4fefada4..c3600d10 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -92,6 +92,10 @@ def build(args): if args.strict: pmb.chroot.zap(args, False) + if args.envkernel: + pmb.build.envkernel.package_kernel(args) + return + # Set src and force src = os.path.realpath(os.path.expanduser(args.src[0])) if args.src else None force = True if src else args.force diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 0d8f5a7f..160daf16 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -493,6 +493,9 @@ def arguments(): " you don't need to build and install the kernel. But it" " is incompatible with how Alpine's abuild handles it.", dest="ignore_depends") + build.add_argument("--envkernel", action="store_true", + help="Create an apk package from the build output of" + " a kernel compiled with envkernel.sh.") for action in [checksum, build, aportgen]: argument_packages = action.add_argument("packages", nargs="+") if argcomplete: diff --git a/test/test_envkernel.py b/test/test_envkernel.py new file mode 100644 index 00000000..865ad800 --- /dev/null +++ b/test/test_envkernel.py @@ -0,0 +1,146 @@ +""" +Copyright 2019 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import sys +import pytest + +# Import from parent directory +pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/..")) +sys.path.insert(0, pmb_src) + +import pmb.aportgen +import pmb.aportgen.core +import pmb.build +import pmb.build.envkernel +import pmb.config +import pmb.helpers.logging + + +@pytest.fixture +def args(tmpdir, request): + import pmb.parse + sys.argv = ["pmbootstrap.py", "init"] + args = pmb.parse.arguments() + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def test_package_kernel_args(args): + args.packages = ["package-one", "package-two"] + with pytest.raises(RuntimeError) as e: + pmb.build.envkernel.package_kernel(args) + assert "--envkernel needs exactly one linux-* package as argument." in \ + str(e.value) + + +def test_find_kbuild_output_dir(args): + # Test parsing an APKBUILD + pkgname = "linux-envkernel-test" + testdata = pmb_src + "/test/testdata" + path = testdata + "/apkbuild/APKBUILD." + pkgname + function_body = pmb.parse.function_body(path, "package") + kbuild_out = pmb.build.envkernel.find_kbuild_output_dir(args, + function_body) + assert kbuild_out == "build" + + # Test full function body + function_body = [ + " install -Dm644 \"$srcdir\"/build/arch/arm/boot/dt.img ", + " \"$pkgdir\"/boot/dt.img", + "", + " install -Dm644 \"$srcdir\"/build/arch/arm/boot/zImage-dtb ", + " \"$pkgdir\"/boot/vmlinuz-$_flavor", + "", + " install -D \"$srcdir\"/build/include/config/kernel.release ", + " \"$pkgdir\"/usr/share/kernel/$_flavor/kernel.release", + "", + " cd \"$srcdir\"/build", + " unset LDFLAGS", + "", + " make ARCH=\"$_carch\" CC=\"${CC:-gcc}\" ", + " KBUILD_BUILD_VERSION=\"$((pkgrel + 1))-Alpine\" ", + " INSTALL_MOD_PATH=\"$pkgdir\" modules_install", + ] + kbuild_out = pmb.build.envkernel.find_kbuild_output_dir(args, + function_body) + assert kbuild_out == "build" + + # Test no kbuild out dir + function_body = [ + " install -Dm644 \"$srcdir\"/arch/arm/boot/zImage ", + " \"$pkgdir\"/boot/vmlinuz-$_flavor", + " install -D \"$srcdir\"/include/config/kernel.release ", + " \"$pkgdir\"/usr/share/kernel/$_flavor/kernel.release", + ] + kbuild_out = pmb.build.envkernel.find_kbuild_output_dir(args, + function_body) + assert kbuild_out == "" + + # Test curly brackets around srcdir + function_body = [ + " install -Dm644 \"${srcdir}\"/build/arch/arm/boot/zImage ", + " \"$pkgdir\"/boot/vmlinuz-$_flavor", + " install -D \"${srcdir}\"/build/include/config/kernel.release ", + " \"$pkgdir\"/usr/share/kernel/$_flavor/kernel.release", + ] + kbuild_out = pmb.build.envkernel.find_kbuild_output_dir(args, + function_body) + assert kbuild_out == "build" + + # Test multiple sub directories + function_body = [ + " install -Dm644 \"${srcdir}\"/sub/dir/arch/arm/boot/zImage-dtb ", + " \"$pkgdir\"/boot/vmlinuz-$_flavor", + " install -D \"${srcdir}\"/sub/dir/include/config/kernel.release ", + " \"$pkgdir\"/usr/share/kernel/$_flavor/kernel.release", + ] + kbuild_out = pmb.build.envkernel.find_kbuild_output_dir(args, + function_body) + assert kbuild_out == "sub/dir" + + # Test no kbuild out dir found + function_body = [ + " install -Dm644 \"$srcdir\"/build/not/found/zImage-dtb ", + " \"$pkgdir\"/boot/vmlinuz-$_flavor", + " install -D \"$srcdir\"/not/found/kernel.release ", + " \"$pkgdir\"/usr/share/kernel/$_flavor/kernel.release", + ] + with pytest.raises(RuntimeError) as e: + kbuild_out = pmb.build.envkernel.find_kbuild_output_dir(args, + function_body) + assert ("Couldn't find a kbuild out directory. Is your APKBUILD messed up?" + " If not, then consider adjusting the patterns in " + "pmb/build/envkernel.py to work with your APKBUILD, or submit an " + "issue.") in str(e.value) + + # Test multiple different kbuild out dirs + function_body = [ + " install -Dm644 \"$srcdir\"/build/arch/arm/boot/zImage-dtb ", + " \"$pkgdir\"/boot/vmlinuz-$_flavor", + " install -D \"$srcdir\"/include/config/kernel.release ", + " \"$pkgdir\"/usr/share/kernel/$_flavor/kernel.release", + ] + with pytest.raises(RuntimeError) as e: + kbuild_out = pmb.build.envkernel.find_kbuild_output_dir(args, + function_body) + assert ("Multiple kbuild out directories found. Can you modify your " + "APKBUILD so it only has one output path? If you can't resolve it," + " please open an issue.") in str(e.value) diff --git a/test/testdata/apkbuild/APKBUILD.linux-envkernel-test b/test/testdata/apkbuild/APKBUILD.linux-envkernel-test new file mode 100644 index 00000000..4166c917 --- /dev/null +++ b/test/testdata/apkbuild/APKBUILD.linux-envkernel-test @@ -0,0 +1,20 @@ +pkgname="linux-envkernel-test" + +package() { + install -Dm644 "$srcdir"/build/arch/arm/boot/dt.img \ + "$pkgdir"/boot/dt.img + + install -Dm644 "$srcdir"/build/arch/arm/boot/zImage-dtb \ + "$pkgdir"/boot/vmlinuz-$_flavor + + install -D "$srcdir"/build/include/config/kernel.release \ + "$pkgdir"/usr/share/kernel/$_flavor/kernel.release + + cd "$srcdir"/build + unset LDFLAGS + + echo "--[ Installing modules ]--" + make ARCH="$_carch" CC="${CC:-gcc}" \ + KBUILD_BUILD_VERSION="$((pkgrel + 1))-Alpine" CONFIG_NO_ERROR_ON_MISMATCH=y \ + INSTALL_MOD_PATH="$pkgdir" INSTALL_MOD_STRIP=1 modules_install +}