Administration/pmbootstrap-meow
2
0
Fork 0
forked from Mirror/pmbootstrap
Code Pull requests Activity Actions

pmb.chroot: fix E501, switch to f-strings (MR 2040)

Browse source
This commit is contained in:
Anri Dellal 2021-03-22 00:19:12 +03:00 • committed by Oliver Smith
parent 87c7859b40
commit 88ec1d1106
No known key found for this signature in database
GPG key ID: 5AE7F5513E0885CB
9 changed files with 179 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

54
pmb/chroot/apk.py
View file

@ -29,7 +29,7 @@ def update_repository_list(args, suffix="native", check=False):
return return
# Read old entries or create folder structure # Read old entries or create folder structure
path = args.work + "/chroot_" + suffix + "/etc/apk/repositories" path = f"{args.work}/chroot_{suffix}/etc/apk/repositories"
lines_old = [] lines_old = []
if os.path.exists(path): if os.path.exists(path):
# Read all old lines # Read all old lines
@ -48,15 +48,15 @@ def update_repository_list(args, suffix="native", check=False):
# Check phase: raise error when still outdated # Check phase: raise error when still outdated
if check: if check:
raise RuntimeError("Failed to update: " + path) raise RuntimeError(f"Failed to update: {path}")
# Update the file # Update the file
logging.debug("(" + suffix + ") update /etc/apk/repositories") logging.debug(f"({suffix}) update /etc/apk/repositories")
if os.path.exists(path): if os.path.exists(path):
pmb.helpers.run.root(args, ["rm", path]) pmb.helpers.run.root(args, ["rm", path])
for line in lines_new: for line in lines_new:
pmb.helpers.run.root(args, ["sh", "-c", "echo " + pmb.helpers.run.root(args, ["sh", "-c", "echo "
shlex.quote(line) + " >> " + path]) f"{shlex.quote(line)} >> {path}"])
update_repository_list(args, suffix, True) update_repository_list(args, suffix, True)
@ -71,9 +71,9 @@ def check_min_version(args, suffix="native"):
return return
# Skip if apk is not installed yet # Skip if apk is not installed yet
if not os.path.exists(args.work + "/chroot_" + suffix + "/sbin/apk"): if not os.path.exists(f"{args.work}/chroot_{suffix}/sbin/apk"):
logging.debug("NOTE: Skipped apk version check for chroot '" + suffix + logging.debug(f"NOTE: Skipped apk version check for chroot '{suffix}'"
"', because it is not installed yet!") ", because it is not installed yet!")
return return
# Compare # Compare
@ -94,7 +94,8 @@ def install_is_necessary(args, build, arch, package, packages_installed):
:param build: Set to true to build the package, if the binary packages are :param build: Set to true to build the package, if the binary packages are
out of date, and it is in the aports folder. out of date, and it is in the aports folder.
:param packages_installed: Return value from installed(). :param packages_installed: Return value from installed().
:returns: True if the package needs to be installed/updated, False otherwise. :returns: True if the package needs to be installed/updated,
False otherwise.
""" """
# User may have disabled buiding packages during "pmbootstrap install" # User may have disabled buiding packages during "pmbootstrap install"
build_disabled = False build_disabled = False
@ -118,8 +119,8 @@ def install_is_necessary(args, build, arch, package, packages_installed):
" 'pmbootstrap install' has been disabled." " 'pmbootstrap install' has been disabled."
" Consider changing this option in" " Consider changing this option in"
" 'pmbootstrap init'.") " 'pmbootstrap init'.")
logging.warning("WARNING: Internal error in pmbootstrap," + logging.warning("WARNING: Internal error in pmbootstrap,"
" package '" + package + "' for " + arch + f" package '{package}' for {arch}"
" has not been built yet, but it should have" " has not been built yet, but it should have"
" been. Rebuilding it with force. Please " " been. Rebuilding it with force. Please "
" report this, if there is no ticket about this" " report this, if there is no ticket about this"
@ -134,10 +135,10 @@ def install_is_necessary(args, build, arch, package, packages_installed):
data_repo["version"]) data_repo["version"])
# a) Installed newer (should not happen normally) # a) Installed newer (should not happen normally)
if compare == 1: if compare == 1:
logging.info("WARNING: " + arch + " package '" + package + logging.info(f"WARNING: {arch} package '{package}'"
"' installed version " + data_installed["version"] + f" installed version {data_installed['version']}"
" is newer, than the version in the repositories: " + " is newer, than the version in the repositories:"
data_repo["version"] + f" {data_repo['version']}"
" See also: <https://postmarketos.org/warning-repo>") " See also: <https://postmarketos.org/warning-repo>")
return False return False
@ -154,9 +155,10 @@ def install_is_necessary(args, build, arch, package, packages_installed):
def replace_aports_packages_with_path(args, packages, suffix, arch): def replace_aports_packages_with_path(args, packages, suffix, arch):
""" """
apk will only re-install packages with the same pkgname, pkgver and pkgrel, apk will only re-install packages with the same pkgname,
when you give it the absolute path to the package. This function replaces pkgver and pkgrel, when you give it the absolute path to the package.
all packages, that were built locally, with the absolute path to the package. This function replaces all packages, that were built locally,
with the absolute path to the package.
""" """
ret = [] ret = []
for package in packages: for package in packages:
@ -164,16 +166,16 @@ def replace_aports_packages_with_path(args, packages, suffix, arch):
if aport: if aport:
data_repo = pmb.parse.apkindex.package(args, package, arch, False) data_repo = pmb.parse.apkindex.package(args, package, arch, False)
if not data_repo: if not data_repo:
raise RuntimeError(package + ": could not find binary" raise RuntimeError(f"{package}: could not find binary"
" package, although it should exist for" " package, although it should exist for"
" sure at this point in the code." " sure at this point in the code."
" Probably an APKBUILD subpackage parsing" " Probably an APKBUILD subpackage parsing"
" bug. Related: https://gitlab.com/" " bug. Related: https://gitlab.com/"
"postmarketOS/build.postmarketos.org/" "postmarketOS/build.postmarketos.org/"
"issues/61") "issues/61")
apk_path = ("/mnt/pmbootstrap-packages/" + arch + "/" + apk_path = (f"/mnt/pmbootstrap-packages/{arch}/"
package + "-" + data_repo["version"] + ".apk") f"{package}-{data_repo['version']}.apk")
if os.path.exists(args.work + "/chroot_" + suffix + apk_path): if os.path.exists(f"{args.work}/chroot_{suffix}{apk_path}"):
package = apk_path package = apk_path
ret.append(package) ret.append(package)
return ret return ret
@ -208,13 +210,13 @@ def install(args, packages, suffix="native", build=True):
# to be passed to apk! # to be passed to apk!
for package in packages_todo: for package in packages_todo:
if package.startswith("-"): if package.startswith("-"):
raise ValueError("Invalid package name: " + package) raise ValueError(f"Invalid package name: {package}")
# Readable install message without dependencies # Readable install message without dependencies
message = "(" + suffix + ") install" message = f"({suffix}) install"
for pkgname in packages: for pkgname in packages:
if pkgname not in packages_installed: if pkgname not in packages_installed:
message += " " + pkgname message += f" {pkgname}"
logging.info(message) logging.info(message)
# Local packages: Using the path instead of pkgname makes apk update # Local packages: Using the path instead of pkgname makes apk update
@ -258,5 +260,5 @@ def installed(args, suffix="native"):
}, ... }, ...
} }
""" """
path = args.work + "/chroot_" + suffix + "/lib/apk/db/installed" path = f"{args.work}/chroot_{suffix}/lib/apk/db/installed"
return pmb.parse.apkindex.parse(args, path, False) return pmb.parse.apkindex.parse(args, path, False)

47
pmb/chroot/apk_static.py
View file

@ -32,19 +32,19 @@ def read_signature_info(tar):
sigfilename = filename sigfilename = filename
break break
if not sigfilename: if not sigfilename:
raise RuntimeError("Could not find signature filename in apk." + raise RuntimeError("Could not find signature filename in apk."
" This means, that your apk file is damaged. Delete it" + " This means, that your apk file is damaged."
" and try again. If the problem persists, fill out a bug" + " Delete it and try again."
" report.") " If the problem persists, fill out a bug report.")
sigkey = sigfilename[len(prefix):] sigkey = sigfilename[len(prefix):]
logging.debug("sigfilename: " + sigfilename) logging.debug(f"sigfilename: {sigfilename}")
logging.debug("sigkey: " + sigkey) logging.debug(f"sigkey: {sigkey}")
# Get path to keyfile on disk # Get path to keyfile on disk
sigkey_path = pmb.config.apk_keys_path + "/" + sigkey sigkey_path = f"{pmb.config.apk_keys_path}/{sigkey}"
if "/" in sigkey or not os.path.exists(sigkey_path): if "/" in sigkey or not os.path.exists(sigkey_path):
logging.debug("sigkey_path: " + sigkey_path) logging.debug(f"sigkey_path: {sigkey_path}")
raise RuntimeError("Invalid signature key: " + sigkey) raise RuntimeError(f"Invalid signature key: {sigkey}")
return (sigfilename, sigkey_path) return (sigfilename, sigkey_path)
@ -71,7 +71,7 @@ def extract_temp(tar, sigfilename):
ret[ftype]["temp_path"] = path ret[ftype]["temp_path"] = path
shutil.copyfileobj(tar.extractfile(member), handle) shutil.copyfileobj(tar.extractfile(member), handle)
logging.debug("extracted: " + path) logging.debug(f"extracted: {path}")
handle.close() handle.close()
return ret return ret
@ -83,7 +83,7 @@ def verify_signature(args, files, sigkey_path):
:param files: return value from extract_temp() :param files: return value from extract_temp()
:raises RuntimeError: when verification failed and removes temp files :raises RuntimeError: when verification failed and removes temp files
""" """
logging.debug("Verify apk.static signature with " + sigkey_path) logging.debug(f"Verify apk.static signature with {sigkey_path}")
try: try:
pmb.helpers.run.user(args, ["openssl", "dgst", "-sha1", "-verify", pmb.helpers.run.user(args, ["openssl", "dgst", "-sha1", "-verify",
sigkey_path, "-signature", files[ sigkey_path, "-signature", files[
@ -115,21 +115,23 @@ def extract(args, version, apk_path):
temp_path = files["apk"]["temp_path"] temp_path = files["apk"]["temp_path"]
# Verify the version, that the extracted binary reports # Verify the version, that the extracted binary reports
logging.debug("Verify the version reported by the apk.static binary" + logging.debug("Verify the version reported by the apk.static binary"
" (must match the package version " + version + ")") f" (must match the package version {version})")
os.chmod(temp_path, os.stat(temp_path).st_mode | stat.S_IEXEC) os.chmod(temp_path, os.stat(temp_path).st_mode | stat.S_IEXEC)
version_bin = pmb.helpers.run.user(args, [temp_path, "--version"], version_bin = pmb.helpers.run.user(args, [temp_path, "--version"],
output_return=True) output_return=True)
version_bin = version_bin.split(" ")[1].split(",")[0] version_bin = version_bin.split(" ")[1].split(",")[0]
if not version.startswith(version_bin + "-r"): if not version.startswith(f"{version_bin}-r"):
os.unlink(temp_path) os.unlink(temp_path)
raise RuntimeError("Downloaded apk-tools-static-" + version + ".apk," raise RuntimeError(f"Downloaded apk-tools-static-{version}.apk,"
" but the apk binary inside that package reports to be" " but the apk binary inside that package reports"
" version: " + version_bin + "! Looks like a downgrade attack" f" to be version: {version_bin}!"
" from a malicious server! Switch the server (-m) and try again.") " Looks like a downgrade attack"
" from a malicious server! Switch the server (-m)"
" and try again.")
# Move it to the right path # Move it to the right path
target_path = args.work + "/apk.static" target_path = f"{args.work}/apk.static"
shutil.move(temp_path, target_path) shutil.move(temp_path, target_path)
@ -140,7 +142,7 @@ def download(args, file):
channel_cfg = pmb.config.pmaports.read_config_channel(args) channel_cfg = pmb.config.pmaports.read_config_channel(args)
mirrordir = channel_cfg["mirrordir_alpine"] mirrordir = channel_cfg["mirrordir_alpine"]
base_url = f"{args.mirror_alpine}{mirrordir}/main/{args.arch_native}" base_url = f"{args.mirror_alpine}{mirrordir}/main/{args.arch_native}"
return pmb.helpers.http.download(args, base_url + "/" + file, file) return pmb.helpers.http.download(args, f"{base_url}/{file}", file)
def init(args): def init(args):
@ -158,7 +160,7 @@ def init(args):
args, version, "Run 'pmbootstrap update', then try again.") args, version, "Run 'pmbootstrap update', then try again.")
# Download, extract, verify apk-tools-static # Download, extract, verify apk-tools-static
apk_name = "apk-tools-static-" + version + ".apk" apk_name = f"apk-tools-static-{version}.apk"
apk_static = download(args, apk_name) apk_static = download(args, apk_name)
extract(args, version, apk_static) extract(args, version, apk_static)
@ -166,4 +168,5 @@ def init(args):
def run(args, parameters): def run(args, parameters):
if args.offline: if args.offline:
parameters = ["--no-network"] + parameters parameters = ["--no-network"] + parameters
pmb.helpers.apk.apk_with_progress(args, [args.work + "/apk.static"] + parameters, chroot=False) pmb.helpers.apk.apk_with_progress(
args, [f"{args.work}/apk.static"] + parameters, chroot=False)

40
pmb/chroot/init.py
View file

@ -22,7 +22,7 @@ def copy_resolv_conf(args, suffix="native"):
If the file doesn't exist, create an empty file with 'touch'. If the file doesn't exist, create an empty file with 'touch'.
""" """
host = "/etc/resolv.conf" host = "/etc/resolv.conf"
chroot = args.work + "/chroot_" + suffix + host chroot = f"{args.work}/chroot_{suffix}{host}"
if os.path.exists(host): if os.path.exists(host):
if not os.path.exists(chroot) or not filecmp.cmp(host, chroot): if not os.path.exists(chroot) or not filecmp.cmp(host, chroot):
pmb.helpers.run.root(args, ["cp", host, chroot]) pmb.helpers.run.root(args, ["cp", host, chroot])
@ -35,24 +35,25 @@ def setup_qemu_emulation(args, suffix):
if not pmb.parse.arch.cpu_emulation_required(args, arch): if not pmb.parse.arch.cpu_emulation_required(args, arch):
return return
chroot = args.work + "/chroot_" + suffix chroot = f"{args.work}/chroot_{suffix}"
arch_qemu = pmb.parse.arch.alpine_to_qemu(arch) arch_qemu = pmb.parse.arch.alpine_to_qemu(arch)
# mount --bind the qemu-user binary # mount --bind the qemu-user binary
pmb.chroot.binfmt.register(args, arch) pmb.chroot.binfmt.register(args, arch)
pmb.helpers.mount.bind_file(args, args.work + "/chroot_native/usr/bin/qemu-" + arch_qemu, pmb.helpers.mount.bind_file(args, f"{args.work}/chroot_native"
chroot + "/usr/bin/qemu-" + arch_qemu + "-static", f"/usr/bin/qemu-{arch_qemu}",
f"{chroot}/usr/bin/qemu-{arch_qemu}-static",
create_folders=True) create_folders=True)
def init(args, suffix="native"): def init(args, suffix="native"):
# When already initialized: just prepare the chroot # When already initialized: just prepare the chroot
chroot = args.work + "/chroot_" + suffix chroot = f"{args.work}/chroot_{suffix}"
arch = pmb.parse.arch.from_chroot_suffix(args, suffix) arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
pmb.chroot.mount(args, suffix) pmb.chroot.mount(args, suffix)
setup_qemu_emulation(args, suffix) setup_qemu_emulation(args, suffix)
if os.path.islink(chroot + "/bin/sh"): if os.path.islink(f"{chroot}/bin/sh"):
pmb.config.workdir.chroot_check_channel(args, suffix) pmb.config.workdir.chroot_check_channel(args, suffix)
copy_resolv_conf(args, suffix) copy_resolv_conf(args, suffix)
pmb.chroot.apk.update_repository_list(args, suffix) pmb.chroot.apk.update_repository_list(args, suffix)
@ -61,17 +62,17 @@ def init(args, suffix="native"):
# Require apk-tools-static # Require apk-tools-static
pmb.chroot.apk_static.init(args) pmb.chroot.apk_static.init(args)
logging.info("(" + suffix + ") install alpine-base") logging.info(f"({suffix}) install alpine-base")
# Initialize cache # Initialize cache
apk_cache = args.work + "/cache_apk_" + arch apk_cache = f"{args.work}/cache_apk_{arch}"
pmb.helpers.run.root(args, ["ln", "-s", "-f", "/var/cache/apk", chroot + pmb.helpers.run.root(args, ["ln", "-s", "-f", "/var/cache/apk",
"/etc/apk/cache"]) f"{chroot}/etc/apk/cache"])
# Initialize /etc/apk/keys/, resolv.conf, repositories # Initialize /etc/apk/keys/, resolv.conf, repositories
for key in glob.glob(pmb.config.apk_keys_path + "/*.pub"): for key in glob.glob(f"{pmb.config.apk_keys_path}/*.pub"):
pmb.helpers.run.root(args, ["cp", key, args.work + pmb.helpers.run.root(args, ["cp", key, f"{args.work}"
"/config_apk_keys/"]) "/config_apk_keys/"])
copy_resolv_conf(args, suffix) copy_resolv_conf(args, suffix)
pmb.chroot.apk.update_repository_list(args, suffix) pmb.chroot.apk.update_repository_list(args, suffix)
@ -80,21 +81,22 @@ def init(args, suffix="native"):
# Install alpine-base # Install alpine-base
pmb.helpers.repo.update(args, arch) pmb.helpers.repo.update(args, arch)
pmb.chroot.apk_static.run(args, ["--root", chroot, pmb.chroot.apk_static.run(args, ["--root", chroot,
"--cache-dir", apk_cache, "--initdb", "--arch", arch, "--cache-dir", apk_cache,
"--initdb", "--arch", arch,
"add", "alpine-base"]) "add", "alpine-base"])
# Building chroots: create "pmos" user, add symlinks to /home/pmos # Building chroots: create "pmos" user, add symlinks to /home/pmos
if not suffix.startswith("rootfs_"): if not suffix.startswith("rootfs_"):
pmb.chroot.root(args, ["adduser", "-D", "pmos", "-u", pmb.chroot.root(args, ["adduser", "-D", "pmos", "-u",
pmb.config.chroot_uid_user], suffix, auto_init=False) pmb.config.chroot_uid_user],
suffix, auto_init=False)
# Create the links (with subfolders if necessary) # Create the links (with subfolders if necessary)
for target, link_name in pmb.config.chroot_home_symlinks.items(): for target, link_name in pmb.config.chroot_home_symlinks.items():
link_dir = os.path.dirname(link_name) link_dir = os.path.dirname(link_name)
if not os.path.exists(chroot + link_dir): if not os.path.exists(f"{chroot}{link_dir}"):
pmb.chroot.user(args, ["mkdir", "-p", link_dir], suffix) pmb.chroot.user(args, ["mkdir", "-p", link_dir], suffix)
if not os.path.exists(chroot + target): if not os.path.exists(f"{chroot}{target}"):
pmb.chroot.root(args, ["mkdir", "-p", target], suffix) pmb.chroot.root(args, ["mkdir", "-p", target], suffix)
pmb.chroot.user(args, ["ln", "-s", target, link_name], suffix) pmb.chroot.user(args, ["ln", "-s", target, link_name], suffix)
pmb.chroot.root(args, ["chown", "pmos:pmos", target], pmb.chroot.root(args, ["chown", "pmos:pmos", target], suffix)
suffix)

34
pmb/chroot/initfs.py
View file

@ -14,12 +14,13 @@ def build(args, flavor, suffix):
pmb.chroot.initfs_hooks.update(args, suffix) pmb.chroot.initfs_hooks.update(args, suffix)
# Call mkinitfs # Call mkinitfs
logging.info("(" + suffix + ") mkinitfs " + flavor) logging.info(f"({suffix}) mkinitfs {flavor}")
release_file = (args.work + "/chroot_" + suffix + "/usr/share/kernel/" + release_file = (f"{args.work}/chroot_{suffix}/usr/share/kernel/"
flavor + "/kernel.release") f"{flavor}/kernel.release")
with open(release_file, "r") as handle: with open(release_file, "r") as handle:
release = handle.read().rstrip() release = handle.read().rstrip()
pmb.chroot.root(args, ["mkinitfs", "-o", "/boot/initramfs-" + flavor, release], pmb.chroot.root(args, ["mkinitfs", "-o",
f"/boot/initramfs-{flavor}", release],
suffix) suffix)
@ -33,27 +34,28 @@ def extract(args, flavor, suffix, extra=False):
if extra: if extra:
inside = "/tmp/initfs-extra-extracted" inside = "/tmp/initfs-extra-extracted"
flavor += "-extra" flavor += "-extra"
outside = args.work + "/chroot_" + suffix + inside outside = f"{args.work}/chroot_{suffix}{inside}"
if os.path.exists(outside): if os.path.exists(outside):
if not pmb.helpers.cli.confirm(args, "Extraction folder " + outside + if not pmb.helpers.cli.confirm(args, f"Extraction folder {outside}"
" already exists. Do you want to overwrite it?"): " already exists."
" Do you want to overwrite it?"):
raise RuntimeError("Aborted!") raise RuntimeError("Aborted!")
pmb.chroot.root(args, ["rm", "-r", inside], suffix) pmb.chroot.root(args, ["rm", "-r", inside], suffix)
# Extraction script (because passing a file to stdin is not allowed # Extraction script (because passing a file to stdin is not allowed
# in pmbootstrap's chroot/shell functions for security reasons) # in pmbootstrap's chroot/shell functions for security reasons)
with open(args.work + "/chroot_" + suffix + "/tmp/_extract.sh", "w") as handle: with open(f"{args.work}/chroot_{suffix}/tmp/_extract.sh", "w") as handle:
handle.write( handle.write(
"#!/bin/sh\n" "#!/bin/sh\n"
"cd " + inside + " && cpio -i < _initfs\n") f"cd {inside} && cpio -i < _initfs\n")
# Extract # Extract
commands = [["mkdir", "-p", inside], commands = [["mkdir", "-p", inside],
["cp", "/boot/initramfs-" + flavor, inside + "/_initfs.gz"], ["cp", f"/boot/initramfs-{flavor}", f"{inside}/_initfs.gz"],
["gzip", "-d", inside + "/_initfs.gz"], ["gzip", "-d", f"{inside}/_initfs.gz"],
["cat", "/tmp/_extract.sh"], # for the log ["cat", "/tmp/_extract.sh"], # for the log
["sh", "/tmp/_extract.sh"], ["sh", "/tmp/_extract.sh"],
["rm", "/tmp/_extract.sh", inside + "/_initfs"] ["rm", "/tmp/_extract.sh", f"{inside}/_initfs"]
] ]
for command in commands: for command in commands:
pmb.chroot.root(args, command, suffix) pmb.chroot.root(args, command, suffix)
@ -73,7 +75,7 @@ def ls(args, flavor, suffix, extra=False):
def frontend(args): def frontend(args):
# Find the appropriate kernel flavor # Find the appropriate kernel flavor
suffix = "rootfs_" + args.device suffix = f"rootfs_{args.device}"
flavors = pmb.chroot.other.kernel_flavors_installed(args, suffix) flavors = pmb.chroot.other.kernel_flavors_installed(args, suffix)
flavor = flavors[0] flavor = flavors[0]
if hasattr(args, "flavor") and args.flavor: if hasattr(args, "flavor") and args.flavor:
@ -85,9 +87,9 @@ def frontend(args):
build(args, flavor, suffix) build(args, flavor, suffix)
elif action == "extract": elif action == "extract":
dir = extract(args, flavor, suffix) dir = extract(args, flavor, suffix)
logging.info("Successfully extracted initramfs to: " + dir) logging.info(f"Successfully extracted initramfs to: {dir}")
dir_extra = extract(args, flavor, suffix, True) dir_extra = extract(args, flavor, suffix, True)
logging.info("Successfully extracted initramfs-extra to: " + dir_extra) logging.info(f"Successfully extracted initramfs-extra to: {dir_extra}")
elif action == "ls": elif action == "ls":
logging.info("*** initramfs ***") logging.info("*** initramfs ***")
ls(args, flavor, suffix) ls(args, flavor, suffix)
@ -109,4 +111,4 @@ def frontend(args):
if action in ["ls", "extract"]: if action in ["ls", "extract"]:
link = "https://wiki.postmarketos.org/wiki/Initramfs_development" link = "https://wiki.postmarketos.org/wiki/Initramfs_development"
logging.info("See also: <" + link + ">") logging.info(f"See also: <{link}>")

15
pmb/chroot/initfs_hooks.py
View file

@ -23,7 +23,7 @@ def list_chroot(args, suffix, remove_prefix=True):
def list_aports(args): def list_aports(args):
ret = [] ret = []
prefix = pmb.config.initfs_hook_prefix prefix = pmb.config.initfs_hook_prefix
for path in glob.glob(args.aports + "/*/" + prefix + "*"): for path in glob.glob(f"{args.aports}/*/{prefix}*"):
ret.append(os.path.basename(path)[len(prefix):]) ret.append(os.path.basename(path)[len(prefix):])
return ret return ret
@ -33,27 +33,24 @@ def ls(args, suffix):
hooks_aports = list_aports(args) hooks_aports = list_aports(args)
for hook in hooks_aports: for hook in hooks_aports:
line = "* " + hook line = f"* {hook} ({'' if hook in hooks_chroot else 'not '}installed)"
if hook in hooks_chroot:
line += " (installed)"
else:
line += " (not installed)"
logging.info(line) logging.info(line)
def add(args, hook, suffix): def add(args, hook, suffix):
if hook not in list_aports(args): if hook not in list_aports(args):
raise RuntimeError("Invalid hook name! Run 'pmbootstrap initfs hook_ls'" raise RuntimeError("Invalid hook name!"
" Run 'pmbootstrap initfs hook_ls'"
" to get a list of all hooks.") " to get a list of all hooks.")
prefix = pmb.config.initfs_hook_prefix prefix = pmb.config.initfs_hook_prefix
pmb.chroot.apk.install(args, [prefix + hook], suffix) pmb.chroot.apk.install(args, [f"{prefix}{hook}"], suffix)
def delete(args, hook, suffix): def delete(args, hook, suffix):
if hook not in list_chroot(args, suffix): if hook not in list_chroot(args, suffix):
raise RuntimeError("There is no such hook installed!") raise RuntimeError("There is no such hook installed!")
prefix = pmb.config.initfs_hook_prefix prefix = pmb.config.initfs_hook_prefix
pmb.chroot.root(args, ["apk", "del", prefix + hook], suffix) pmb.chroot.root(args, ["apk", "del", f"{prefix}{hook}"], suffix)
def update(args, suffix): def update(args, suffix):

23
pmb/chroot/root.py
View file

@ -18,14 +18,16 @@ def executables_absolute_path():
for binary in ["sh", "chroot"]: for binary in ["sh", "chroot"]:
path = shutil.which(binary, path=pmb.config.chroot_host_path) path = shutil.which(binary, path=pmb.config.chroot_host_path)
if not path: if not path:
raise RuntimeError("Could not find the '" + binary + raise RuntimeError(f"Could not find the '{binary}'"
"' executable. Make sure, that it is in" " your current user's PATH.") " executable. Make sure, that it is in"
" your current user's PATH.")
ret[binary] = path ret[binary] = path
return ret return ret
def root(args, cmd, suffix="native", working_dir="/", output="log", def root(args, cmd, suffix="native", working_dir="/", output="log",
output_return=False, check=None, env={}, auto_init=True, disable_timeout=False): output_return=False, check=None, env={}, auto_init=True,
disable_timeout=False):
""" """
Run a command inside a chroot as root. Run a command inside a chroot as root.
@ -37,18 +39,18 @@ def root(args, cmd, suffix="native", working_dir="/", output="log",
arguments and the return value. arguments and the return value.
""" """
# Initialize chroot # Initialize chroot
chroot = args.work + "/chroot_" + suffix chroot = f"{args.work}/chroot_{suffix}"
if not auto_init and not os.path.islink(chroot + "/bin/sh"): if not auto_init and not os.path.islink(f"{chroot}/bin/sh"):
raise RuntimeError("Chroot does not exist: " + chroot) raise RuntimeError(f"Chroot does not exist: {chroot}")
if auto_init: if auto_init:
pmb.chroot.init(args, suffix) pmb.chroot.init(args, suffix)
# Readable log message (without all the escaping) # Readable log message (without all the escaping)
msg = "(" + suffix + ") % " msg = f"({suffix}) % "
for key, value in env.items(): for key, value in env.items():
msg += key + "=" + value + " " msg += f"{key}={value} "
if working_dir != "/": if working_dir != "/":
msg += "cd " + working_dir + "; " msg += f"cd {working_dir}; "
msg += " ".join(cmd) msg += " ".join(cmd)
# Merge env with defaults into env_all # Merge env with defaults into env_all
@ -72,4 +74,5 @@ def root(args, cmd, suffix="native", working_dir="/", output="log",
cmd_sudo = ["sudo", "env", "-i", executables["sh"], "-c", cmd_sudo = ["sudo", "env", "-i", executables["sh"], "-c",
pmb.helpers.run.flat_cmd(cmd_chroot, env=env_all)] pmb.helpers.run.flat_cmd(cmd_chroot, env=env_all)]
return pmb.helpers.run_core.core(args, msg, cmd_sudo, None, output, return pmb.helpers.run_core.core(args, msg, cmd_sudo, None, output,
output_return, check, True, disable_timeout) output_return, check, True,
disable_timeout)

46
pmb/chroot/zap.py
View file

@ -23,10 +23,10 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
:param dry: Only show what would be deleted, do not delete for real :param dry: Only show what would be deleted, do not delete for real
:param pkgs_local: Remove *all* self-compiled packages (!) :param pkgs_local: Remove *all* self-compiled packages (!)
:param http: Clear the http cache (used e.g. for the initial apk download) :param http: Clear the http cache (used e.g. for the initial apk download)
:param pkgs_local_mismatch: Remove the packages, that have a different version :param pkgs_local_mismatch: Remove the packages, that have
compared to what is in the aports folder. a different version compared to what is in the aports folder.
:param pkgs_online_mismatch: Clean out outdated binary packages downloaded from :param pkgs_online_mismatch: Clean out outdated binary packages
mirrors (e.g. from Alpine) downloaded from mirrors (e.g. from Alpine)
:param distfiles: Clear the downloaded files cache :param distfiles: Clear the downloaded files cache
:param rust: Remove rust related caches :param rust: Remove rust related caches
@ -39,7 +39,8 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
logging.debug("Calculate work folder size") logging.debug("Calculate work folder size")
size_old = pmb.helpers.other.folder_size(args, args.work) size_old = pmb.helpers.other.folder_size(args, args.work)
# Delete packages with a different version compared to aports, then re-index # Delete packages with a different version compared to aports,
# then re-index
if pkgs_local_mismatch: if pkgs_local_mismatch:
zap_pkgs_local_mismatch(args, confirm, dry) zap_pkgs_local_mismatch(args, confirm, dry)
@ -67,11 +68,12 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
# Delete everything matching the patterns # Delete everything matching the patterns
for pattern in patterns: for pattern in patterns:
pattern = os.path.realpath(args.work + "/" + pattern) pattern = os.path.realpath(f"{args.work}/{pattern}")
matches = glob.glob(pattern) matches = glob.glob(pattern)
for match in matches: for match in matches:
if not confirm or pmb.helpers.cli.confirm(args, "Remove " + match + "?"): if (not confirm or
logging.info("% rm -rf " + match) pmb.helpers.cli.confirm(args, f"Remove {match}?")):
logging.info(f"% rm -rf {match}")
if not dry: if not dry:
pmb.helpers.run.root(args, ["rm", "-rf", match]) pmb.helpers.run.root(args, ["rm", "-rf", match])
@ -87,7 +89,7 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
else: else:
size_new = pmb.helpers.other.folder_size(args, args.work) size_new = pmb.helpers.other.folder_size(args, args.work)
mb = (size_old - size_new) / 1024 / 1024 mb = (size_old - size_new) / 1024 / 1024
logging.info("Cleared up ~" + str(math.ceil(mb)) + " MB of space") logging.info(f"Cleared up ~{math.ceil(mb)} MB of space")
def zap_pkgs_local_mismatch(args, confirm=True, dry=False): def zap_pkgs_local_mismatch(args, confirm=True, dry=False):
@ -112,29 +114,29 @@ def zap_pkgs_local_mismatch(args, confirm=True, dry=False):
arch = block["arch"] arch = block["arch"]
# Apk path # Apk path
apk_path_short = arch + "/" + pkgname + "-" + version + ".apk" apk_path_short = f"{arch}/{pkgname}-{version}.apk"
apk_path = f"{args.work}/packages/{channel}/{apk_path_short}" apk_path = f"{args.work}/packages/{channel}/{apk_path_short}"
if not os.path.exists(apk_path): if not os.path.exists(apk_path):
logging.info("WARNING: Package mentioned in index not" logging.info("WARNING: Package mentioned in index not"
" found: " + apk_path_short) f" found: {apk_path_short}")
continue continue
# Aport path # Aport path
aport_path = pmb.helpers.pmaports.find(args, origin, False) aport_path = pmb.helpers.pmaports.find(args, origin, False)
if not aport_path: if not aport_path:
logging.info("% rm " + apk_path_short + " (" + origin + logging.info(f"% rm {apk_path_short}"
" aport not found)") f" ({origin} aport not found)")
if not dry: if not dry:
pmb.helpers.run.root(args, ["rm", apk_path]) pmb.helpers.run.root(args, ["rm", apk_path])
reindex = True reindex = True
continue continue
# Clear out any binary apks that do not match what is in aports # Clear out any binary apks that do not match what is in aports
apkbuild = pmb.parse.apkbuild(args, aport_path + "/APKBUILD") apkbuild = pmb.parse.apkbuild(args, f"{aport_path}/APKBUILD")
version_aport = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] version_aport = f"{apkbuild['pkgver']}-r{apkbuild['pkgrel']}"
if version != version_aport: if version != version_aport:
logging.info("% rm " + apk_path_short + " (" + origin + logging.info(f"% rm {apk_path_short}"
" aport: " + version_aport + ")") f" ({origin} aport: {version_aport})")
if not dry: if not dry:
pmb.helpers.run.root(args, ["rm", apk_path]) pmb.helpers.run.root(args, ["rm", apk_path])
reindex = True reindex = True
@ -145,18 +147,20 @@ def zap_pkgs_local_mismatch(args, confirm=True, dry=False):
def zap_pkgs_online_mismatch(args, confirm=True, dry=False): def zap_pkgs_online_mismatch(args, confirm=True, dry=False):
# Check whether we need to do anything # Check whether we need to do anything
paths = glob.glob(args.work + "/cache_apk_*") paths = glob.glob(f"{args.work}/cache_apk_*")
if not len(paths): if not len(paths):
return return
if confirm and not pmb.helpers.cli.confirm(args, "Remove outdated binary packages?"): if (confirm and not pmb.helpers.cli.confirm(args,
"Remove outdated"
" binary packages?")):
return return
# Iterate over existing apk caches # Iterate over existing apk caches
for path in paths: for path in paths:
arch = os.path.basename(path).split("_", 2)[2] arch = os.path.basename(path).split("_", 2)[2]
suffix = "native" if arch == args.arch_native else "buildroot_" + arch suffix = "native" if arch == args.arch_native else f"buildroot_{arch}"
# Clean the cache with apk # Clean the cache with apk
logging.info("(" + suffix + ") apk -v cache clean") logging.info(f"({suffix}) apk -v cache clean")
if not dry: if not dry:
pmb.chroot.root(args, ["apk", "-v", "cache", "clean"], suffix) pmb.chroot.root(args, ["apk", "-v", "cache", "clean"], suffix)

65
pmb/export/odin.py
View file

@ -12,8 +12,9 @@ import pmb.helpers.file
def odin(args, flavor, folder): def odin(args, flavor, folder):
""" """
Create Odin flashable tar file with kernel and initramfs for devices configured with Create Odin flashable tar file with kernel and initramfs
the flasher method 'heimdall-isorec' and with boot.img for devices with 'heimdall-bootimg' for devices configured with the flasher method 'heimdall-isorec'
and with boot.img for devices with 'heimdall-bootimg'
""" """
pmb.flasher.init(args) pmb.flasher.init(args)
suffix = "rootfs_" + args.device suffix = "rootfs_" + args.device
@ -21,49 +22,55 @@ def odin(args, flavor, folder):
# Validate method # Validate method
method = args.deviceinfo["flash_method"] method = args.deviceinfo["flash_method"]
if not method.startswith("heimdall-"): if not method.startswith("heimdall-"):
raise RuntimeError("An odin flashable tar is not supported for the flash" raise RuntimeError("An odin flashable tar is not supported"
" method '" + method + "' specified in the current configuration." f" for the flash method '{method}' specified"
" in the current configuration."
" Only 'heimdall' methods are supported.") " Only 'heimdall' methods are supported.")
# Partitions # Partitions
partition_kernel = args.deviceinfo["flash_heimdall_partition_kernel"] or "KERNEL" partition_kernel = \
partition_initfs = args.deviceinfo["flash_heimdall_partition_initfs"] or "RECOVERY" args.deviceinfo["flash_heimdall_partition_kernel"] or "KERNEL"
partition_initfs = \
args.deviceinfo["flash_heimdall_partition_initfs"] or "RECOVERY"
# Temporary folder # Temporary folder
temp_folder = "/tmp/odin-flashable-tar" temp_folder = "/tmp/odin-flashable-tar"
if os.path.exists(args.work + "/chroot_native" + temp_folder): if os.path.exists(f"{args.work}/chroot_native{temp_folder}"):
pmb.chroot.root(args, ["rm", "-rf", temp_folder]) pmb.chroot.root(args, ["rm", "-rf", temp_folder])
# Odin flashable tar generation script (because redirecting stdin/stdout is not allowed # Odin flashable tar generation script
# (because redirecting stdin/stdout is not allowed
# in pmbootstrap's chroot/shell functions for security reasons) # in pmbootstrap's chroot/shell functions for security reasons)
with open(args.work + "/chroot_rootfs_" + args.device + "/tmp/_odin.sh", "w") as handle: odin_script = f"{args.work}/chroot_rootfs_{args.device}/tmp/_odin.sh"
odin_kernel_md5 = partition_kernel + ".bin.md5" with open(odin_script, "w") as handle:
odin_initfs_md5 = partition_initfs + ".bin.md5" odin_kernel_md5 = f"{partition_kernel}.bin.md5"
odin_device_tar = args.device + ".tar" odin_initfs_md5 = f"{partition_initfs}.bin.md5"
odin_device_tar_md5 = args.device + ".tar.md5" odin_device_tar = f"{args.device}.tar"
odin_device_tar_md5 = f"{args.device}.tar.md5"
handle.write( handle.write(
"#!/bin/sh\n" "#!/bin/sh\n"
"cd " + temp_folder + "\n") f"cd {temp_folder}\n")
if method == "heimdall-isorec": if method == "heimdall-isorec":
handle.write( handle.write(
# Kernel: copy and append md5 # Kernel: copy and append md5
"cp /boot/vmlinuz-" + flavor + " " + odin_kernel_md5 + "\n" f"cp /boot/vmlinuz-{flavor} {odin_kernel_md5}\n"
"md5sum -t " + odin_kernel_md5 + " >> " + odin_kernel_md5 + "\n" f"md5sum -t {odin_kernel_md5} >> {odin_kernel_md5}\n"
# Initramfs: recompress with lzop, append md5 # Initramfs: recompress with lzop, append md5
"gunzip -c /boot/initramfs-" + flavor + " | lzop > " + odin_initfs_md5 + "\n" f"gunzip -c /boot/initramfs-{flavor}"
"md5sum -t " + odin_initfs_md5 + " >> " + odin_initfs_md5 + "\n") f" | lzop > {odin_initfs_md5}\n"
f"md5sum -t {odin_initfs_md5} >> {odin_initfs_md5}\n")
elif method == "heimdall-bootimg": elif method == "heimdall-bootimg":
handle.write( handle.write(
# boot.img: copy and append md5 # boot.img: copy and append md5
"cp /boot/boot.img-" + flavor + " " + odin_kernel_md5 + "\n" f"cp /boot/boot.img-{flavor} {odin_kernel_md5}\n"
"md5sum -t " + odin_kernel_md5 + " >> " + odin_kernel_md5 + "\n") f"md5sum -t {odin_kernel_md5} >> {odin_kernel_md5}\n")
handle.write( handle.write(
# Create tar, remove included files and append md5 # Create tar, remove included files and append md5
"tar -c -f " + odin_device_tar + " *.bin.md5\n" f"tar -c -f {odin_device_tar} *.bin.md5\n"
"rm *.bin.md5\n" "rm *.bin.md5\n"
"md5sum -t " + odin_device_tar + " >> " + odin_device_tar + "\n" f"md5sum -t {odin_device_tar} >> {odin_device_tar}\n"
"mv " + odin_device_tar + " " + odin_device_tar_md5 + "\n") f"mv {odin_device_tar} {odin_device_tar_md5}\n")
commands = [["mkdir", "-p", temp_folder], commands = [["mkdir", "-p", temp_folder],
["cat", "/tmp/_odin.sh"], # for the log ["cat", "/tmp/_odin.sh"], # for the log
@ -75,19 +82,19 @@ def odin(args, flavor, folder):
# Move Odin flashable tar to native chroot and cleanup temp folder # Move Odin flashable tar to native chroot and cleanup temp folder
pmb.chroot.user(args, ["mkdir", "-p", "/home/pmos/rootfs"]) pmb.chroot.user(args, ["mkdir", "-p", "/home/pmos/rootfs"])
pmb.chroot.root(args, ["mv", "/mnt/rootfs_" + args.device + temp_folder + pmb.chroot.root(args, ["mv", f"/mnt/rootfs_{args.device}{temp_folder}"
"/" + odin_device_tar_md5, "/home/pmos/rootfs/"]), f"/{odin_device_tar_md5}", "/home/pmos/rootfs/"]),
pmb.chroot.root(args, ["chown", "pmos:pmos", pmb.chroot.root(args, ["chown", "pmos:pmos",
"/home/pmos/rootfs/" + odin_device_tar_md5]) f"/home/pmos/rootfs/{odin_device_tar_md5}"])
pmb.chroot.root(args, ["rmdir", temp_folder], suffix) pmb.chroot.root(args, ["rmdir", temp_folder], suffix)
# Create the symlink # Create the symlink
file = args.work + "/chroot_native/home/pmos/rootfs/" + odin_device_tar_md5 file = f"{args.work}/chroot_native/home/pmos/rootfs/{odin_device_tar_md5}"
link = folder + "/" + odin_device_tar_md5 link = f"{folder}/{odin_device_tar_md5}"
pmb.helpers.file.symlink(args, file, link) pmb.helpers.file.symlink(args, file, link)
# Display a readable message # Display a readable message
msg = " * " + odin_device_tar_md5 msg = f" * {odin_device_tar_md5}"
if method == "heimdall-isorec": if method == "heimdall-isorec":
msg += " (Odin flashable file, contains initramfs and kernel)" msg += " (Odin flashable file, contains initramfs and kernel)"
elif method == "heimdall-bootimg": elif method == "heimdall-bootimg":

7
test/static_code_analysis.sh
View file

@ -60,12 +60,19 @@ py_files="
pmb/build/menuconfig.py pmb/build/menuconfig.py
pmb/build/newapkbuild.py pmb/build/newapkbuild.py
pmb/chroot/__init__.py pmb/chroot/__init__.py
pmb/chroot/apk.py
pmb/chroot/apk_static.py
pmb/chroot/binfmt.py pmb/chroot/binfmt.py
pmb/chroot/distccd.py pmb/chroot/distccd.py
pmb/chroot/init.py
pmb/chroot/initfs.py
pmb/chroot/initfs_hooks.py
pmb/chroot/mount.py pmb/chroot/mount.py
pmb/chroot/other.py pmb/chroot/other.py
pmb/chroot/root.py
pmb/chroot/shutdown.py pmb/chroot/shutdown.py
pmb/chroot/user.py pmb/chroot/user.py
pmb/chroot/zap.py
pmb/config/pmaports.py pmb/config/pmaports.py
pmb/config/save.py pmb/config/save.py
pmb/config/workdir.py pmb/config/workdir.py