diff --git a/pmb/__init__.py b/pmb/__init__.py index 96dac007..d2d11254 100644 --- a/pmb/__init__.py +++ b/pmb/__init__.py @@ -45,7 +45,7 @@ def print_log_hint() -> None: context = get_context(allow_failure=True) if context and context.details_to_stdout: return - log = context.log if context else Config().work / "log.txt" + log = context.log if context else Config().cache / "log.txt" # Hints about the log file (print to stdout only) log_hint = "Run 'pmbootstrap log' for details." if not os.path.exists(log): @@ -103,7 +103,7 @@ def main(*, original_uid: int) -> int: # Migrate work folder if necessary if args.action not in ["shutdown", "zap", "log"]: - other.migrate_work_folder() + other.migrate_localdir() # Run the function with the action's name (in pmb/helpers/frontend.py) if args.action: diff --git a/pmb/aportgen/__init__.py b/pmb/aportgen/__init__.py index f4437d67..02a36280 100644 --- a/pmb/aportgen/__init__.py +++ b/pmb/aportgen/__init__.py @@ -79,7 +79,7 @@ def generate( if not pmb.helpers.cli.confirm("Continue and overwrite?"): raise RuntimeError("Aborted.") - aportgen = config.work / "aportgen" + aportgen = config.cache / "aportgen" if os.path.exists(aportgen): pmb.helpers.run.user(["rm", "-r", aportgen]) diff --git a/pmb/aportgen/core.py b/pmb/aportgen/core.py index 13a44b3a..a6aa24cf 100644 --- a/pmb/aportgen/core.py +++ b/pmb/aportgen/core.py @@ -103,7 +103,7 @@ def rewrite( lines_new += line.rstrip() + "\n" # Copy/modify lines, skip Maintainer/Contributor - path = get_context().config.work / "aportgen/APKBUILD" + path = get_context().config.cache / "aportgen/APKBUILD" with open(path, "r+", encoding="utf-8") as handle: skip_in_func = False for line in handle.readlines(): @@ -173,7 +173,7 @@ def get_upstream_aport(pkgname: str, arch: Arch | None = None, retain_branch: bo """ # APKBUILD pmb.helpers.git.clone("aports_upstream") - aports_upstream_path = get_context().config.work / "cache_git/aports_upstream" + aports_upstream_path = get_context().config.cache / "git/aports_upstream" if retain_branch: logging.info("Not changing aports branch as --fork-alpine-retain-branch was used.") @@ -252,7 +252,7 @@ def prepare_tempdir() -> Path: chroot = Chroot.native() pmb.chroot.init(chroot) tempdir = Path("/tmp/aportgen") - aportgen = get_context().config.work / "aportgen" + aportgen = get_context().config.cache / "aportgen" pmb.chroot.root(["rm", "-rf", tempdir], chroot) pmb.helpers.run.user(["mkdir", "-p", aportgen, chroot / tempdir]) @@ -265,7 +265,7 @@ def generate_checksums(tempdir: Path, apkbuild_path: Path) -> None: :param tempdir: Temporary directory as provided by prepare_tempdir(). :param apkbuild_path: APKBUILD to generate new checksums for. """ - aportgen = get_context().config.work / "aportgen" + aportgen = get_context().config.cache / "aportgen" pmb.build.init_abuild_minimal() pmb.chroot.root(["chown", "-R", "pmos:pmos", tempdir]) diff --git a/pmb/aportgen/device.py b/pmb/aportgen/device.py index 337df357..871cdc9d 100644 --- a/pmb/aportgen/device.py +++ b/pmb/aportgen/device.py @@ -248,7 +248,7 @@ def generate_deviceinfo( content += content_uuu # Write to file - work = get_context().config.work + work = get_context().config.cache pmb.helpers.run.user(["mkdir", "-p", work / "aportgen"]) path = work / "aportgen/deviceinfo" with open(path, "w", encoding="utf-8") as handle: @@ -270,7 +270,7 @@ def generate_modules_initfs() -> None: """ # Write to file - work = get_context().config.work + work = get_context().config.cache pmb.helpers.run.user(["mkdir", "-p", work / "aportgen"]) path = work / "aportgen/modules-initfs" with open(path, "w", encoding="utf-8") as handle: @@ -320,7 +320,7 @@ def generate_apkbuild(pkgname: str, name: str, arch: Arch, flash_method: str) -> """ # Write the file - work = get_context().config.work + work = get_context().config.cache pmb.helpers.run.user(["mkdir", "-p", work / "aportgen"]) path = work / "aportgen/APKBUILD" with open(path, "w", encoding="utf-8") as handle: diff --git a/pmb/aportgen/gcc.py b/pmb/aportgen/gcc.py index efb091b7..fcc6ce30 100644 --- a/pmb/aportgen/gcc.py +++ b/pmb/aportgen/gcc.py @@ -24,7 +24,7 @@ def generate(pkgname: str) -> None: based_on = "main/gcc6 (from postmarketOS)" else: raise ValueError(f"Invalid prefix '{prefix}', expected gcc, gcc4 or gcc6.") - pmb.helpers.run.user(["cp", "-r", upstream, context.config.work / "aportgen"]) + pmb.helpers.run.user(["cp", "-r", upstream, context.config.cache / "aportgen"]) # Rewrite APKBUILD fields = { diff --git a/pmb/aportgen/linux.py b/pmb/aportgen/linux.py index f7724766..006f956d 100644 --- a/pmb/aportgen/linux.py +++ b/pmb/aportgen/linux.py @@ -112,7 +112,7 @@ def generate_apkbuild(pkgname: str, deviceinfo: Deviceinfo, patches: list[str]) """ # Write the file - with (get_context().config.work / "aportgen/APKBUILD").open("w", encoding="utf-8") as hndl: + with (get_context().config.cache / "aportgen/APKBUILD").open("w", encoding="utf-8") as hndl: for line in content.rstrip().split("\n"): hndl.write(line[8:].replace(" " * 4, "\t") + "\n") @@ -120,7 +120,7 @@ def generate_apkbuild(pkgname: str, deviceinfo: Deviceinfo, patches: list[str]) def generate(pkgname: str) -> None: device = "-".join(pkgname.split("-")[1:]) deviceinfo = pmb.parse.deviceinfo(device) - work = get_context().config.work + work = get_context().config.cache # Symlink commonly used patches pmb.helpers.run.user(["mkdir", "-p", work / "aportgen"]) diff --git a/pmb/build/_package.py b/pmb/build/_package.py index a3e4b12d..bc713fd1 100644 --- a/pmb/build/_package.py +++ b/pmb/build/_package.py @@ -134,7 +134,7 @@ def finish( ) -> None: """Various finishing tasks that need to be done after a build.""" # Verify output file - out_dir = get_context().config.work / "packages" / channel + out_dir = get_context().config.cache / "packages" / channel if not (out_dir / output).exists(): raise RuntimeError(f"Package not found after build: {(out_dir / output)}") @@ -522,7 +522,7 @@ def packages( f"A binary package for {name} has a newer version ({index_data.version})" f" than the source ({pkgver}-{apkbuild['pkgrel']}). Please ensure your pmaports branch is up" " to date and that you don't have a newer version of the package in your local" - f" binary repo ({context.config.work / 'packages' / channel / pkg_arch})." + f" binary repo ({context.config.cache / 'packages' / channel / pkg_arch})." ) build_queue.append( { diff --git a/pmb/build/backend.py b/pmb/build/backend.py index 2795118a..9a847184 100644 --- a/pmb/build/backend.py +++ b/pmb/build/backend.py @@ -34,7 +34,7 @@ def override_source( return # Mount source in chroot - mount_path = "mnt/pmbootstrap/source-override/" + mount_path = "work/source-override/" mount_path_outside = chroot / mount_path pmb.helpers.mount.bind(src, mount_path_outside, umount=True) @@ -227,7 +227,7 @@ def run_abuild( ) pmb.mount.bind(hostchroot.path, buildchroot / "/mnt/sysroot", umount=True) - pkgdir = context.config.work / "packages" / channel + pkgdir = context.config.cache / "packages" / channel if not pkgdir.exists(): pmb.helpers.run.root(["mkdir", "-p", pkgdir]) pmb.helpers.run.root( @@ -243,7 +243,7 @@ def run_abuild( [ ["mkdir", "-p", "/home/pmos/packages"], ["rm", "-f", "/home/pmos/packages/pmos"], - ["ln", "-sf", f"/mnt/pmbootstrap/packages/{channel}", "/home/pmos/packages/pmos"], + ["ln", "-sf", f"/cache/packages/{channel}", "/home/pmos/packages/pmos"], ], buildchroot, ) diff --git a/pmb/build/envkernel.py b/pmb/build/envkernel.py index afe817ae..d365186b 100644 --- a/pmb/build/envkernel.py +++ b/pmb/build/envkernel.py @@ -100,7 +100,7 @@ def find_kbuild_output_dir(function_body: list[str]) -> str: def modify_apkbuild(pkgname: str, aport: Path) -> None: """Modify kernel APKBUILD to package build output from envkernel.sh.""" - work = get_context().config.work + work = get_context().config.cache apkbuild_path = aport / "APKBUILD" apkbuild = pmb.parse.apkbuild(apkbuild_path) if os.path.exists(work / "aportgen"): @@ -121,7 +121,7 @@ def modify_apkbuild(pkgname: str, aport: Path) -> None: def overmount_makefile(context: Context, kbuild_out: str) -> None: - makefile = context.config.work / "tmp_kernel.mk" + makefile = context.config.cache / "tmp_kernel.mk" makefile.open("w").write(""" # Automatically generated by pmbootstrap to use relative include paths include ../Makefile @@ -183,7 +183,7 @@ def run_abuild( # abuild and shouldn't really be done here. channel = pmb.config.pmaports.read_config(pmaports_path)["channel"] print(f"Building for channel: {channel}") - pkgdir = context.config.work / "packages" / channel + pkgdir = context.config.cache / "packages" / channel if not pkgdir.exists(): pmb.helpers.run.root(["mkdir", "-p", pkgdir]) pmb.helpers.run.root( @@ -199,7 +199,7 @@ def run_abuild( [ ["mkdir", "-p", "/home/pmos/packages"], ["rm", "-f", "/home/pmos/packages/pmos"], - ["ln", "-sf", f"/mnt/pmbootstrap/packages/{channel}", "/home/pmos/packages/pmos"], + ["ln", "-sf", f"/cache/packages/{channel}", "/home/pmos/packages/pmos"], ], chroot, ) @@ -242,7 +242,7 @@ def package_kernel(args: PmbArgs) -> None: context = get_context() modify_apkbuild(pkgname, aport) - apkbuild_path = context.config.work / "aportgen/APKBUILD" + apkbuild_path = context.config.cache / "aportgen/APKBUILD" arch = pmb.parse.deviceinfo().arch apkbuild = pmb.parse.apkbuild(apkbuild_path, check_pkgname=False) diff --git a/pmb/build/init.py b/pmb/build/init.py index 9592bf1b..92973d77 100644 --- a/pmb/build/init.py +++ b/pmb/build/init.py @@ -5,6 +5,7 @@ from pmb.core.context import Context from pmb.helpers import logging import os import pathlib +import shutil import pmb.build import pmb.config @@ -54,16 +55,15 @@ def init(chroot: Chroot = Chroot.native()) -> bool: init_abuild_minimal(chroot) # Generate package signing keys - if not os.path.exists(get_context().config.work / "config_abuild/abuild.conf"): + if not os.path.exists(get_context().config.cache / "abuild-config/abuild.conf"): logging.info(f"({chroot}) generate abuild keys") pmb.chroot.user( ["abuild-keygen", "-n", "-q", "-a"], chroot, env={"PACKAGER": "pmos "} ) # Copy package signing key to /etc/apk/keys - for key in (chroot / "mnt/pmbootstrap/abuild-config").glob("*.pub"): - key = key.relative_to(chroot.path) - pmb.chroot.root(["cp", key, "/etc/apk/keys/"], chroot) + for key in (get_context().config.cache / "abuild-config").glob("*.pub"): + shutil.copy(key, chroot / "etc/apk/keys") apk_arch = chroot.arch diff --git a/pmb/build/other.py b/pmb/build/other.py index 4fdb8290..805c7213 100644 --- a/pmb/build/other.py +++ b/pmb/build/other.py @@ -56,7 +56,7 @@ def copy_to_buildpath( def abuild_overrides(apkbuild: Path) -> None: """Override some abuild functions by patching the APKBUILD file.""" - if apkbuild.is_relative_to(get_context().config.work / "cache_git"): + if apkbuild.is_relative_to(get_context().config.cache / "git"): raise ValueError(f"Refusing to patch file in pmaports repo: {apkbuild}") # Patch the APKBUILD file @@ -144,7 +144,7 @@ def index_repo(arch: Arch | None = None) -> None: paths: list[Path] = [] for channel in pmb.config.pmaports.all_channels(): - pkgdir: Path = get_context().config.work / "packages" / channel + pkgdir: Path = get_context().config.cache / "packages" / channel if arch: paths.append(pkgdir / arch) else: @@ -153,7 +153,7 @@ def index_repo(arch: Arch | None = None) -> None: for path in paths: if path.is_dir(): path_channel, path_arch = path.parts[-2:] - path_repo_chroot = Path("/mnt/pmbootstrap/packages") / path_channel / path_arch + path_repo_chroot = Path("/cache/packages") / path_channel / path_arch logging.debug(f"(native) index {path_channel}/{path_arch} repository") description = str(datetime.datetime.now()) commands = [ @@ -210,7 +210,7 @@ def configure_ccache(chroot: Chroot = Chroot.native(), verify: bool = False) -> # Check if the settings have been set already arch = chroot.arch config = get_context().config - path = config.work / f"cache_ccache_{arch}" / "ccache.conf" + path = config.cache / f"ccache_{arch}" / "ccache.conf" if os.path.exists(path): with open(path, encoding="utf-8") as handle: for line in handle: diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index 81ebb4fb..756b1420 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -89,7 +89,7 @@ def packages_get_locally_built_apks(package_list: list[str], arch: Arch) -> list :param arch: architecture that the locally built packages should have :returns: Pair of lists, the first is the input packages with local apks removed. the second is a list of apk file paths that are valid inside the chroots, e.g. - ["/mnt/pmbootstrap/packages/x86_64/hello-world-1-r6.apk", ...] + ["/cache/packages/x86_64/hello-world-1-r6.apk", ...] """ channels: list[str] = pmb.config.pmaports.all_channels() local: list[Path] = [] @@ -108,7 +108,7 @@ def packages_get_locally_built_apks(package_list: list[str], arch: Arch) -> list # this will have weird behaviour if you build gnome-shell for edge and # then checkout out the systemd branch... But there isn't for channel in channels: - apk_path = get_context().config.work / "packages" / channel / arch / apk_file + apk_path = get_context().config.cache / "packages" / channel / arch / apk_file if apk_path.exists(): local.append(apk_path) break @@ -173,7 +173,7 @@ def install_run_apk( # FIXME: use /mnt/pmb… until MR 2351 is reverted (pmb#2388) user_repo: list[PathString] = [] for channel in pmb.config.pmaports.all_channels(): - user_repo += ["--repository", context.config.work / "packages" / channel] + user_repo += ["--repository", context.config.cache / "packages" / channel] for i, command in enumerate(commands): command = [*user_repo, *command] diff --git a/pmb/chroot/init.py b/pmb/chroot/init.py index a6cedb8c..4701ef43 100644 --- a/pmb/chroot/init.py +++ b/pmb/chroot/init.py @@ -56,21 +56,22 @@ def mark_in_chroot(chroot: Chroot = Chroot.native()) -> None: pmb.helpers.run.root(["touch", in_chroot_file]) -def init_keys() -> None: +def init_keys(chroot: Chroot) -> None: """ All Alpine and postmarketOS repository keys are shipped with pmbootstrap. - Copy them into $WORK/config_apk_keys, which gets mounted inside the various + Copy them into $WORK/keys, which gets mounted inside the various chroots as /etc/apk/keys. This is done before installing any package, so apk can verify APKINDEX files of binary repositories even though alpine-keys/postmarketos-keys are not installed yet. """ + target = chroot / "etc/apk/keys/" + target.mkdir(exist_ok=True, parents=True) for key in pmb.config.apk_keys_path.glob("*.pub"): - target = get_context().config.work / "config_apk_keys" / key.name - if not target.exists(): - # Copy as root, so the resulting files in chroots are owned by root - pmb.helpers.run.root(["cp", key, target]) + shutil.copy(key, target) + for key in (get_context().config.cache / "keys").glob("*.pub"): + shutil.copy(key, target) def init_usr_merge(chroot: Chroot) -> None: @@ -100,6 +101,13 @@ def warn_if_chroots_outdated() -> None: ) +def setup_cache_path(chroot: Chroot): + # Set up the apk cache to point to the working cache + cache_target = chroot / "etc/apk/cache" + if not cache_target.is_symlink(): + cache_target.symlink_to(f"/cache/apk_{chroot.arch}") + + @Cache("chroot") def init(chroot: Chroot, usr_merge: UsrMerge = UsrMerge.AUTO) -> None: """ @@ -132,9 +140,10 @@ def init(chroot: Chroot, usr_merge: UsrMerge = UsrMerge.AUTO) -> None: pmb.chroot.mount(chroot) mark_in_chroot(chroot) if chroot.exists(): - copy_resolv_conf(chroot) pmb.helpers.apk.update_repository_list(chroot.path) + copy_resolv_conf(chroot) warn_if_chroots_outdated() + setup_cache_path(chroot) return # Fetch apk.static @@ -143,9 +152,11 @@ def init(chroot: Chroot, usr_merge: UsrMerge = UsrMerge.AUTO) -> None: logging.info(f"({chroot}) Creating chroot") # Initialize /etc/apk/keys/, resolv.conf, repositories - init_keys() - copy_resolv_conf(chroot) + init_keys(chroot) + # Also creates /etc pmb.helpers.apk.update_repository_list(chroot.path) + copy_resolv_conf(chroot) + setup_cache_path(chroot) pmb.config.workdir.chroot_save_init(chroot) diff --git a/pmb/chroot/mount.py b/pmb/chroot/mount.py index cdf9d459..9d7768cb 100644 --- a/pmb/chroot/mount.py +++ b/pmb/chroot/mount.py @@ -42,7 +42,7 @@ def mount(chroot: Chroot): channel = pmb.config.pmaports.read_config(pkgrepo_default_path())["channel"] mountpoints: dict[Path, Path] = {} for src_template, target_template in pmb.config.chroot_mount_bind.items(): - src_template = src_template.replace("$WORK", os.fspath(get_context().config.work)) + src_template = src_template.replace("$CACHE", os.fspath(get_context().config.cache)) src_template = src_template.replace("$ARCH", str(arch)) src_template = src_template.replace("$CHANNEL", channel) mountpoints[Path(src_template).resolve()] = Path(target_template) @@ -81,13 +81,13 @@ def mount_native_into_foreign(chroot: Chroot) -> None: def remove_mnt_pmbootstrap(chroot: Chroot) -> None: - """Safely remove /mnt/pmbootstrap directories from the chroot, without + """Safely remove /cache directories from the chroot, without running rm -r as root and potentially removing data inside the mountpoint in case it was still mounted (bug in pmbootstrap, or user ran pmbootstrap 2x in parallel). This is similar to running 'rm -r -d', but we don't assume that the host's rm has the -d flag (busybox does not).""" - mnt_dir = chroot / "mnt/pmbootstrap" + mnt_dir = chroot / "work" if not mnt_dir.exists(): return @@ -96,7 +96,7 @@ def remove_mnt_pmbootstrap(chroot: Chroot) -> None: path.rmdir() if mnt_dir.exists(): - raise RuntimeError("Failed to remove /work!") + raise RuntimeError("Failed to remove /cache!") def umount(chroot: Chroot) -> None: diff --git a/pmb/chroot/shutdown.py b/pmb/chroot/shutdown.py index 0ea47918..ecee342b 100644 --- a/pmb/chroot/shutdown.py +++ b/pmb/chroot/shutdown.py @@ -44,7 +44,7 @@ def shutdown(only_install_related: bool = False) -> None: # the chroots, but we want it gone afterwards (e.g. when the chroot # contents get copied to a rootfs / installer image, or if creating an # android recovery zip from its contents). - for marker in get_context().config.localdir.glob("chroot_*/in-pmbootstrap"): + for marker in get_context().config.work.glob("chroot_*/in-pmbootstrap"): pmb.helpers.run.root(["rm", marker]) logging.debug("Shutdown complete") diff --git a/pmb/chroot/zap.py b/pmb/chroot/zap.py index 70cced0e..bd415ffc 100644 --- a/pmb/chroot/zap.py +++ b/pmb/chroot/zap.py @@ -82,11 +82,11 @@ def zap( if pkgs_local: patterns += ["packages"] if http: - patterns += ["cache_http"] + patterns += ["http"] if distfiles: - patterns += ["cache_distfiles"] + patterns += ["distfiles"] if rust: - patterns += ["cache_rust"] + patterns += ["rust"] if netboot: patterns += ["images_netboot"] @@ -96,7 +96,7 @@ def zap( # Delete everything matching the patterns for pattern in patterns: logging.debug(f"Deleting {pattern}") - pattern = os.path.realpath(f"{get_context().config.work}/{pattern}") + pattern = os.path.realpath(f"{get_context().config.cache}/{pattern}") matches = glob.glob(pattern) for match in matches: if not confirm or pmb.helpers.cli.confirm(f"Remove {match}?"): @@ -119,7 +119,7 @@ def zap( def zap_pkgs_local_mismatch(confirm: bool = True, dry: bool = False) -> None: channel = pmb.config.pmaports.read_config()["channel"] - if not os.path.exists(f"{get_context().config.work}/packages/{channel}"): + if not os.path.exists(f"{get_context().config.cache}/packages/{channel}"): return question = ( @@ -130,7 +130,7 @@ def zap_pkgs_local_mismatch(confirm: bool = True, dry: bool = False) -> None: return reindex = False - for apkindex_path in (get_context().config.work / "packages" / channel).glob( + for apkindex_path in (get_context().config.cache / "packages" / channel).glob( "*/APKINDEX.tar.gz" ): # Delete packages without same version in aports @@ -143,7 +143,7 @@ def zap_pkgs_local_mismatch(confirm: bool = True, dry: bool = False) -> None: # Apk path apk_path_short = f"{arch}/{pkgname}-{version}.apk" - apk_path = f"{get_context().config.work}/packages/{channel}/{apk_path_short}" + apk_path = f"{get_context().config.cache}/packages/{channel}/{apk_path_short}" if not os.path.exists(apk_path): logging.info(f"WARNING: Package mentioned in index not found: {apk_path_short}") continue @@ -175,7 +175,7 @@ def zap_pkgs_local_mismatch(confirm: bool = True, dry: bool = False) -> None: def zap_pkgs_online_mismatch(confirm: bool = True, dry: bool = False) -> None: # Check whether we need to do anything - paths = list(get_context().config.work.glob("cache_apk_*")) + paths = list(get_context().config.cache.glob("apk_*")) if not len(paths): return if confirm and not pmb.helpers.cli.confirm("Remove outdated binary packages?"): diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index 0e0dd6e5..e865fc88 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -150,24 +150,13 @@ chroot_path = ":".join( host_path = os.environ["PATH"] + ":/usr/sbin/" # Folders that get mounted inside the chroot -# $WORK gets replaced with get_context().config.work +# $CACHE gets replaced with get_context().config.cache # $ARCH gets replaced with the chroot architecture (eg. x86_64, armhf) # $CHANNEL gets replaced with the release channel (e.g. edge, v21.03) -# Use no more than one dir after /mnt/pmbootstrap, see remove_mnt_pmbootstrap. +# Use no more than one dir after /cache, see remove_mnt_pmbootstrap. chroot_mount_bind = { "/proc": "/proc", - "$WORK/cache_apk_$ARCH": "/var/cache/apk", - "$WORK/cache_appstream/$ARCH/$CHANNEL": "/mnt/appstream-data", - "$WORK/cache_ccache_$ARCH": "/mnt/pmbootstrap/ccache", - "$WORK/cache_distfiles": "/var/cache/distfiles", - "$WORK/cache_git": "/mnt/pmbootstrap/git", - "$WORK/cache_go": "/mnt/pmbootstrap/go", - "$WORK/cache_rust": "/mnt/pmbootstrap/rust", - "$WORK/config_abuild": "/mnt/pmbootstrap/abuild-config", - "$WORK/config_apk_keys": "/etc/apk/keys", - "$WORK/cache_sccache": "/mnt/pmbootstrap/sccache", - "$WORK/images_netboot": "/mnt/pmbootstrap/netboot", - "$WORK/packages/": "/mnt/pmbootstrap/packages", + "$CACHE": "/cache", } # Building chroots (all chroots, except for the rootfs_ chroot) get symlinks in @@ -183,15 +172,14 @@ chroot_mount_bind = { # rust depends caching described above) and to cache build artifacts (GOCACHE, # similar to ccache). chroot_home_symlinks = { - "/mnt/pmbootstrap/abuild-config": "/home/pmos/.abuild", - "/mnt/pmbootstrap/ccache": "/home/pmos/.ccache", - "/mnt/pmbootstrap/go/gocache": "/home/pmos/.cache/go-build", - "/mnt/pmbootstrap/go/gomodcache": "/home/pmos/go/pkg/mod", - # "/mnt/pmbootstrap/packages": "/home/pmos/packages/pmos", - "/mnt/pmbootstrap/rust/git/db": "/home/pmos/.cargo/git/db", - "/mnt/pmbootstrap/rust/registry/cache": "/home/pmos/.cargo/registry/cache", - "/mnt/pmbootstrap/rust/registry/index": "/home/pmos/.cargo/registry/index", - "/mnt/pmbootstrap/sccache": "/home/pmos/.cache/sccache", + "/cache/abuild-config": "/home/pmos/.abuild", + "/cache/ccache_$ARCH": "/home/pmos/.ccache", + "/cache/go/gocache": "/home/pmos/.cache/go-build", + "/cache/go/gomodcache": "/home/pmos/go/pkg/mod", + "/cache/rust/git/db": "/home/pmos/.cargo/git/db", + "/cache/rust/registry/cache": "/home/pmos/.cargo/registry/cache", + "/cache/rust/registry/index": "/home/pmos/.cargo/registry/index", + "/cache/sccache": "/home/pmos/.cache/sccache", } # Age in hours that we keep the APKINDEXes before downloading them again. diff --git a/pmb/config/init.py b/pmb/config/init.py index a429f381..e240632d 100644 --- a/pmb/config/init.py +++ b/pmb/config/init.py @@ -773,14 +773,14 @@ def frontend(args: PmbArgs) -> None: # Work folder (needs to be first, so we can create chroots early) config = get_context().config - using_default_pmaports = config.aports[-1].is_relative_to(config.work) + using_default_pmaports = config.aports[-1].is_relative_to(config.cache) config.work, work_exists = ask_for_work_path(config.work) # If the work dir changed then we need to update the pmaports path # to be relative to the new workdir if using_default_pmaports: - config.aports = [config.work / "cache_git/pmaports"] + config.aports = [config.cache / "git/pmaports"] config.aports[-1] = ask_for_pmaports_path(config.aports[-1]) @@ -789,7 +789,7 @@ def frontend(args: PmbArgs) -> None: pmb.config.save(args.config, config) # Migrate work dir if necessary - pmb.helpers.other.migrate_work_folder() + pmb.helpers.other.migrate_localdir() # Clone pmaports pmb.config.pmaports.init() diff --git a/pmb/core/chroot.py b/pmb/core/chroot.py index 05decfc4..743c29d5 100644 --- a/pmb/core/chroot.py +++ b/pmb/core/chroot.py @@ -79,7 +79,7 @@ class Chroot: return (self / "bin/sh").is_symlink() def is_mounted(self) -> bool: - return self.exists() and pmb.helpers.mount.ismount(self.path / "etc/apk/keys") + return self.exists() and (self.path / "etc/apk/cache").is_symlink() @property def arch(self) -> Arch: diff --git a/pmb/core/config.py b/pmb/core/config.py index b950d011..0312d499 100644 --- a/pmb/core/config.py +++ b/pmb/core/config.py @@ -51,7 +51,7 @@ class Config: # This is a class variable that gets treated as an instance variable. It's wrong, but since we # only ever have one config (for now?) it doesn't cause any issues. Would be good to fix though. aports: list[Path] = [ # noqa: RUF012 - Path(os.path.expanduser("~") + "/.local/var/pmbootstrap/cache_git/pmaports") + Path(os.path.expanduser("~") + "/.local/var/pmbootstrap/cache/git/pmaports") ] boot_size: int = 256 build_default_device_arch: bool = False @@ -84,6 +84,7 @@ class Config: ui_extras: bool = False user: str = "user" work: Path = Path(os.path.expanduser("~") + "/.local/var/pmbootstrap") + cache: Path = Path(os.path.expanduser("~") + "/.local/var/pmbootstrap/cache") # automatically zap chroots that are for the wrong channel auto_zap_misconfigured_chroots: AutoZapConfig = AutoZapConfig.NO diff --git a/pmb/core/context.py b/pmb/core/context.py index ce0b9ff7..4d439b18 100644 --- a/pmb/core/context.py +++ b/pmb/core/context.py @@ -39,7 +39,7 @@ class Context: config: Config def __init__(self, config: Config): - self.log = config.work / "log.txt" + self.log = config.cache / "log.txt" self.config = config diff --git a/pmb/helpers/apk.py b/pmb/helpers/apk.py index c932cffe..b70efcdf 100644 --- a/pmb/helpers/apk.py +++ b/pmb/helpers/apk.py @@ -49,13 +49,13 @@ def update_repository_list( for line in handle: lines_old.append(line[:-1]) else: - pmb.helpers.run.root(["mkdir", "-p", path.parent]) + path.parent.mkdir(exist_ok=True, parents=True) user_repo_dir: Path | None if isinstance(user_repository, Path): user_repo_dir = user_repository else: - user_repo_dir = Path("/mnt/pmbootstrap/packages") if user_repository else None + user_repo_dir = Path("/cache/packages") if user_repository else None # Up to date: Save cache, return lines_new = pmb.helpers.repo.urls( @@ -93,8 +93,8 @@ def _prepare_fifo() -> Path: path of the fifo as needed by cat to read from it (always relative to the host) """ - pmb.helpers.run.root(["mkdir", "-p", get_context().config.work / "tmp"]) - fifo = get_context().config.work / "tmp/apk_progress_fifo" + pmb.helpers.run.root(["mkdir", "-p", get_context().config.cache / "tmp"]) + fifo = get_context().config.cache / "tmp/apk_progress_fifo" if os.path.exists(fifo): pmb.helpers.run.root(["rm", "-f", fifo]) @@ -164,9 +164,9 @@ def _prepare_cmd(command: Sequence[PathString], chroot: Chroot | None) -> list[s # Our _apk_with_progress() wrapper also need --no-progress, since all that does is # prevent apk itself from rendering progress bars. We instead want it to tell us # the progress so we can render it. So we always set --no-progress. - _command: list[str] = [str(config.work / "apk.static"), "--no-progress"] + _command: list[str] = [str(config.cache / "apk.static"), "--no-progress"] if chroot: - cache_dir = config.work / f"cache_apk_{chroot.arch}" + cache_dir = config.cache / f"apk_{chroot.arch}" _command.extend( [ "--root", @@ -182,7 +182,7 @@ def _prepare_cmd(command: Sequence[PathString], chroot: Chroot | None) -> list[s _command.extend(["--cache-dir", str(cache_dir)]) local_repos = pmb.helpers.repo.urls( - user_repository=config.work / "packages", mirrors_exclude=True + user_repository=config.cache / "packages", mirrors_exclude=True ) for repo in local_repos: _command.extend(["--repository", repo]) @@ -233,8 +233,8 @@ def run(command: Sequence[PathString], chroot: Chroot, with_progress: bool = Tru def cache_clean(arch: Arch) -> None: """Clean the APK cache for a specific architecture.""" - work = get_context().config.work - cache_dir = work / f"cache_apk_{arch}" + work = get_context().config.cache + cache_dir = work / f"apk_{arch}" if not cache_dir.exists(): return @@ -257,7 +257,7 @@ def cache_clean(arch: Arch) -> None: (tmproot / "lib/apk/db/installed").touch(exist_ok=True) (tmproot / "lib/apk/db/triggers").touch(exist_ok=True) - (tmproot / "etc/apk/keys").symlink_to(work / "config_apk_keys") + (tmproot / "etc/apk/keys").symlink_to(work / "keys") # Our fake rootfs needs a valid repositories file for apk # to have something to compare the cache against diff --git a/pmb/helpers/apk_static.py b/pmb/helpers/apk_static.py index bfd14f90..cc829ed8 100644 --- a/pmb/helpers/apk_static.py +++ b/pmb/helpers/apk_static.py @@ -148,7 +148,7 @@ def extract(version: str, apk_path: Path) -> None: ) # Move it to the right path - target_path = get_context().config.work / "apk.static" + target_path = get_context().config.cache / "apk.static" shutil.move(temp_path, target_path) diff --git a/pmb/helpers/args.py b/pmb/helpers/args.py index 70fbb849..51442c71 100644 --- a/pmb/helpers/args.py +++ b/pmb/helpers/args.py @@ -32,7 +32,7 @@ import pmb.helpers.args with "pmbootstrap init". Examples: - args.aports ("$WORK/cache_git/pmaports", override with --aports) + args.aports ("$WORK/git/pmaports", override with --aports) args.device ("samsung-i9100", "qemu-amd64" etc.) get_context().config.work ("/home/user/.local/var/pmbootstrap", override with --work) diff --git a/pmb/helpers/git.py b/pmb/helpers/git.py index 6c7d9bd2..8ec1748f 100644 --- a/pmb/helpers/git.py +++ b/pmb/helpers/git.py @@ -30,7 +30,7 @@ def get_path(name_repo: str) -> Path: :returns: full path to repository """ if name_repo == "aports_upstream": - return get_context().config.work / "cache_git" / name_repo + return get_context().config.cache / "git" / name_repo return pkgrepo_path(name_repo) @@ -55,7 +55,7 @@ def clone(name_repo: str) -> None: # Create parent dir and clone logging.info(f"Clone git repository: {url}") - (get_context().config.work / "cache_git").mkdir(exist_ok=True) + (get_context().config.cache / "git").mkdir(exist_ok=True) pmb.helpers.run.user(command, output="stdout") # FETCH_HEAD does not exist after initial clone. Create it, so diff --git a/pmb/helpers/http.py b/pmb/helpers/http.py index 21eaf087..abf6b3dd 100644 --- a/pmb/helpers/http.py +++ b/pmb/helpers/http.py @@ -69,11 +69,11 @@ def download( """ # Create cache folder context = get_context() - if not os.path.exists(context.config.work / "cache_http"): - pmb.helpers.run.user(["mkdir", "-p", context.config.work / "cache_http"]) + if not os.path.exists(context.config.cache / "http"): + pmb.helpers.run.user(["mkdir", "-p", context.config.cache / "http"]) # Check if file exists in cache - path = context.config.work / "cache_http" / cache_file(prefix, url) + path = context.config.cache / "http" / cache_file(prefix, url) if os.path.exists(path): if cache: return path diff --git a/pmb/helpers/other.py b/pmb/helpers/other.py index 05ebb86d..0e278d50 100644 --- a/pmb/helpers/other.py +++ b/pmb/helpers/other.py @@ -47,13 +47,13 @@ def check_grsec() -> None: ) -def migrate_success(work: Path, version: int) -> None: +def migrate_success(localdir: Path, version: int) -> None: logging.info("Migration to version " + str(version) + " done") - with open(work / "version", "w") as handle: + with open(localdir / "version", "w") as handle: handle.write(str(version) + "\n") -def migrate_work_folder() -> None: +def migrate_localdir() -> None: # Read current version context = get_context() current = 0 @@ -131,7 +131,7 @@ def migrate_work_folder() -> None: raise NonBugError( "Sorry, we can't migrate that automatically. Please" " run 'pmbootstrap shutdown', then delete your" - " current work folder manually ('sudo rm -rf " + " current localdir manually ('sudo rm -rf " f"{context.config.work}') and start over with 'pmbootstrap" " init'. All your binary packages and caches will" " be lost." diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index 1b4b1bb4..2306d5dc 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -60,10 +60,10 @@ def urls( ) -> list[str]: """Get a list of repository URLs, as they are in /etc/apk/repositories. - :param user_repository: add /mnt/pmbootstrap/packages + :param user_repository: add /cache/packages :param mirrors_exclude: mirrors to exclude (see pmb.core.config.Mirrors) or true to exclude all mirrors and only return the local repos - :returns: list of mirror strings, like ["/mnt/pmbootstrap/packages", + :returns: list of mirror strings, like ["/cache/packages", "http://...", ...] """ ret: list[str] = [] @@ -141,11 +141,11 @@ def apkindex_files( # Local user repository (for packages compiled with pmbootstrap) if user_repository: for channel in pmb.config.pmaports.all_channels(): - ret.append(get_context().config.work / "packages" / channel / arch / "APKINDEX.tar.gz") + ret.append(get_context().config.cache / "packages" / channel / arch / "APKINDEX.tar.gz") # Resolve the APKINDEX.$HASH.tar.gz files for url in urls(False, exclude_mirrors): - ret.append(get_context().config.work / f"cache_apk_{arch}" / apkindex_hash(url)) + ret.append(get_context().config.cache / f"apk_{arch}" / apkindex_hash(url)) return ret @@ -181,7 +181,7 @@ def update(arch: Arch | None = None, force: bool = False, existing_only: bool = for arch in architectures: # APKINDEX file name from the URL url_full = f"{url}/{arch}/APKINDEX.tar.gz" - cache_apk_outside = get_context().config.work / f"cache_apk_{arch}" + cache_apk_outside = get_context().config.cache / f"apk_{arch}" apkindex = cache_apk_outside / f"{apkindex_hash(url)}" # Find update reason, possibly skip non-existing or known 404 files @@ -258,5 +258,5 @@ def alpine_apkindex_path(repo: str = "main", arch: Arch | None = None) -> Path: # Find it on disk channel_cfg = pmb.config.pmaports.read_config_channel() repo_link = f"{get_context().config.mirrors['alpine']}{channel_cfg['mirrordir_alpine']}/{repo}" - cache_folder = get_context().config.work / (f"cache_apk_{arch}") + cache_folder = get_context().config.cache / (f"apk_{arch}") return cache_folder / apkindex_hash(repo_link) diff --git a/pmb/init/sandbox.py b/pmb/init/sandbox.py index 90bb533b..8b5ebfa0 100644 --- a/pmb/init/sandbox.py +++ b/pmb/init/sandbox.py @@ -862,9 +862,9 @@ class OverlayOperation(FSOperation): with umask(~mode): os.mkdir(f"{dst}/upper", mode=mode) with umask(~0o755): - os.mkdir(f"{dst}/work") + os.mkdir(f"{dst}/cache") - options += [f"upperdir={dst}/upper", f"workdir={dst}/work"] + options += [f"upperdir={dst}/upper", f"workdir={dst}/cache"] else: if upperdir: options += [f"upperdir={upperdir}"] diff --git a/pmb/install/_install.py b/pmb/install/_install.py index bd195593..20376e67 100644 --- a/pmb/install/_install.py +++ b/pmb/install/_install.py @@ -132,13 +132,13 @@ def configure_apk(args: PmbArgs, rootfs: Path) -> None: """ Copy over all official keys, and the keys used to compile local packages (unless --no-local-pkgs is set). Then copy the corresponding APKINDEX files - and remove the /mnt/pmbootstrap/packages repository. + and remove the /cache/packages repository. """ # Official keys keys_dir = pmb.config.apk_keys_path # Official keys + local keys - keys_dir = get_context().config.work / "config_apk_keys" + keys_dir = get_context().config.cache / "keys" # Copy over keys for key in keys_dir.glob("*.pub"): @@ -417,16 +417,19 @@ def setup_appstream(offline: bool, chroot: Chroot) -> None: if "alpine-appstream-downloader" not in installed_pkgs or offline: return - if not pmb.chroot.root( - ["alpine-appstream-downloader", "/mnt/appstream-data"], chroot, check=False - ): + target_dir = Path("/cache/appstream") / chroot.arch / chroot.channel + logging.info(f"appstream target dir: {target_dir}") + + # FIXME: it would be great to run this on the host and not potentially + # through QEMU! + if not pmb.chroot.root(["alpine-appstream-downloader", target_dir], chroot, check=False): pmb.chroot.root(["mkdir", "-p", "/var/lib/swcatalog"], chroot) pmb.chroot.root( [ "cp", "-r", - "/mnt/appstream-data/icons", - "/mnt/appstream-data/xml", + target_dir / "icons", + target_dir / "xml", "-t", "/var/lib/swcatalog", ], diff --git a/pmb/install/blockdevice.py b/pmb/install/blockdevice.py index 34ab0992..42d54d3f 100644 --- a/pmb/install/blockdevice.py +++ b/pmb/install/blockdevice.py @@ -91,7 +91,7 @@ def create_and_mount_image( # Make sure there is enough free space size_full = round(layout.boot.size + layout.root.size) - disk_data = os.statvfs(get_context().config.work) + disk_data = os.statvfs(get_context().config.cache) free = disk_data.f_bsize * disk_data.f_bavail if size_full > free: raise RuntimeError( diff --git a/pmb/netboot/__init__.py b/pmb/netboot/__init__.py index 8485d44d..6c3b0e64 100644 --- a/pmb/netboot/__init__.py +++ b/pmb/netboot/__init__.py @@ -22,7 +22,7 @@ def start_nbd_server(device: str, replace: bool, ip: str = "172.16.42.2", port: chroot = Chroot.native() pmb.chroot.init(chroot) - rootfs_path = Path("/mnt/pmbootstrap/netboot") / f"{device}.img" + rootfs_path = Path("/cache/netboot") / f"{device}.img" if not (chroot / rootfs_path).exists() or replace: rootfs_path2 = Path("/home/pmos/rootfs") / f"{device}.img" if not (chroot / rootfs_path2).exists(): @@ -35,7 +35,7 @@ def start_nbd_server(device: str, replace: bool, ip: str = "172.16.42.2", port: return pmb.chroot.root(["cp", rootfs_path2, rootfs_path]) logging.info( - f"NOTE: Copied device image to {get_context().config.work}" + f"NOTE: Copied device image to {get_context().config.cache}" f'/images_netboot/. The image will persist "pmbootstrap ' f'zap" for your convenience. Use "pmbootstrap netboot ' f'serve --help" for more options.' diff --git a/pmb/parse/_apkbuild.py b/pmb/parse/_apkbuild.py index 62d07fc5..4a5e593f 100644 --- a/pmb/parse/_apkbuild.py +++ b/pmb/parse/_apkbuild.py @@ -348,7 +348,7 @@ def apkbuild(path: Path, check_pkgver: bool = True, check_pkgname: bool = True) path = path / "APKBUILD" if not path.exists(): - raise FileNotFoundError(f"{path.relative_to(get_context().config.work)} not found!") + raise FileNotFoundError(f"{path.relative_to(get_context().config.cache)} not found!") # Read the file and check line endings lines = read_file(path) diff --git a/pmb/sideload/__init__.py b/pmb/sideload/__init__.py index 29f9ebbd..5ccf5bae 100644 --- a/pmb/sideload/__init__.py +++ b/pmb/sideload/__init__.py @@ -24,7 +24,7 @@ def scp_abuild_key(args: PmbArgs, user: str, host: str, port: str) -> None: :param host: target device ssh hostname :param port: target device ssh port""" - keys = list((get_context().config.work / "config_abuild").glob("*.pub")) + keys = list((get_context().config.cache / "config_abuild").glob("*.pub")) key = keys[0] key_name = os.path.basename(key) @@ -124,7 +124,7 @@ def sideload( channel = pmb.config.pmaports.read_config(base_aports)["channel"] apk_file = f"{pkgname}-{data_repo.version}.apk" - host_path = context.config.work / "packages" / channel / arch / apk_file + host_path = context.config.cache / "packages" / channel / arch / apk_file if not host_path.is_file(): to_build.append(pkgname)