Fix pmbootstrap zap -m / various zap improvements (#1166)

zap -m:
* APKINDEX parsing: parse the "origin" field as well, so we know
  where a subpackage comes from
* pmbootstrap zap -m: properly delete all packages, that do not
  have an aport or where the aport has another version. This also
  works with subpackages now,
  we use the origin field to resolve it.
* Only reindex when packages have been deleted in "zap -m"

zap in general:
* Show the amount of cleared up space after the deletion instead
  of "Done"
* Print "Shutdown complete" to "pmbootstrap log" instead of stdout
  (we need to call it twice during zap now to get the space
  calculation right)
* Add `--dry` argument to `pmbootstrap zap` (this was very useful
  for debugging) to list the packages/chroots that would get
  deleted
* Roughly output the command that would get executed to delete
  files, so it's obvious what's going on in --dry mode. (% rm ...)
This commit is contained in:
Oliver Smith 2018-01-31 19:34:02 +00:00 committed by GitHub
parent 8f099d3506
commit 0f5056f6b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 97 additions and 48 deletions

View file

@ -94,4 +94,4 @@ def shutdown(args, only_install_related=False):
for arch in pmb.config.build_device_architectures: for arch in pmb.config.build_device_architectures:
if pmb.parse.arch.cpu_emulation_required(args, arch): if pmb.parse.arch.cpu_emulation_required(args, arch):
pmb.chroot.binfmt.unregister(args, arch) pmb.chroot.binfmt.unregister(args, arch)
logging.info("Shutdown complete") logging.debug("Shutdown complete")

View file

@ -16,39 +16,46 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>. along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
""" """
import os
import glob import glob
import logging import logging
import math
import os
import pmb.chroot import pmb.chroot
import pmb.helpers.run import pmb.helpers.run
def zap(args, confirm=True, packages=False, http=False, mismatch_bins=False, def zap(args, confirm=True, dry=False, packages=False, http=False,
old_bins=False, distfiles=False): mismatch_bins=False, old_bins=False, distfiles=False):
""" """
Shutdown everything inside the chroots (e.g. distccd, adb), umount Shutdown everything inside the chroots (e.g. distccd, adb), umount
everything and then safely remove folders from the work-directory. everything and then safely remove folders from the work-directory.
:arg packages: Remove *all* self-compiled packages (!) :param dry: Only show what would be deleted, do not delete for real
:arg http: Clear the http cache (used e.g. for the initial apk download) :param packages: Remove *all* self-compiled packages (!)
:arg mismatch_bins: Remove the packages, that have a different version :param http: Clear the http cache (used e.g. for the initial apk download)
compared to what is in the abuilds folder. :param mismatch_bins: Remove the packages, that have a different version
:arg old_bins: Clean out outdated binary packages downloaded from compared to what is in the aports folder.
:param old_bins: Clean out outdated binary packages downloaded from
mirrors (e.g. from Alpine) mirrors (e.g. from Alpine)
:arg distfiles: Clear the downloaded files cache :param distfiles: Clear the downloaded files cache
NOTE: This function gets called in pmb/config/init.py, with only args.work NOTE: This function gets called in pmb/config/init.py, with only args.work
and args.device set! and args.device set!
""" """
# Get current work folder size
if not dry:
pmb.chroot.shutdown(args)
logging.debug("Calculate work folder size")
size_old = pmb.helpers.other.folder_size(args, args.work)
# Delete packages with a different version compared to aports, then re-index # Delete packages with a different version compared to aports, then re-index
if mismatch_bins: if mismatch_bins:
zap_mismatch_bins(args, confirm) zap_mismatch_bins(args, confirm, dry)
# Delete outdated binary packages # Delete outdated binary packages
if old_bins: if old_bins:
zap_old_bins(args, confirm) zap_old_bins(args, confirm, dry)
pmb.chroot.shutdown(args) pmb.chroot.shutdown(args)
@ -71,52 +78,87 @@ def zap(args, confirm=True, packages=False, http=False, mismatch_bins=False,
matches = glob.glob(pattern) matches = glob.glob(pattern)
for match in matches: for match in matches:
if not confirm or pmb.helpers.cli.confirm(args, "Remove " + match + "?"): if not confirm or pmb.helpers.cli.confirm(args, "Remove " + match + "?"):
logging.info("% rm -rf " + match)
if not dry:
pmb.helpers.run.root(args, ["rm", "-rf", match]) pmb.helpers.run.root(args, ["rm", "-rf", match])
# Chroots were zapped, so no repo lists exist anymore # Chroots were zapped, so no repo lists exist anymore
args.cache["apk_repository_list_updated"].clear() args.cache["apk_repository_list_updated"].clear()
# Print amount of cleaned up space
if dry:
logging.info("Dry run: nothing has been deleted")
else:
size_new = pmb.helpers.other.folder_size(args, args.work)
mb = (size_old - size_new) / 1024 / 1024
logging.info("Cleared up ~" + str(math.ceil(mb)) + " MB of space")
def zap_mismatch_bins(args, confirm=True):
def zap_mismatch_bins(args, confirm=True, dry=False):
if not os.path.exists(args.work + "/packages/"): if not os.path.exists(args.work + "/packages/"):
return return
if confirm and not pmb.helpers.cli.confirm(args, "Remove packages that are newer than" if confirm and not pmb.helpers.cli.confirm(args, "Remove packages that are newer than"
" the corresponding package in aports?"): " the corresponding package in aports?"):
return return
for arch in os.listdir(os.path.realpath(args.work + "/packages/")):
arch_pkg_path = os.path.realpath(args.work) + "/packages/" + arch
# Delete all broken symbolic links
pmb.helpers.run.root(args, ["find", "-L", arch_pkg_path, "-maxdepth", "1",
"-type", "l", "-delete"])
bin_apks = pmb.parse.apkindex.parse(args, arch_pkg_path + "/APKINDEX.tar.gz")
for bin_apk in bin_apks:
bin_pkgname = bin_apks[bin_apk]["pkgname"]
bin_version = bin_apks[bin_apk]["version"]
bin_apk_path = arch_pkg_path + "/" + bin_pkgname + "-" + bin_version + ".apk"
# Do not fail if unable to find aport reindex = False
aport = pmb.build.other.find_aport(args, bin_pkgname, False) for apkindex_path in glob.glob(args.work + "/packages/*/APKINDEX.tar.gz"):
if not aport: apkindex = pmb.parse.apkindex.parse(args, apkindex_path)
logging.warning("WARNING: Could not resolve aport for package " + bin_apk_path) for pkgname, bin_data in apkindex.items():
# Only real packages have apks, provided packages do not exist
# (e.g. "so:libtest.so.1.2")
if pkgname != bin_data["pkgname"]:
continue
origin = bin_data["origin"]
version = bin_data["version"]
arch = bin_data["arch"]
# Apk path
apk_path_short = arch + "/" + pkgname + "-" + version + ".apk"
apk_path = args.work + "/packages/" + apk_path_short
if not os.path.exists(apk_path):
logging.info("WARNING: Package mentioned in index not"
" found: " + apk_path_short)
continue
# Aport path
aport_path = pmb.build.other.find_aport(args, origin, False)
if not aport_path:
logging.info("% rm " + apk_path_short + " (" + origin +
" aport not found)")
if not dry:
pmb.helpers.run.root(args, ["rm", apk_path])
reindex = True
continue continue
apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
aport_version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
# Clear out any binary apks that do not match what is in aports # Clear out any binary apks that do not match what is in aports
if pmb.parse.version.compare(bin_version, aport_version) and os.path.exists(bin_apk_path): apkbuild = pmb.parse.apkbuild(args, aport_path + "/APKBUILD")
logging.info("Remove mismatched binary package (aports version: " + version_aport = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
aport_version + "): " + arch + "/" + bin_pkgname + "-" + if version != version_aport:
bin_version + ".apk") logging.info("% rm " + apk_path_short + " (" + origin +
pmb.helpers.run.root(args, ["rm", bin_apk_path]) " aport: " + version_aport + ")")
if not dry:
pmb.helpers.run.root(args, ["rm", apk_path])
reindex = True
if reindex:
pmb.build.other.index_repo(args) pmb.build.other.index_repo(args)
def zap_old_bins(args, confirm=True): def zap_old_bins(args, confirm=True, dry=False):
# Check whether we need to do anything
paths = glob.glob(args.work + "/cache_apk_*")
if not len(paths):
return
if confirm and not pmb.helpers.cli.confirm(args, "Remove outdated binary packages?"): if confirm and not pmb.helpers.cli.confirm(args, "Remove outdated binary packages?"):
return return
arch_native = pmb.parse.arch.alpine_native()
if os.path.exists(args.work + "/cache_apk_" + arch_native): # Iterate over existing apk caches
pmb.chroot.root(args, ["apk", "-v", "cache", "clean"]) for path in paths:
for arch in pmb.config.build_device_architectures: arch = os.path.basename(path).split("_", 2)[2]
if arch != arch_native and os.path.exists(args.work + "/cache_apk_" + arch): chroot = "native" if arch == args.arch_native else "buildroot_" + arch
pmb.chroot.root(args, ["apk", "-v", "cache", "clean"], "buildroot_" + arch)
# Clean the cache with apk
logging.info("(" + chroot + ") apk -v cache clean")
if not dry:
pmb.chroot.root(args, ["apk", "-v", "cache", "clean"], chroot)

View file

@ -329,10 +329,13 @@ def log_distccd(args):
def zap(args): def zap(args):
pmb.chroot.zap(args, packages=args.packages, http=args.http, pmb.chroot.zap(args, dry=args.dry, packages=args.packages, http=args.http,
mismatch_bins=args.mismatch_bins, old_bins=args.old_bins, mismatch_bins=args.mismatch_bins, old_bins=args.old_bins,
distfiles=args.distfiles) distfiles=args.distfiles)
# Don't write the "Done" message
pmb.helpers.logging.disable()
def bootimg_analyze(args): def bootimg_analyze(args):
bootimg = pmb.parse.bootimg(args, args.path) bootimg = pmb.parse.bootimg(args, args.path)

View file

@ -47,11 +47,12 @@ def parse_next_block(args, path, lines, start):
ret = {} ret = {}
mapping = { mapping = {
"A": "arch", "A": "arch",
"P": "pkgname",
"V": "version",
"D": "depends", "D": "depends",
"o": "origin",
"P": "pkgname",
"p": "provides", "p": "provides",
"t": "timestamp" "t": "timestamp",
"V": "version",
} }
end_of_block_found = False end_of_block_found = False
for i in range(start[0], len(lines)): for i in range(start[0], len(lines)):

View file

@ -226,6 +226,9 @@ def arguments():
# Action: zap # Action: zap
zap = sub.add_parser("zap", help="safely delete chroot folders") zap = sub.add_parser("zap", help="safely delete chroot folders")
zap.add_argument("--dry", action="store_true", help="instead of actually"
" deleting anything, print out what would have been"
" deleted")
zap.add_argument("-p", "--packages", action="store_true", help="also delete" zap.add_argument("-p", "--packages", action="store_true", help="also delete"
" the precious, self-compiled packages") " the precious, self-compiled packages")
zap.add_argument("-hc", "--http", action="store_true", help="also delete http" zap.add_argument("-hc", "--http", action="store_true", help="also delete http"