pmbootstrap/pmb/helpers/repo.py

# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
"""
Functions that work with binary package repos.

See also:
- pmb/helpers/pmaports.py (work with pmaports)
- pmb/helpers/package.py (work with both)
"""
import os
import hashlib
import logging
from pathlib import Path
from typing import List

import pmb.config.pmaports
from pmb.core.types import PmbArgs
import pmb.helpers.http
import pmb.helpers.run


def hash(url, length=8):
    r"""Generate the hash that APK adds to the APKINDEX and apk packages in its apk cache folder.

    It is the "12345678" part in this example:
    "APKINDEX.12345678.tar.gz".

    :param length: The length of the hash in the output file.

    See also: official implementation in apk-tools:
    <https://git.alpinelinux.org/cgit/apk-tools/>

    blob.c: apk_blob_push_hexdump(), "const char \\*xd"
    apk_defines.h: APK_CACHE_CSUM_BYTES
    database.c: apk_repo_format_cache_index()
    """
    binary = hashlib.sha1(url.encode("utf-8")).digest()
    xd = "0123456789abcdefghijklmnopqrstuvwxyz"
    csum_bytes = int(length / 2)

    ret = ""
    for i in range(csum_bytes):
        ret += xd[(binary[i] >> 4) & 0xf]
        ret += xd[binary[i] & 0xf]

    return ret


def urls(args, user_repository=True, postmarketos_mirror=True, alpine=True):
    """Get a list of repository URLs, as they are in /etc/apk/repositories.

    :param user_repository: add /mnt/pmbootstrap/packages
    :param postmarketos_mirror: add postmarketos mirror URLs
    :param alpine: add alpine mirror URLs
    :returns: list of mirror strings, like ["/mnt/pmbootstrap/packages",
                                            "http://...", ...]
    """
    ret: List[str] = []

    # Get mirrordirs from channels.cfg (postmarketOS mirrordir is the same as
    # the pmaports branch of the channel, no need to make it more complicated)
    channel_cfg = pmb.config.pmaports.read_config_channel(args)
    mirrordir_pmos = channel_cfg["branch_pmaports"]
    mirrordir_alpine = channel_cfg["mirrordir_alpine"]

    # Local user repository (for packages compiled with pmbootstrap)
    if user_repository:
        ret.append("/mnt/pmbootstrap/packages")

    # Upstream postmarketOS binary repository
    if postmarketos_mirror:
        for mirror in args.mirrors_postmarketos:
            # Remove "master" mirrordir to avoid breakage until bpo is adjusted
            # (build.postmarketos.org#63) and to give potential other users of
            # this flag a heads up.
            if mirror.endswith("/master"):
                logging.warning("WARNING: 'master' at the end of"
                                " --mirror-pmOS is deprecated, the branch gets"
                                " added automatically now!")
                mirror = mirror[:-1 * len("master")]
            ret.append(f"{mirror}{mirrordir_pmos}")

    # Upstream Alpine Linux repositories
    if alpine:
        directories = ["main", "community"]
        if mirrordir_alpine == "edge":
            directories.append("testing")
        for dir in directories:
            ret.append(f"{args.mirror_alpine}{mirrordir_alpine}/{dir}")

    return ret


def apkindex_files(args: PmbArgs, arch=None, user_repository=True, pmos=True,
                   alpine=True) -> List[Path]:
    """Get a list of outside paths to all resolved APKINDEX.tar.gz files for a specific arch.

    :param arch: defaults to native
    :param user_repository: add path to index of locally built packages
    :param pmos: add paths to indexes of postmarketos mirrors
    :param alpine: add paths to indexes of alpine mirrors
    :returns: list of absolute APKINDEX.tar.gz file paths
    """
    if not arch:
        arch = pmb.config.arch_native

    ret = []
    # Local user repository (for packages compiled with pmbootstrap)
    if user_repository:
        channel = pmb.config.pmaports.read_config(args)["channel"]
        ret = [pmb.config.work / "packages" / channel / arch / "APKINDEX.tar.gz"]

    # Resolve the APKINDEX.$HASH.tar.gz files
    for url in urls(args, False, pmos, alpine):
        ret.append(pmb.config.work / f"cache_apk_{arch}" / f"APKINDEX.{hash(url)}.tar.gz")

    return ret


def update(args, arch=None, force=False, existing_only=False):
    """Download the APKINDEX files for all URLs depending on the architectures.

    :param arch: * one Alpine architecture name ("x86_64", "armhf", ...)
                 * None for all architectures
    :param force: even update when the APKINDEX file is fairly recent
    :param existing_only: only update the APKINDEX files that already exist,
                          this is used by "pmbootstrap update"

    :returns: True when files have been downloaded, False otherwise
    """
    # Skip in offline mode, only show once
    cache_key = "pmb.helpers.repo.update"
    if args.offline:
        if not pmb.helpers.other.cache[cache_key]["offline_msg_shown"]:
            logging.info("NOTE: skipping package index update (offline mode)")
            pmb.helpers.other.cache[cache_key]["offline_msg_shown"] = True
        return False

    # Architectures and retention time
    architectures = [arch] if arch else pmb.config.build_device_architectures
    retention_hours = pmb.config.apkindex_retention_time
    retention_seconds = retention_hours * 3600

    # Find outdated APKINDEX files. Formats:
    # outdated: {URL: apkindex_path, ... }
    # outdated_arches: ["armhf", "x86_64", ... ]
    outdated = {}
    outdated_arches = []
    for url in urls(args, False):
        for arch in architectures:
            # APKINDEX file name from the URL
            url_full = url + "/" + arch + "/APKINDEX.tar.gz"
            cache_apk_outside = pmb.config.work / f"cache_apk_{arch}"
            apkindex = cache_apk_outside / f"APKINDEX.{hash(url)}.tar.gz"

            # Find update reason, possibly skip non-existing or known 404 files
            reason = None
            if url_full in pmb.helpers.other.cache[cache_key]["404"]:
                # We already attempted to download this file once in this
                # session
                continue
            elif not os.path.exists(apkindex):
                if existing_only:
                    continue
                reason = "file does not exist yet"
            elif force:
                reason = "forced update"
            elif pmb.helpers.file.is_older_than(apkindex, retention_seconds):
                reason = "older than " + str(retention_hours) + "h"
            if not reason:
                continue

            # Update outdated and outdated_arches
            logging.debug("APKINDEX outdated (" + reason + "): " + url_full)
            outdated[url_full] = apkindex
            if arch not in outdated_arches:
                outdated_arches.append(arch)

    # Bail out or show log message
    if not len(outdated):
        return False
    logging.info("Update package index for " + ", ".join(outdated_arches) +
                 " (" + str(len(outdated)) + " file(s))")

    # Download and move to right location
    for (i, (url, target)) in enumerate(outdated.items()):
        pmb.helpers.cli.progress_print(args, i / len(outdated))
        temp = pmb.helpers.http.download(args, url, "APKINDEX", False,
                                         logging.DEBUG, True)
        if not temp:
            pmb.helpers.other.cache[cache_key]["404"].append(url)
            continue
        target_folder = os.path.dirname(target)
        if not os.path.exists(target_folder):
            pmb.helpers.run.root(args, ["mkdir", "-p", target_folder])
        pmb.helpers.run.root(args, ["cp", temp, target])
    pmb.helpers.cli.progress_flush(args)

    return True


def alpine_apkindex_path(args, repo="main", arch=None):
    """Get the path to a specific Alpine APKINDEX file on disk and download it if necessary.

    :param repo: Alpine repository name (e.g. "main")
    :param arch: Alpine architecture (e.g. "armhf"), defaults to native arch.
    :returns: full path to the APKINDEX file
    """
    # Repo sanity check
    if repo not in ["main", "community", "testing", "non-free"]:
        raise RuntimeError("Invalid Alpine repository: " + repo)

    # Download the file
    arch = arch or pmb.config.arch_native
    update(args, arch)

    # Find it on disk
    channel_cfg = pmb.config.pmaports.read_config_channel(args)
    repo_link = f"{args.mirror_alpine}{channel_cfg['mirrordir_alpine']}/{repo}"
    cache_folder = pmb.config.work / "cache_apk_" + arch
    return cache_folder + "/APKINDEX." + hash(repo_link) + ".tar.gz"