diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index a1fab083..591acfaa 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -51,7 +51,8 @@ def update_repository_list(suffix: Chroot, postmarketos_mirror=True, pmb.helpers.run.root(["mkdir", "-p", path.parent]) # Up to date: Save cache, return - lines_new = pmb.helpers.repo.urls(postmarketos_mirror=postmarketos_mirror) + exclude = ["pmaports"] if not postmarketos_mirror else [] + lines_new = pmb.helpers.repo.urls(mirrors_exclude=exclude) if lines_old == lines_new: pmb.helpers.other.cache["apk_repository_list_updated"].append(suffix) return diff --git a/pmb/config/load.py b/pmb/config/load.py index 022791d3..637d8a5c 100644 --- a/pmb/config/load.py +++ b/pmb/config/load.py @@ -30,9 +30,15 @@ def load(path: Path) -> Config: # default values won't be set in the config file if key not in cfg["pmbootstrap"]: continue + elif key == "mirror_alpine": + # DEPRCATED: We have special handling for this below. + continue # Handle whacky type conversions elif key == "mirrors_postmarketos": - config.mirrors_postmarketos = cfg["pmbootstrap"]["mirrors_postmarketos"].split(",") + mirrors = cfg["pmbootstrap"]["mirrors_postmarketos"].split(",") + if len(mirrors) > 1: + logging.warning("Multiple mirrors are not supported, using the last one") + config.mirrors_postmarketos = [mirrors[-1].strip("/master")] # Convert strings to paths elif type(getattr(Config, key)) == PosixPath: setattr(config, key, Path(cfg["pmbootstrap"][key])) @@ -45,6 +51,10 @@ def load(path: Path) -> Config: elif key in cfg["pmbootstrap"]: setattr(config, key, cfg["pmbootstrap"][key]) + # One time migration "mirror_alpine" -> mirrors.alpine + if "mirror_alpine" in cfg["pmbootstrap"]: + config.mirrors["alpine"] = cfg["pmbootstrap"]["mirror_alpine"] + save(path, config) return config @@ -69,6 +79,9 @@ def serialize(config: Config, skip_defaults=True) -> configparser.ConfigParser: # we wouldn't be able to tell if the user overwrote it. if skip_defaults and Config.get_default(key) == getattr(config, key): continue + if key == "mirror_alpine" or key == "mirrors_postmarketos": + # DEPRECATED: skip these + continue if key == "providers": cfg["providers"] = config.providers elif key.startswith("mirrors."): diff --git a/pmb/core/config.py b/pmb/core/config.py index 97cfa735..4c3aeb7d 100644 --- a/pmb/core/config.py +++ b/pmb/core/config.py @@ -47,11 +47,6 @@ class Config(): "pmaports": "http://mirror.postmarketos.org/postmarketos/", "systemd": "http://mirror.postmarketos.org/postmarketos/staging/systemd/" } - # NOTE: mirrors use http by default to leverage caching - mirror_alpine: str = "http://dl-cdn.alpinelinux.org/alpine/" - # NOTE: mirrors_postmarketos variable type is supposed to be - # comma-separated string, not a python list or any other type! - mirrors_postmarketos: List[str] = ["http://mirror.postmarketos.org/postmarketos/"] qemu_redir_stdio: bool = False ssh_key_glob: str = "~/.ssh/id_*.pub" ssh_keys: bool = False diff --git a/pmb/helpers/pkgrel_bump.py b/pmb/helpers/pkgrel_bump.py index 519a3a8f..0b475395 100644 --- a/pmb/helpers/pkgrel_bump.py +++ b/pmb/helpers/pkgrel_bump.py @@ -106,7 +106,7 @@ def auto(args: PmbArgs, dry=False): """:returns: list of aport names, where the pkgrel needed to be changed""" ret = [] for arch in Arch.supported(): - paths = pmb.helpers.repo.apkindex_files(args, arch, alpine=False) + paths = pmb.helpers.repo.apkindex_files(arch, exclude_mirrors=["alpine"]) for path in paths: logging.info(f"scan {path}") index = pmb.parse.apkindex.parse(path, False) diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index ccc310e3..b6bcc25b 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -11,10 +11,10 @@ import os import hashlib from pmb.core import get_context from pmb.core.arch import Arch -from pmb.core.pkgrepo import pkgrepo_paths +from pmb.core.pkgrepo import pkgrepo_names, pkgrepo_paths from pmb.helpers import logging from pathlib import Path -from typing import List, Optional +from typing import List, Optional, Set import pmb.config.pmaports from pmb.types import PmbArgs @@ -50,17 +50,18 @@ def apkindex_hash(url: str, length: int=8) -> Path: return Path(f"APKINDEX.{ret}.tar.gz") -def urls(user_repository=True, postmarketos_mirror=True, alpine=True): +# FIXME: this function gets called way too much, needs to be +# cached +def urls(user_repository=True, mirrors_exclude: List[str] = []): """Get a list of repository URLs, as they are in /etc/apk/repositories. :param user_repository: add /mnt/pmbootstrap/packages - :param postmarketos_mirror: add postmarketos mirror URLs - :param alpine: add alpine mirror URLs + :param mirrors_exclude: mirrors to exclude (see pmb.core.config.Mirrors) :returns: list of mirror strings, like ["/mnt/pmbootstrap/packages", "http://...", ...] """ ret: List[str] = [] - context = get_context() + config = get_context().config # Get mirrordirs from channels.cfg (postmarketOS mirrordir is the same as # the pmaports branch of the channel, no need to make it more complicated) @@ -70,42 +71,43 @@ def urls(user_repository=True, postmarketos_mirror=True, alpine=True): # Local user repository (for packages compiled with pmbootstrap) if user_repository: - # FIXME: We shouldn't hardcod this here for channel in pmb.config.pmaports.all_channels(): ret.append(f"/mnt/pmbootstrap/packages/{channel}") - # Upstream postmarketOS binary repository - if postmarketos_mirror: - for mirror in context.config.mirrors_postmarketos: - # Remove "master" mirrordir to avoid breakage until bpo is adjusted - # (build.postmarketos.org#63) and to give potential other users of - # this flag a heads up. - if mirror.endswith("/master"): - logging.warning("WARNING: 'master' at the end of" - " --mirror-pmOS is deprecated, the branch gets" - " added automatically now!") - mirror = mirror[:-1 * len("master")] - ret.append(f"{mirror}{mirrordir_pmos}") + # Don't add the systemd mirror if systemd is disabled + if not pmb.config.is_systemd_selected(config): + mirrors_exclude.append("systemd") - # Upstream Alpine Linux repositories - if alpine: - directories = ["main", "community"] - if mirrordir_alpine == "edge": - directories.append("testing") - for dir in directories: - ret.append(f"{context.config.mirror_alpine}{mirrordir_alpine}/{dir}") + # ["pmaports", "systemd", "alpine", "plasma-nightly"] + # print(f"Mirrors: repos: {pkgrepo_names()} exclude: {mirrors_exclude}") + for repo in pkgrepo_names() + ["alpine"]: + if repo in mirrors_exclude: + continue + mirror = config.mirrors[repo] # mypy: disable-error-code="literal-required" + mirrordirs = [] + if repo == "alpine": + # FIXME: This is a bit of a mess + mirrordirs = [f"{mirrordir_alpine}/main", f"{mirrordir_alpine}/community"] + if mirrordir_alpine == "edge": + mirrordirs.append(f"{mirrordir_alpine}/testing") + else: + mirrordirs = [mirrordir_pmos] + + for mirrordir in mirrordirs: + url = os.path.join(mirror, mirrordir) + if url not in ret: + ret.append(url) return ret -def apkindex_files(arch: Optional[Arch]=None, user_repository=True, pmos=True, - alpine=True) -> List[Path]: +def apkindex_files(arch: Optional[Arch]=None, user_repository=True, + exclude_mirrors: List[str] = []) -> List[Path]: """Get a list of outside paths to all resolved APKINDEX.tar.gz files for a specific arch. :param arch: defaults to native :param user_repository: add path to index of locally built packages - :param pmos: add paths to indexes of postmarketos mirrors - :param alpine: add paths to indexes of alpine mirrors + :param exclude_mirrors: list of mirrors to exclude (e.g. ["alpine", "pmaports"]) :returns: list of absolute APKINDEX.tar.gz file paths """ if not arch: @@ -118,7 +120,7 @@ def apkindex_files(arch: Optional[Arch]=None, user_repository=True, pmos=True, ret.append(get_context().config.work / "packages" / channel / arch / "APKINDEX.tar.gz") # Resolve the APKINDEX.$HASH.tar.gz files - for url in urls(False, pmos, alpine): + for url in urls(False, exclude_mirrors): ret.append(get_context().config.work / f"cache_apk_{arch}" / apkindex_hash(url)) return ret @@ -223,6 +225,6 @@ def alpine_apkindex_path(repo="main", arch: Optional[Arch]=None): # Find it on disk channel_cfg = pmb.config.pmaports.read_config_channel() - repo_link = f"{get_context().config.mirror_alpine}{channel_cfg['mirrordir_alpine']}/{repo}" + repo_link = f"{get_context().config.mirrors["alpine"]}{channel_cfg['mirrordir_alpine']}/{repo}" cache_folder = get_context().config.work / (f"cache_apk_{arch}") return cache_folder / apkindex_hash(repo_link)