# Copyright 2023 Luca Weiss # SPDX-License-Identifier: GPL-3.0-or-later import datetime import fnmatch from pmb.helpers import logging import os import re import urllib.parse from typing import Optional from pmb.types import PmbArgs import pmb.helpers.file import pmb.helpers.http import pmb.helpers.pmaports req_headers: dict[str, str] = {} req_headers_github: dict[str, str] = {} ANITYA_API_BASE = "https://release-monitoring.org/api/v2" GITHUB_API_BASE = "https://api.github.com" GITLAB_HOSTS = [ "https://gitlab.com", "https://gitlab.freedesktop.org", "https://gitlab.gnome.org", "https://invent.kde.org", "https://source.puri.sm", ] def init_req_headers() -> None: global req_headers global req_headers_github # Only initialize them once if req_headers and req_headers_github: return # Generic request headers req_headers = {"User-Agent": f"pmbootstrap/{pmb.__version__} aportupgrade"} # Request headers specific to GitHub req_headers_github = dict(req_headers) if os.getenv("GITHUB_TOKEN") is not None: token = os.getenv("GITHUB_TOKEN") req_headers_github["Authorization"] = f"token {token}" else: logging.info( "NOTE: Consider using a GITHUB_TOKEN environment variable" " to increase your rate limit" ) def get_package_version_info_github(repo_name: str, ref: Optional[str]): logging.debug(f"Trying GitHub repository: {repo_name}") # Get the URL argument to request a special ref, if needed ref_arg = "" if ref is not None: ref_arg = f"?sha={ref}" # Get the commits for the repository commits = pmb.helpers.http.retrieve_json( f"{GITHUB_API_BASE}/repos/{repo_name}/commits{ref_arg}", headers=req_headers_github ) latest_commit = commits[0] commit_date = latest_commit["commit"]["committer"]["date"] # Extract the time from the field date = datetime.datetime.strptime(commit_date, "%Y-%m-%dT%H:%M:%SZ") return { "sha": latest_commit["sha"], "date": date, } def get_package_version_info_gitlab(gitlab_host: str, repo_name: str, ref: Optional[str]): logging.debug(f"Trying GitLab repository: {repo_name}") repo_name_safe = urllib.parse.quote(repo_name, safe="") # Get the URL argument to request a special ref, if needed ref_arg = "" if ref is not None: ref_arg = f"?ref_name={ref}" # Get the commits for the repository commits = pmb.helpers.http.retrieve_json( f"{gitlab_host}/api/v4/projects/{repo_name_safe}/repository" f"/commits{ref_arg}", headers=req_headers, ) latest_commit = commits[0] commit_date = latest_commit["committed_date"] # Extract the time from the field # 2019-10-14T09:32:00.000Z / 2019-12-27T07:58:53.000-05:00 date = datetime.datetime.strptime(commit_date, "%Y-%m-%dT%H:%M:%S.000%z") return { "sha": latest_commit["id"], "date": date, } def upgrade_git_package(args: PmbArgs, pkgname: str, package) -> None: """Update _commit/pkgver/pkgrel in a git-APKBUILD (or pretend to do it if args.dry is set). :param pkgname: the package name :param package: a dict containing package information """ # Get the wanted source line source = package["source"][0] source = re.split(r"::", source) if 1 <= len(source) <= 2: source = source[-1] else: raise RuntimeError( "Unhandled number of source elements. Please open" f" a bug report: {source}" ) verinfo = None github_match = re.match(r"https://github\.com/(.+)/(?:archive|releases)", source) gitlab_match = re.match(rf"({'|'.join(GITLAB_HOSTS)})/(.+)/-/archive/", source) if github_match: verinfo = get_package_version_info_github(github_match.group(1), args.ref) elif gitlab_match: verinfo = get_package_version_info_gitlab( gitlab_match.group(1), gitlab_match.group(2), args.ref ) if verinfo is None: # ignore for now logging.warning(f"{pkgname}: source not handled: {source}") return # Get the new commit sha sha = package["_commit"] sha_new = verinfo["sha"] # Format the new pkgver, keep the value before _git the same if package["pkgver"] == "9999": pkgver = package["_pkgver"] else: pkgver = package["pkgver"] pkgver_match = re.match(r"([\d.]+)_git", pkgver) if pkgver_match is None: msg = "pkgver did not match the expected pattern!" raise RuntimeError(msg) date_pkgver = verinfo["date"].strftime("%Y%m%d") pkgver_new = f"{pkgver_match.group(1)}_git{date_pkgver}" # pkgrel will be zero pkgrel = int(package["pkgrel"]) pkgrel_new = 0 if sha == sha_new: logging.info(f"{pkgname}: up-to-date") return logging.info(f"{pkgname}: upgrading pmaport") if args.dry: logging.info(f" Would change _commit from {sha} to {sha_new}") logging.info(f" Would change pkgver from {pkgver} to {pkgver_new}") logging.info(f" Would change pkgrel from {pkgrel} to {pkgrel_new}") return if package["pkgver"] == "9999": pmb.helpers.file.replace_apkbuild(args, pkgname, "_pkgver", pkgver_new) else: pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new) pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgrel", pkgrel_new) pmb.helpers.file.replace_apkbuild(args, pkgname, "_commit", sha_new, True) return def upgrade_stable_package(args: PmbArgs, pkgname: str, package) -> None: """ Update _commit/pkgver/pkgrel in an APKBUILD (or pretend to do it if args.dry is set). :param pkgname: the package name :param package: a dict containing package information """ # Looking up if there's a custom mapping from postmarketOS package name # to Anitya project name. mappings = pmb.helpers.http.retrieve_json( f"{ANITYA_API_BASE}/packages/?distribution=postmarketOS" f"&name={pkgname}", headers=req_headers, ) if mappings["total_items"] < 1: projects = pmb.helpers.http.retrieve_json( f"{ANITYA_API_BASE}/projects/?name={pkgname}", headers=req_headers ) if projects["total_items"] < 1: logging.warning(f"{pkgname}: failed to get Anitya project") return else: project_name = mappings["items"][0]["project"] ecosystem = mappings["items"][0]["ecosystem"] projects = pmb.helpers.http.retrieve_json( f"{ANITYA_API_BASE}/projects/?name={project_name}&" f"ecosystem={ecosystem}", headers=req_headers, ) if projects["total_items"] < 1: logging.warning(f"{pkgname}: didn't find any projects, can't upgrade!") return if projects["total_items"] > 1: logging.warning( f"{pkgname}: found more than one project, can't " f"upgrade! Please create an explicit mapping of " f'"project" to the package name.' ) return # Get the first, best-matching item project = projects["items"][0] # Check that we got a version number if len(project["stable_versions"]) < 1: logging.warning(f"{pkgname}: got no version number, ignoring") return version = project["stable_versions"][0] # Compare the pmaports version with the project version if package["pkgver"] == version: logging.info(f"{pkgname}: up-to-date") return if package["pkgver"] == "9999": pkgver = package["_pkgver"] else: pkgver = package["pkgver"] pkgver_new = version pkgrel = package["pkgrel"] pkgrel_new = 0 if not pmb.parse.version.validate(pkgver_new): logging.warning(f"{pkgname}: would upgrade to invalid pkgver:" f" {pkgver_new}, ignoring") return logging.info(f"{pkgname}: upgrading pmaport") if args.dry: logging.info(f" Would change pkgver from {pkgver} to {pkgver_new}") logging.info(f" Would change pkgrel from {pkgrel} to {pkgrel_new}") return if package["pkgver"] == "9999": pmb.helpers.file.replace_apkbuild(args, pkgname, "_pkgver", pkgver_new) else: pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new) pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgrel", pkgrel_new) return def upgrade(args: PmbArgs, pkgname, git=True, stable=True) -> None: """Find new versions of a single package and upgrade it. :param pkgname: the name of the package :param git: True if git packages should be upgraded :param stable: True if stable packages should be upgraded """ # Initialize request headers init_req_headers() package = pmb.helpers.pmaports.get(pkgname) # Run the correct function if "_git" in package["pkgver"]: if git: upgrade_git_package(args, pkgname, package) else: if stable: upgrade_stable_package(args, pkgname, package) def upgrade_all(args: PmbArgs) -> None: """Upgrade all packages, based on args.all, args.all_git and args.all_stable.""" for pkgname in pmb.helpers.pmaports.get_list(): # Always ignore postmarketOS-specific packages that have no upstream # source skip = False for pattern in pmb.config.upgrade_ignore: if fnmatch.fnmatch(pkgname, pattern): skip = True if skip: continue upgrade(args, pkgname, args.all or args.all_git, args.all or args.all_stable)