forked from Mirror/pmbootstrap
31cc898dd5
With the new chroot type, we can now write fancy paths in the pythonic way. Convert most of the codebase over, as well as adding various other type hints. Signed-off-by: Caleb Connolly <caleb@postmarketos.org>
289 lines
9.6 KiB
Python
289 lines
9.6 KiB
Python
# Copyright 2023 Luca Weiss
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
import datetime
|
|
import fnmatch
|
|
from pmb.helpers import logging
|
|
import os
|
|
import re
|
|
import urllib.parse
|
|
from typing import Dict, Optional
|
|
|
|
from pmb.core.types import PmbArgs
|
|
import pmb.helpers.file
|
|
import pmb.helpers.http
|
|
import pmb.helpers.pmaports
|
|
|
|
req_headers: Dict[str, str]
|
|
req_headers_github: Dict[str, str]
|
|
|
|
ANITYA_API_BASE = "https://release-monitoring.org/api/v2"
|
|
GITHUB_API_BASE = "https://api.github.com"
|
|
GITLAB_HOSTS = [
|
|
"https://gitlab.com",
|
|
"https://gitlab.freedesktop.org",
|
|
"https://gitlab.gnome.org",
|
|
"https://invent.kde.org",
|
|
"https://source.puri.sm",
|
|
]
|
|
|
|
|
|
def init_req_headers() -> None:
|
|
global req_headers
|
|
global req_headers_github
|
|
# Only initialize them once
|
|
if req_headers is not None and req_headers_github is not None:
|
|
return
|
|
# Generic request headers
|
|
req_headers = {
|
|
'User-Agent': f'pmbootstrap/{pmb.__version__} aportupgrade'}
|
|
|
|
# Request headers specific to GitHub
|
|
req_headers_github = dict(req_headers)
|
|
if os.getenv("GITHUB_TOKEN") is not None:
|
|
token = os.getenv("GITHUB_TOKEN")
|
|
req_headers_github['Authorization'] = f'token {token}'
|
|
else:
|
|
logging.info("NOTE: Consider using a GITHUB_TOKEN environment variable"
|
|
" to increase your rate limit")
|
|
|
|
|
|
def get_package_version_info_github(repo_name: str, ref: Optional[str]):
|
|
logging.debug("Trying GitHub repository: {}".format(repo_name))
|
|
|
|
# Get the URL argument to request a special ref, if needed
|
|
ref_arg = ""
|
|
if ref is not None:
|
|
ref_arg = f"?sha={ref}"
|
|
|
|
# Get the commits for the repository
|
|
commits = pmb.helpers.http.retrieve_json(
|
|
f"{GITHUB_API_BASE}/repos/{repo_name}/commits{ref_arg}",
|
|
headers=req_headers_github)
|
|
latest_commit = commits[0]
|
|
commit_date = latest_commit["commit"]["committer"]["date"]
|
|
# Extract the time from the field
|
|
date = datetime.datetime.strptime(commit_date, "%Y-%m-%dT%H:%M:%SZ")
|
|
return {
|
|
"sha": latest_commit["sha"],
|
|
"date": date,
|
|
}
|
|
|
|
|
|
def get_package_version_info_gitlab(gitlab_host: str, repo_name: str,
|
|
ref: Optional[str]):
|
|
logging.debug("Trying GitLab repository: {}".format(repo_name))
|
|
|
|
repo_name_safe = urllib.parse.quote(repo_name, safe='')
|
|
|
|
# Get the URL argument to request a special ref, if needed
|
|
ref_arg = ""
|
|
if ref is not None:
|
|
ref_arg = f"?ref_name={ref}"
|
|
|
|
# Get the commits for the repository
|
|
commits = pmb.helpers.http.retrieve_json(
|
|
f"{gitlab_host}/api/v4/projects/{repo_name_safe}/repository"
|
|
f"/commits{ref_arg}",
|
|
headers=req_headers)
|
|
latest_commit = commits[0]
|
|
commit_date = latest_commit["committed_date"]
|
|
# Extract the time from the field
|
|
# 2019-10-14T09:32:00.000Z / 2019-12-27T07:58:53.000-05:00
|
|
date = datetime.datetime.strptime(commit_date, "%Y-%m-%dT%H:%M:%S.000%z")
|
|
return {
|
|
"sha": latest_commit["id"],
|
|
"date": date,
|
|
}
|
|
|
|
|
|
def upgrade_git_package(args: PmbArgs, pkgname: str, package) -> None:
|
|
"""Update _commit/pkgver/pkgrel in a git-APKBUILD (or pretend to do it if args.dry is set).
|
|
|
|
:param pkgname: the package name
|
|
:param package: a dict containing package information
|
|
"""
|
|
# Get the wanted source line
|
|
source = package["source"][0]
|
|
source = re.split(r"::", source)
|
|
if 1 <= len(source) <= 2:
|
|
source = source[-1]
|
|
else:
|
|
raise RuntimeError("Unhandled number of source elements. Please open"
|
|
f" a bug report: {source}")
|
|
|
|
verinfo = None
|
|
|
|
github_match = re.match(
|
|
r"https://github\.com/(.+)/(?:archive|releases)", source)
|
|
gitlab_match = re.match(
|
|
fr"({'|'.join(GITLAB_HOSTS)})/(.+)/-/archive/", source)
|
|
if github_match:
|
|
verinfo = get_package_version_info_github(
|
|
github_match.group(1), args.ref)
|
|
elif gitlab_match:
|
|
verinfo = get_package_version_info_gitlab(
|
|
gitlab_match.group(1), gitlab_match.group(2), args.ref)
|
|
|
|
if verinfo is None:
|
|
# ignore for now
|
|
logging.warning("{}: source not handled: {}".format(pkgname, source))
|
|
return
|
|
|
|
# Get the new commit sha
|
|
sha = package["_commit"]
|
|
sha_new = verinfo["sha"]
|
|
|
|
# Format the new pkgver, keep the value before _git the same
|
|
if package["pkgver"] == "9999":
|
|
pkgver = package["_pkgver"]
|
|
else:
|
|
pkgver = package["pkgver"]
|
|
|
|
pkgver_match = re.match(r"([\d.]+)_git", pkgver)
|
|
if pkgver_match is None:
|
|
msg = "pkgver did not match the expected pattern!"
|
|
raise RuntimeError(msg)
|
|
|
|
date_pkgver = verinfo["date"].strftime("%Y%m%d")
|
|
pkgver_new = f"{pkgver_match.group(1)}_git{date_pkgver}"
|
|
|
|
# pkgrel will be zero
|
|
pkgrel = int(package["pkgrel"])
|
|
pkgrel_new = 0
|
|
|
|
if sha == sha_new:
|
|
logging.info("{}: up-to-date".format(pkgname))
|
|
return
|
|
|
|
logging.info("{}: upgrading pmaport".format(pkgname))
|
|
if args.dry:
|
|
logging.info(f" Would change _commit from {sha} to {sha_new}")
|
|
logging.info(f" Would change pkgver from {pkgver} to {pkgver_new}")
|
|
logging.info(f" Would change pkgrel from {pkgrel} to {pkgrel_new}")
|
|
return
|
|
|
|
if package["pkgver"] == "9999":
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "_pkgver", pkgver_new)
|
|
else:
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new)
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgrel", pkgrel_new)
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "_commit", sha_new, True)
|
|
return
|
|
|
|
|
|
def upgrade_stable_package(args: PmbArgs, pkgname: str, package) -> None:
|
|
"""
|
|
Update _commit/pkgver/pkgrel in an APKBUILD (or pretend to do it if
|
|
args.dry is set).
|
|
|
|
:param pkgname: the package name
|
|
:param package: a dict containing package information
|
|
"""
|
|
|
|
# Looking up if there's a custom mapping from postmarketOS package name
|
|
# to Anitya project name.
|
|
mappings = pmb.helpers.http.retrieve_json(
|
|
f"{ANITYA_API_BASE}/packages/?distribution=postmarketOS"
|
|
f"&name={pkgname}", headers=req_headers)
|
|
if mappings["total_items"] < 1:
|
|
projects = pmb.helpers.http.retrieve_json(
|
|
f"{ANITYA_API_BASE}/projects/?name={pkgname}", headers=req_headers)
|
|
if projects["total_items"] < 1:
|
|
logging.warning(f"{pkgname}: failed to get Anitya project")
|
|
return
|
|
else:
|
|
project_name = mappings["items"][0]["project"]
|
|
ecosystem = mappings["items"][0]["ecosystem"]
|
|
projects = pmb.helpers.http.retrieve_json(
|
|
f"{ANITYA_API_BASE}/projects/?name={project_name}&"
|
|
f"ecosystem={ecosystem}",
|
|
headers=req_headers)
|
|
|
|
if projects["total_items"] < 1:
|
|
logging.warning(f"{pkgname}: didn't find any projects, can't upgrade!")
|
|
return
|
|
if projects["total_items"] > 1:
|
|
logging.warning(f"{pkgname}: found more than one project, can't "
|
|
f"upgrade! Please create an explicit mapping of "
|
|
f"\"project\" to the package name.")
|
|
return
|
|
|
|
# Get the first, best-matching item
|
|
project = projects["items"][0]
|
|
|
|
# Check that we got a version number
|
|
if len(project["stable_versions"]) < 1:
|
|
logging.warning("{}: got no version number, ignoring".format(pkgname))
|
|
return
|
|
|
|
version = project["stable_versions"][0]
|
|
|
|
# Compare the pmaports version with the project version
|
|
if package["pkgver"] == version:
|
|
logging.info("{}: up-to-date".format(pkgname))
|
|
return
|
|
|
|
if package["pkgver"] == "9999":
|
|
pkgver = package["_pkgver"]
|
|
else:
|
|
pkgver = package["pkgver"]
|
|
|
|
pkgver_new = version
|
|
|
|
pkgrel = package["pkgrel"]
|
|
pkgrel_new = 0
|
|
|
|
if not pmb.parse.version.validate(pkgver_new):
|
|
logging.warning(f"{pkgname}: would upgrade to invalid pkgver:"
|
|
f" {pkgver_new}, ignoring")
|
|
return
|
|
|
|
logging.info("{}: upgrading pmaport".format(pkgname))
|
|
if args.dry:
|
|
logging.info(f" Would change pkgver from {pkgver} to {pkgver_new}")
|
|
logging.info(f" Would change pkgrel from {pkgrel} to {pkgrel_new}")
|
|
return
|
|
|
|
if package["pkgver"] == "9999":
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "_pkgver", pkgver_new)
|
|
else:
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new)
|
|
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgrel", pkgrel_new)
|
|
return
|
|
|
|
|
|
def upgrade(args: PmbArgs, pkgname, git=True, stable=True) -> None:
|
|
"""Find new versions of a single package and upgrade it.
|
|
|
|
:param pkgname: the name of the package
|
|
:param git: True if git packages should be upgraded
|
|
:param stable: True if stable packages should be upgraded
|
|
"""
|
|
# Initialize request headers
|
|
init_req_headers()
|
|
|
|
package = pmb.helpers.pmaports.get(args, pkgname)
|
|
# Run the correct function
|
|
if "_git" in package["pkgver"]:
|
|
if git:
|
|
upgrade_git_package(args, pkgname, package)
|
|
else:
|
|
if stable:
|
|
upgrade_stable_package(args, pkgname, package)
|
|
|
|
|
|
def upgrade_all(args: PmbArgs) -> None:
|
|
"""Upgrade all packages, based on args.all, args.all_git and args.all_stable."""
|
|
for pkgname in pmb.helpers.pmaports.get_list(args):
|
|
# Always ignore postmarketOS-specific packages that have no upstream
|
|
# source
|
|
skip = False
|
|
for pattern in pmb.config.upgrade_ignore:
|
|
if fnmatch.fnmatch(pkgname, pattern):
|
|
skip = True
|
|
if skip:
|
|
continue
|
|
|
|
upgrade(args, pkgname, args.all or args.all_git,
|
|
args.all or args.all_stable)
|