config: clean up parsing and add mirrors (MR 2252)

Add a new config section "mirrors", to replace the mirrors_alpine and
mirrors_postmarketos options. This will allow for more flexibility since
we can then handle the systemd staging repo (and others like plasma
nightly) with relative ease.

The loading/saving is fixed and now properly avoids writing out default
values, this way if the defaults are changed the user won't be stuck
with old values in their pmbootstrap.cfg.

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>

pmb/config/__init__.py

@@ -11,7 +11,7 @@ from typing import Dict, List, Sequence
 #
 # FIXME (#2324): this sucks, we should re-organise this and not rely on "lifting"
 # this functions this way
-from pmb.config.load import load, sanity_checks, save
+from pmb.config.load import load, sanity_checks, save, serialize
 from pmb.config.sudo import which_sudo
 from pmb.config.other import is_systemd_selected
 

pmb/config/load.py

@@ -1,7 +1,7 @@
 # Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pathlib import Path, PosixPath
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Mapping, Optional
 from pmb.helpers import logging
 import configparser
 import os
@@ -45,6 +45,13 @@ def load(path: Path) -> Config:
     for key in Config.__dict__.keys():
         if key == "providers":
             setattr(config, key, cfg["providers"])
+        if key == "mirrors" and key in cfg:
+            for subkey in Config.mirrors.keys():
+                if subkey in cfg["mirrors"]:
+                    setattr(config, f"mirrors.{subkey}", cfg["mirrors"][subkey])
+        # default values won't be set in the config file
+        if key not in cfg["pmbootstrap"]:
+            continue
         # Handle whacky type conversions
         elif key == "mirrors_postmarketos":
             config.mirrors_postmarketos = cfg["pmbootstrap"]["mirrors_postmarketos"].split(",")
@@ -64,18 +71,32 @@ def load(path: Path) -> Config:
 
     return config
 
-def save(output: Path, config: Config):
+
-    logging.debug(f"Save config: {output}")
+def serialize(config: Config, skip_defaults=True) -> configparser.ConfigParser:
-    output.parent.mkdir(parents=True, exist_ok=True)
+    """Serialize the config object into a ConfigParser to write it out
-    output.touch(0o700, exist_ok=True)
+    in the pmbootstrap.cfg INI format.
     
+    :param config: The config object to serialize
+    :param skip_defaults: Skip writing out default values
+    """
     cfg = configparser.ConfigParser()
     cfg["pmbootstrap"] = {}
     cfg["providers"] = {}
+    cfg["mirrors"] = {}
 
-    for key in Config.__annotations__.keys():
+    # .keys() flat maps dictionaries like config.mirrors with
+    # dotted notation
+    for key in Config.keys():
+        # If the default value hasn't changed then don't write out,
+        # this makes it possible to update the default, otherwise
+        # we wouldn't be able to tell if the user overwrote it.
+        if skip_defaults and Config.get_default(key) == getattr(config, key):
+            continue
         if key == "providers":
             cfg["providers"] = config.providers
+        elif key.startswith("mirrors."):
+            _key = key.split(".")[1]
+            cfg["mirrors"][_key] = getattr(config, key)
         # Handle whacky type conversions
         elif key == "mirrors_postmarketos":
             cfg["pmbootstrap"]["mirrors_postmarketos"] = ",".join(config.mirrors_postmarketos)
@@ -87,7 +108,16 @@ def save(output: Path, config: Config):
         elif isinstance(getattr(Config, key), bool):
             cfg["pmbootstrap"][key] = str(getattr(config, key))
         else:
-            cfg["pmbootstrap"][key] = getattr(config, key)
+            cfg["pmbootstrap"][key] = str(getattr(config, key))
+
+    return cfg
+
+def save(output: Path, config: Config):
+    logging.debug(f"Save config: {output}")
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.touch(0o700, exist_ok=True)
+    
+    cfg = serialize(config)
 
     with output.open("w") as handle:
         cfg.write(handle)

pmb/core/config.py

@@ -1,9 +1,16 @@
 
+from copy import deepcopy
 import multiprocessing
-from typing import List, Dict
+from typing import Any, List, Dict, TypedDict
 from pathlib import Path
 import os
 
+class Mirrors(TypedDict):
+    alpine: str
+    pmaports: str
+    systemd: str
+
+
 class Config():
     aports: List[Path] = [Path(os.path.expanduser("~") +
                         "/.local/var/pmbootstrap/cache_git/pmaports")]
@@ -20,6 +27,11 @@ class Config():
     kernel: str = "stable"
     keymap: str = ""
     locale: str = "en_US.UTF-8"
+    mirrors: Mirrors = {
+        "alpine": "http://dl-cdn.alpinelinux.org/alpine/",
+        "pmaports": "http://mirror.postmarketos.org/postmarketos/",
+        "systemd": "http://mirror.postmarketos.org/postmarketos/staging/systemd/"
+    }
     # NOTE: mirrors use http by default to leverage caching
     mirror_alpine: str = "http://dl-cdn.alpinelinux.org/alpine/"
     # NOTE: mirrors_postmarketos variable type is supposed to be
@@ -37,3 +49,57 @@ class Config():
     work: Path = Path(os.path.expanduser("~") + "/.local/var/pmbootstrap")
 
     providers: Dict[str, str] = { }
+
+
+    def __init__(self):
+        # Make sure we aren't modifying the class defaults
+        for key in Config.__annotations__.keys():
+            setattr(self, key, deepcopy(Config.get_default(key)))
+    
+
+    @staticmethod
+    def keys() -> List[str]:
+        keys = list(Config.__annotations__.keys())
+        keys.remove("mirrors")
+        keys += [f"mirrors.{k}" for k in Mirrors.__annotations__.keys()]
+        return sorted(keys)
+
+
+    @staticmethod
+    def get_default(dotted_key: str) -> Any:
+        """Get the default value for a config option, supporting
+        nested dictionaries (e.g. "mirrors.alpine")."""
+        keys = dotted_key.split(".")
+        if len(keys) == 1:
+            return getattr(Config, keys[0])
+        elif len(keys) == 2:
+            return getattr(Config, keys[0])[keys[1]]
+        else:
+            raise ValueError(f"Invalid dotted key: {dotted_key}")
+
+
+    def __setattr__(self, key: str, value: str):
+        """Allow for setattr() to be used with a dotted key
+        to set nested dictionaries (e.g. "mirrors.alpine")."""
+        keys = key.split(".")
+        if len(keys) == 1:
+            super(Config, self).__setattr__(key, value)
+        elif len(keys) == 2:
+            #print(f"cfgset, before: {super(Config, self).__getattribute__(keys[0])[keys[1]]}")
+            super(Config, self).__getattribute__(keys[0])[keys[1]] = value
+            #print(f"cfgset, after: {super(Config, self).__getattribute__(keys[0])[keys[1]]}")
+        else:
+            raise ValueError(f"Invalid dotted key: {key}")
+
+
+    def __getattribute__(self, key: str) -> str:
+        #print(repr(self))
+        """Allow for getattr() to be used with a dotted key
+        to get nested dictionaries (e.g. "mirrors.alpine")."""
+        keys = key.split(".")
+        if len(keys) == 1:
+            return super(Config, self).__getattribute__(key)
+        elif len(keys) == 2:
+            return super(Config, self).__getattribute__(keys[0])[keys[1]]
+        else:
+            raise ValueError(f"Invalid dotted key: {key}")

pmb/helpers/frontend.py

@@ -211,7 +211,7 @@ def chroot(args: PmbArgs):
 
 
 def config(args: PmbArgs):
-    keys = pmb.config.config_keys
+    keys = Config.keys()
     if args.name and args.name not in keys:
         logging.info("NOTE: Valid config keys: " + ", ".join(keys))
         raise RuntimeError("Invalid config key: " + args.name)
@@ -222,7 +222,7 @@ def config(args: PmbArgs):
     if args.reset:
         if args.name is None:
             raise RuntimeError("config --reset requires a name to be given.")
-        def_value = getattr(Config(), args.name)
+        def_value = Config.get_default(args.name)
         setattr(config, args.name, def_value)
         logging.info(f"Config changed to default: {args.name}='{def_value}'")
         pmb.config.save(args.config, config)
@@ -238,7 +238,11 @@ def config(args: PmbArgs):
             value = ""
         print(value)
     else:
-        print(open(args.config).read())
+        # Serialize the entire config including default values for
+        # the user. Even though the defaults aren't actually written
+        # to disk.
+        cfg = pmb.config.serialize(config, skip_defaults=False)
+        cfg.write(sys.stdout)
 
     # Don't write the "Done" message
     pmb.helpers.logging.disable()

pmb/parse/arguments.py

@@ -924,8 +924,8 @@ def get_parser():
                         help="Reset config options with the given name to it's"
                         " default.")
     config.add_argument("name", nargs="?", help="variable name, one of: " +
-                        ", ".join(sorted(pmb.config.config_keys)),
+                        ", ".join(sorted(Config.keys())),
-                        choices=pmb.config.config_keys, metavar="name")
+                        choices=Config.keys(), metavar="name")
     config.add_argument("value", nargs="?", help="set variable to value")
 
     # Action: bootimg_analyze