diff --git a/pmb/build/backend.py b/pmb/build/backend.py index ad426039..2795118a 100644 --- a/pmb/build/backend.py +++ b/pmb/build/backend.py @@ -34,7 +34,7 @@ def override_source( return # Mount source in chroot - mount_path = "/mnt/pmbootstrap/source-override/" + mount_path = "mnt/pmbootstrap/source-override/" mount_path_outside = chroot / mount_path pmb.helpers.mount.bind(src, mount_path_outside, umount=True) diff --git a/pmb/chroot/mount.py b/pmb/chroot/mount.py index b4c3d4a9..39fbfa62 100644 --- a/pmb/chroot/mount.py +++ b/pmb/chroot/mount.py @@ -20,9 +20,11 @@ def mount_dev_tmpfs(chroot: Chroot = Chroot.native()) -> None: it. """ # Do nothing when it is already mounted - dev = chroot / "dev" - if pmb.helpers.mount.ismount(dev): - return + # dev = chroot / "dev" + # if pmb.helpers.mount.ismount(dev): + # return + + logging.info(f"mount_dev_tmpfs({chroot})") # Use sandbox to set up /dev inside the chroot ttyname = os.ttyname(2) if os.isatty(2) else "" @@ -32,6 +34,7 @@ def mount_dev_tmpfs(chroot: Chroot = Chroot.native()) -> None: def mount(chroot: Chroot): # Mount tmpfs as the chroot's /dev + chroot.path.mkdir(exist_ok=True) mount_dev_tmpfs(chroot) # Get all mountpoints diff --git a/pmb/chroot/shutdown.py b/pmb/chroot/shutdown.py index 5971b9f7..ff1dbc1e 100644 --- a/pmb/chroot/shutdown.py +++ b/pmb/chroot/shutdown.py @@ -7,7 +7,6 @@ from contextlib import closing import pmb.chroot import pmb.helpers.mount -import pmb.install.losetup from pmb.core import Chroot, ChrootType from pmb.core.context import get_context @@ -70,9 +69,6 @@ def shutdown(only_install_related: bool = False) -> None: pmb.helpers.mount.umount_all(chroot / "mnt/install") shutdown_cryptsetup_device("pm_crypt") - # Umount all losetup mounted images - pmb.install.losetup.detach_all() - # Remove "in-pmbootstrap" marker from all chroots. This marker indicates # that pmbootstrap has set up all mount points etc. to run programs inside # the chroots, but we want it gone afterwards (e.g. when the chroot diff --git a/pmb/commands/flasher.py b/pmb/commands/flasher.py index e0ec07cc..76077305 100644 --- a/pmb/commands/flasher.py +++ b/pmb/commands/flasher.py @@ -5,8 +5,10 @@ import pmb.parse.deviceinfo from pmb import commands from pmb.core.context import get_context +from pmb.core.chroot import Chroot from pmb.flasher.frontend import flash_lk2nd, kernel, list_flavors, rootfs, sideload from pmb.helpers import logging +import pmb.chroot class Flasher(commands.Command): @@ -39,6 +41,9 @@ class Flasher(commands.Command): logging.info("This device doesn't support any flash method.") return + # Ensure the chroot is ready + pmb.chroot.init(Chroot.native()) + if action in ["boot", "flash_kernel"]: kernel( deviceinfo, diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index a3fc20bd..0e0dd6e5 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -26,6 +26,10 @@ apk_keys_path: Path = pmb_src / "pmb/data/keys" # In the mount namespace this is where we mount our own binfmt_misc dir binfmt_misc = "/tmp/pmb_binfmt_misc" +# This is the sector size we align to when creating partition tables, since +# it works on all disks of smaller sector sizes too +block_size = 4096 + # apk-tools minimum version # https://pkgs.alpinelinux.org/packages?name=apk-tools&branch=edge # Update this frequently to prevent a MITM attack with an outdated version diff --git a/pmb/core/context.py b/pmb/core/context.py index 07bd33ac..ce0b9ff7 100644 --- a/pmb/core/context.py +++ b/pmb/core/context.py @@ -33,6 +33,9 @@ class Context: ccache: bool = False go_mod_cache: bool = False + # Disk image sector size (not filesystem block size!) + sector_size: int | None = None + config: Config def __init__(self, config: Config): diff --git a/pmb/helpers/args.py b/pmb/helpers/args.py index 1233bed2..70fbb849 100644 --- a/pmb/helpers/args.py +++ b/pmb/helpers/args.py @@ -93,6 +93,18 @@ def init(args: PmbArgs) -> PmbArgs: pmb.helpers.logging.init(context.log, args.verbose, context.details_to_stdout) pmb.helpers.logging.debug(f"Pmbootstrap v{pmb.__version__} (Python {sys.version})") + # Now we go round-about to set context based on deviceinfo hahaha + if args.action != "config": + context.sector_size = context.sector_size or pmb.parse.deviceinfo().rootfs_image_sector_size + if context.sector_size is None: + context.sector_size = 512 + + valid_sector_size = [512, 2048, 4096] + if context.sector_size not in valid_sector_size: + raise ValueError( + f"Invalid sector size from cmdline or deviceinfo file! Must be one of {valid_sector_size} but got {context.sector_size}" + ) + # Initialization code which may raise errors if args.action not in [ "init", diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 901a4012..97a636c9 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -90,6 +90,9 @@ def build(args: PmbArgs) -> None: pmb.build.envkernel.package_kernel(args) return + # Ensure native chroot is initialized + pmb.chroot.init(Chroot.native()) + # Set src and force src = os.path.realpath(os.path.expanduser(args.src[0])) if args.src else None force = True if src else get_context().force diff --git a/pmb/helpers/mount.py b/pmb/helpers/mount.py index 15162cdd..45197860 100644 --- a/pmb/helpers/mount.py +++ b/pmb/helpers/mount.py @@ -67,12 +67,14 @@ def bind_file(source: Path, destination: Path, create_folders: bool = False) -> if create_folders: dest_dir: Path = destination.parent if not dest_dir.is_dir(): - pmb.helpers.run.root(["mkdir", "-p", dest_dir]) + os.makedirs(dest_dir, exist_ok=True) - pmb.helpers.run.root(["touch", destination]) + with sandbox.umask(~0o644): + os.close(os.open(destination, os.O_CREAT | os.O_CLOEXEC | os.O_EXCL)) # Mount - pmb.helpers.run.root(["mount", "--bind", source, destination]) + pmb.logging.info(f"% mount --bind {source} {destination}") + sandbox.mount_rbind(str(source), str(destination), 0) def umount_all_list(prefix: Path, source: Path = Path("/proc/mounts")) -> list[Path]: @@ -101,6 +103,7 @@ def umount_all(folder: Path) -> None: """Umount all folders that are mounted inside a given folder.""" for mountpoint in umount_all_list(folder): if mountpoint.name != "binfmt_misc": + pmb.logging.info(f"% umount {mountpoint}") sandbox.umount2(str(mountpoint), sandbox.MNT_DETACH) diff --git a/pmb/install/__init__.py b/pmb/install/__init__.py index 852c45f2..47835cad 100644 --- a/pmb/install/__init__.py +++ b/pmb/install/__init__.py @@ -5,5 +5,4 @@ from pmb.install._install import get_kernel_package as get_kernel_package from pmb.install.partition import partition as partition from pmb.install.partition import partition_cgpt as partition_cgpt from pmb.install.format import format as format -from pmb.install.format import get_root_filesystem as get_root_filesystem -from pmb.install.partition import partitions_mount as partitions_mount +from pmb.install._install import get_root_filesystem as get_root_filesystem diff --git a/pmb/install/_install.py b/pmb/install/_install.py index e77d4014..eb4c4f3d 100644 --- a/pmb/install/_install.py +++ b/pmb/install/_install.py @@ -32,6 +32,7 @@ import pmb.install.ui import pmb.install from pmb.core import Chroot, ChrootType from pmb.core.context import get_context +from pmb.types import DiskPartition # Keep track of the packages we already visited in get_recommends() to avoid # infinite recursion @@ -45,17 +46,17 @@ def get_subpartitions_size(chroot: Chroot) -> tuple[int, int]: :param suffix: the chroot suffix, e.g. "rootfs_qemu-amd64" :returns: (boot, root) the size of the boot and root - partition as integer in MiB + partition as integer in bytes """ config = get_context().config - boot = int(config.boot_size) + boot = int(config.boot_size) * 1024 * 1024 # Estimate root partition size, then add some free space. The size # calculation is not as trivial as one may think, and depending on the # file system etc it seems to be just impossible to get it right. - root = pmb.helpers.other.folder_size(chroot.path) / 1024 + root = pmb.helpers.other.folder_size(chroot.path) * 1024 root *= 1.20 - root += 50 + int(config.extra_space) + root += 50 + int(config.extra_space) * 1024 * 1024 return (boot, round(root)) @@ -111,73 +112,23 @@ def get_kernel_package(config: Config) -> list[str]: return ["device-" + config.device + "-kernel-" + config.kernel] -def copy_files_from_chroot(args: PmbArgs, chroot: Chroot) -> None: - """ - Copy all files from the rootfs chroot to /mnt/install, except - for the home folder (because /home will contain some empty - mountpoint folders). - - :param suffix: the chroot suffix, e.g. "rootfs_qemu-amd64" - """ - # Mount the device rootfs - logging.info(f"(native) copy {chroot} to /mnt/install/") - mountpoint = mount_device_rootfs(chroot) - mountpoint_outside = Chroot.native() / mountpoint - - # Remove empty qemu-user binary stub (where the binary was bind-mounted) - arch_qemu = pmb.parse.deviceinfo().arch.qemu() - qemu_binary = mountpoint_outside / f"usr/bin/qemu-{arch_qemu}-static" - if os.path.exists(qemu_binary): - pmb.helpers.run.root(["rm", qemu_binary]) - - # Remove apk progress fifo - fifo = chroot / "tmp/apk_progress_fifo" - if os.path.exists(fifo): - pmb.helpers.run.root(["rm", fifo]) - - # Get all folders inside the device rootfs (except for home) - folders: list[str] = [] - for path in mountpoint_outside.glob("*"): - if path.name == "home": - continue - folders.append(path.name) - - # Update or copy all files - if args.rsync: - pmb.chroot.apk.install(["rsync"], Chroot.native()) - rsync_flags = "-a" - if args.verbose: - rsync_flags += "vP" - pmb.chroot.root( - ["rsync", rsync_flags, "--delete", *folders, "/mnt/install/"], working_dir=mountpoint - ) - pmb.chroot.root(["rm", "-rf", "/mnt/install/home"]) - else: - pmb.chroot.root(["cp", "-a", *folders, "/mnt/install/"], working_dir=mountpoint) - - # Log how much space and inodes we have used - pmb.chroot.user(["df", "-h", "/mnt/install"]) - pmb.chroot.user(["df", "-i", "/mnt/install"]) - - -def create_home_from_skel(filesystem: str, user: str) -> None: +def create_home_from_skel(filesystem: str, user: str, rootfs: Path) -> None: """ Create /home/{user} from /etc/skel """ - rootfs = Chroot.native() / "mnt/install" # In btrfs, home subvol & home dir is created in format.py if filesystem != "btrfs": - pmb.helpers.run.root(["mkdir", rootfs / "home"]) + (rootfs / "home").mkdir(exist_ok=True) - home = rootfs / "home" / user + user_home = rootfs / "home" / user if (rootfs / "etc/skel").exists(): - pmb.helpers.run.root(["cp", "-a", (rootfs / "etc/skel"), home]) + pmb.helpers.run.root(["cp", "-a", (rootfs / "etc/skel"), user_home]) else: - pmb.helpers.run.root(["mkdir", home]) - pmb.helpers.run.root(["chown", "-R", "10000", home]) + user_home.mkdir(exist_ok=True) + pmb.helpers.run.root(["chown", "-R", "10000:10000", user_home]) -def configure_apk(args: PmbArgs) -> None: +def configure_apk(args: PmbArgs, rootfs: Path) -> None: """ Copy over all official keys, and the keys used to compile local packages (unless --no-local-pkgs is set). Then copy the corresponding APKINDEX files @@ -187,11 +138,9 @@ def configure_apk(args: PmbArgs) -> None: keys_dir = pmb.config.apk_keys_path # Official keys + local keys - if args.install_local_pkgs: - keys_dir = get_context().config.work / "config_apk_keys" + keys_dir = get_context().config.work / "config_apk_keys" # Copy over keys - rootfs = Chroot.native() / "mnt/install" for key in keys_dir.glob("*.pub"): pmb.helpers.run.root(["cp", key, rootfs / "etc/apk/keys/"]) @@ -202,11 +151,10 @@ def configure_apk(args: PmbArgs) -> None: for f in index_files: pmb.helpers.run.root(["cp", f, rootfs / "var/cache/apk/"]) - # Disable pmbootstrap repository - pmb.chroot.root( - ["sed", "-i", r"/\/mnt\/pmbootstrap\/packages/d", "/mnt/install/etc/apk/repositories"] + # Populate repositories + open(rootfs / "etc/apk/repositories", "w").write( + open(Chroot.native() / "etc/apk/repositories").read() ) - pmb.helpers.run.user(["cat", rootfs / "etc/apk/repositories"]) def set_user(config: Config) -> None: @@ -298,7 +246,7 @@ def setup_login(args: PmbArgs, config: Config, chroot: Chroot) -> None: pmb.chroot.root(["passwd", "-l", "root"], chroot) -def copy_ssh_keys(config: Config) -> None: +def copy_ssh_keys(config: Config, rootfs: Path) -> None: """ If requested, copy user's SSH public keys to the device if they exist """ @@ -329,19 +277,18 @@ def copy_ssh_keys(config: Config) -> None: outfile.write(f"{key}") outfile.close() - target = Chroot.native() / "mnt/install/home/" / config.user / ".ssh" - pmb.helpers.run.root(["mkdir", target]) + target = rootfs / "home/" / config.user / ".ssh" + target.mkdir(exist_ok=True) pmb.helpers.run.root(["chmod", "700", target]) pmb.helpers.run.root(["cp", authorized_keys, target / "authorized_keys"]) pmb.helpers.run.root(["rm", authorized_keys]) pmb.helpers.run.root(["chown", "-R", "10000:10000", target]) -def setup_keymap(config: Config) -> None: +def setup_keymap(config: Config, chroot: Chroot) -> None: """ Set the keymap with the setup-keymap utility if the device requires it """ - chroot = Chroot(ChrootType.ROOTFS, config.device) deviceinfo = pmb.parse.deviceinfo(device=config.device) if not deviceinfo.keymaps or deviceinfo.keymaps.strip() == "": logging.info("NOTE: No valid keymap specified for device") @@ -609,8 +556,8 @@ def generate_binary_list(args: PmbArgs, chroot: Chroot, step: int) -> list[tuple ) # Insure that embedding the firmware will not overrun the # first partition - boot_part_start = pmb.parse.deviceinfo().boot_part_start or "2048" - max_size = (int(boot_part_start) * 512) - (offset * step) + boot_part_start = pmb.parse.deviceinfo().boot_part_start + max_size = (boot_part_start * pmb.config.block_size) - (offset * step) binary_size = os.path.getsize(binary_path) if binary_size > max_size: raise RuntimeError( @@ -725,7 +672,7 @@ def sanity_check_disk_size(args: PmbArgs) -> None: with open(sysfs) as handle: raw = handle.read() - # Size is in 512-byte blocks + # Size is in 512-byte blocks some of the time... size = int(raw.strip()) human = f"{size / 2 / 1024 / 1024:.2f} GiB" @@ -741,70 +688,62 @@ def sanity_check_disk_size(args: PmbArgs) -> None: raise RuntimeError("Aborted.") -def get_partition_layout(kernel: bool) -> PartitionLayout: +def get_partition_layout(chroot: Chroot, kernel: bool, split: bool, single_partition: bool) -> PartitionLayout: """ :param kernel: create a separate kernel partition before all other partitions, e.g. for the ChromeOS devices with cgpt :returns: the partition layout, e.g. without reserve and kernel: {"kernel": None, "boot": 1, "reserve": None, "root": 2} """ - ret: PartitionLayout = { - "kernel": None, - "boot": 1, - "reserve": None, - "root": 2, - } - + layout: PartitionLayout = PartitionLayout("/dev/install", split) + if kernel: - ret["kernel"] = 1 - ret["boot"] += 1 - ret["root"] += 1 + layout.append(DiskPartition("kernel", pmb.parse.deviceinfo().cgpt_kpart_size)) - if reserve: - ret["reserve"] = ret["root"] - ret["root"] += 1 - return ret + (size_boot, size_root) = get_subpartitions_size(chroot) + + if single_partition: + if kernel: + # FIXME: check this way earlier! + raise RuntimeError("--single-partition is not supported on Chromebooks, sorry!") + layout.append(DiskPartition("root", size_root)) + return layout + + layout.append(DiskPartition("boot", size_boot)) + layout.append(DiskPartition("root", size_root)) + + if split: + layout.boot.path = "/dev/installp1" + layout.root.path = "/dev/installp2" + else: + # Both partitions are in the same disk image and we access + # them with offsets + layout.boot.path = "/dev/install" + layout.root.path = "/dev/install" + + return layout -def get_uuid(args: PmbArgs, partition: Path) -> str: - """ - Get UUID of a partition - - :param partition: block device for getting UUID from - """ - return pmb.chroot.root( - [ - "blkid", - "-s", - "UUID", - "-o", - "value", - partition, - ], - output_return=True, - ).rstrip() +def get_uuid(args: PmbArgs, disk: Path, partition: str) -> str: + pass -def create_crypttab(args: PmbArgs, layout: PartitionLayout | None, chroot: Chroot) -> None: +def create_crypttab(args: PmbArgs, layout: PartitionLayout, disk: Path, chroot: Chroot) -> None: """ Create /etc/crypttab config :param layout: partition layout from get_partition_layout() or None :param suffix: of the chroot, which crypttab will be created to """ - if layout: - root_dev = Path(f"/dev/installp{layout['root']}") - else: - root_dev = Path("/dev/install") - luks_uuid = get_uuid(args, root_dev) + luks_uuid = layout.root.uuid crypttab = f"root UUID={luks_uuid} none luks\n" (chroot / "tmp/crypttab").open("w").write(crypttab) pmb.chroot.root(["mv", "/tmp/crypttab", "/etc/crypttab"], chroot) -def create_fstab(args: PmbArgs, layout: PartitionLayout | None, chroot: Chroot) -> None: +def create_fstab(args: PmbArgs, layout: PartitionLayout, disk: Path, chroot: Chroot) -> None: """ Create /etc/fstab config @@ -812,15 +751,8 @@ def create_fstab(args: PmbArgs, layout: PartitionLayout | None, chroot: Chroot) :param chroot: of the chroot, which fstab will be created to """ - if layout: - boot_dev = Path(f"/dev/installp{layout['boot']}") - root_dev = Path(f"/dev/installp{layout['root']}") - else: - boot_dev = None - root_dev = Path("/dev/install") - root_mount_point = ( - "/dev/mapper/root" if args.full_disk_encryption else f"UUID={get_uuid(args, root_dev)}" + "/dev/mapper/root" if args.full_disk_encryption else f"UUID={layout.root.uuid}" ) root_filesystem = pmb.install.get_root_filesystem(args) @@ -842,8 +774,9 @@ def create_fstab(args: PmbArgs, layout: PartitionLayout | None, chroot: Chroot) {root_mount_point} / {root_filesystem} defaults 0 0 """.lstrip() - if boot_dev: - boot_mount_point = f"UUID={get_uuid(args, boot_dev)}" + # FIXME: need a better way to check if we have a boot partition... + if len(layout) > 1: + boot_mount_point = f"UUID={layout.boot.uuid}" boot_options = "nodev,nosuid,noexec" boot_filesystem = pmb.parse.deviceinfo().boot_filesystem or "ext2" if boot_filesystem in ("fat16", "fat32"): @@ -853,22 +786,38 @@ def create_fstab(args: PmbArgs, layout: PartitionLayout | None, chroot: Chroot) with (chroot / "tmp/fstab").open("w") as f: f.write(fstab) + print(fstab) pmb.chroot.root(["mv", "/tmp/fstab", "/etc/fstab"], chroot) +def get_root_filesystem(args: PmbArgs) -> str: + ret = args.filesystem or pmb.parse.deviceinfo().root_filesystem or "ext4" + pmaports_cfg = pmb.config.pmaports.read_config() + + supported = pmaports_cfg.get("supported_root_filesystems", "ext4") + supported_list = supported.split(",") + + if ret not in supported_list: + raise ValueError( + f"Root filesystem {ret} is not supported by your" + " currently checked out pmaports branch. Update your" + " branch ('pmbootstrap pull'), change it" + " ('pmbootstrap init'), or select one of these" + f" filesystems: {', '.join(supported_list)}" + ) + return ret + + def install_system_image( args: PmbArgs, chroot: Chroot, step: int, steps: int, - boot_label: str = "pmOS_boot", - root_label: str = "pmOS_root", split: bool = False, single_partition: bool = False, disk: Path | None = None, ) -> None: """ - :param size_reserve: empty partition between root and boot in MiB (pma#463) :param suffix: the chroot suffix, where the rootfs that will be installed on the device has been created (e.g. "rootfs_qemu-amd64") :param step: next installation step @@ -880,42 +829,47 @@ def install_system_image( """ config = get_context().config device = chroot.name + deviceinfo = pmb.parse.deviceinfo() # Partition and fill image file/disk block device logging.info(f"*** ({step}/{steps}) PREPARE INSTALL BLOCKDEVICE ***") pmb.helpers.mount.umount_all(chroot.path) - (size_boot, size_root) = get_subpartitions_size(chroot) - if not single_partition: - layout = get_partition_layout( - size_reserve, bool(pmb.parse.deviceinfo().cgpt_kpart and args.install_cgpt) - ) - else: - layout = None + layout = get_partition_layout(chroot, + bool(deviceinfo.cgpt_kpart and args.install_cgpt), + split, + single_partition + ) + logging.info(f"split: {split}") + logging.info("Using partition layout:") + logging.info(", ".join([str(x) for x in layout])) if not args.rsync: - pmb.install.blockdevice.create(args, size_boot, size_root, split, disk) - if not split and layout: - if pmb.parse.deviceinfo().cgpt_kpart and args.install_cgpt: - pmb.install.partition_cgpt(layout, size_boot) + pmb.install.blockdevice.create(args, layout, split, disk) + if not split and not single_partition: + if deviceinfo.cgpt_kpart and args.install_cgpt: + pmb.install.partition_cgpt(layout) else: - pmb.install.partition(layout, size_boot) + pmb.install.partition(layout) + else: + layout.root.offset = 0 + if not single_partition: + layout.boot.offset = 0 - # Inform kernel about changed partition table in case parted couldn't - pmb.chroot.root(["partprobe", "/dev/install"], check=False) + # if not split and not single_partition: + # assert layout # Initialized above for not single_partition case (mypy needs this) + # pmb.install.partitions_mount(device, layout, disk) - if not split and not single_partition: - assert layout # Initialized above for not single_partition case (mypy needs this) - pmb.install.partitions_mount(device, layout, disk) - - pmb.install.format(args, layout, boot_label, root_label, disk) + layout.root.filesystem = get_root_filesystem(args) + layout.boot.filesystem = deviceinfo.boot_filesystem or "ext2" # Since we shut down the chroot we need to mount it again pmb.chroot.mount(chroot) # Create /etc/fstab and /etc/crypttab logging.info("(native) create /etc/fstab") - create_fstab(args, layout, chroot) + # FIXME: don't hardcode /dev/install everywhere! + create_fstab(args, layout, "/dev/install", chroot) if args.full_disk_encryption: logging.info("(native) create /etc/crypttab") - create_crypttab(args, layout, chroot) + create_crypttab(args, layout, "/dev/install", chroot) # Run mkinitfs to pass UUIDs to cmdline logging.info(f"({chroot}) mkinitfs") @@ -927,11 +881,13 @@ def install_system_image( pmb.chroot.remove_mnt_pmbootstrap(chroot) # Just copy all the files - logging.info(f"*** ({step + 1}/{steps}) FILL INSTALL BLOCKDEVICE ***") - copy_files_from_chroot(args, chroot) - create_home_from_skel(args.filesystem, config.user) - configure_apk(args) - copy_ssh_keys(config) + logging.info(f"*** ({step + 1}/{steps}) FORMAT AND COPY BLOCKDEVICE ***") + create_home_from_skel(args.filesystem, config.user, chroot.path) + configure_apk(args, chroot.path) + copy_ssh_keys(config, chroot.path) + + # The formatting step also copies files into the disk image + pmb.install.format(args, layout, chroot.path, disk) # Don't try to embed firmware and cgpt on split images since there's no # place to put it and it will end up in /dev of the chroot instead @@ -947,7 +903,7 @@ def install_system_image( # Convert rootfs to sparse using img2simg sparse = args.sparse if sparse is None: - sparse = pmb.parse.deviceinfo().flash_sparse == "true" + sparse = deviceinfo.flash_sparse == "true" if sparse and not split and not disk: workdir = Path("/home/pmos/rootfs") @@ -959,7 +915,7 @@ def install_system_image( pmb.chroot.user(["mv", "-f", sys_image_sparse, sys_image], working_dir=workdir) # patch sparse image for Samsung devices if specified - samsungify_strategy = pmb.parse.deviceinfo().flash_sparse_samsung_format + samsungify_strategy = deviceinfo.flash_sparse_samsung_format if samsungify_strategy: logging.info("(native) convert sparse image into Samsung's sparse image format") pmb.chroot.apk.install(["sm-sparse-image-tool"], Chroot.native()) @@ -1277,7 +1233,7 @@ def create_device_rootfs(args: PmbArgs, step: int, steps: int) -> None: setup_login(args, config, chroot) # Set the keymap if the device requires it - setup_keymap(config) + setup_keymap(config, chroot) # Set timezone setup_timezone(chroot, config.timezone) @@ -1305,8 +1261,6 @@ def install(args: PmbArgs) -> None: if not args.android_recovery_zip and args.disk: sanity_check_disk(args) sanity_check_disk_size(args) - if args.on_device_installer: - sanity_check_ondev_version(args) # --single-partition implies --no-split. There is nothing to split if # there is only a single partition. @@ -1343,7 +1297,6 @@ def install(args: PmbArgs) -> None: else: install_system_image( args, - 0, chroot, step, steps, diff --git a/pmb/install/blockdevice.py b/pmb/install/blockdevice.py index 5a4b4f66..34ab0992 100644 --- a/pmb/install/blockdevice.py +++ b/pmb/install/blockdevice.py @@ -3,10 +3,10 @@ from pmb.helpers import logging import os from pathlib import Path -from pmb.types import PmbArgs +from pmb.types import PmbArgs, PartitionLayout import pmb.helpers.mount -import pmb.install.losetup import pmb.helpers.cli +import pmb.helpers.run import pmb.config from pmb.core import Chroot from pmb.core.context import get_context @@ -62,20 +62,22 @@ def mount_disk(path: Path) -> None: def create_and_mount_image( - args: PmbArgs, size_boot: int, size_root: int, split: bool = False + args: PmbArgs, + layout: PartitionLayout, + split: bool = False, ) -> None: """ Create a new image file, and mount it as /dev/install. - :param size_boot: size of the boot partition in MiB - :param size_root: size of the root partition in MiB + :param size_boot: size of the boot partition in bytes + :param size_root: size of the root partition in bytes :param split: create separate images for boot and root partitions """ # Short variables for paths chroot = Chroot.native() config = get_context().config - img_path_prefix = Path("/home/pmos/rootfs") + img_path_prefix = chroot / "home/pmos/rootfs" img_path_full = img_path_prefix / f"{config.device}.img" img_path_boot = img_path_prefix / f"{config.device}-boot.img" img_path_root = img_path_prefix / f"{config.device}-root.img" @@ -85,45 +87,54 @@ def create_and_mount_image( outside = chroot / img_path if os.path.exists(outside): pmb.helpers.mount.umount_all(chroot / "mnt") - pmb.install.losetup.umount(img_path) pmb.chroot.root(["rm", img_path]) # Make sure there is enough free space - size_mb = round(size_boot + size_root) + size_full = round(layout.boot.size + layout.root.size) disk_data = os.statvfs(get_context().config.work) - free = round((disk_data.f_bsize * disk_data.f_bavail) / (1024**2)) - if size_mb > free: + free = disk_data.f_bsize * disk_data.f_bavail + if size_full > free: raise RuntimeError( - f"Not enough free space to create rootfs image! (free: {free}M, required: {size_mb}M)" + f"Not enough free space to create rootfs image! (free: {round(free / (1024**2))}M, required: {round(size_full / (1024**2))}M)" ) # Create empty image files - pmb.chroot.user(["mkdir", "-p", "/home/pmos/rootfs"]) - size_mb_full = str(size_mb) + "M" - size_mb_boot = str(size_boot) + "M" - size_mb_root = str(size_root) + "M" - images = {img_path_full: size_mb_full} + rootfs_dir = chroot / "home/pmos/rootfs" + rootfs_dir.mkdir(exist_ok=True) + os.chown(rootfs_dir, int(pmb.config.chroot_uid_user), int(pmb.config.chroot_uid_user)) if split: - images = {img_path_boot: size_mb_boot, img_path_root: size_mb_root} - for img_path, image_size_mb in images.items(): - logging.info(f"(native) create {img_path.name} ({image_size_mb})") - pmb.chroot.root(["truncate", "-s", image_size_mb, img_path]) + images = {img_path_boot: layout.boot.size, img_path_root: layout.root.size} + else: + # Account for the partition table + size_full += pmb.parse.deviceinfo().boot_part_start * pmb.config.block_size + # Add 4 sectors for alignment and the backup header + size_full += pmb.config.block_size * 4 + # Round to sector size + size_full = int((size_full + 512) / pmb.config.block_size) * pmb.config.block_size + images = {img_path_full: size_full} + + for img_path, image_size in images.items(): + img_path.unlink(missing_ok=True) + logging.info(f"(native) create {img_path.name} ({round(image_size / (1024**2))}M)") + pmb.helpers.run.user(["truncate", "-s", f"{image_size}", f"{img_path}"]) + os.chown(img_path, int(pmb.config.chroot_uid_user), int(pmb.config.chroot_uid_user)) + # pmb.helpers.run.root(["dd", "if=/dev/zero", f"of={img_path}", f"bs={image_size}", "count=1"]) # Mount to /dev/install - mount_image_paths = {img_path_full: "/dev/install"} - if split: - mount_image_paths = {img_path_boot: "/dev/installp1", img_path_root: "/dev/installp2"} + if not split: + layout.boot.path = layout.root.path = layout.path = "/dev/install" + mount_image_paths = {img_path_full: layout.path} + else: + layout.boot.path = "/dev/installp1" + layout.root.path = "/dev/installp2" + mount_image_paths = {img_path_boot: layout.boot.path, img_path_root: layout.root.path} for img_path, mount_point in mount_image_paths.items(): - logging.info(f"(native) mount {mount_point} ({img_path.name})") - pmb.install.losetup.mount(img_path, args.sector_size) - device = pmb.install.losetup.device_by_back_file(img_path) - pmb.helpers.mount.bind_file(device, Chroot.native() / mount_point) + # logging.info(f"(native) mount {mount_point} ({img_path.name})") + pmb.helpers.mount.bind_file(img_path, chroot / mount_point) -def create( - args: PmbArgs, size_boot: int, size_root: int, split: bool, disk: Path | None -) -> None: +def create(args: PmbArgs, layout: PartitionLayout, split: bool, disk: Path | None) -> None: """ Create /dev/install (the "install blockdevice"). @@ -136,4 +147,4 @@ def create( if disk: mount_disk(disk) else: - create_and_mount_image(args, size_boot, size_root, split) + create_and_mount_image(args, layout, split) diff --git a/pmb/install/format.py b/pmb/install/format.py index e598f7fc..6bc82764 100644 --- a/pmb/install/format.py +++ b/pmb/install/format.py @@ -3,11 +3,12 @@ from pmb.helpers import logging from pmb.helpers.devices import get_device_category_by_name import pmb.chroot +import pmb.chroot.apk from pmb.core import Chroot from pmb.core.context import get_context from pmb.types import PartitionLayout, PmbArgs, PathString import os -import tempfile +from pathlib import Path def install_fsprogs(filesystem: str) -> None: @@ -18,33 +19,89 @@ def install_fsprogs(filesystem: str) -> None: pmb.chroot.apk.install([fsprogs], Chroot.native()) -def format_and_mount_boot(args: PmbArgs, device: str, boot_label: str) -> None: +def format_and_mount_boot(layout: PartitionLayout) -> None: """ :param device: boot partition on install block device (e.g. /dev/installp1) - :param boot_label: label of the root partition (e.g. "pmOS_boot") When adjusting this function, make sure to also adjust ondev-prepare-internal-storage.sh in postmarketos-ondev.git! """ - mountpoint = "/mnt/install/boot" - filesystem = pmb.parse.deviceinfo().boot_filesystem or "ext2" + pmb.chroot.apk.install(["mtools"], Chroot.native(), build=False, quiet=True) + deviceinfo = pmb.parse.deviceinfo() + filesystem = deviceinfo.boot_filesystem or "ext2" + layout.boot.filesystem = filesystem + offset_sectors = deviceinfo.boot_part_start + offset_bytes = layout.boot.offset + boot_path = "/mnt/rootfs/boot" install_fsprogs(filesystem) - logging.info(f"(native) format {device} (boot, {filesystem}), mount to {mountpoint}") + logging.info(f"(native) format {layout.boot.path} (boot, {filesystem})") + # mkfs.fat takes offset in sectors! wtf... if filesystem == "fat16": - pmb.chroot.root(["mkfs.fat", "-F", "16", "-n", boot_label, device]) + pmb.chroot.root( + [ + "mkfs.fat", + "-F", + "16", + "-i", + layout.boot.uuid.replace("-", ""), + "--offset", + str(offset_sectors), + "-n", + layout.boot.partition_label, + layout.boot.path, + ] + ) elif filesystem == "fat32": - pmb.chroot.root(["mkfs.fat", "-F", "32", "-n", boot_label, device]) + pmb.chroot.root( + [ + "mkfs.fat", + "-F", + "32", + "-i", + layout.boot.uuid.replace("-", ""), + "--offset", + str(offset_sectors), + "-n", + layout.boot.partition_label, + layout.boot.path, + ] + ) elif filesystem == "ext2": - pmb.chroot.root(["mkfs.ext2", "-F", "-q", "-L", boot_label, device]) + pmb.chroot.root( + [ + "mkfs.ext2", + "-d", + boot_path, + "-U", + layout.boot.uuid, + "-F", + "-q", + "-E", + f"offset={offset_bytes}", + "-L", + layout.boot.partition_label, + layout.boot.path, + f"{round(layout.boot.size / 1024)}k", + ] + ) elif filesystem == "btrfs": - pmb.chroot.root(["mkfs.btrfs", "-f", "-q", "-L", boot_label, device]) + raise ValueError("BTRFS not yet supported with new sandbox") + pmb.chroot.root(["mkfs.btrfs", "-f", "-q", "-L", layout.boot.partition_label, layout.boot.path]) else: raise RuntimeError("Filesystem " + filesystem + " is not supported!") - pmb.chroot.root(["mkdir", "-p", mountpoint]) - pmb.chroot.root(["mount", device, mountpoint]) + + # Copy in the filesystem + if filesystem.startswith("fat"): + contents = [ + path.relative_to(Chroot.native().path) + for path in (Chroot.native() / boot_path).glob("*") + ] + pmb.chroot.root( + ["mcopy", "-i", f"{layout.boot.path}@@{offset_bytes}", "-s", *contents, "::"] + ) -def format_luks_root(args: PmbArgs, device: str) -> None: +def format_luks_root(args: PmbArgs, layout: PartitionLayout, device: str) -> None: """ :param device: root partition on install block device (e.g. /dev/installp2) """ @@ -65,6 +122,8 @@ def format_luks_root(args: PmbArgs, device: str) -> None: "--iter-time", args.iter_time, "--use-random", + "--offset", + str(layout.root.offset), device, ] open_cmd = ["cryptsetup", "luksOpen"] @@ -91,24 +150,6 @@ def format_luks_root(args: PmbArgs, device: str) -> None: raise RuntimeError("Failed to open cryptdevice!") -def get_root_filesystem(args: PmbArgs) -> str: - ret = args.filesystem or pmb.parse.deviceinfo().root_filesystem or "ext4" - pmaports_cfg = pmb.config.pmaports.read_config() - - supported = pmaports_cfg.get("supported_root_filesystems", "ext4") - supported_list = supported.split(",") - - if ret not in supported_list: - raise ValueError( - f"Root filesystem {ret} is not supported by your" - " currently checked out pmaports branch. Update your" - " branch ('pmbootstrap pull'), change it" - " ('pmbootstrap init'), or select one of these" - f" filesystems: {', '.join(supported_list)}" - ) - return ret - - def prepare_btrfs_subvolumes(args: PmbArgs, device: str, mountpoint: str) -> None: """ Create separate subvolumes if root filesystem is btrfs. @@ -162,17 +203,26 @@ def prepare_btrfs_subvolumes(args: PmbArgs, device: str, mountpoint: str) -> Non pmb.chroot.root(["chattr", "+C", f"{mountpoint}/var"]) -def format_and_mount_root( - args: PmbArgs, device: str, root_label: str, disk: PathString | None -) -> None: +def format_and_mount_root(args: PmbArgs, layout: PartitionLayout) -> None: """ - :param device: root partition on install block device (e.g. /dev/installp2) - :param root_label: label of the root partition (e.g. "pmOS_root") - :param disk: path to disk block device (e.g. /dev/mmcblk0) or None + :param layout: disk image layout """ # Format if not args.rsync: - filesystem = get_root_filesystem(args) + filesystem = layout.root.filesystem + layout.root.filesystem = filesystem + rootfs = Path("/mnt/rootfs") + + # Bind mount an empty path over /boot so we don't include it in the root partition + # FIXME: better way to check if running with --single-partition + if len(layout) > 1: + empty_dir = Path("/tmp/empty") + pmb.mount.bind(empty_dir, Chroot.native() / rootfs / "boot") + + if filesystem != "ext4": + raise RuntimeError( + "Only EXT4 supports offset parameter for writing directly to disk image!" + ) if filesystem == "ext4": device_category = get_device_category_by_name(get_context().config.device) @@ -184,39 +234,50 @@ def format_and_mount_root( category_opts = ["-O", "^metadata_csum"] else: category_opts = [] - - mkfs_root_args = ["mkfs.ext4", *category_opts, "-F", "-q", "-L", root_label] - if not disk: - # pmb#2568: tell mkfs.ext4 to make a filesystem with enough - # indoes that we don't run into "out of space" errors - mkfs_root_args = [*mkfs_root_args, "-i", "16384"] + # Some downstream kernels don't support metadata_csum (#1364). + # When changing the options of mkfs.ext4, also change them in the + # recovery zip code (see 'grep -r mkfs\.ext4')! + mkfs_root_args = [ + "mkfs.ext4", + "-d", + rootfs, + "-F", + "-q", + "-L", + layout.root.partition_label, + "-U", + layout.root.uuid, + *category_opts, + ] + # pmb#2568: tell mkfs.ext4 to make a filesystem with enough + # indoes that we don't run into "out of space" errors + mkfs_root_args = [*mkfs_root_args, "-i", "16384"] + if not layout.split: + mkfs_root_args = [*mkfs_root_args, "-E", f"offset={layout.root.offset}"] elif filesystem == "f2fs": - mkfs_root_args = ["mkfs.f2fs", "-f", "-l", root_label] + mkfs_root_args = ["mkfs.f2fs", "-f", "-l", layout.root.partition_label] elif filesystem == "btrfs": - mkfs_root_args = ["mkfs.btrfs", "-f", "-L", root_label] + mkfs_root_args = ["mkfs.btrfs", "-f", "-L", layout.root.partition_label] else: raise RuntimeError(f"Don't know how to format {filesystem}!") install_fsprogs(filesystem) - logging.info(f"(native) format {device} (root, {filesystem})") - pmb.chroot.root([*mkfs_root_args, device]) + logging.info(f"(native) format {layout.root.path} (root, {filesystem})") + pmb.chroot.root([*mkfs_root_args, layout.root.path, f"{round(layout.root.size / 1024)}k"]) - # Mount - mountpoint = "/mnt/install" - logging.info("(native) mount " + device + " to " + mountpoint) - pmb.chroot.root(["mkdir", "-p", mountpoint]) - pmb.chroot.root(["mount", device, mountpoint]) + # Unmount the empty dir we mounted over /boot + pmb.mount.umount_all(Chroot.native() / rootfs / "boot") - if not args.rsync and filesystem == "btrfs": - # Make flat btrfs subvolume layout - prepare_btrfs_subvolumes(args, device, mountpoint) + # FIXME: btrfs borked + # if not args.rsync and filesystem == "btrfs": + # # Make flat btrfs subvolume layout + # prepare_btrfs_subvolumes(args, device, mountpoint) def format( args: PmbArgs, layout: PartitionLayout | None, - boot_label: str, - root_label: str, + rootfs: Path, disk: PathString | None, ) -> None: """ @@ -225,17 +286,13 @@ def format( :param root_label: label of the root partition (e.g. "pmOS_root") :param disk: path to disk block device (e.g. /dev/mmcblk0) or None """ - if layout: - root_dev = f"/dev/installp{layout['root']}" - boot_dev = f"/dev/installp{layout['boot']}" - else: - root_dev = "/dev/install" - boot_dev = None + # FIXME: do this elsewhere? + pmb.mount.bind(rootfs, Chroot.native().path / "mnt/rootfs") - if args.full_disk_encryption: - format_luks_root(args, root_dev) - root_dev = "/dev/mapper/pm_crypt" + # FIXME: probably broken because luksOpen uses loop under the hood, needs testing... + # root_dev = "/dev/mapper/pm_crypt" - format_and_mount_root(args, root_dev, root_label, disk) - if boot_dev: - format_and_mount_boot(args, boot_dev, boot_label) + format_and_mount_root(args, layout) + # FIXME: better way to check if we are running with --single-partition + if len(layout) > 1: + format_and_mount_boot(layout) diff --git a/pmb/install/losetup.py b/pmb/install/losetup.py deleted file mode 100644 index 179137ed..00000000 --- a/pmb/install/losetup.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2023 Oliver Smith -# SPDX-License-Identifier: GPL-3.0-or-later -import json -from pathlib import Path -from pmb.core.context import get_context -from pmb.helpers import logging -import time - -from pmb.types import PathString -import pmb.helpers.mount -import pmb.helpers.run -import pmb.chroot -from pmb.core import Chroot - - -def init() -> None: - if not Path("/sys/module/loop").is_dir(): - pmb.helpers.run.root(["modprobe", "loop"]) - for loopdevice in Path("/dev/").glob("loop*"): - if loopdevice.is_dir(): - continue - pmb.helpers.mount.bind_file(loopdevice, Chroot.native() / loopdevice) - - -def mount(img_path: Path, _sector_size: int | None = None) -> Path: - """ - :param img_path: Path to the img file inside native chroot. - """ - logging.debug(f"(native) mount {img_path} (loop)") - - # Try to mount multiple times (let the kernel module initialize #1594) - for i in range(5): - # Retry - if i > 0: - logging.debug("loop module might not be initialized yet, retry in one second...") - time.sleep(1) - - # Mount and return on success - init() - - sector_size = None - if _sector_size: - sector_size = str(_sector_size) - sector_size = sector_size or pmb.parse.deviceinfo().rootfs_image_sector_size - - losetup_cmd: list[PathString] = ["losetup", "-f", img_path] - if sector_size: - losetup_cmd += ["-b", str(int(sector_size))] - - pmb.chroot.root(losetup_cmd, check=False) - try: - device_by_back_file(img_path) - return - except RuntimeError: - pass - - try: - return device_by_back_file(img_path) - except RuntimeError as e: - if i == 4: - raise e - - raise AssertionError("This should never be reached") - - -def device_by_back_file(back_file: Path) -> Path: - """ - Get the /dev/loopX device that points to a specific image file. - """ - - # Get list from losetup - losetup_output = pmb.chroot.root(["losetup", "--json", "--list"], output_return=True) - if not losetup_output: - raise RuntimeError("losetup failed") - - # Find the back_file - losetup = json.loads(losetup_output) - for loopdevice in losetup["loopdevices"]: - if loopdevice["back-file"] is not None and Path(loopdevice["back-file"]) == back_file: - return Path(loopdevice["name"]) - raise RuntimeError(f"Failed to find loop device for {back_file}") - - -def umount(img_path: Path) -> None: - """ - :param img_path: Path to the img file inside native chroot. - """ - device: Path - try: - device = device_by_back_file(img_path) - except RuntimeError: - return - logging.debug(f"(native) umount {device}") - pmb.chroot.root(["losetup", "-d", device]) - - -def detach_all() -> None: - """ - Detach all loop devices used by pmbootstrap - """ - losetup_output = pmb.helpers.run.root(["losetup", "--json", "--list"], output_return=True) - if not losetup_output: - return - losetup = json.loads(losetup_output) - work = get_context().config.work - for loopdevice in losetup["loopdevices"]: - if Path(loopdevice["back-file"]).is_relative_to(work): - pmb.chroot.root(["losetup", "-d", loopdevice["name"]]) - return diff --git a/pmb/install/partition.py b/pmb/install/partition.py index 4cf698e5..dcaadf2a 100644 --- a/pmb/install/partition.py +++ b/pmb/install/partition.py @@ -5,118 +5,129 @@ from pmb.helpers import logging import os import time import pmb.chroot +import pmb.chroot.apk import pmb.config -import pmb.install.losetup from pmb.core import Chroot from pmb.types import PartitionLayout import pmb.core.dps +from functools import lru_cache +from pmb.core.context import get_context +import subprocess -# FIXME (#2324): this function drops disk to a string because it's easier -# to manipulate, this is probably bad. -def partitions_mount(device: str, layout: PartitionLayout, disk: Path | None) -> None: +@lru_cache +def get_partition_layout(partition: str, disk: str) -> tuple[int, int]: """ - Mount blockdevices of partitions inside native chroot - :param layout: partition layout from get_partition_layout() - :param disk: path to disk block device (e.g. /dev/mmcblk0) or None + Get the size of a partition in a disk image in bytes """ - if not disk: - img_path = Path("/home/pmos/rootfs") / f"{device}.img" - disk = pmb.install.losetup.device_by_back_file(img_path) + out = pmb.chroot.root( + [ + "fdisk", + "--list-details", + "--noauto-pt", + "--sector-size", + str(get_context().sector_size), + "--output", + "Name,Start,End", + disk, + ], + output_return=True, + ).rstrip() - logging.info(f"Mounting partitions of {disk} inside the chroot") - - tries = 20 - - # Devices ending with a number have a "p" before the partition number, - # /dev/sda1 has no "p", but /dev/mmcblk0p1 has. See add_partition() in - # block/partitions/core.c of linux.git. - partition_prefix = str(disk) - if str.isdigit(disk.name[-1:]): - partition_prefix = f"{disk}p" - - found = False - for i in range(tries): - if os.path.exists(f"{partition_prefix}1"): - found = True + start_end: list[str] | None = None + for line in out.splitlines(): + # FIXME: really ugly matching lmao + if line.startswith(partition): + start_end = list( + filter(lambda x: bool(x), line.replace(f"{partition} ", "").strip().split(" ")) + ) break - logging.debug(f"NOTE: ({i + 1}/{tries}) failed to find the install partition. Retrying...") - time.sleep(0.1) + if not start_end: + raise ValueError(f"Can't find partition {partition} in {disk}") - if not found: - raise RuntimeError( - f"Unable to find the first partition of {disk}, " - f"expected it to be at {partition_prefix}1!" - ) + start = int(start_end[0]) + end = int(start_end[1]) - partitions = [layout["boot"], layout["root"]] - - if layout["kernel"]: - partitions += [layout["kernel"]] - - for i in partitions: - source = Path(f"{partition_prefix}{i}") - target = Chroot.native() / "dev" / f"installp{i}" - pmb.helpers.mount.bind_file(source, target) + return (start, end) -def partition(layout: PartitionLayout, size_boot: int) -> None: +def partition(layout: PartitionLayout) -> None: """ - Partition /dev/install and create /dev/install{p1,p2,p3}: - * /dev/installp1: boot - * /dev/installp2: root (or reserved space) - * /dev/installp3: (root, if reserved space > 0) + Partition /dev/install with boot and root partitions + + NOTE: this function modifies "layout" to set the offset properties + of each partition, these offsets are then used when formatting + and populating the partitions so that we can access the disk image + directly without loop mounting. :param layout: partition layout from get_partition_layout() - :param size_boot: size of the boot partition in MiB """ + # Install sgdisk, gptfdisk is also useful for debugging + pmb.chroot.apk.install(["sgdisk", "gptfdisk"], Chroot.native(), build=False, quiet=True) + + deviceinfo = pmb.parse.deviceinfo() + # Convert to MB and print info - mb_boot = f"{size_boot}M" - mb_reserved = f"{size_reserve}M" - mb_root_start = f"{size_boot + size_reserve}M" - logging.info( - f"(native) partition /dev/install (boot: {mb_boot}," - f" reserved: {mb_reserved}, root: the rest)" + logging.info(f"(native) partition /dev/install (boot: {layout.boot.size_mb}M)") + + boot_offset_sectors = deviceinfo.boot_part_start or "2048" + # For MBR we use to --gpt-to-mbr flag of sgdisk + # FIXME: test MBR support + partition_type = deviceinfo.partition_type or "gpt" + if partition_type == "msdos": + partition_type = "dos" + + sector_size = get_context().sector_size + + boot_size_sectors = layout.boot.size_sectors(sector_size) + root_offset_sectors = boot_offset_sectors + boot_size_sectors + 1 + + # Align to 2048-sector boundaries (round UP) + root_offset_sectors = int((root_offset_sectors + 2047) / 2048) * 2048 + + arch = str(deviceinfo.arch) + root_type_guid = pmb.core.dps.root[arch][1] + + proc = subprocess.Popen( + [ + "chroot", + os.fspath(Chroot.native().path), + "sh", + "-c", + f"sfdisk --no-tell-kernel --sector-size {sector_size} {layout.path}", + ], + stdin=subprocess.PIPE, + ) + proc.stdin.write( + ( + f"label: {partition_type}\n" + f"start={boot_offset_sectors},size={boot_size_sectors},name={layout.boot.partition_label},type=U\n" + f"start={root_offset_sectors},size=+,name={layout.root.partition_label},type={root_type_guid}\n" + ).encode() + ) + proc.stdin.flush() + proc.stdin.close() + while proc.poll() is None: + if proc.stdout is not None: + print(proc.stdout.readline().decode("utf-8")) + if proc.returncode != 0: + raise RuntimeError(f"Disk partitioning failed! sfdisk exited with code {proc.returncode}") + + # Configure the partition offsets and final sizes based on sgdisk + boot_start_sect, _boot_end_sect = get_partition_layout( + layout.boot.partition_label, "/dev/install" + ) + root_start_sect, root_end_sect = get_partition_layout( + layout.root.partition_label, "/dev/install" ) - filesystem = pmb.parse.deviceinfo().boot_filesystem or "vfat" - - # Actual partitioning with 'parted'. Using check=False, because parted - # sometimes "fails to inform the kernel". In case it really failed with - # partitioning, the follow-up mounting/formatting will not work, so it - # will stop there (see #463). - boot_part_start = pmb.parse.deviceinfo().boot_part_start or "2048" - - partition_type = pmb.parse.deviceinfo().partition_type or "gpt" - - commands = [ - ["mktable", partition_type], - ["mkpart", "primary", filesystem, boot_part_start + "s", mb_boot], - ] - - if size_reserve: - mb_reserved_end = f"{round(size_reserve + size_boot)}M" - commands += [["mkpart", "primary", mb_boot, mb_reserved_end]] - - arch = str(pmb.parse.deviceinfo().arch) - - commands += [["mkpart", "primary", mb_root_start, "100%"]] - if partition_type.lower() == "gpt": - commands += [ - ["type", str(layout["root"]), pmb.core.dps.root[arch][1]], - # esp is an alias for boot on GPT - ["set", str(layout["boot"]), "esp", "on"], - ["type", str(layout["boot"]), pmb.core.dps.boot["esp"][1]], - ] - # Some devices still use MBR and will not work with only esp set - elif partition_type.lower() == "msdos": - commands += [["set", str(layout["boot"]), "boot", "on"]] - - for command in commands: - pmb.chroot.root(["parted", "-s", "/dev/install", *command], check=False) + layout.boot.offset = boot_start_sect * sector_size + layout.root.offset = root_start_sect * sector_size + layout.root.size = (root_end_sect - root_start_sect) * sector_size -def partition_cgpt(layout: PartitionLayout, size_boot: int) -> None: +# FIXME: sgdisk? +def partition_cgpt(layout: PartitionLayout, size_boot: int = 0) -> None: """ This function does similar functionality to partition(), but this one is for ChromeOS devices which use special GPT. We don't follow diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 2aa94fa7..0fd11e8e 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -120,7 +120,7 @@ def arguments_install(subparser: argparse._SubParsersAction) -> None: help="create combined boot and root image file", dest="split", action="store_false", - default=None, + default=False, ) group.add_argument( "--split", help="create separate boot and root image files", action="store_true" diff --git a/pmb/parse/deviceinfo.py b/pmb/parse/deviceinfo.py index 6e5c5335..337c5503 100644 --- a/pmb/parse/deviceinfo.py +++ b/pmb/parse/deviceinfo.py @@ -161,7 +161,7 @@ class Deviceinfo: flash_fastboot_max_size: str | None = "" flash_sparse: str | None = "" flash_sparse_samsung_format: str | None = "" - rootfs_image_sector_size: str | None = "" + rootfs_image_sector_size: int | None = 512 sd_embed_firmware: str | None = "" sd_embed_firmware_step_size: str | None = "" partition_blacklist: str | None = "" diff --git a/pmb/qemu/run.py b/pmb/qemu/run.py index ed5412cd..ad74d97b 100644 --- a/pmb/qemu/run.py +++ b/pmb/qemu/run.py @@ -23,7 +23,6 @@ import pmb.chroot.other import pmb.chroot.initfs import pmb.config import pmb.config.pmaports -import pmb.install.losetup from pmb.types import Env, PathString, PmbArgs import pmb.helpers.run import pmb.parse.cpuinfo diff --git a/pmb/types.py b/pmb/types.py index 1af1124a..9c5c8d92 100644 --- a/pmb/types.py +++ b/pmb/types.py @@ -10,6 +10,8 @@ from typing import Any, Literal, TypedDict from pmb.core.arch import Arch from pmb.core.chroot import Chroot +import uuid + class CrossCompile(enum.Enum): # Cross compilation isn't needed for this package: @@ -58,6 +60,70 @@ class CrossCompile(enum.Enum): return Chroot.native() +class DiskPartition: + name: str + size: int # in bytes + filesystem: str | None + # offset into the disk image! + offset: int # bytes + path: str # e.g. /dev/install or /dev/installp1 for --split + _uuid: str + + def __init__(self, name: str, size: int): + self.name = name + self.size = size + self.filesystem = None + self.offset = 0 + self.path = "" + self._uuid = "" + + @property + def uuid(self) -> str: + """ + We generate a UUID the first time we're called. The length + depends on which filesystem, since FAT only supported short + volume IDs. + """ + if self.filesystem is None: + raise ValueError("Can't get UUID when filesystem not set") + + if self._uuid: + return self._uuid + + if self.filesystem.startswith("fat"): + # FAT UUIDs are only 8 bytes and are always uppercase + self._uuid = ("-".join(str(uuid.uuid4()).split("-")[1:3])).upper() + else: + self._uuid = str(uuid.uuid4()) + + return self._uuid + + @property + def size_mb(self) -> int: + return round(self.size / (1024**2)) + + @property + def partition_label(self) -> str: + return f"pmOS_{self.name}" + + def offset_sectors(self, sector_size: int) -> int: + if self.offset % sector_size != 0: + raise ValueError( + f"Partition {self.name} offset not a multiple of sector size {sector_size}!" + ) + return int(self.offset / sector_size) + + def size_sectors(self, sector_size: int) -> int: + ss = int((self.size + sector_size) / sector_size) + # sgdisk requires aligning to 2048-sector boundaries. + # It conservatively rounds down but we want to round up... + ss = int((ss + 2047) / 2048) * 2048 + return ss + + def __str__(self) -> str: + return f"DiskPartition {{name: {self.name}, size: {self.size_mb}M, offset: {self.offset / 1024 / 1024}M{', path: ' + self.path if self.path else ''}{', fs: ' + self.filesystem if self.filesystem else ''}}}" + + RunOutputTypeDefault = Literal["log", "stdout", "interactive", "tui", "null"] RunOutputTypePopen = Literal["background", "pipe"] RunOutputType = RunOutputTypeDefault | RunOutputTypePopen @@ -72,11 +138,55 @@ WithExtraRepos = Literal["default", "enabled", "disabled"] # future refactoring efforts easier. -class PartitionLayout(TypedDict): - kernel: int | None - boot: int - reserve: int | None - root: int +class PartitionLayout(list[DiskPartition]): + """ + Subclass list to provide easy accessors without relying on + fragile indexes while still allowing the partitions to be + iterated over for simplicity. This is not a good design tbh + """ + path: str # path to disk image + split: bool # image per partition + + def __init__(self, path: str, split: bool): + super().__init__(self) + # Path to the disk image + self.path = path + self.split = split + + @property + def kernel(self): + """ + Get the kernel partition (specific to Chromebooks). + """ + if self[0].name != "kernel": + raise ValueError("First partition not kernel partition!") + return self[0] + + @property + def boot(self): + """ + Get the boot partition, must be the first or second if we have + a kernel partition + """ + if self[0].name == "boot": + return self[0] + if self[0].name == "kernel" and self[1].name == "boot": + return self[1] + + raise ValueError("First partition not boot partition!") + + @property + def root(self): + """ + Get the root partition, must be the second or third if we have + a kernel partition + """ + if self[1].name == "root": + return self[1] + if self[0].name == "kernel" and self[2].name == "root": + return self[2] + + raise ValueError("First partition not root partition!") class AportGenEntry(TypedDict): @@ -241,3 +351,6 @@ class PmbArgs(Namespace): xauth: bool xconfig: bool zap: bool + + +# type: ignore