Message ID | 20220421085232.1949-3-henning.schild@siemens.com |
---|---|
State | Superseded, archived |
Headers | show |
Series | bump bitbake and wic for python 3.10 support | expand |
Am Thu, 21 Apr 2022 10:52:32 +0200 schrieb Henning Schild <henning.schild@siemens.com>: > Update to the latest wic from OE-core. > > OE-core Revision: 712552b5cc427d7be1258c45886de9b57f7272c9 > > Signed-off-by: Henning Schild <henning.schild@siemens.com> > --- > .../wic/plugins/source/bootimg-efi-isar.py | 77 > ++++++++++++++++--- .../wic/plugins/source/bootimg-pcbios-isar.py | > 6 +- scripts/lib/wic/canned-wks/common.wks.inc | 2 +- > scripts/lib/wic/canned-wks/directdisk-gpt.wks | 2 +- > scripts/lib/wic/canned-wks/mkefidisk.wks | 2 +- > scripts/lib/wic/engine.py | 6 +- > scripts/lib/wic/help.py | 10 ++- > scripts/lib/wic/ksparser.py | 8 +- > scripts/lib/wic/misc.py | 4 +- > scripts/lib/wic/partition.py | 25 ++++-- > scripts/lib/wic/pluginbase.py | 8 +- > scripts/lib/wic/plugins/imager/direct.py | 11 ++- > scripts/lib/wic/plugins/source/bootimg-efi.py | 74 +++++++++++++++--- > .../lib/wic/plugins/source/bootimg-pcbios.py | 6 +- > scripts/lib/wic/plugins/source/rawcopy.py | 35 ++++++++- > scripts/lib/wic/plugins/source/rootfs.py | 2 +- > scripts/wic | 9 ++- > 17 files changed, 231 insertions(+), 56 deletions(-) > > diff --git a/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py > b/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py index > 5ba0777ad244..a24e04f309da 100644 --- > a/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py +++ > b/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py @@ -12,6 > +12,7 @@ > import logging > import os > +import tempfile > import shutil > import re > > @@ -129,12 +130,13 @@ class BootimgEFIPlugin(SourcePlugin): > bootloader = creator.ks.bootloader > > loader_conf = "" > - loader_conf += "default boot\n" > + if source_params.get('create-unified-kernel-image') != > "true": > + loader_conf += "default boot\n" > loader_conf += "timeout %d\n" % bootloader.timeout > > initrd = source_params.get('initrd') > > - if initrd: > + if initrd and > source_params.get('create-unified-kernel-image') != "true": # > obviously we need to have a common common deploy var bootimg_dir = > get_bitbake_var("DEPLOY_DIR_IMAGE") if not bootimg_dir: > @@ -195,11 +197,12 @@ class BootimgEFIPlugin(SourcePlugin): > for rd in initrds: > boot_conf += "initrd /%s\n" % rd > > - logger.debug("Writing systemd-boot config " > - "%s/hdd/boot/loader/entries/boot.conf", > cr_workdir) > - cfg = open("%s/hdd/boot/loader/entries/boot.conf" % > cr_workdir, "w") > - cfg.write(boot_conf) > - cfg.close() > + if source_params.get('create-unified-kernel-image') != > "true": > + logger.debug("Writing systemd-boot config " > + "%s/hdd/boot/loader/entries/boot.conf", > cr_workdir) > + cfg = open("%s/hdd/boot/loader/entries/boot.conf" % > cr_workdir, "w") > + cfg.write(boot_conf) > + cfg.close() > > > @classmethod > @@ -300,11 +303,63 @@ class BootimgEFIPlugin(SourcePlugin): > kernel = "%s-%s.bin" % \ > (get_bitbake_var("KERNEL_IMAGETYPE"), > get_bitbake_var("INITRAMFS_LINK_NAME")) > - install_cmd = "install -m 0644 %s/%s %s/%s" % \ > - (staging_kernel_dir, kernel, hdddir, kernel) > + if source_params.get('create-unified-kernel-image') == > "true": > + initrd = source_params.get('initrd') > + if not initrd: > + raise WicError("initrd= must be specified when > create-unified-kernel-image=true, exiting") + > + deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") > + efi_stub = glob("%s/%s" % (deploy_dir, > "linux*.efi.stub")) > + if len(efi_stub) == 0: > + raise WicError("Unified Kernel Image EFI stub not > found, exiting") > + efi_stub = efi_stub[0] > + > + with tempfile.TemporaryDirectory() as tmp_dir: > + label = source_params.get('label') > + label_conf = "root=%s" % creator.rootdev > + if label: > + label_conf = "LABEL=%s" % label > + > + bootloader = creator.ks.bootloader > + cmdline = open("%s/cmdline" % tmp_dir, "w") > + cmdline.write("%s %s" % (label_conf, > bootloader.append)) > + cmdline.close() > + > + initrds = initrd.split(';') > + initrd = open("%s/initrd" % tmp_dir, "wb") > + for f in initrds: > + with open("%s/%s" % (deploy_dir, f), 'rb') as > in_file: > + shutil.copyfileobj(in_file, initrd) > + initrd.close() > + > + # Searched by systemd-boot: > + # > https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images > + install_cmd = "install -d %s/EFI/Linux" % hdddir > + exec_cmd(install_cmd) > + > + staging_dir_host = > get_bitbake_var("STAGING_DIR_HOST") + > + # > https://www.freedesktop.org/software/systemd/man/systemd-stub.html > + objcopy_cmd = "objcopy \ > + --add-section .osrel=%s --change-section-vma > .osrel=0x20000 \ > + --add-section .cmdline=%s --change-section-vma > .cmdline=0x30000 \ > + --add-section .linux=%s --change-section-vma > .linux=0x2000000 \ > + --add-section .initrd=%s --change-section-vma > .initrd=0x3000000 \ > + %s %s" % \ > + ("%s/usr/lib/os-release" % staging_dir_host, > + cmdline.name, > + "%s/%s" % (staging_kernel_dir, kernel), > + initrd.name, > + efi_stub, > + "%s/EFI/Linux/linux.efi" % hdddir) > + exec_cmd(objcopy_cmd) > + else: > + install_cmd = "install -m 0644 %s/%s %s/%s" % \ > + (staging_kernel_dir, kernel, hdddir, kernel) > + > + install_cmd = > isar_populate_boot_cmd(rootfs_dir['ROOTFS_DIR'], hdddir) > + exec_cmd(install_cmd) > > - install_cmd = > isar_populate_boot_cmd(rootfs_dir['ROOTFS_DIR'], hdddir) > - exec_cmd(install_cmd) > > if get_bitbake_var("IMAGE_EFI_BOOT_FILES"): > for src_path, dst_path in cls.install_task: > diff --git > a/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py > b/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py index > 9136d4f215ca..7fb0f2a87394 100644 --- > a/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py +++ > b/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py @@ > -213,8 +213,10 @@ class BootimgPcbiosIsarPlugin(SourcePlugin): # > dosfs image, created by mkdosfs bootimg = "%s/boot%s.img" % > (cr_workdir, part.lineno) > - dosfs_cmd = "mkdosfs -n boot -i %s -S 512 -C %s %d" % \ > - (part.fsuuid, bootimg, blocks) > + label = part.label if part.label else "boot" > + > + dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \ > + (label, part.fsuuid, bootimg, blocks) > exec_native_cmd(dosfs_cmd, native_sysroot) > > mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir) > diff --git a/scripts/lib/wic/canned-wks/common.wks.inc > b/scripts/lib/wic/canned-wks/common.wks.inc index > 4fd29fa8c119..89880b417b6e 100644 --- > a/scripts/lib/wic/canned-wks/common.wks.inc +++ > b/scripts/lib/wic/canned-wks/common.wks.inc @@ -1,3 +1,3 @@ > # This file is included into 3 canned wks files from this directory > part /boot --source bootimg-pcbios --ondisk sda --label boot > --active --align 1024 -part / --source rootfs --use-uuid > --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align > 1024 +part / --source rootfs --use-uuid --fstype=ext4 --label > platform --align 1024 diff --git This is https://github.com/openembedded/openembedded-core/commit/7e8017208bed98b6c90735cb641fc9d7aedf9140 and it was not taken for our forks in meta-isar/scripts/lib/wic/canned-wks/ Because we can not be sure to have a brandnew e2fsprogs always. With scripts/wic_fakeroot we have a way to centrally hook into wic plugins and play with arguments. But i think having our forked wks files is nicer and we might eventually be able to drop wic_fakeroot, at the moment it only deals with one remaining hack for old e2fsck Henning > a/scripts/lib/wic/canned-wks/directdisk-gpt.wks > b/scripts/lib/wic/canned-wks/directdisk-gpt.wks index > cf16c0c30bbd..8d7d8de6ea7e 100644 --- > a/scripts/lib/wic/canned-wks/directdisk-gpt.wks +++ > b/scripts/lib/wic/canned-wks/directdisk-gpt.wks @@ -4,7 +4,7 @@ > part /boot --source bootimg-pcbios --ondisk sda --label boot > --active --align 1024 -part / --source rootfs --ondisk sda > --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align > 1024 --use-uuid +part / --source rootfs --ondisk sda --fstype=ext4 > --label platform --align 1024 --use-uuid bootloader --ptable gpt > --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb > vga=0x318 console=tty0 console=ttyS0,115200n8" diff --git > a/scripts/lib/wic/canned-wks/mkefidisk.wks > b/scripts/lib/wic/canned-wks/mkefidisk.wks index > d1878e23e5a3..9f534fe18471 100644 --- > a/scripts/lib/wic/canned-wks/mkefidisk.wks +++ > b/scripts/lib/wic/canned-wks/mkefidisk.wks @@ -4,7 +4,7 @@ > part /boot --source bootimg-efi --sourceparams="loader=grub-efi" > --ondisk sda --label msdos --active --align 1024 > -part / --source rootfs --ondisk sda --fstype=ext4 --mkfs-extraopts > "-T default" --label platform --align 1024 --use-uuid +part / > --source rootfs --ondisk sda --fstype=ext4 --label platform --align > 1024 --use-uuid part swap --ondisk sda --size 44 --label swap1 > --fstype=swap > diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py > index 018815b96688..674ccfc24418 100644 > --- a/scripts/lib/wic/engine.py > +++ b/scripts/lib/wic/engine.py > @@ -19,10 +19,10 @@ import os > import tempfile > import json > import subprocess > +import shutil > import re > > from collections import namedtuple, OrderedDict > -from distutils.spawn import find_executable > > from wic import WicError > from wic.filemap import sparse_copy > @@ -245,7 +245,7 @@ class Disk: > for path in pathlist.split(':'): > self.paths = "%s%s:%s" % (native_sysroot, path, > self.paths) > - self.parted = find_executable("parted", self.paths) > + self.parted = shutil.which("parted", path=self.paths) > if not self.parted: > raise WicError("Can't find executable parted") > > @@ -283,7 +283,7 @@ class Disk: > "resize2fs", "mkswap", "mkdosfs", > "debugfs","blkid"): aname = "_%s" % name > if aname not in self.__dict__: > - setattr(self, aname, find_executable(name, > self.paths)) > + setattr(self, aname, shutil.which(name, > path=self.paths)) if aname not in self.__dict__ or > self.__dict__[aname] is None: raise WicError("Can't find executable > '{}'".format(name)) return self.__dict__[aname] > diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py > index bd3a2b97dfaf..4ff7470a6a79 100644 > --- a/scripts/lib/wic/help.py > +++ b/scripts/lib/wic/help.py > @@ -637,7 +637,7 @@ DESCRIPTION > oe-core: directdisk.bbclass and mkefidisk.sh. The difference > between wic and those examples is that with wic the functionality > of those scripts is implemented by a general-purpose partitioning > - 'language' based on Redhat kickstart syntax). > + 'language' based on Red Hat kickstart syntax). > > The initial motivation and design considerations that lead to the > current tool are described exhaustively in Yocto Bug #3847 > @@ -840,8 +840,8 @@ DESCRIPTION > meanings. The commands are based on the Fedora kickstart > documentation but with modifications to reflect wic capabilities. > > - > http://fedoraproject.org/wiki/Anaconda/Kickstart#part_or_partition > - http://fedoraproject.org/wiki/Anaconda/Kickstart#bootloader > + > https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#part-or-partition > + > https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#bootloader > Commands > > @@ -930,6 +930,7 @@ DESCRIPTION > ext4 > btrfs > squashfs > + erofs > swap > > --fsoptions: Specifies a free-form string of options to be > @@ -990,6 +991,9 @@ DESCRIPTION > multiple partitions and we want to keep > the right permissions and usernames in all the partitions. > > + --no-fstab-update: This option is specific to wic. It does > not update the > + '/etc/fstab' stock file for the given > partition. + > --extra-space: This option is specific to wic. It adds extra > space after the space filled by the content > of the partition. The final size can go > diff --git a/scripts/lib/wic/ksparser.py b/scripts/lib/wic/ksparser.py > index 3eb669da39ca..0df9eb0d057d 100644 > --- a/scripts/lib/wic/ksparser.py > +++ b/scripts/lib/wic/ksparser.py > @@ -157,7 +157,8 @@ class KickStart(): > part.add_argument('--fsoptions', dest='fsopts') > part.add_argument('--fstype', default='vfat', > choices=('ext2', 'ext3', 'ext4', 'btrfs', > - 'squashfs', 'vfat', 'msdos', > 'swap')) > + 'squashfs', 'vfat', 'msdos', > 'erofs', > + 'swap')) > part.add_argument('--mkfs-extraopts', default='') > part.add_argument('--label') > part.add_argument('--use-label', action='store_true') > @@ -184,6 +185,7 @@ class KickStart(): > part.add_argument('--use-uuid', action='store_true') > part.add_argument('--uuid') > part.add_argument('--fsuuid') > + part.add_argument('--no-fstab-update', action='store_true') > > bootloader = subparsers.add_parser('bootloader') > bootloader.add_argument('--append') > @@ -229,6 +231,10 @@ class KickStart(): > err = "%s:%d: SquashFS does not > support LABEL" \ % (confpath, lineno) > raise KickStartError(err) > + # erofs does not support filesystem labels > + if parsed.fstype == 'erofs' and parsed.label: > + err = "%s:%d: erofs does not support > LABEL" % (confpath, lineno) > + raise KickStartError(err) > if parsed.fstype == 'msdos' or parsed.fstype > == 'vfat': if parsed.fsuuid: > if > parsed.fsuuid.upper().startswith('0X'): diff --git > a/scripts/lib/wic/misc.py b/scripts/lib/wic/misc.py index > 57c042c503e6..3e118229960b 100644 --- a/scripts/lib/wic/misc.py > +++ b/scripts/lib/wic/misc.py > @@ -16,9 +16,9 @@ import logging > import os > import re > import subprocess > +import shutil > > from collections import defaultdict > -from distutils import spawn > > from wic import WicError > > @@ -122,7 +122,7 @@ def find_executable(cmd, paths): > if provided and "%s-native" % recipe in provided: > return True > > - return spawn.find_executable(cmd, paths) > + return shutil.which(cmd, path=paths) > > def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): > """ > diff --git a/scripts/lib/wic/partition.py > b/scripts/lib/wic/partition.py index 76d144d12d30..09e491dd494d 100644 > --- a/scripts/lib/wic/partition.py > +++ b/scripts/lib/wic/partition.py > @@ -54,6 +54,7 @@ class Partition(): > self.uuid = args.uuid > self.fsuuid = args.fsuuid > self.type = args.type > + self.no_fstab_update = args.no_fstab_update > self.updated_fstab_path = None > self.has_fstab = False > self.update_fstab_in_rootfs = False > @@ -104,7 +105,7 @@ class Partition(): > extra_blocks = self.extra_space > > rootfs_size = actual_rootfs_size + extra_blocks > - rootfs_size *= self.overhead_factor > + rootfs_size = int(rootfs_size * self.overhead_factor) > > logger.debug("Added %d extra blocks to %s to get to %d > total blocks", extra_blocks, self.mountpoint, rootfs_size) > @@ -141,9 +142,9 @@ class Partition(): > native_sysroot) > self.source_file = "%s/fs.%s" % (cr_workdir, > self.fstype) else: > - if self.fstype == 'squashfs': > - raise WicError("It's not possible to create > empty squashfs " > - "partition '%s'" % > (self.mountpoint)) > + if self.fstype in ('squashfs', 'erofs'): > + raise WicError("It's not possible to create > empty %s " > + "partition '%s'" % (self.fstype, > self.mountpoint)) > rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label, > self.lineno, > self.fstype) @@ -170,7 +171,7 @@ class Partition(): > # Split sourceparams string of the form > key1=val1[,key2=val2,...] # into a dict. Also accepts valueless keys > i.e. without = splitted = self.sourceparams.split(',') > - srcparams_dict = dict(par.split('=', 1) for par in > splitted if par) > + srcparams_dict = dict((par.split('=', 1) + [None])[:2] > for par in splitted if par) > plugin = PluginMgr.get_plugins('source')[self.source] > plugin.do_configure_partition(self, srcparams_dict, creator, > @@ -286,7 +287,7 @@ class Partition(): > (self.fstype, extraopts, rootfs, label_str, self.fsuuid, > rootfs_dir) exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) > > - if self.updated_fstab_path and self.has_fstab: > + if self.updated_fstab_path and self.has_fstab and not > self.no_fstab_update: debugfs_script_path = os.path.join(cr_workdir, > "debugfs_script") with open(debugfs_script_path, "w") as f: > f.write("cd etc\n") > @@ -350,7 +351,7 @@ class Partition(): > mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir) > exec_native_cmd(mcopy_cmd, native_sysroot) > > - if self.updated_fstab_path and self.has_fstab: > + if self.updated_fstab_path and self.has_fstab and not > self.no_fstab_update: mcopy_cmd = "mcopy -i %s %s ::/etc/fstab" % > (rootfs, self.updated_fstab_path) exec_native_cmd(mcopy_cmd, > native_sysroot) > @@ -369,6 +370,16 @@ class Partition(): > (rootfs_dir, rootfs, extraopts) > exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo) > > + def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, > rootfs_dir, > + native_sysroot, pseudo): > + """ > + Prepare content for a erofs rootfs partition. > + """ > + extraopts = self.mkfs_extraopts or '' > + erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \ > + (extraopts, self.fsuuid, rootfs, rootfs_dir) > + exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo) > + > def prepare_empty_partition_ext(self, rootfs, oe_builddir, > native_sysroot): > """ > diff --git a/scripts/lib/wic/pluginbase.py > b/scripts/lib/wic/pluginbase.py index d9b4e57747e7..b64568339b1b > 100644 --- a/scripts/lib/wic/pluginbase.py > +++ b/scripts/lib/wic/pluginbase.py > @@ -9,9 +9,11 @@ __all__ = ['ImagerPlugin', 'SourcePlugin'] > > import os > import logging > +import types > > from collections import defaultdict > -from importlib.machinery import SourceFileLoader > +import importlib > +import importlib.util > > from wic import WicError > from wic.misc import get_bitbake_var > @@ -54,7 +56,9 @@ class PluginMgr: > mname = fname[:-3] > mpath = os.path.join(ppath, fname) > logger.debug("loading plugin module %s", > mpath) > - SourceFileLoader(mname, > mpath).load_module() > + spec = > importlib.util.spec_from_file_location(mname, mpath) > + module = > importlib.util.module_from_spec(spec) > + spec.loader.exec_module(module) > > return PLUGINS.get(ptype) > > diff --git a/scripts/lib/wic/plugins/imager/direct.py > b/scripts/lib/wic/plugins/imager/direct.py index > ea709e8c545d..35fff7c10242 100644 --- > a/scripts/lib/wic/plugins/imager/direct.py +++ > b/scripts/lib/wic/plugins/imager/direct.py @@ -77,7 +77,8 @@ class > DirectPlugin(ImagerPlugin): > image_path = self._full_path(self.workdir, > self.parts[0].disk, "direct") self._image = > PartitionedImage(image_path, self.ptable_format, > - self.parts, > self.native_sysroot) > + self.parts, > self.native_sysroot, > + options.extra_space) > > def setup_workdir(self, workdir): > if workdir: > @@ -116,7 +117,7 @@ class DirectPlugin(ImagerPlugin): > updated = False > for part in self.parts: > if not part.realnum or not part.mountpoint \ > - or part.mountpoint == "/": > + or part.mountpoint == "/" or not > part.mountpoint.startswith('/'): continue > > if part.use_uuid: > @@ -258,6 +259,8 @@ class DirectPlugin(ImagerPlugin): > if part.mountpoint == "/": > if part.uuid: > return "PARTUUID=%s" % part.uuid > + elif part.label: > + return "PARTLABEL=%s" % part.label > else: > suffix = 'p' if part.disk.startswith('mmcblk') > else '' return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) > @@ -293,7 +296,7 @@ class PartitionedImage(): > Partitioned image in a file. > """ > > - def __init__(self, path, ptable_format, partitions, > native_sysroot=None): > + def __init__(self, path, ptable_format, partitions, > native_sysroot=None, extra_space=0): self.path = path # Path to the > image file self.numpart = 0 # Number of allocated partitions > self.realpart = 0 # Number of partitions in the partition > table @@ -314,6 +317,7 @@ class PartitionedImage(): > self.sector_size = SECTOR_SIZE > self.native_sysroot = native_sysroot > num_real_partitions = len([p for p in self.partitions if not > p.no_table]) > + self.extra_space = extra_space > > # calculate the real partition number, accounting for > partitions not # in the partition table and logical partitions > @@ -483,6 +487,7 @@ class PartitionedImage(): > self.min_size += GPT_OVERHEAD > > self.min_size *= self.sector_size > + self.min_size += self.extra_space > > def _create_partition(self, device, parttype, fstype, start, > size): """ Create a partition on an image described by the 'device' > object. """ diff --git > a/scripts/lib/wic/plugins/source/bootimg-efi.py > b/scripts/lib/wic/plugins/source/bootimg-efi.py index > cdc72543c200..0391aebdc840 100644 --- > a/scripts/lib/wic/plugins/source/bootimg-efi.py +++ > b/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -12,6 +12,7 @@ > import logging > import os > +import tempfile > import shutil > import re > > @@ -119,12 +120,13 @@ class BootimgEFIPlugin(SourcePlugin): > bootloader = creator.ks.bootloader > > loader_conf = "" > - loader_conf += "default boot\n" > + if source_params.get('create-unified-kernel-image') != > "true": > + loader_conf += "default boot\n" > loader_conf += "timeout %d\n" % bootloader.timeout > > initrd = source_params.get('initrd') > > - if initrd: > + if initrd and > source_params.get('create-unified-kernel-image') != "true": # > obviously we need to have a common common deploy var bootimg_dir = > get_bitbake_var("DEPLOY_DIR_IMAGE") if not bootimg_dir: > @@ -183,11 +185,12 @@ class BootimgEFIPlugin(SourcePlugin): > for rd in initrds: > boot_conf += "initrd /%s\n" % rd > > - logger.debug("Writing systemd-boot config " > - "%s/hdd/boot/loader/entries/boot.conf", > cr_workdir) > - cfg = open("%s/hdd/boot/loader/entries/boot.conf" % > cr_workdir, "w") > - cfg.write(boot_conf) > - cfg.close() > + if source_params.get('create-unified-kernel-image') != > "true": > + logger.debug("Writing systemd-boot config " > + "%s/hdd/boot/loader/entries/boot.conf", > cr_workdir) > + cfg = open("%s/hdd/boot/loader/entries/boot.conf" % > cr_workdir, "w") > + cfg.write(boot_conf) > + cfg.close() > > > @classmethod > @@ -288,9 +291,60 @@ class BootimgEFIPlugin(SourcePlugin): > kernel = "%s-%s.bin" % \ > (get_bitbake_var("KERNEL_IMAGETYPE"), > get_bitbake_var("INITRAMFS_LINK_NAME")) > - install_cmd = "install -m 0644 %s/%s %s/%s" % \ > - (staging_kernel_dir, kernel, hdddir, kernel) > - exec_cmd(install_cmd) > + if source_params.get('create-unified-kernel-image') == > "true": > + initrd = source_params.get('initrd') > + if not initrd: > + raise WicError("initrd= must be specified when > create-unified-kernel-image=true, exiting") + > + deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") > + efi_stub = glob("%s/%s" % (deploy_dir, > "linux*.efi.stub")) > + if len(efi_stub) == 0: > + raise WicError("Unified Kernel Image EFI stub not > found, exiting") > + efi_stub = efi_stub[0] > + > + with tempfile.TemporaryDirectory() as tmp_dir: > + label = source_params.get('label') > + label_conf = "root=%s" % creator.rootdev > + if label: > + label_conf = "LABEL=%s" % label > + > + bootloader = creator.ks.bootloader > + cmdline = open("%s/cmdline" % tmp_dir, "w") > + cmdline.write("%s %s" % (label_conf, > bootloader.append)) > + cmdline.close() > + > + initrds = initrd.split(';') > + initrd = open("%s/initrd" % tmp_dir, "wb") > + for f in initrds: > + with open("%s/%s" % (deploy_dir, f), 'rb') as > in_file: > + shutil.copyfileobj(in_file, initrd) > + initrd.close() > + > + # Searched by systemd-boot: > + # > https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images > + install_cmd = "install -d %s/EFI/Linux" % hdddir > + exec_cmd(install_cmd) > + > + staging_dir_host = > get_bitbake_var("STAGING_DIR_HOST") + > + # > https://www.freedesktop.org/software/systemd/man/systemd-stub.html > + objcopy_cmd = "objcopy \ > + --add-section .osrel=%s --change-section-vma > .osrel=0x20000 \ > + --add-section .cmdline=%s --change-section-vma > .cmdline=0x30000 \ > + --add-section .linux=%s --change-section-vma > .linux=0x2000000 \ > + --add-section .initrd=%s --change-section-vma > .initrd=0x3000000 \ > + %s %s" % \ > + ("%s/usr/lib/os-release" % staging_dir_host, > + cmdline.name, > + "%s/%s" % (staging_kernel_dir, kernel), > + initrd.name, > + efi_stub, > + "%s/EFI/Linux/linux.efi" % hdddir) > + exec_cmd(objcopy_cmd) > + else: > + install_cmd = "install -m 0644 %s/%s %s/%s" % \ > + (staging_kernel_dir, kernel, hdddir, kernel) > + exec_cmd(install_cmd) > > if get_bitbake_var("IMAGE_EFI_BOOT_FILES"): > for src_path, dst_path in cls.install_task: > diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py > b/scripts/lib/wic/plugins/source/bootimg-pcbios.py index > f2639e700493..32e47f183146 100644 --- > a/scripts/lib/wic/plugins/source/bootimg-pcbios.py +++ > b/scripts/lib/wic/plugins/source/bootimg-pcbios.py @@ -186,8 +186,10 > @@ class BootimgPcbiosPlugin(SourcePlugin): # dosfs image, created by > mkdosfs bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno) > > - dosfs_cmd = "mkdosfs -n boot -i %s -S 512 -C %s %d" % \ > - (part.fsuuid, bootimg, blocks) > + label = part.label if part.label else "boot" > + > + dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \ > + (label, part.fsuuid, bootimg, blocks) > exec_native_cmd(dosfs_cmd, native_sysroot) > > mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir) > diff --git a/scripts/lib/wic/plugins/source/rawcopy.py > b/scripts/lib/wic/plugins/source/rawcopy.py index > 3c4997d8ba5e..7c90cd3cf82b 100644 --- > a/scripts/lib/wic/plugins/source/rawcopy.py +++ > b/scripts/lib/wic/plugins/source/rawcopy.py @@ -4,6 +4,8 @@ > > import logging > import os > +import signal > +import subprocess > > from wic import WicError > from wic.pluginbase import SourcePlugin > @@ -29,15 +31,34 @@ class RawCopyPlugin(SourcePlugin): > cmd = 'btrfs filesystem label %s %s' % (dst, label) > elif fstype == 'swap': > cmd = 'mkswap -L %s %s' % (label, dst) > - elif fstype == 'squashfs': > - raise WicError("It's not possible to update a squashfs " > - "filesystem label '%s'" % (label)) > + elif fstype in ('squashfs', 'erofs'): > + raise WicError("It's not possible to update a %s " > + "filesystem label '%s'" % (fstype, label)) > else: > raise WicError("Cannot update filesystem label: " > "Unknown fstype: '%s'" % (fstype)) > > exec_cmd(cmd) > > + @staticmethod > + def do_image_uncompression(src, dst, workdir): > + def subprocess_setup(): > + # Python installs a SIGPIPE handler by default. This is > usually not what > + # non-Python subprocesses expect. > + # SIGPIPE errors are known issues with gzip/bash > + signal.signal(signal.SIGPIPE, signal.SIG_DFL) > + > + extension = os.path.splitext(src)[1] > + decompressor = { > + ".bz2": "bzip2", > + ".gz": "gzip", > + ".xz": "xz" > + }.get(extension) > + if not decompressor: > + raise WicError("Not supported compressor filename > extension: %s" % extension) > + cmd = "%s -dc %s > %s" % (decompressor, src, dst) > + subprocess.call(cmd, preexec_fn=subprocess_setup, > shell=True, cwd=workdir) + > @classmethod > def do_prepare_partition(cls, part, source_params, cr, > cr_workdir, oe_builddir, bootimg_dir, kernel_dir, > @@ -56,7 +77,13 @@ class RawCopyPlugin(SourcePlugin): > if 'file' not in source_params: > raise WicError("No file specified") > > - src = os.path.join(kernel_dir, source_params['file']) > + if 'unpack' in source_params: > + img = os.path.join(kernel_dir, source_params['file']) > + src = os.path.join(cr_workdir, > os.path.splitext(source_params['file'])[0]) > + RawCopyPlugin.do_image_uncompression(img, src, > cr_workdir) > + else: > + src = os.path.join(kernel_dir, source_params['file']) > + > dst = os.path.join(cr_workdir, "%s.%s" % > (os.path.basename(source_params['file']), part.lineno)) > if not os.path.exists(os.path.dirname(dst)): > diff --git a/scripts/lib/wic/plugins/source/rootfs.py > b/scripts/lib/wic/plugins/source/rootfs.py index > 96d940a91d61..2e34e715ca73 100644 --- > a/scripts/lib/wic/plugins/source/rootfs.py +++ > b/scripts/lib/wic/plugins/source/rootfs.py @@ -218,7 +218,7 @@ class > RootfsPlugin(SourcePlugin): # Update part.has_fstab here as fstab may > have been added or # removed by the above modifications. > part.has_fstab = os.path.exists(os.path.join(new_rootfs, > "etc/fstab")) > - if part.update_fstab_in_rootfs and part.has_fstab: > + if part.update_fstab_in_rootfs and part.has_fstab and > not part.no_fstab_update: fstab_path = os.path.join(new_rootfs, > "etc/fstab") # Assume that fstab should always be owned by root with > fixed permissions install_cmd = "install -m 0644 %s %s" % > (part.updated_fstab_path, fstab_path) diff --git a/scripts/wic > b/scripts/wic index a741aed364bf..aee63a45aacf 100755 > --- a/scripts/wic > +++ b/scripts/wic > @@ -22,9 +22,9 @@ import sys > import argparse > import logging > import subprocess > +import shutil > > from collections import namedtuple > -from distutils import spawn > > # External modules > scripts_path = os.path.dirname(os.path.realpath(__file__)) > @@ -47,7 +47,7 @@ if os.environ.get('SDKTARGETSYSROOT'): > break > sdkroot = os.path.dirname(sdkroot) > > -bitbake_exe = spawn.find_executable('bitbake') > +bitbake_exe = shutil.which('bitbake') > if bitbake_exe: > bitbake_path = scriptpath.add_bitbake_lib_path() > import bb > @@ -159,6 +159,9 @@ def wic_create_subcommand(options, usage_str): > "(Use -e/--image-name to specify it)") > native_sysroot = options.native_sysroot > > + if options.kernel_dir: > + kernel_dir = options.kernel_dir > + > if not options.vars_dir and (not native_sysroot or not > os.path.isdir(native_sysroot)): logger.info("Building wic-tools...\n") > subprocess.check_call(["bitbake", "wic-tools"]) > @@ -346,6 +349,8 @@ def wic_init_parser_create(subparser): > default=False, help="output debug information") > subparser.add_argument("-i", "--imager", dest="imager", > default="direct", help="the wic imager plugin") > + subparser.add_argument("--extra-space", type=int, > dest="extra_space", > + default=0, help="additional free disk space to > add to the image") return > >
diff --git a/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py b/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py index 5ba0777ad244..a24e04f309da 100644 --- a/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py +++ b/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py @@ -12,6 +12,7 @@ import logging import os +import tempfile import shutil import re @@ -129,12 +130,13 @@ class BootimgEFIPlugin(SourcePlugin): bootloader = creator.ks.bootloader loader_conf = "" - loader_conf += "default boot\n" + if source_params.get('create-unified-kernel-image') != "true": + loader_conf += "default boot\n" loader_conf += "timeout %d\n" % bootloader.timeout initrd = source_params.get('initrd') - if initrd: + if initrd and source_params.get('create-unified-kernel-image') != "true": # obviously we need to have a common common deploy var bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") if not bootimg_dir: @@ -195,11 +197,12 @@ class BootimgEFIPlugin(SourcePlugin): for rd in initrds: boot_conf += "initrd /%s\n" % rd - logger.debug("Writing systemd-boot config " - "%s/hdd/boot/loader/entries/boot.conf", cr_workdir) - cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") - cfg.write(boot_conf) - cfg.close() + if source_params.get('create-unified-kernel-image') != "true": + logger.debug("Writing systemd-boot config " + "%s/hdd/boot/loader/entries/boot.conf", cr_workdir) + cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") + cfg.write(boot_conf) + cfg.close() @classmethod @@ -300,11 +303,63 @@ class BootimgEFIPlugin(SourcePlugin): kernel = "%s-%s.bin" % \ (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) - install_cmd = "install -m 0644 %s/%s %s/%s" % \ - (staging_kernel_dir, kernel, hdddir, kernel) + if source_params.get('create-unified-kernel-image') == "true": + initrd = source_params.get('initrd') + if not initrd: + raise WicError("initrd= must be specified when create-unified-kernel-image=true, exiting") + + deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + efi_stub = glob("%s/%s" % (deploy_dir, "linux*.efi.stub")) + if len(efi_stub) == 0: + raise WicError("Unified Kernel Image EFI stub not found, exiting") + efi_stub = efi_stub[0] + + with tempfile.TemporaryDirectory() as tmp_dir: + label = source_params.get('label') + label_conf = "root=%s" % creator.rootdev + if label: + label_conf = "LABEL=%s" % label + + bootloader = creator.ks.bootloader + cmdline = open("%s/cmdline" % tmp_dir, "w") + cmdline.write("%s %s" % (label_conf, bootloader.append)) + cmdline.close() + + initrds = initrd.split(';') + initrd = open("%s/initrd" % tmp_dir, "wb") + for f in initrds: + with open("%s/%s" % (deploy_dir, f), 'rb') as in_file: + shutil.copyfileobj(in_file, initrd) + initrd.close() + + # Searched by systemd-boot: + # https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images + install_cmd = "install -d %s/EFI/Linux" % hdddir + exec_cmd(install_cmd) + + staging_dir_host = get_bitbake_var("STAGING_DIR_HOST") + + # https://www.freedesktop.org/software/systemd/man/systemd-stub.html + objcopy_cmd = "objcopy \ + --add-section .osrel=%s --change-section-vma .osrel=0x20000 \ + --add-section .cmdline=%s --change-section-vma .cmdline=0x30000 \ + --add-section .linux=%s --change-section-vma .linux=0x2000000 \ + --add-section .initrd=%s --change-section-vma .initrd=0x3000000 \ + %s %s" % \ + ("%s/usr/lib/os-release" % staging_dir_host, + cmdline.name, + "%s/%s" % (staging_kernel_dir, kernel), + initrd.name, + efi_stub, + "%s/EFI/Linux/linux.efi" % hdddir) + exec_cmd(objcopy_cmd) + else: + install_cmd = "install -m 0644 %s/%s %s/%s" % \ + (staging_kernel_dir, kernel, hdddir, kernel) + + install_cmd = isar_populate_boot_cmd(rootfs_dir['ROOTFS_DIR'], hdddir) + exec_cmd(install_cmd) - install_cmd = isar_populate_boot_cmd(rootfs_dir['ROOTFS_DIR'], hdddir) - exec_cmd(install_cmd) if get_bitbake_var("IMAGE_EFI_BOOT_FILES"): for src_path, dst_path in cls.install_task: diff --git a/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py b/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py index 9136d4f215ca..7fb0f2a87394 100644 --- a/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py +++ b/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py @@ -213,8 +213,10 @@ class BootimgPcbiosIsarPlugin(SourcePlugin): # dosfs image, created by mkdosfs bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno) - dosfs_cmd = "mkdosfs -n boot -i %s -S 512 -C %s %d" % \ - (part.fsuuid, bootimg, blocks) + label = part.label if part.label else "boot" + + dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \ + (label, part.fsuuid, bootimg, blocks) exec_native_cmd(dosfs_cmd, native_sysroot) mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir) diff --git a/scripts/lib/wic/canned-wks/common.wks.inc b/scripts/lib/wic/canned-wks/common.wks.inc index 4fd29fa8c119..89880b417b6e 100644 --- a/scripts/lib/wic/canned-wks/common.wks.inc +++ b/scripts/lib/wic/canned-wks/common.wks.inc @@ -1,3 +1,3 @@ # This file is included into 3 canned wks files from this directory part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024 -part / --source rootfs --use-uuid --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024 +part / --source rootfs --use-uuid --fstype=ext4 --label platform --align 1024 diff --git a/scripts/lib/wic/canned-wks/directdisk-gpt.wks b/scripts/lib/wic/canned-wks/directdisk-gpt.wks index cf16c0c30bbd..8d7d8de6ea7e 100644 --- a/scripts/lib/wic/canned-wks/directdisk-gpt.wks +++ b/scripts/lib/wic/canned-wks/directdisk-gpt.wks @@ -4,7 +4,7 @@ part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024 -part / --source rootfs --ondisk sda --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024 --use-uuid +part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid bootloader --ptable gpt --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8" diff --git a/scripts/lib/wic/canned-wks/mkefidisk.wks b/scripts/lib/wic/canned-wks/mkefidisk.wks index d1878e23e5a3..9f534fe18471 100644 --- a/scripts/lib/wic/canned-wks/mkefidisk.wks +++ b/scripts/lib/wic/canned-wks/mkefidisk.wks @@ -4,7 +4,7 @@ part /boot --source bootimg-efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024 -part / --source rootfs --ondisk sda --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024 --use-uuid +part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid part swap --ondisk sda --size 44 --label swap1 --fstype=swap diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py index 018815b96688..674ccfc24418 100644 --- a/scripts/lib/wic/engine.py +++ b/scripts/lib/wic/engine.py @@ -19,10 +19,10 @@ import os import tempfile import json import subprocess +import shutil import re from collections import namedtuple, OrderedDict -from distutils.spawn import find_executable from wic import WicError from wic.filemap import sparse_copy @@ -245,7 +245,7 @@ class Disk: for path in pathlist.split(':'): self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) - self.parted = find_executable("parted", self.paths) + self.parted = shutil.which("parted", path=self.paths) if not self.parted: raise WicError("Can't find executable parted") @@ -283,7 +283,7 @@ class Disk: "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"): aname = "_%s" % name if aname not in self.__dict__: - setattr(self, aname, find_executable(name, self.paths)) + setattr(self, aname, shutil.which(name, path=self.paths)) if aname not in self.__dict__ or self.__dict__[aname] is None: raise WicError("Can't find executable '{}'".format(name)) return self.__dict__[aname] diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py index bd3a2b97dfaf..4ff7470a6a79 100644 --- a/scripts/lib/wic/help.py +++ b/scripts/lib/wic/help.py @@ -637,7 +637,7 @@ DESCRIPTION oe-core: directdisk.bbclass and mkefidisk.sh. The difference between wic and those examples is that with wic the functionality of those scripts is implemented by a general-purpose partitioning - 'language' based on Redhat kickstart syntax). + 'language' based on Red Hat kickstart syntax). The initial motivation and design considerations that lead to the current tool are described exhaustively in Yocto Bug #3847 @@ -840,8 +840,8 @@ DESCRIPTION meanings. The commands are based on the Fedora kickstart documentation but with modifications to reflect wic capabilities. - http://fedoraproject.org/wiki/Anaconda/Kickstart#part_or_partition - http://fedoraproject.org/wiki/Anaconda/Kickstart#bootloader + https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#part-or-partition + https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#bootloader Commands @@ -930,6 +930,7 @@ DESCRIPTION ext4 btrfs squashfs + erofs swap --fsoptions: Specifies a free-form string of options to be @@ -990,6 +991,9 @@ DESCRIPTION multiple partitions and we want to keep the right permissions and usernames in all the partitions. + --no-fstab-update: This option is specific to wic. It does not update the + '/etc/fstab' stock file for the given partition. + --extra-space: This option is specific to wic. It adds extra space after the space filled by the content of the partition. The final size can go diff --git a/scripts/lib/wic/ksparser.py b/scripts/lib/wic/ksparser.py index 3eb669da39ca..0df9eb0d057d 100644 --- a/scripts/lib/wic/ksparser.py +++ b/scripts/lib/wic/ksparser.py @@ -157,7 +157,8 @@ class KickStart(): part.add_argument('--fsoptions', dest='fsopts') part.add_argument('--fstype', default='vfat', choices=('ext2', 'ext3', 'ext4', 'btrfs', - 'squashfs', 'vfat', 'msdos', 'swap')) + 'squashfs', 'vfat', 'msdos', 'erofs', + 'swap')) part.add_argument('--mkfs-extraopts', default='') part.add_argument('--label') part.add_argument('--use-label', action='store_true') @@ -184,6 +185,7 @@ class KickStart(): part.add_argument('--use-uuid', action='store_true') part.add_argument('--uuid') part.add_argument('--fsuuid') + part.add_argument('--no-fstab-update', action='store_true') bootloader = subparsers.add_parser('bootloader') bootloader.add_argument('--append') @@ -229,6 +231,10 @@ class KickStart(): err = "%s:%d: SquashFS does not support LABEL" \ % (confpath, lineno) raise KickStartError(err) + # erofs does not support filesystem labels + if parsed.fstype == 'erofs' and parsed.label: + err = "%s:%d: erofs does not support LABEL" % (confpath, lineno) + raise KickStartError(err) if parsed.fstype == 'msdos' or parsed.fstype == 'vfat': if parsed.fsuuid: if parsed.fsuuid.upper().startswith('0X'): diff --git a/scripts/lib/wic/misc.py b/scripts/lib/wic/misc.py index 57c042c503e6..3e118229960b 100644 --- a/scripts/lib/wic/misc.py +++ b/scripts/lib/wic/misc.py @@ -16,9 +16,9 @@ import logging import os import re import subprocess +import shutil from collections import defaultdict -from distutils import spawn from wic import WicError @@ -122,7 +122,7 @@ def find_executable(cmd, paths): if provided and "%s-native" % recipe in provided: return True - return spawn.find_executable(cmd, paths) + return shutil.which(cmd, path=paths) def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): """ diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py index 76d144d12d30..09e491dd494d 100644 --- a/scripts/lib/wic/partition.py +++ b/scripts/lib/wic/partition.py @@ -54,6 +54,7 @@ class Partition(): self.uuid = args.uuid self.fsuuid = args.fsuuid self.type = args.type + self.no_fstab_update = args.no_fstab_update self.updated_fstab_path = None self.has_fstab = False self.update_fstab_in_rootfs = False @@ -104,7 +105,7 @@ class Partition(): extra_blocks = self.extra_space rootfs_size = actual_rootfs_size + extra_blocks - rootfs_size *= self.overhead_factor + rootfs_size = int(rootfs_size * self.overhead_factor) logger.debug("Added %d extra blocks to %s to get to %d total blocks", extra_blocks, self.mountpoint, rootfs_size) @@ -141,9 +142,9 @@ class Partition(): native_sysroot) self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype) else: - if self.fstype == 'squashfs': - raise WicError("It's not possible to create empty squashfs " - "partition '%s'" % (self.mountpoint)) + if self.fstype in ('squashfs', 'erofs'): + raise WicError("It's not possible to create empty %s " + "partition '%s'" % (self.fstype, self.mountpoint)) rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label, self.lineno, self.fstype) @@ -170,7 +171,7 @@ class Partition(): # Split sourceparams string of the form key1=val1[,key2=val2,...] # into a dict. Also accepts valueless keys i.e. without = splitted = self.sourceparams.split(',') - srcparams_dict = dict(par.split('=', 1) for par in splitted if par) + srcparams_dict = dict((par.split('=', 1) + [None])[:2] for par in splitted if par) plugin = PluginMgr.get_plugins('source')[self.source] plugin.do_configure_partition(self, srcparams_dict, creator, @@ -286,7 +287,7 @@ class Partition(): (self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir) exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) - if self.updated_fstab_path and self.has_fstab: + if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update: debugfs_script_path = os.path.join(cr_workdir, "debugfs_script") with open(debugfs_script_path, "w") as f: f.write("cd etc\n") @@ -350,7 +351,7 @@ class Partition(): mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir) exec_native_cmd(mcopy_cmd, native_sysroot) - if self.updated_fstab_path and self.has_fstab: + if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update: mcopy_cmd = "mcopy -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path) exec_native_cmd(mcopy_cmd, native_sysroot) @@ -369,6 +370,16 @@ class Partition(): (rootfs_dir, rootfs, extraopts) exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo) + def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, + native_sysroot, pseudo): + """ + Prepare content for a erofs rootfs partition. + """ + extraopts = self.mkfs_extraopts or '' + erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \ + (extraopts, self.fsuuid, rootfs, rootfs_dir) + exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo) + def prepare_empty_partition_ext(self, rootfs, oe_builddir, native_sysroot): """ diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py index d9b4e57747e7..b64568339b1b 100644 --- a/scripts/lib/wic/pluginbase.py +++ b/scripts/lib/wic/pluginbase.py @@ -9,9 +9,11 @@ __all__ = ['ImagerPlugin', 'SourcePlugin'] import os import logging +import types from collections import defaultdict -from importlib.machinery import SourceFileLoader +import importlib +import importlib.util from wic import WicError from wic.misc import get_bitbake_var @@ -54,7 +56,9 @@ class PluginMgr: mname = fname[:-3] mpath = os.path.join(ppath, fname) logger.debug("loading plugin module %s", mpath) - SourceFileLoader(mname, mpath).load_module() + spec = importlib.util.spec_from_file_location(mname, mpath) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) return PLUGINS.get(ptype) diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py index ea709e8c545d..35fff7c10242 100644 --- a/scripts/lib/wic/plugins/imager/direct.py +++ b/scripts/lib/wic/plugins/imager/direct.py @@ -77,7 +77,8 @@ class DirectPlugin(ImagerPlugin): image_path = self._full_path(self.workdir, self.parts[0].disk, "direct") self._image = PartitionedImage(image_path, self.ptable_format, - self.parts, self.native_sysroot) + self.parts, self.native_sysroot, + options.extra_space) def setup_workdir(self, workdir): if workdir: @@ -116,7 +117,7 @@ class DirectPlugin(ImagerPlugin): updated = False for part in self.parts: if not part.realnum or not part.mountpoint \ - or part.mountpoint == "/": + or part.mountpoint == "/" or not part.mountpoint.startswith('/'): continue if part.use_uuid: @@ -258,6 +259,8 @@ class DirectPlugin(ImagerPlugin): if part.mountpoint == "/": if part.uuid: return "PARTUUID=%s" % part.uuid + elif part.label: + return "PARTLABEL=%s" % part.label else: suffix = 'p' if part.disk.startswith('mmcblk') else '' return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) @@ -293,7 +296,7 @@ class PartitionedImage(): Partitioned image in a file. """ - def __init__(self, path, ptable_format, partitions, native_sysroot=None): + def __init__(self, path, ptable_format, partitions, native_sysroot=None, extra_space=0): self.path = path # Path to the image file self.numpart = 0 # Number of allocated partitions self.realpart = 0 # Number of partitions in the partition table @@ -314,6 +317,7 @@ class PartitionedImage(): self.sector_size = SECTOR_SIZE self.native_sysroot = native_sysroot num_real_partitions = len([p for p in self.partitions if not p.no_table]) + self.extra_space = extra_space # calculate the real partition number, accounting for partitions not # in the partition table and logical partitions @@ -483,6 +487,7 @@ class PartitionedImage(): self.min_size += GPT_OVERHEAD self.min_size *= self.sector_size + self.min_size += self.extra_space def _create_partition(self, device, parttype, fstype, start, size): """ Create a partition on an image described by the 'device' object. """ diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg-efi.py index cdc72543c200..0391aebdc840 100644 --- a/scripts/lib/wic/plugins/source/bootimg-efi.py +++ b/scripts/lib/wic/plugins/source/bootimg-efi.py @@ -12,6 +12,7 @@ import logging import os +import tempfile import shutil import re @@ -119,12 +120,13 @@ class BootimgEFIPlugin(SourcePlugin): bootloader = creator.ks.bootloader loader_conf = "" - loader_conf += "default boot\n" + if source_params.get('create-unified-kernel-image') != "true": + loader_conf += "default boot\n" loader_conf += "timeout %d\n" % bootloader.timeout initrd = source_params.get('initrd') - if initrd: + if initrd and source_params.get('create-unified-kernel-image') != "true": # obviously we need to have a common common deploy var bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") if not bootimg_dir: @@ -183,11 +185,12 @@ class BootimgEFIPlugin(SourcePlugin): for rd in initrds: boot_conf += "initrd /%s\n" % rd - logger.debug("Writing systemd-boot config " - "%s/hdd/boot/loader/entries/boot.conf", cr_workdir) - cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") - cfg.write(boot_conf) - cfg.close() + if source_params.get('create-unified-kernel-image') != "true": + logger.debug("Writing systemd-boot config " + "%s/hdd/boot/loader/entries/boot.conf", cr_workdir) + cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") + cfg.write(boot_conf) + cfg.close() @classmethod @@ -288,9 +291,60 @@ class BootimgEFIPlugin(SourcePlugin): kernel = "%s-%s.bin" % \ (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) - install_cmd = "install -m 0644 %s/%s %s/%s" % \ - (staging_kernel_dir, kernel, hdddir, kernel) - exec_cmd(install_cmd) + if source_params.get('create-unified-kernel-image') == "true": + initrd = source_params.get('initrd') + if not initrd: + raise WicError("initrd= must be specified when create-unified-kernel-image=true, exiting") + + deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE") + efi_stub = glob("%s/%s" % (deploy_dir, "linux*.efi.stub")) + if len(efi_stub) == 0: + raise WicError("Unified Kernel Image EFI stub not found, exiting") + efi_stub = efi_stub[0] + + with tempfile.TemporaryDirectory() as tmp_dir: + label = source_params.get('label') + label_conf = "root=%s" % creator.rootdev + if label: + label_conf = "LABEL=%s" % label + + bootloader = creator.ks.bootloader + cmdline = open("%s/cmdline" % tmp_dir, "w") + cmdline.write("%s %s" % (label_conf, bootloader.append)) + cmdline.close() + + initrds = initrd.split(';') + initrd = open("%s/initrd" % tmp_dir, "wb") + for f in initrds: + with open("%s/%s" % (deploy_dir, f), 'rb') as in_file: + shutil.copyfileobj(in_file, initrd) + initrd.close() + + # Searched by systemd-boot: + # https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images + install_cmd = "install -d %s/EFI/Linux" % hdddir + exec_cmd(install_cmd) + + staging_dir_host = get_bitbake_var("STAGING_DIR_HOST") + + # https://www.freedesktop.org/software/systemd/man/systemd-stub.html + objcopy_cmd = "objcopy \ + --add-section .osrel=%s --change-section-vma .osrel=0x20000 \ + --add-section .cmdline=%s --change-section-vma .cmdline=0x30000 \ + --add-section .linux=%s --change-section-vma .linux=0x2000000 \ + --add-section .initrd=%s --change-section-vma .initrd=0x3000000 \ + %s %s" % \ + ("%s/usr/lib/os-release" % staging_dir_host, + cmdline.name, + "%s/%s" % (staging_kernel_dir, kernel), + initrd.name, + efi_stub, + "%s/EFI/Linux/linux.efi" % hdddir) + exec_cmd(objcopy_cmd) + else: + install_cmd = "install -m 0644 %s/%s %s/%s" % \ + (staging_kernel_dir, kernel, hdddir, kernel) + exec_cmd(install_cmd) if get_bitbake_var("IMAGE_EFI_BOOT_FILES"): for src_path, dst_path in cls.install_task: diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/scripts/lib/wic/plugins/source/bootimg-pcbios.py index f2639e700493..32e47f183146 100644 --- a/scripts/lib/wic/plugins/source/bootimg-pcbios.py +++ b/scripts/lib/wic/plugins/source/bootimg-pcbios.py @@ -186,8 +186,10 @@ class BootimgPcbiosPlugin(SourcePlugin): # dosfs image, created by mkdosfs bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno) - dosfs_cmd = "mkdosfs -n boot -i %s -S 512 -C %s %d" % \ - (part.fsuuid, bootimg, blocks) + label = part.label if part.label else "boot" + + dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \ + (label, part.fsuuid, bootimg, blocks) exec_native_cmd(dosfs_cmd, native_sysroot) mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir) diff --git a/scripts/lib/wic/plugins/source/rawcopy.py b/scripts/lib/wic/plugins/source/rawcopy.py index 3c4997d8ba5e..7c90cd3cf82b 100644 --- a/scripts/lib/wic/plugins/source/rawcopy.py +++ b/scripts/lib/wic/plugins/source/rawcopy.py @@ -4,6 +4,8 @@ import logging import os +import signal +import subprocess from wic import WicError from wic.pluginbase import SourcePlugin @@ -29,15 +31,34 @@ class RawCopyPlugin(SourcePlugin): cmd = 'btrfs filesystem label %s %s' % (dst, label) elif fstype == 'swap': cmd = 'mkswap -L %s %s' % (label, dst) - elif fstype == 'squashfs': - raise WicError("It's not possible to update a squashfs " - "filesystem label '%s'" % (label)) + elif fstype in ('squashfs', 'erofs'): + raise WicError("It's not possible to update a %s " + "filesystem label '%s'" % (fstype, label)) else: raise WicError("Cannot update filesystem label: " "Unknown fstype: '%s'" % (fstype)) exec_cmd(cmd) + @staticmethod + def do_image_uncompression(src, dst, workdir): + def subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + # SIGPIPE errors are known issues with gzip/bash + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + extension = os.path.splitext(src)[1] + decompressor = { + ".bz2": "bzip2", + ".gz": "gzip", + ".xz": "xz" + }.get(extension) + if not decompressor: + raise WicError("Not supported compressor filename extension: %s" % extension) + cmd = "%s -dc %s > %s" % (decompressor, src, dst) + subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=workdir) + @classmethod def do_prepare_partition(cls, part, source_params, cr, cr_workdir, oe_builddir, bootimg_dir, kernel_dir, @@ -56,7 +77,13 @@ class RawCopyPlugin(SourcePlugin): if 'file' not in source_params: raise WicError("No file specified") - src = os.path.join(kernel_dir, source_params['file']) + if 'unpack' in source_params: + img = os.path.join(kernel_dir, source_params['file']) + src = os.path.join(cr_workdir, os.path.splitext(source_params['file'])[0]) + RawCopyPlugin.do_image_uncompression(img, src, cr_workdir) + else: + src = os.path.join(kernel_dir, source_params['file']) + dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno)) if not os.path.exists(os.path.dirname(dst)): diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py index 96d940a91d61..2e34e715ca73 100644 --- a/scripts/lib/wic/plugins/source/rootfs.py +++ b/scripts/lib/wic/plugins/source/rootfs.py @@ -218,7 +218,7 @@ class RootfsPlugin(SourcePlugin): # Update part.has_fstab here as fstab may have been added or # removed by the above modifications. part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab")) - if part.update_fstab_in_rootfs and part.has_fstab: + if part.update_fstab_in_rootfs and part.has_fstab and not part.no_fstab_update: fstab_path = os.path.join(new_rootfs, "etc/fstab") # Assume that fstab should always be owned by root with fixed permissions install_cmd = "install -m 0644 %s %s" % (part.updated_fstab_path, fstab_path) diff --git a/scripts/wic b/scripts/wic index a741aed364bf..aee63a45aacf 100755 --- a/scripts/wic +++ b/scripts/wic @@ -22,9 +22,9 @@ import sys import argparse import logging import subprocess +import shutil from collections import namedtuple -from distutils import spawn # External modules scripts_path = os.path.dirname(os.path.realpath(__file__)) @@ -47,7 +47,7 @@ if os.environ.get('SDKTARGETSYSROOT'): break sdkroot = os.path.dirname(sdkroot) -bitbake_exe = spawn.find_executable('bitbake') +bitbake_exe = shutil.which('bitbake') if bitbake_exe: bitbake_path = scriptpath.add_bitbake_lib_path() import bb @@ -159,6 +159,9 @@ def wic_create_subcommand(options, usage_str): "(Use -e/--image-name to specify it)") native_sysroot = options.native_sysroot + if options.kernel_dir: + kernel_dir = options.kernel_dir + if not options.vars_dir and (not native_sysroot or not os.path.isdir(native_sysroot)): logger.info("Building wic-tools...\n") subprocess.check_call(["bitbake", "wic-tools"]) @@ -346,6 +349,8 @@ def wic_init_parser_create(subparser): default=False, help="output debug information") subparser.add_argument("-i", "--imager", dest="imager", default="direct", help="the wic imager plugin") + subparser.add_argument("--extra-space", type=int, dest="extra_space", + default=0, help="additional free disk space to add to the image") return
Update to the latest wic from OE-core. OE-core Revision: 712552b5cc427d7be1258c45886de9b57f7272c9 Signed-off-by: Henning Schild <henning.schild@siemens.com> --- .../wic/plugins/source/bootimg-efi-isar.py | 77 ++++++++++++++++--- .../wic/plugins/source/bootimg-pcbios-isar.py | 6 +- scripts/lib/wic/canned-wks/common.wks.inc | 2 +- scripts/lib/wic/canned-wks/directdisk-gpt.wks | 2 +- scripts/lib/wic/canned-wks/mkefidisk.wks | 2 +- scripts/lib/wic/engine.py | 6 +- scripts/lib/wic/help.py | 10 ++- scripts/lib/wic/ksparser.py | 8 +- scripts/lib/wic/misc.py | 4 +- scripts/lib/wic/partition.py | 25 ++++-- scripts/lib/wic/pluginbase.py | 8 +- scripts/lib/wic/plugins/imager/direct.py | 11 ++- scripts/lib/wic/plugins/source/bootimg-efi.py | 74 +++++++++++++++--- .../lib/wic/plugins/source/bootimg-pcbios.py | 6 +- scripts/lib/wic/plugins/source/rawcopy.py | 35 ++++++++- scripts/lib/wic/plugins/source/rootfs.py | 2 +- scripts/wic | 9 ++- 17 files changed, 231 insertions(+), 56 deletions(-)