diff --git a/RECIPE-API-CHANGELOG.md b/RECIPE-API-CHANGELOG.md index 3798a9d1..42641d9a 100644 --- a/RECIPE-API-CHANGELOG.md +++ b/RECIPE-API-CHANGELOG.md @@ -1225,3 +1225,20 @@ the rootfs. This has been replaced by the `exclude-docs` rootfs feature. The When enabled, the package changelogs are now removed as well (copyright is still kept for legal reasons). + +### WIC runs on the build host, add imager_run_host + +WIC's Python driver now runs on the build host instead of inside the imaging +chroot. Only the image utilities (e.g. mkfs tools, mtools, bmaptool) still run +in the chroot. By that, the chroot distribution no longer needs a Python +interpreter that is compatible with BitBake, which is required to update +BitBake beyond Debian buster's Python version. + +Recipes can use the new `imager_run_host` instead of `imager_run` to run a +script on the host. It provides the same privilege handling and mounts and +sets `IMAGER_CHROOT` to the path of the prepared imaging chroot. + +Custom WIC plugins that start subprocesses directly must use `exec_cmd()` or +`exec_native_cmd()` from `wic.misc` to still run their tools in the chroot. +Native sysroot paths passed to such tools must be converted with +`get_native_path()`. The WIC work directory is now `${WORKDIR}/wic`. diff --git a/doc/technical_overview.md b/doc/technical_overview.md index 5d2e1b8a..d3d4e827 100644 --- a/doc/technical_overview.md +++ b/doc/technical_overview.md @@ -243,6 +243,23 @@ This process contains the following steps: 1.1. `IMAGE_CMD:ext4`: target filesystem is packed to extfs image. 1.2. `IMAGE_CMD:wic`: a bootable disk image gets created for the platform +WIC runs with the build host's Python interpreter. Image utilities and their +data remain in the imaging chroot, so older target distributions do not need +to provide a Python interpreter compatible with BitBake. + +`imager_run_host` prepares the same tool environment as `imager_run`, but +executes the supplied shell script outside the chroot. It still uses Isar's +privilege handling and, in rootless mode, the mapped user and mount namespaces. +`IMAGER_CHROOT` identifies the prepared chroot for the script. + +WIC uses `WIC_TOOL_RUNNER` to execute utility commands in that environment and +`WIC_TOOL_SYSROOT` to locate native tool data from the host. Native paths are +converted explicitly with `get_native_path` when constructing utility commands; +command text and non-path arguments are not rewritten. Rootfs copy helpers use +an optional command executor to route their utilities through the same runner, +without changing the default behavior for other callers. Plugins that launch +subprocesses directly must integrate with this runner to retain tool isolation. + ## 3.8 Cross-compilation Isar provides the possibility to cross-compile all or individual packages. If diff --git a/meta/classes-recipe/image-tools-extension.bbclass b/meta/classes-recipe/image-tools-extension.bbclass index 8f666444..9532fa47 100644 --- a/meta/classes-recipe/image-tools-extension.bbclass +++ b/meta/classes-recipe/image-tools-extension.bbclass @@ -33,6 +33,10 @@ imager_run() { imager_run_${ISAR_CHROOT_MODE} "$@" } +imager_run_host() { + IMAGER_HOST_RUN=1 imager_run "$@" +} + imager_run_schroot() { local_install="${@(d.getVar("INSTALL_%s" % d.getVar("BB_CURRENTTASK")) or '').strip()}" local_bom="${@(d.getVar("BOM_%s" % d.getVar("BB_CURRENTTASK")) or '').strip()}" @@ -85,7 +89,11 @@ EOAPT ${local_install}" fi - schroot -r -c ${session_id} "$@" + if [ "${IMAGER_HOST_RUN:-0}" = "1" ]; then + IMAGER_CHROOT="${schroot_dir}" run_privileged_heredoc "$@" + else + schroot -r -c ${session_id} "$@" + fi if [ -n "${local_bom}" ]; then schroot -r -c ${session_id} -d / -- \ @@ -198,7 +206,12 @@ EOF set -e mkdir -p ${ROOTFS_IMAGETOOLS}/${SCRIPTSDIR} ${@insert_isar_mounts(d, d.getVar('ROOTFS_IMAGETOOLS'), d.getVar('IMAGER_UNSHARE_MOUNTS'))} - chroot ${ROOTFS_IMAGETOOLS} "$@" <&3 + if [ "${IMAGER_HOST_RUN:-0}" = "1" ]; then + export IMAGER_CHROOT="${ROOTFS_IMAGETOOLS}" + "$@" <&3 + else + chroot ${ROOTFS_IMAGETOOLS} "$@" <&3 + fi EOF if [ -n "${local_bom}" ]; then diff --git a/meta/classes-recipe/imagetypes_wic.bbclass b/meta/classes-recipe/imagetypes_wic.bbclass index 6cf7bd8e..25ff3095 100644 --- a/meta/classes-recipe/imagetypes_wic.bbclass +++ b/meta/classes-recipe/imagetypes_wic.bbclass @@ -144,7 +144,23 @@ check_for_wic_warnings() { } do_image_wic[depends] += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-sbom', 'sbom-chroot:do_build', '', d)}" -do_image_wic[file-checksums] += "${WKS_FILE_CHECKSUM}" +do_image_wic[file-checksums] += " \ + ${WKS_FILE_CHECKSUM} \ + ${SCRIPTSDIR}/wic:True \ + ${SCRIPTSDIR}/wic-chroot:True \ + ${SCRIPTSDIR}/wic_fakeroot:True \ + ${SCRIPTSDIR}/lib/wic/misc.py:True \ + ${SCRIPTSDIR}/lib/wic/partition.py:True \ + ${SCRIPTSDIR}/lib/wic/plugins/imager/direct.py:True \ + ${SCRIPTSDIR}/lib/wic/plugins/source/bootimg-pcbios.py:True \ + ${SCRIPTSDIR}/lib/wic/plugins/source/isoimage-isohybrid.py:True \ + ${SCRIPTSDIR}/lib/wic/plugins/source/rawcopy.py:True \ + ${SCRIPTSDIR}/lib/wic/plugins/source/rootfs.py:True \ + ${LAYERDIR_core}/lib/oe/path.py:True \ + ${LAYERDIR_core}/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py:True \ + ${LAYERDIR_core}/scripts/lib/wic/plugins/source/isoimage-isohybrid-isar.py:True \ +" +do_image_wic[cleandirs] += "${WORKDIR}/wic" IMAGE_CMD:wic() { generate_wic_image check_for_wic_warnings @@ -162,9 +178,17 @@ generate_wic_image() { mkdir -p ${IMAGE_ROOTFS}/../pseudo touch ${IMAGE_ROOTFS}/../pseudo/files.db - imager_run -p -d ${PP_WORK} -u root <<'EOIMAGER' + imager_run_host <<'EOIMAGER' set -e + export WIC_TOOL_RUNNER="${SCRIPTSDIR}/wic-chroot" + export WIC_TOOL_SYSROOT="$IMAGER_CHROOT" + export TMPDIR="${WORKDIR}/wic/tmp" + mkdir -p "$TMPDIR" + cd "${WORKDIR}" + + "$WIC_TOOL_RUNNER" /bin/sh -s <<'EOTOOLS' + set -e # The python path is hard-coded as /usr/bin/python3-native/python3 in wic. Handle that. mkdir -p /usr/bin/python3-native/ if [ $(head -1 $(which bmaptool) | grep python3) ];then @@ -172,24 +196,27 @@ generate_wic_image() { else ln -fs /usr/bin/python2 /usr/bin/python3-native/python3 fi +EOTOOLS export PATH="${BITBAKEDIR}/bin:$PATH" "${SCRIPTSDIR}"/wic create "${WKS_FULL_PATH}" \ --vars "${STAGING_DIR}/${MACHINE}/imgdata/" \ - -o "/tmp/${IMAGE_FULLNAME}.wic/" \ + --native-sysroot "$IMAGER_CHROOT" \ + -o "${WORKDIR}/wic/" \ --bmap \ -e "${IMAGE_BASENAME}" ${WIC_CREATE_EXTRA_ARGS} - WIC_DIRECT=$(ls -t -1 /tmp/${IMAGE_FULLNAME}.wic/*.direct | head -1) - mv -f ${WIC_DIRECT} ${PP_DEPLOY}/${IMAGE_FULLNAME}.wic - mv -f ${WIC_DIRECT}.bmap ${PP_DEPLOY}/${IMAGE_FULLNAME}.wic.bmap + WIC_DIRECT=$(ls -t -1 ${WORKDIR}/wic/*.direct | head -1) + mv -f ${WIC_DIRECT} ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic + mv -f ${WIC_DIRECT}.bmap ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic.bmap # deploy partition files if requested (ending with .p) if [ "${WIC_DEPLOY_PARTITIONS}" -eq "1" ]; then # locate *.direct.p partition files - find "/tmp/${IMAGE_FULLNAME}.wic/" -type f -regextype sed -regex ".*\.direct.*\.p[0-9]\{1,\}" | while read f; do + find "${WORKDIR}/wic/" -type f -regextype sed \ + -regex ".*\.direct.*\.p[0-9]\{1,\}" | while read f; do suffix=$(basename $f | sed 's/.*\.direct\(.*\)/\1/') - mv -f ${f} ${PP_DEPLOY}/${IMAGE_FULLNAME}.wic${suffix} + mv -f ${f} ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.wic${suffix} done fi EOIMAGER diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py index 5d21cdcb..76f1b0cc 100644 --- a/meta/lib/oe/path.py +++ b/meta/lib/oe/path.py @@ -10,6 +10,8 @@ import shutil import subprocess import os.path +import bb + def join(*paths): """Like os.path.join but doesn't treat absolute RHS specially""" return os.path.normpath("/".join(paths)) @@ -85,18 +87,22 @@ def format_display(path, metadata): else: return rel -def copytree(src, dst): +def copytree(src, dst, command_runner=None): # We could use something like shutil.copytree here but it turns out to # to be slow. It takes twice as long copying to an empty directory. # If dst already has contents performance can be 15 time slower # This way we also preserve hardlinks between files in the tree. + if command_runner is None: + command_runner = subprocess.check_output bb.utils.mkdirhier(dst) cmd = "tar --xattrs --xattrs-include='*' -cf - -S -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst) - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + command_runner(cmd, shell=True, stderr=subprocess.STDOUT) -def copyhardlinktree(src, dst): +def copyhardlinktree(src, dst, command_runner=None): """Make a tree of hard links when possible, otherwise copy.""" + if command_runner is None: + command_runner = subprocess.check_output bb.utils.mkdirhier(dst) if os.path.isdir(src) and not len(os.listdir(src)): return @@ -120,7 +126,7 @@ def copyhardlinktree(src, dst): # Need to copy directories only with tar first since cp will error if two # writers try and create a directory at the same time cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -S -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst) - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + command_runner(cmd, shell=True, stderr=subprocess.STDOUT) source = '' if os.path.isdir(src): if len(glob.glob('%s/.??*' % src)) > 0: @@ -132,9 +138,9 @@ def copyhardlinktree(src, dst): source = src s_dir = os.getcwd() cmd = 'cp -afl --preserve=xattr %s %s' % (source, os.path.realpath(dst)) - subprocess.check_output(cmd, shell=True, cwd=s_dir, stderr=subprocess.STDOUT) + command_runner(cmd, shell=True, cwd=s_dir, stderr=subprocess.STDOUT) else: - copytree(src, dst) + copytree(src, dst, command_runner=command_runner) def copyhardlink(src, dst): """Make a hard link when possible, otherwise copy.""" diff --git a/meta/scripts/lib/wic/plugins/source/bootimg-biosplusefi-isar.py b/meta/scripts/lib/wic/plugins/source/bootimg-biosplusefi-isar.py index fb6b7fcc..df37f97b 100644 --- a/meta/scripts/lib/wic/plugins/source/bootimg-biosplusefi-isar.py +++ b/meta/scripts/lib/wic/plugins/source/bootimg-biosplusefi-isar.py @@ -18,6 +18,7 @@ # AUTHORS # William Bourque +import os import types from wic.pluginbase import SourcePlugin diff --git a/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py b/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py index a2592986..7fb5f448 100644 --- a/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py +++ b/meta/scripts/lib/wic/plugins/source/bootimg-efi-isar.py @@ -521,7 +521,13 @@ class BootimgEFIPlugin(SourcePlugin): kernel_dir_orig = kernel_dir kernel_dir = "/usr/lib/systemd/boot/efi/" - for mod in [x for x in os.listdir(kernel_dir) if x.startswith("systemd-")]: + efi_dir = kernel_dir + if os.environ.get('WIC_TOOL_RUNNER'): + tool_sysroot = os.environ.get('WIC_TOOL_SYSROOT') + if tool_sysroot: + efi_dir = os.path.join(tool_sysroot, kernel_dir.lstrip('/')) + + for mod in [x for x in os.listdir(efi_dir) if x.startswith("systemd-")]: target = mod[8:] if target.endswith('.signed'): target = target[:-7] diff --git a/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py b/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py index d5040b72..7569c7b1 100644 --- a/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py +++ b/meta/scripts/lib/wic/plugins/source/bootimg-pcbios-isar.py @@ -18,7 +18,7 @@ from wic import WicError from wic.engine import get_custom_config from wic.pluginbase import SourcePlugin from wic.misc import (exec_cmd, exec_native_cmd, - get_bitbake_var, BOOTDD_EXTRA_SPACE) + get_bitbake_var, get_native_path, BOOTDD_EXTRA_SPACE) # allow plugins to import from isarpluginbase if '__file__' in globals(): @@ -83,7 +83,8 @@ class BootimgPcbiosIsarPlugin(SourcePlugin): logger.debug("Installing MBR on disk %s as %s with size %s bytes", disk_name, full_path, disk.min_size) - dd_cmd = "dd if=%s of=%s conv=notrunc" % (mbrfile, full_path) + dd_cmd = "dd if=%s of=%s conv=notrunc" % (get_native_path(mbrfile), + full_path) exec_cmd(dd_cmd, native_sysroot) @classmethod @@ -167,6 +168,7 @@ class BootimgPcbiosIsarPlugin(SourcePlugin): bootimg_dir = cls._get_bootimg_dir(bootimg_dir, 'syslinux') staging_kernel_dir = kernel_dir + bootimg_dir = get_native_path(bootimg_dir) hdddir = "%s/hdd/boot" % cr_workdir diff --git a/meta/scripts/lib/wic/plugins/source/isoimage-isohybrid-isar.py b/meta/scripts/lib/wic/plugins/source/isoimage-isohybrid-isar.py index 6aa58870..9a7817e7 100644 --- a/meta/scripts/lib/wic/plugins/source/isoimage-isohybrid-isar.py +++ b/meta/scripts/lib/wic/plugins/source/isoimage-isohybrid-isar.py @@ -16,7 +16,8 @@ import shutil from wic import WicError from wic.engine import get_custom_config from wic.pluginbase import SourcePlugin -from wic.misc import exec_cmd, exec_native_cmd, get_bitbake_var +from wic.misc import (exec_cmd, exec_native_cmd, + get_bitbake_var, get_native_path) # allow plugins to import from isarpluginbase if '__file__' in globals(): @@ -477,6 +478,7 @@ class IsoImagePlugin(SourcePlugin): syslinux_dir = get_bitbake_var("STAGING_DATADIR") if not syslinux_dir: raise WicError("Couldn't find STAGING_DATADIR, exiting.") + syslinux_dir = get_native_path(syslinux_dir) if os.path.exists("%s/isolinux" % isodir): shutil.rmtree("%s/isolinux" % isodir) diff --git a/meta/scripts/lib/wic/plugins/source/rootfs-u-boot.py b/meta/scripts/lib/wic/plugins/source/rootfs-u-boot.py index 5611f93a..ab10e391 100644 --- a/meta/scripts/lib/wic/plugins/source/rootfs-u-boot.py +++ b/meta/scripts/lib/wic/plugins/source/rootfs-u-boot.py @@ -17,6 +17,8 @@ import glob import logging import re import os +import shlex +import shutil from wic import WicError from wic.plugins.source.rootfs import RootfsPlugin @@ -98,13 +100,14 @@ class RootfsUBootPlugin(RootfsPlugin): results = glob.glob(os.path.join("/usr/bin/qemu-*-static")) qemu_static = results[0] if len(results) > 0 else None if qemu_static: - cp_cmd = "cp -L %s %s/usr/bin" % (qemu_static, real_rootfs_dir) - exec_cmd(cp_cmd) - update_cmd = "chroot %s sh -c update-u-boot-script" % real_rootfs_dir - exec_cmd(update_cmd) + qemu_target = os.path.join(real_rootfs_dir, "usr/bin", + os.path.basename(qemu_static)) + shutil.copy(qemu_static, qemu_target) + update_cmd = "TMPDIR=/tmp chroot %s sh -c update-u-boot-script" % \ + shlex.quote(real_rootfs_dir) + exec_cmd(update_cmd, as_shell=True) if qemu_static: - rm_cmd = "rm -f %s/usr/bin/%s" % (real_rootfs_dir, qemu_static) - exec_cmd(rm_cmd) + os.remove(qemu_target) # For reproducibility set the time stamp of newly updated files if os.getenv('SOURCE_DATE_EPOCH'): diff --git a/scripts/lib/wic/misc.py b/scripts/lib/wic/misc.py index 1a7c140f..dc6114a2 100644 --- a/scripts/lib/wic/misc.py +++ b/scripts/lib/wic/misc.py @@ -15,6 +15,7 @@ import logging import os import re +import shlex import subprocess import shutil @@ -51,6 +52,17 @@ NATIVE_RECIPES = {"bmaptool": "bmaptool", "tar": "tar" } +def get_native_path(path): + """Translate a host sysroot path to its matching path in the chroot.""" + tool_sysroot = os.environ.get('WIC_TOOL_SYSROOT') + if tool_sysroot and os.environ.get('WIC_TOOL_RUNNER'): + tool_sysroot = os.path.normpath(tool_sysroot) + if path == tool_sysroot: + return '/' + if path.startswith(tool_sysroot + '/'): + return path[len(tool_sysroot):] + return path + def runtool(cmdln_or_args): """ wrapper for most of the subprocess calls input: @@ -62,10 +74,18 @@ def runtool(cmdln_or_args): cmd = cmdln_or_args[0] shell = False else: - import shlex cmd = shlex.split(cmdln_or_args)[0] shell = True + tool_runner = os.environ.get('WIC_TOOL_RUNNER') + if tool_runner and os.path.basename(cmd) != 'bitbake': + runner_args = shlex.split(tool_runner) + if shell: + cmdln_or_args = runner_args + ['/bin/sh', '-c', cmdln_or_args] + else: + cmdln_or_args = runner_args + cmdln_or_args + shell = False + sout = subprocess.PIPE serr = subprocess.STDOUT @@ -123,6 +143,14 @@ def find_executable(cmd, paths): if provided and "%s-native" % recipe in provided: return True + if os.environ.get('WIC_TOOL_RUNNER'): + paths = os.pathsep.join(get_native_path(path) + for path in paths.split(os.pathsep)) + ret, _ = runtool(['/bin/sh', '-c', + 'PATH=%s command -v %s' % + (shlex.quote(paths), shlex.quote(cmd))]) + return ret == 0 + return shutil.which(cmd, path=paths) def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): @@ -133,6 +161,7 @@ def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): Always need to execute native commands as_shell """ + native_sysroot = get_native_path(native_sysroot) # The reason -1 is used is because there may be "export" commands. args = cmd_and_args.split(';')[-1].split() logger.debug(args) @@ -263,4 +292,9 @@ def get_bitbake_var(var, image=None, cache=True): Provide old get_bitbake_var API by wrapping get_var method of BB_VARS singleton. """ - return BB_VARS.get_var(var, image, cache) + value = BB_VARS.get_var(var, image, cache) + tool_sysroot = os.environ.get('WIC_TOOL_SYSROOT') + if tool_sysroot and value and var in ('RECIPE_SYSROOT_NATIVE', + 'STAGING_DATADIR', 'STAGING_LIBDIR'): + return os.path.join(tool_sysroot, value.lstrip('/')) + return value diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py index bf2c34d5..7c6b83f3 100644 --- a/scripts/lib/wic/partition.py +++ b/scripts/lib/wic/partition.py @@ -15,7 +15,8 @@ import os import uuid from wic import WicError -from wic.misc import exec_cmd, exec_native_cmd, get_bitbake_var +from wic.misc import (exec_cmd, exec_native_cmd, + get_bitbake_var, get_native_path) from wic.pluginbase import PluginMgr logger = logging.getLogger('wic') @@ -219,6 +220,7 @@ class Partition(): os.remove(rootfs) p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot) + p_prefix = get_native_path(p_prefix) if (pseudo_dir): # Canonicalize the ignore paths. This corresponds to # calling oe.path.canonicalize(), which is used in bitbake.conf. @@ -548,4 +550,3 @@ class Partition(): logger.warn("%s Inodes (of size %d) are too small." % (get_err_str(self), size)) break - diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py index a65ae36a..60f2dbf7 100644 --- a/scripts/lib/wic/plugins/imager/direct.py +++ b/scripts/lib/wic/plugins/imager/direct.py @@ -25,7 +25,8 @@ from wic import WicError from wic.filemap import sparse_copy from wic.ksparser import KickStart, KickStartError from wic.pluginbase import PluginMgr, ImagerPlugin -from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd +from wic.misc import (get_bitbake_var, get_native_path, + exec_cmd, exec_native_cmd) logger = logging.getLogger('wic') @@ -212,8 +213,10 @@ class DirectPlugin(ImagerPlugin): # Generate .bmap if self.bmap: logger.debug("Generating bmap file for %s", disk_name) - python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3') - bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool') + tool_sysroot = get_native_path(self.native_sysroot) + python = os.path.join(tool_sysroot, + 'usr/bin/python3-native/python3') + bmaptool = os.path.join(tool_sysroot, 'usr/bin/bmaptool') exec_native_cmd("%s %s create %s -o %s.bmap" % \ (python, bmaptool, full_path, full_path), self.native_sysroot) # Compress the image diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/scripts/lib/wic/plugins/source/bootimg-pcbios.py index a207a835..77e2e52b 100644 --- a/scripts/lib/wic/plugins/source/bootimg-pcbios.py +++ b/scripts/lib/wic/plugins/source/bootimg-pcbios.py @@ -18,7 +18,7 @@ from wic import WicError from wic.engine import get_custom_config from wic.pluginbase import SourcePlugin from wic.misc import (exec_cmd, exec_native_cmd, - get_bitbake_var, BOOTDD_EXTRA_SPACE) + get_bitbake_var, get_native_path, BOOTDD_EXTRA_SPACE) logger = logging.getLogger('wic') @@ -73,7 +73,8 @@ class BootimgPcbiosPlugin(SourcePlugin): logger.debug("Installing MBR on disk %s as %s with size %s bytes", disk_name, full_path, disk.min_size) - dd_cmd = "dd if=%s of=%s conv=notrunc" % (mbrfile, full_path) + dd_cmd = "dd if=%s of=%s conv=notrunc" % (get_native_path(mbrfile), + full_path) exec_cmd(dd_cmd, native_sysroot) @classmethod @@ -146,6 +147,7 @@ class BootimgPcbiosPlugin(SourcePlugin): bootimg_dir = cls._get_bootimg_dir(bootimg_dir, 'syslinux') staging_kernel_dir = kernel_dir + bootimg_dir = get_native_path(bootimg_dir) hdddir = "%s/hdd/boot" % cr_workdir diff --git a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py index 607356ad..4b8815b6 100644 --- a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py +++ b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py @@ -18,7 +18,8 @@ import shutil from wic import WicError from wic.engine import get_custom_config from wic.pluginbase import SourcePlugin -from wic.misc import exec_cmd, exec_native_cmd, get_bitbake_var +from wic.misc import (exec_cmd, exec_native_cmd, + get_bitbake_var, get_native_path) logger = logging.getLogger('wic') @@ -385,6 +386,7 @@ class IsoImagePlugin(SourcePlugin): syslinux_dir = get_bitbake_var("STAGING_DATADIR") if not syslinux_dir: raise WicError("Couldn't find STAGING_DATADIR, exiting.") + syslinux_dir = get_native_path(syslinux_dir) if os.path.exists("%s/isolinux" % isodir): shutil.rmtree("%s/isolinux" % isodir) diff --git a/scripts/lib/wic/plugins/source/rawcopy.py b/scripts/lib/wic/plugins/source/rawcopy.py index 21903c2f..a5e9b1d6 100644 --- a/scripts/lib/wic/plugins/source/rawcopy.py +++ b/scripts/lib/wic/plugins/source/rawcopy.py @@ -6,8 +6,7 @@ import logging import os -import signal -import subprocess +import shlex from wic import WicError from wic.pluginbase import SourcePlugin @@ -48,12 +47,6 @@ class RawCopyPlugin(SourcePlugin): @staticmethod def do_image_uncompression(src, dst, workdir): - def subprocess_setup(): - # Python installs a SIGPIPE handler by default. This is usually not what - # non-Python subprocesses expect. - # SIGPIPE errors are known issues with gzip/bash - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - extension = os.path.splitext(src)[1] decompressor = { ".bz2": "bzip2", @@ -63,8 +56,9 @@ class RawCopyPlugin(SourcePlugin): }.get(extension) if not decompressor: raise WicError("Not supported compressor filename extension: %s" % extension) - cmd = "%s -dc %s > %s" % (decompressor, src, dst) - subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=workdir) + cmd = "cd %s && %s -dc %s > %s" % (shlex.quote(workdir), decompressor, + shlex.quote(src), shlex.quote(dst)) + exec_cmd(cmd, as_shell=True) @classmethod def do_prepare_partition(cls, part, source_params, cr, cr_workdir, @@ -90,6 +84,12 @@ class RawCopyPlugin(SourcePlugin): RawCopyPlugin.do_image_uncompression(img, src, cr_workdir) else: src = os.path.join(kernel_dir, source_params['file']) + if os.environ.get('WIC_TOOL_RUNNER'): + tool_sysroot = os.environ.get('WIC_TOOL_SYSROOT') + if tool_sysroot and os.path.isabs(src): + host_src = os.path.join(tool_sysroot, src.lstrip('/')) + if os.path.lexists(host_src): + src = host_src dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno)) diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py index e29f3a4c..d9629b9c 100644 --- a/scripts/lib/wic/plugins/source/rootfs.py +++ b/scripts/lib/wic/plugins/source/rootfs.py @@ -13,6 +13,7 @@ import logging import os +import shlex import shutil import sys @@ -21,7 +22,8 @@ from pathlib import Path from wic import WicError from wic.pluginbase import SourcePlugin -from wic.misc import get_bitbake_var, exec_native_cmd +from wic.misc import (exec_cmd, exec_native_cmd, + get_bitbake_var, get_native_path) logger = logging.getLogger('wic') @@ -63,6 +65,7 @@ class RootfsPlugin(SourcePlugin): @staticmethod def __get_pseudo(native_sysroot, rootfs, pseudo_dir): + native_sysroot = get_native_path(native_sysroot) pseudo = "export PSEUDO_PREFIX=%s/usr;" % native_sysroot pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir pseudo += "export PSEUDO_PASSWD=%s;" % rootfs @@ -70,6 +73,12 @@ class RootfsPlugin(SourcePlugin): pseudo += "%s " % get_bitbake_var("FAKEROOTCMD") return pseudo + @staticmethod + def __copy_rootfs_command(command, cwd=None, **kwargs): + if cwd: + command = "cd %s && %s" % (shlex.quote(cwd), command) + return exec_cmd(command, as_shell=True) + @classmethod def do_prepare_partition(cls, part, source_params, cr, cr_workdir, oe_builddir, bootimg_dir, kernel_dir, @@ -122,7 +131,11 @@ class RootfsPlugin(SourcePlugin): orig_dir = cls.__validate_path("--change-directory", part.rootfs_dir, cd) else: orig_dir = part.rootfs_dir - copyhardlinktree(orig_dir, new_rootfs) + command_runner = None + if os.environ.get('WIC_TOOL_RUNNER'): + command_runner = cls.__copy_rootfs_command + copyhardlinktree(orig_dir, new_rootfs, + command_runner=command_runner) # Convert the pseudo directory to its new location if (pseudo_dir): diff --git a/scripts/wic-chroot b/scripts/wic-chroot new file mode 100755 index 00000000..19be0c00 --- /dev/null +++ b/scripts/wic-chroot @@ -0,0 +1,18 @@ +#!/bin/sh +# This software is a part of Isar. +# Copyright (C) Siemens AG, 2026 +# +# SPDX-License-Identifier: MIT + +set -eu + +: "${IMAGER_CHROOT:?No imaging chroot configured}" + +# Expand the variables in the chroot shell, not on the host +# shellcheck disable=SC2016 +exec chroot "$IMAGER_CHROOT" /bin/sh -c ' + cd "$1" || exit + shift + export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + exec "$@" +' sh "$PWD" "$@"