@@ -1201,3 +1201,9 @@ To opt out, remove the feature explicitly:
```
ROOTFS_FEATURES:remove = "clean-apt-credentials"
```
+
+### do_rootfs_postprocess is folded into do_rootfs_install
+
+All `ROOTFS_POSTPROCESS_COMMAND` steps are now executed within the `do_rootfs_install`
+task to avoid idempotency issues on partial rebuilds. Task that previously had a
+dependency to `do_rootfs_postprocess` shall now be changed to run after `do_rootfs_install`.
@@ -14,14 +14,11 @@ python() {
for entry in (d.getVar("GROUPS") or "").split():
group_entry = "GROUP_{}".format(entry)
d.appendVarFlag("image_postprocess_accounts", "vardeps", " {}".format(group_entry))
- d.appendVarFlag("do_rootfs_install", "vardeps", " {}".format(group_entry))
for entry in (d.getVar("USERS") or "").split():
user_entry = "USER_{}".format(entry)
d.appendVarFlag("image_postprocess_accounts", "vardeps", " {}".format(user_entry))
- d.appendVarFlag("do_rootfs_install", "vardeps", " {}".format(user_entry))
}
-do_rootfs_install[vardeps] += "GROUPS USERS"
def image_create_groups(d: "DataSmart") -> None:
"""Creates the groups defined in the ``GROUPS`` bitbake variable.
@@ -408,7 +408,7 @@ do_copy_boot_files() {
cp -f "$dtb" "${DEPLOYDIR}/"
done
}
-addtask copy_boot_files before do_rootfs_postprocess after do_rootfs_install
+addtask copy_boot_files before do_rootfs do_generate_initramfs after do_rootfs_install
python do_copy_boot_files_setscene () {
sstate_setscene(d)
@@ -492,7 +492,7 @@ EOSUDO
-exec touch '{}' -h -d@${SOURCE_DATE_EPOCH} ';'
}
do_rootfs_finalize[network] = "${TASK_USE_SUDO}"
-addtask rootfs_finalize before do_rootfs after do_rootfs_postprocess
+addtask rootfs_finalize before do_rootfs after do_rootfs_install
ROOTFS_QA_FIND_ARGS ?= ""
@@ -78,7 +78,7 @@ rootfs_purge_initramfs() {
# If an external initrd shall be used, run it as a task
# instead of as part of the rootfs install.
-addtask do_generate_initramfs before do_rootfs_postprocess after do_rootfs_install
+addtask do_generate_initramfs before do_rootfs after do_rootfs_install
addtask do_generate_initramfs_setscene
ROOTFS_INSTALL_COMMAND:remove = "rootfs_generate_initramfs"
@@ -430,19 +430,21 @@ do_rootfs_install[root_cleandirs] = "${ROOTFSDIR}"
do_rootfs_install[cleandirs] += "${DEPLOYDIR}"
do_rootfs_install[sstate-inputdirs] = "${DEPLOYDIR}"
do_rootfs_install[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
-do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND} ${ROOTFS_VARDEPS}"
+do_rootfs_install[vardeps] += "${ROOTFS_CONFIGURE_COMMAND} ${ROOTFS_INSTALL_COMMAND} ${ROOTFS_POSTPROCESS_COMMAND} ${ROOTFS_VARDEPS}"
do_rootfs_install[vardepsexclude] += "IMAGE_ROOTFS"
do_rootfs_install[depends] = "bootstrap-${@'target' if d.getVar('ROOTFS_ARCH') == d.getVar('DISTRO_ARCH') else 'host'}:do_build"
+do_rootfs_install[depends] += "base-apt:do_cache isar-apt:do_cache_config"
do_rootfs_install[deptask] = "do_deploy_deb"
do_rootfs_install[rdeptask] = "do_deploy_deb"
do_rootfs_install[network] = "${TASK_USE_SUDO}"
python do_rootfs_install() {
configure_cmds = (d.getVar("ROOTFS_CONFIGURE_COMMAND") or "").split()
install_cmds = (d.getVar("ROOTFS_INSTALL_COMMAND") or "").split()
+ postprocess_cmds = (d.getVar("ROOTFS_POSTPROCESS_COMMAND") or "").split()
# Mount after configure commands, so that they have time to copy
# 'isar-apt' (sdkchroot):
- cmds = ['rootfs_prepare'] + configure_cmds + ['rootfs_do_mounts'] + install_cmds
+ cmds = ['rootfs_prepare'] + configure_cmds + ['rootfs_do_mounts'] + install_cmds + postprocess_cmds
# NOTE: The weights specify how long each task takes in seconds and are used
# by the MultiStageProgressReporter to render a progress bar for this task.
@@ -470,7 +472,7 @@ python do_rootfs_install() {
progress_reporter.finish()
bb.build.exec_func('rootfs_do_umounts', d)
}
-addtask rootfs_install before do_rootfs_postprocess after do_unpack
+addtask rootfs_install before do_rootfs after do_unpack
do_cache_deb_src[network] = "${TASK_USE_SUDO}"
do_cache_deb_src() {
@@ -611,33 +613,6 @@ image_postprocess_populate_systemd_preset() {
EOSH
}
-do_rootfs_postprocess[vardeps] = "${ROOTFS_POSTPROCESS_COMMAND}"
-do_rootfs_postprocess[network] = "${TASK_USE_SUDO}"
-do_rootfs_postprocess[depends] = "base-apt:do_cache isar-apt:do_cache_config"
-python do_rootfs_postprocess() {
- # Take care that its correctly mounted:
- bb.build.exec_func('rootfs_do_mounts', d)
- # Take care that qemu-*-static is available, since it could have been
- # removed on a previous execution of this task:
- bb.build.exec_func('rootfs_do_qemu', d)
-
- progress_reporter = bb.progress.ProgressHandler(d)
- progress_reporter.update(0)
-
- cmds = d.getVar("ROOTFS_POSTPROCESS_COMMAND")
- if cmds is None or not cmds.strip():
- return
- cmds = cmds.split()
-
- try:
- for i, cmd in enumerate(cmds):
- bb.build.exec_func(cmd, d)
- progress_reporter.update(int(i / len(cmds) * 100))
- finally:
- bb.build.exec_func('rootfs_do_umounts', d)
-}
-addtask rootfs_postprocess before do_rootfs after do_unpack
-
ROOTFS_INSTALL_COMMAND += "${@bb.utils.contains('ROOTFS_FEATURES', 'generate-initrd', 'rootfs_generate_initramfs', '', d)}"
rootfs_generate_initramfs[weight] = "1000"
rootfs_generate_initramfs[progress] = "custom:rootfs_progress.InitrdProgressHandler"
@@ -682,7 +657,7 @@ rootfs_install_sstate_prepare() {
run_privileged_heredoc <<'EOF'
mount -o bind,private '${ROOTFSDIR}' '${WORKDIR}/mnt/rootfs' -o ro
- lopts="--one-file-system --exclude=var/cache/apt/archives"
+ lopts="--one-file-system"
tar -C ${WORKDIR}/mnt/rootfs -cpSf rootfs.tar $lopts ${SSTATE_TAR_ATTR_FLAGS} .
umount -q ${WORKDIR}/mnt/rootfs
EOF
@@ -715,3 +690,8 @@ python do_rootfs_install_setscene() {
sstate_setscene(d)
}
addtask do_rootfs_install_setscene
+
+python do_rootfs_postprocess() {
+ bb.warn("task do_rootfs_postprocess was folded into do_rootfs_install, please order your task after do_rootfs_install instead")
+}
+addtask do_rootfs_postprocess after do_rootfs_install
The do_rootfs_postprocess task is not idempotent and contains steps which can only run on an unmodified rootfs from do_rootfs_install (e.g. user-creation, steps that need the apt cache). To make both steps atomic, we move them into the same task, similar to how OE models it. This further improves build performance on rebuilds from the sstate cache, as the whole artifact can be taken and the postprocess steps do not need to be re-executed. The sstate artifact is also smaller, as the rootfs is stored in clean state. By that, we further drop the manual registration of the USERS / GROUPS vardeps on the do_rootfs_install task. These are now propagated via the corresponding ROOTFS_POSTPROCESS_COMMAND. Signed-off-by: Felix Moessbauer <felix.moessbauer@siemens.com> --- RECIPE-API-CHANGELOG.md | 6 +++ .../image-account-extension.bbclass | 3 -- meta/classes-recipe/image.bbclass | 4 +- meta/classes-recipe/initramfs.bbclass | 2 +- meta/classes-recipe/rootfs.bbclass | 42 +++++-------------- 5 files changed, 20 insertions(+), 37 deletions(-)