| Message ID | 3b7b8dbdde7fa3a4184daa3f8d567e72c8b50d2d.1721139489.git.jan.kiszka@siemens.com |
|---|---|
| State | Superseded, archived |
| Headers | show |
| Series | Introduce container fetcher and pre-loader | expand |
On Tue, 2024-07-16 at 16:18 +0200, Jan Kiszka wrote: > From: Jan Kiszka <jan.kiszka@siemens.com> > > This allows to write dpkg-raw recipes which packages archived container > images and load them into a local docker or podman registry on boot. The > scenario behind this is to pre-fill local registries in a way that still > permits live updates during runtime. > > The loader script only process images which are not yet available under > the same name and tag in the local registry. Also after loading, the > archived images stay on the local file system. This allows to perform > reloading in case the local registry should be emptied (e.g. reset to > factory state). To reduce the space those original images need, they are > compressed, by default with zstd. > > Separate include files are available to cater the main container > engines, one for docker and one for podman. > > Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com> > --- > .../container-loader/container-loader.inc | 101 ++++++++++++++++++ > .../container-loader/docker-loader.inc | 10 ++ > .../files/container-loader.service.tmpl | 12 +++ > .../files/container-loader.sh.tmpl | 18 ++++ > .../container-loader/podman-loader.inc | 10 ++ > 5 files changed, 151 insertions(+) > create mode 100644 meta/recipes-support/container-loader/container-loader.inc > create mode 100644 meta/recipes-support/container-loader/docker-loader.inc > create mode 100644 meta/recipes-support/container-loader/files/container-loader.service.tmpl > create mode 100755 meta/recipes-support/container-loader/files/container-loader.sh.tmpl > create mode 100644 meta/recipes-support/container-loader/podman-loader.inc > > diff --git a/meta/recipes-support/container-loader/container-loader.inc b/meta/recipes- > support/container-loader/container-loader.inc > new file mode 100644 > index 00000000..e97e829b > --- /dev/null > +++ b/meta/recipes-support/container-loader/container-loader.inc > @@ -0,0 +1,101 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +FILESPATH:append := ":${FILE_DIRNAME}/files" > + > +inherit dpkg-raw > + > +SRC_URI += " \ > + file://container-loader.service.tmpl \ > + file://container-loader.sh.tmpl" > + > +CONTAINER_COMPRESSION ?= "zst" > +CONTAINER_DELETE_AFTER_LOAD ?= "0" > + > +DEBIAN_DEPENDS += " \ > + ${CONTAINER_ENGINE_PACKAGES} \ > + ${@', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ > + ', zstd' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \ > + ', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ > + ''}" > + > +CONTAINER_COMPRESSOR_CMD = "${@ \ > + 'gzip -f -9 -n --rsyncable' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ > + 'xz -f ${XZ_DEFAULTS}' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ > + 'zstd -f --rm ${ZSTD_DEFAULTS}' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \ > + ''}" > + > +CONTAINER_DECOMPRESSOR_CMD = "${@ \ > + 'gzip -c -d -n' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ > + 'xz -c -d -T0' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ > + 'pzstd -c -d' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \ > + ''}" > + > +TEMPLATE_FILES += " \ > + container-loader.service.tmpl \ > + container-loader.sh.tmpl" > +TEMPLATE_VARS += " \ > + CONTAINER_ENGINE \ > + CONTAINER_DECOMPRESSOR_CMD \ > + CONTAINER_DELETE_AFTER_LOAD" > + > +do_install() { > + install -m 755 ${WORKDIR}/container-loader.sh ${D}/usr/share/${BPN} > +} > +do_install[cleandirs] += " \ > + ${D}/usr/share/${BPN} \ > + ${D}/usr/share/${BPN}/images" > + > +python do_install_fetched_containers() { > + import os > + > + workdir = d.getVar('WORKDIR') > + D = d.getVar('D') > + BPN = d.getVar('BPN') > + > + image_list = open(D + "/usr/share/" + BPN + "/image.list", "w") > + > + src_uri = d.getVar('SRC_URI').split() > + for uri in src_uri: > + scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri) > + if scheme != "docker": > + continue > + > + image_name = host + (path if path != "/" else "") > + unpacked_image = workdir + "/" + image_name.replace('/', '.') > + dest_dir = D + "/usr/share/" + BPN + "/images" > + tar_image = dest_dir + "/" + image_name.replace('/', '.') + ".tar" > + docker_ref = ":" + parm["tag"] if "tag" in parm else "latest" > + > + bb.utils.remove(tar_image) > + cmd = f"skopeo copy dir:{unpacked_image} " \ > + f"docker-archive:{tar_image}:{image_name}{docker_ref}" > + bb.note(f"running: {cmd}") > + bb.process.run(cmd) > + > + cmd = f"{d.getVar('CONTAINER_COMPRESSOR_CMD')} {tar_image}" > + bb.note(f"running: {cmd}") > + bb.process.run(cmd) > + > + line = f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " + \ > + image_name + docker_ref > + bb.note(f"adding '{line}' to image.list") > + image_list.write(line + "\n") > + > + image_list.close() > +} > + > +addtask install_fetched_containers after do_install before do_prepare_build Add do_install_fetched_containers[vardeps] += "CONTAINER_COMPRESSOR_CMD" here. By the way, for this to work the implementation in bitbake.conf needs to be fixed as well. The vardepsexclude for ZSTD_DEFAULTS should not contain "ZSTD_LEVEL". I could sent out a patch for this. Benedikt > + > +do_prepare_build:append() { > + install -v -m 644 ${WORKDIR}/container-loader.service ${S}/debian/${BPN}.service > + > + # Do not compress the package, most of its payload is already, and trying > + # nevertheless will only cost time without any gain. > + cat <<EOF >> ${S}/debian/rules > +override_dh_builddeb: > + dh_builddeb -- -Znone > +EOF > +} > diff --git a/meta/recipes-support/container-loader/docker-loader.inc b/meta/recipes- > support/container-loader/docker-loader.inc > new file mode 100644 > index 00000000..b864c854 > --- /dev/null > +++ b/meta/recipes-support/container-loader/docker-loader.inc > @@ -0,0 +1,10 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +require container-loader.inc > + > +CONTAINER_ENGINE = "docker" > + > +CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor" > diff --git a/meta/recipes-support/container-loader/files/container-loader.service.tmpl > b/meta/recipes-support/container-loader/files/container-loader.service.tmpl > new file mode 100644 > index 00000000..1638eaf2 > --- /dev/null > +++ b/meta/recipes-support/container-loader/files/container-loader.service.tmpl > @@ -0,0 +1,12 @@ > +[Unit] > +Description=Load archived container images on boot > +After=${CONTAINER_ENGINE}.service > +Requires=${CONTAINER_ENGINE}.service > + > +[Service] > +Type=oneshot > +ExecStart=/usr/share/${BPN}/container-loader.sh > +RemainAfterExit=true > + > +[Install] > +WantedBy=multi-user.target > diff --git a/meta/recipes-support/container-loader/files/container-loader.sh.tmpl b/meta/recipes- > support/container-loader/files/container-loader.sh.tmpl > new file mode 100755 > index 00000000..b6abec92 > --- /dev/null > +++ b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl > @@ -0,0 +1,18 @@ > +#!/bin/sh > +# > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +set -eu > + > +while read -r image ref; do > + if [ -e /usr/share/${BPN}/images/"$image" ] && \ > + [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then > + ${CONTAINER_DECOMPRESSOR_CMD} /usr/share/${BPN}/images/"$image" | \ > + ${CONTAINER_ENGINE} load > + if [ "${CONTAINER_DELETE_AFTER_LOAD}" = "1" ]; then > + rm -f /usr/share/${BPN}/images/"$image" > + fi > + fi > +done < /usr/share/${BPN}/image.list > diff --git a/meta/recipes-support/container-loader/podman-loader.inc b/meta/recipes- > support/container-loader/podman-loader.inc > new file mode 100644 > index 00000000..d2c9a12d > --- /dev/null > +++ b/meta/recipes-support/container-loader/podman-loader.inc > @@ -0,0 +1,10 @@ > +# This software is a part of ISAR. > +# Copyright (c) Siemens AG, 2024 > +# > +# SPDX-License-Identifier: MIT > + > +require container-loader.inc > + > +CONTAINER_ENGINE = "podman" > + > +CONTAINER_ENGINE_PACKAGES ?= "podman"
diff --git a/meta/recipes-support/container-loader/container-loader.inc b/meta/recipes-support/container-loader/container-loader.inc new file mode 100644 index 00000000..e97e829b --- /dev/null +++ b/meta/recipes-support/container-loader/container-loader.inc @@ -0,0 +1,101 @@ +# This software is a part of ISAR. +# Copyright (c) Siemens AG, 2024 +# +# SPDX-License-Identifier: MIT + +FILESPATH:append := ":${FILE_DIRNAME}/files" + +inherit dpkg-raw + +SRC_URI += " \ + file://container-loader.service.tmpl \ + file://container-loader.sh.tmpl" + +CONTAINER_COMPRESSION ?= "zst" +CONTAINER_DELETE_AFTER_LOAD ?= "0" + +DEBIAN_DEPENDS += " \ + ${CONTAINER_ENGINE_PACKAGES} \ + ${@', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ + ', zstd' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \ + ', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ + ''}" + +CONTAINER_COMPRESSOR_CMD = "${@ \ + 'gzip -f -9 -n --rsyncable' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ + 'xz -f ${XZ_DEFAULTS}' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ + 'zstd -f --rm ${ZSTD_DEFAULTS}' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \ + ''}" + +CONTAINER_DECOMPRESSOR_CMD = "${@ \ + 'gzip -c -d -n' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \ + 'xz -c -d -T0' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \ + 'pzstd -c -d' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \ + ''}" + +TEMPLATE_FILES += " \ + container-loader.service.tmpl \ + container-loader.sh.tmpl" +TEMPLATE_VARS += " \ + CONTAINER_ENGINE \ + CONTAINER_DECOMPRESSOR_CMD \ + CONTAINER_DELETE_AFTER_LOAD" + +do_install() { + install -m 755 ${WORKDIR}/container-loader.sh ${D}/usr/share/${BPN} +} +do_install[cleandirs] += " \ + ${D}/usr/share/${BPN} \ + ${D}/usr/share/${BPN}/images" + +python do_install_fetched_containers() { + import os + + workdir = d.getVar('WORKDIR') + D = d.getVar('D') + BPN = d.getVar('BPN') + + image_list = open(D + "/usr/share/" + BPN + "/image.list", "w") + + src_uri = d.getVar('SRC_URI').split() + for uri in src_uri: + scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri) + if scheme != "docker": + continue + + image_name = host + (path if path != "/" else "") + unpacked_image = workdir + "/" + image_name.replace('/', '.') + dest_dir = D + "/usr/share/" + BPN + "/images" + tar_image = dest_dir + "/" + image_name.replace('/', '.') + ".tar" + docker_ref = ":" + parm["tag"] if "tag" in parm else "latest" + + bb.utils.remove(tar_image) + cmd = f"skopeo copy dir:{unpacked_image} " \ + f"docker-archive:{tar_image}:{image_name}{docker_ref}" + bb.note(f"running: {cmd}") + bb.process.run(cmd) + + cmd = f"{d.getVar('CONTAINER_COMPRESSOR_CMD')} {tar_image}" + bb.note(f"running: {cmd}") + bb.process.run(cmd) + + line = f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " + \ + image_name + docker_ref + bb.note(f"adding '{line}' to image.list") + image_list.write(line + "\n") + + image_list.close() +} + +addtask install_fetched_containers after do_install before do_prepare_build + +do_prepare_build:append() { + install -v -m 644 ${WORKDIR}/container-loader.service ${S}/debian/${BPN}.service + + # Do not compress the package, most of its payload is already, and trying + # nevertheless will only cost time without any gain. + cat <<EOF >> ${S}/debian/rules +override_dh_builddeb: + dh_builddeb -- -Znone +EOF +} diff --git a/meta/recipes-support/container-loader/docker-loader.inc b/meta/recipes-support/container-loader/docker-loader.inc new file mode 100644 index 00000000..b864c854 --- /dev/null +++ b/meta/recipes-support/container-loader/docker-loader.inc @@ -0,0 +1,10 @@ +# This software is a part of ISAR. +# Copyright (c) Siemens AG, 2024 +# +# SPDX-License-Identifier: MIT + +require container-loader.inc + +CONTAINER_ENGINE = "docker" + +CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor" diff --git a/meta/recipes-support/container-loader/files/container-loader.service.tmpl b/meta/recipes-support/container-loader/files/container-loader.service.tmpl new file mode 100644 index 00000000..1638eaf2 --- /dev/null +++ b/meta/recipes-support/container-loader/files/container-loader.service.tmpl @@ -0,0 +1,12 @@ +[Unit] +Description=Load archived container images on boot +After=${CONTAINER_ENGINE}.service +Requires=${CONTAINER_ENGINE}.service + +[Service] +Type=oneshot +ExecStart=/usr/share/${BPN}/container-loader.sh +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/meta/recipes-support/container-loader/files/container-loader.sh.tmpl b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl new file mode 100755 index 00000000..b6abec92 --- /dev/null +++ b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl @@ -0,0 +1,18 @@ +#!/bin/sh +# +# Copyright (c) Siemens AG, 2024 +# +# SPDX-License-Identifier: MIT + +set -eu + +while read -r image ref; do + if [ -e /usr/share/${BPN}/images/"$image" ] && \ + [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then + ${CONTAINER_DECOMPRESSOR_CMD} /usr/share/${BPN}/images/"$image" | \ + ${CONTAINER_ENGINE} load + if [ "${CONTAINER_DELETE_AFTER_LOAD}" = "1" ]; then + rm -f /usr/share/${BPN}/images/"$image" + fi + fi +done < /usr/share/${BPN}/image.list diff --git a/meta/recipes-support/container-loader/podman-loader.inc b/meta/recipes-support/container-loader/podman-loader.inc new file mode 100644 index 00000000..d2c9a12d --- /dev/null +++ b/meta/recipes-support/container-loader/podman-loader.inc @@ -0,0 +1,10 @@ +# This software is a part of ISAR. +# Copyright (c) Siemens AG, 2024 +# +# SPDX-License-Identifier: MIT + +require container-loader.inc + +CONTAINER_ENGINE = "podman" + +CONTAINER_ENGINE_PACKAGES ?= "podman"