diff mbox series

[v2,2/5] container-loader: Introduce helper to load container images into local registry

Message ID 27306a537db0da8d51a0ca709b9113248331d340.1721038111.git.jan.kiszka@siemens.com
State Superseded, archived
Headers show
Series Introduce container fetcher and pre-loader | expand

Commit Message

Jan Kiszka July 15, 2024, 10:08 a.m. UTC 
From: Jan Kiszka <jan.kiszka@siemens.com>

This allows to write dpkg-raw recipes which packages archived container
images and load them into a local docker or podman registry on boot. The
scenario behind this is to pre-fill local registries in a way that still
permits live updates during runtime.

The loader script only process images which are not yet available under
the same name and tag in the local registry. Also after loading, the
archived images stay on the local file system. This allows to perform
reloading in case the local registry should be emptied (e.g. reset to
factory state). To reduce the space those original images need, they are
compressed, by default with zstd.

Separate include files are available to cater the main container
engines, one for docker and one for podman.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
---
 .../container-loader/container-loader.inc     | 94 +++++++++++++++++++
 .../container-loader/docker-loader.inc        | 10 ++
 .../files/container-loader.service.tmpl       | 12 +++
 .../files/container-loader.sh.tmpl            | 18 ++++
 .../container-loader/podman-loader.inc        | 10 ++
 5 files changed, 144 insertions(+)
 create mode 100644 meta/recipes-support/container-loader/container-loader.inc
 create mode 100644 meta/recipes-support/container-loader/docker-loader.inc
 create mode 100644 meta/recipes-support/container-loader/files/container-loader.service.tmpl
 create mode 100755 meta/recipes-support/container-loader/files/container-loader.sh.tmpl
 create mode 100644 meta/recipes-support/container-loader/podman-loader.inc
diff mbox series

Patch

diff --git a/meta/recipes-support/container-loader/container-loader.inc b/meta/recipes-support/container-loader/container-loader.inc
new file mode 100644
index 00000000..a0c2ddb3
--- /dev/null
+++ b/meta/recipes-support/container-loader/container-loader.inc
@@ -0,0 +1,94 @@ 
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+FILESPATH:append := ":${FILE_DIRNAME}/files"
+
+inherit dpkg-raw
+
+SRC_URI += " \
+    file://container-loader.service.tmpl \
+    file://container-loader.sh.tmpl"
+
+CONTAINER_COMPRESSION ?= "zst"
+CONTAINER_DELETE_AFTER_LOAD ?= "0"
+
+DEBIAN_DEPENDS += " \
+    ${CONTAINER_ENGINE_PACKAGES} \
+    ${@', gzip' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \
+       ', zstd' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \
+       ', xz-utils' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \
+       ''}"
+
+CONTAINER_COMPRESSOR_CMD = "${@ \
+    'gzip -f -9 -n --rsyncable' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \
+    'xz -f ${XZ_DEFAULTS}' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \
+    'zstd -f --rm ${ZSTD_DEFAULTS}' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \
+    ''}"
+
+CONTAINER_DECOMPRESSOR_CMD = "${@ \
+    'gzip -c -d -n' if d.getVar('CONTAINER_COMPRESSION') == 'gz' else \
+    'xz -c -d -T0' if d.getVar('CONTAINER_COMPRESSION') == 'xz' else \
+    'pzstd -c -d' if d.getVar('CONTAINER_COMPRESSION') == 'zst' else \
+    ''}"
+
+TEMPLATE_FILES += " \
+    container-loader.service.tmpl \
+    container-loader.sh.tmpl"
+TEMPLATE_VARS += " \
+    CONTAINER_ENGINE \
+    CONTAINER_DECOMPRESSOR_CMD \
+    CONTAINER_DELETE_AFTER_LOAD"
+
+do_install() {
+    install -m 755 ${WORKDIR}/container-loader.sh ${D}/usr/share/${BPN}
+}
+do_install[cleandirs] += " \
+    ${D}/usr/share/${BPN} \
+    ${D}/usr/share/${BPN}/images"
+
+python do_install_fetched_containers() {
+    import os
+
+    workdir = d.getVar('WORKDIR')
+    D = d.getVar('D')
+    BPN = d.getVar('BPN')
+
+    image_list = open(D + "/usr/share/" + BPN + "/image.list", "w")
+
+    src_uri = d.getVar('SRC_URI').split()
+    for uri in src_uri:
+        scheme, host, path, _, _, parm = bb.fetch.decodeurl(uri)
+        if scheme != "docker":
+            continue
+
+        image_name = host + (path if path != "/" else "")
+        unpacked_image = workdir + "/" + image_name.replace('/', '.')
+        dest_dir = D + "/usr/share/" + BPN + "/images"
+        tar_image = dest_dir + "/" + image_name.replace('/', '.') + ".tar"
+        docker_ref = ":" + parm["tag"] if "tag" in parm else "latest"
+
+        bb.utils.remove(tar_image)
+        cmd = f"skopeo copy dir:{unpacked_image} " \
+            f"docker-archive:{tar_image}:{image_name}{docker_ref}"
+        bb.note(f"running: {cmd}")
+        bb.process.run(cmd)
+
+        cmd = f"{d.getVar('CONTAINER_COMPRESSOR_CMD')} {tar_image}"
+        bb.note(f"running: {cmd}")
+        bb.process.run(cmd)
+
+        line = f"{os.path.basename(tar_image)}.{d.getVar('CONTAINER_COMPRESSION')} " + \
+            image_name + docker_ref
+        bb.note(f"adding '{line}' to image.list")
+        image_list.write(line + "\n")
+
+    image_list.close()
+}
+
+addtask install_fetched_containers after do_install before do_prepare_build
+
+do_prepare_build:append() {
+    install -v -m 644 ${WORKDIR}/container-loader.service ${S}/debian/${BPN}.service
+}
diff --git a/meta/recipes-support/container-loader/docker-loader.inc b/meta/recipes-support/container-loader/docker-loader.inc
new file mode 100644
index 00000000..b864c854
--- /dev/null
+++ b/meta/recipes-support/container-loader/docker-loader.inc
@@ -0,0 +1,10 @@ 
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+require container-loader.inc
+
+CONTAINER_ENGINE = "docker"
+
+CONTAINER_ENGINE_PACKAGES ?= "docker.io, apparmor"
diff --git a/meta/recipes-support/container-loader/files/container-loader.service.tmpl b/meta/recipes-support/container-loader/files/container-loader.service.tmpl
new file mode 100644
index 00000000..1638eaf2
--- /dev/null
+++ b/meta/recipes-support/container-loader/files/container-loader.service.tmpl
@@ -0,0 +1,12 @@ 
+[Unit]
+Description=Load archived container images on boot
+After=${CONTAINER_ENGINE}.service
+Requires=${CONTAINER_ENGINE}.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/share/${BPN}/container-loader.sh
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target
diff --git a/meta/recipes-support/container-loader/files/container-loader.sh.tmpl b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl
new file mode 100755
index 00000000..b6abec92
--- /dev/null
+++ b/meta/recipes-support/container-loader/files/container-loader.sh.tmpl
@@ -0,0 +1,18 @@ 
+#!/bin/sh
+#
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+set -eu
+
+while read -r image ref; do
+    if [ -e /usr/share/${BPN}/images/"$image" ] && \
+       [ -z "$(${CONTAINER_ENGINE} images -q "$ref")" ]; then
+        ${CONTAINER_DECOMPRESSOR_CMD} /usr/share/${BPN}/images/"$image" | \
+            ${CONTAINER_ENGINE} load
+        if [ "${CONTAINER_DELETE_AFTER_LOAD}" = "1" ]; then
+            rm -f /usr/share/${BPN}/images/"$image"
+        fi
+    fi
+done < /usr/share/${BPN}/image.list
diff --git a/meta/recipes-support/container-loader/podman-loader.inc b/meta/recipes-support/container-loader/podman-loader.inc
new file mode 100644
index 00000000..d2c9a12d
--- /dev/null
+++ b/meta/recipes-support/container-loader/podman-loader.inc
@@ -0,0 +1,10 @@ 
+# This software is a part of ISAR.
+# Copyright (c) Siemens AG, 2024
+#
+# SPDX-License-Identifier: MIT
+
+require container-loader.inc
+
+CONTAINER_ENGINE = "podman"
+
+CONTAINER_ENGINE_PACKAGES ?= "podman"